chore: mark next point release

Only support x86_64 for now as I'm moving fast.

.chglog/CHANGELOG.tpl.md

@@ -0,0 +1,40 @@
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }}
+
+> {{ datetime "2006-01-02" .Tag.Date }}
+
+{{ range .CommitGroups -}}
+### {{ .Title }}
+
+{{ range .Commits -}}
+* {{ .Subject }}
+{{ end }}
+{{ end -}}
+
+{{- if .RevertCommits -}}
+### Reverts
+
+{{ range .RevertCommits -}}
+* {{ .Revert.Header }}
+{{ end }}
+{{ end -}}
+
+{{- if .MergeCommits -}}
+### Pull Requests
+
+{{ range .MergeCommits -}}
+* {{ .Header }}
+{{ end }}
+{{ end -}}
+
+{{- if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+
+{{ range .Notes }}
+{{ .Body }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}

.chglog/config.yml

@@ -0,0 +1,27 @@
+style: github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://git.autonomic.zone:2222/coop-cloud/go-abra
+options:
+  commits:
+    # filters:
+    #   Type:
+    #     - feat
+    #     - fix
+    #     - perf
+    #     - refactor
+  commit_groups:
+    # title_maps:
+    #   feat: Features
+    #   fix: Bug Fixes
+    #   perf: Performance Improvements
+    #   refactor: Code Refactoring
+  header:
+    pattern: "^(\\w*)\\:\\s(.*)$"
+    pattern_maps:
+      - Type
+      - Subject
+  notes:
+    keywords:
+      - BREAKING CHANGE

.drone.yml

@@ -1,49 +1,74 @@
 ---
 kind: pipeline
-name: linters
+name: coopcloud.tech/abra
 steps:
-  - name: run shellcheck
-    image: koalaman/shellcheck-alpine:v0.7.1
+  - name: make check
+    image: golang:1.17
     commands:
-      - shellcheck abra
-      - shellcheck bin/*.sh
+      - make check
 
-  - name: run flake8
-    image: alpine/flake8:3.9.0
+  - name: make static
+    image: golang:1.17
+    ignore: true # until we decide we all want this check
+    environment:
+      STATIC_CHECK_URL: honnef.co/go/tools/cmd/staticcheck
+      STATIC_CHECK_VERSION: v0.2.0
     commands:
-      - flake8 --max-line-length 100 bin/app-json.py
+      - go install $STATIC_CHECK_URL@$STATIC_CHECK_VERSION
+      - make static
 
-  - name: run unit tests
-    image: decentral1se/docker-dind-bats-kcov
+  - name: make build
+    image: golang:1.17
     commands:
-      - bats tests
+      - make build
 
-  - name: collect code coverage
-    failure: ignore # until we fix this
-    image: decentral1se/docker-dind-bats-kcov
+  - name: make test
+    image: golang:1.17
     commands:
-      - kcov . bats tests || true
+      - make test
 
-  - name: send code coverage report to codecov
-    failure: ignore # until we fix this
-    image: plugins/codecov
+  - name: notify on failure
+    image: plugins/matrix
     settings:
-      token:
-        from_secret: codecov_token
-        required: true
-
-  - name: notify rocket chat
-    image: plugins/slack
-    settings:
-      webhook:
-        from_secret: rc_builds_url
-      username: comradebritney
-      channel: "internal.builds"
-      template: "{{repo.owner}}/{{repo.name}} build failed: {{build.link}}"
+      homeserver: https://matrix.autonomic.zone
+      roomid: "IFazIpLtxiScqbHqoa:autonomic.zone"
+      userid: "@autono-bot:autonomic.zone"
+      accesstoken:
+        from_secret: autono_bot_access_token
+    depends_on:
+      - make check
+      - make build
+      - make test
     when:
       status:
         - failure
 
-trigger:
-  branch:
-    - main
+  - name: fetch
+    image: docker:git
+    commands:
+      - git fetch --tags
+    depends_on:
+      - make check
+      - make build
+      - make test
+    when:
+      event: tag
+
+  - name: release
+    image: golang:1.17
+    environment:
+      GITEA_TOKEN:
+        from_secret: goreleaser_gitea_token
+    volumes:
+      - name: deps
+        path: /go
+    commands:
+      - curl -sL https://git.io/goreleaser | bash
+    depends_on:
+      - fetch
+    when:
+      event: tag
+
+volumes:
+  - name: deps
+    temp: {}

.envrc.sample

@@ -0,0 +1,6 @@
+go env -w GOPRIVATE=coopcloud.tech
+
+# export PASSWORD_STORE_DIR=$(pwd)/../../autonomic/passwords/passwords/
+# export HCLOUD_TOKEN=$(pass show logins/hetzner/cicd/api_key)
+# export CAPSUL_TOKEN=...
+# export GITEA_TOKEN=...

.gitignore

@@ -1,2 +1,5 @@
-/.venv
-coverage/
+abra
+.vscode/
+vendor/
+.envrc
+dist/

.goreleaser.yml

@@ -0,0 +1,35 @@
+---
+project_name: abra
+gitea_urls:
+  api: https://git.coopcloud.tech/api/v1
+  download: https://git.coopcloud.tech/
+  skip_tls_verify: false
+before:
+  hooks:
+    - go mod tidy
+    - go generate ./...
+builds:
+  - env:
+      - CGO_ENABLED=0
+    dir: cmd/abra
+    goos:
+      - linux
+    ldflags:
+      - "-X 'main.Commit={{ .Commit }}'"
+      - "-X 'main.Version={{ .Version }}'"
+archives:
+  - replacements:
+      linux: Linux
+      386: i386
+      amd64: x86_64
+    format: binary
+checksum:
+  name_template: "checksums.txt"
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - "^docs:"
+      - "^test:"

CHANGELOG.md

@@ -1,125 +0,0 @@
-> 🔥 🔥 🔥 Please note, while we are still in [public
-> alpha](https://docs.cloud.autonomic.zone/roadmap/), the `abra` release
-> versioning scheme is not following [semver](https://semver.org/) conventions
-> because we are still in the exploratory phases of building this tool. Please
-> read the changes before upgrading your `abra` installation as there are
-> **most likely** breaking changes coming each release. Sorry for any
-> inconvenience caused, we're working hard to make this tool stable. Semver
-> will be respected when we reach public beta. 🔥 🔥 🔥
-
-# abra x.x.x (UNRELEASED)
-
-# abra 0.7.2 (2021-04-07)
-
-- Fix installation script development installs (again! Thanks Bash!) ([4747d9b7](https://git.autonomic.zone/coop-cloud/abra/commit/4747d9b7fb5fba914f210b6570bfe2db0b53da23))
-
-# abra 0.7.1 (2021-04-07)
-
-- Fix installation script development installs ([8f2fadb3c](https://git.autonomic.zone/coop-cloud/abra/commit/8f2fadb3c43c5915520f5ea531ea3815c2ba8531))
-
-# abra 0.7.0 (2021-04-07)
-
-- Add `--force` to the `deploy` command to allow overriding deployment logic ([#105](https://git.autonomic.zone/coop-cloud/abra/issues/105))
-- Handle undeployed apps in version summaries when deploying ([#104](https://git.autonomic.zone/coop-cloud/abra/issues/104))
-- Add `--force` to `undeploy` command ([e5e98d5](https://git.autonomic.zone/coop-cloud/abra/commit/e5e98d5))
-- Rename "app type" back to "stack" in the deployment overview ([54b6acc](https://git.autonomic.zone/coop-cloud/abra/commit/54b6acc))
-- Show context connection details on `abra server ls` ([#110](https://git.autonomic.zone/coop-cloud/abra/issues/110))
-- Allow to debug the SSH connection details on swarm init ([#109](https://git.autonomic.zone/coop-cloud/abra/issues/109))
-- Show correct status for apps deployed on servers with missing context ([#99](https://git.autonomic.zone/coop-cloud/abra/issues/99))
-- Search for subcommands in descending order of how many components there are ([#108](https://git.autonomic.zone/coop-cloud/abra/issues/108))
-- Add specific app version checking command (`abra app <app> version`) ([#108](https://git.autonomic.zone/coop-cloud/abra/issues/108))
-- Add docker version check (guestimating < v19 is a bad idea) ([#15](https://git.autonomic.zone/coop-cloud/abra/issues/15))
-- Fix git branch handling when not passing `-b <branch>` ([#122](https://git.autonomic.zone/coop-cloud/abra/issues/122))
-- Add work-around to correctly git clone non-master default branch app repositories ([#122](https://git.autonomic.zone/coop-cloud/abra/issues/122))
-- Replace `--force` (except for the `deploy` command) with a global `--no-prompt` for avoiding interactive questions ([#118](https://git.autonomic.zone/coop-cloud/abra/issues/118))
-- Use [docker-stack-wait-deploy](https://github.com/vitalets/docker-stack-wait-deploy) inspired logic to deploy apps ([#116](https://git.autonomic.zone/coop-cloud/abra/issues/116))
-- Add a domain polling check when deploying apps ([#113](https://git.autonomic.zone/coop-cloud/abra/issues/113))
-- Recognise when apps are already undeployed with `abra app <app> undeploy` ([#123](https://git.autonomic.zone/coop-cloud/abra/issues/123))
-- Add `abra doctor` command to help diagnose setup issues ([#119](https://git.autonomic.zone/coop-cloud/abra/issues/119))
-- Add apps version and feature catalogue generation script ([#121](https://git.autonomic.zone/coop-cloud/abra/issues/121))
-- New `--skip-version-check` option to `deploy` ([df4e504](https://git.autonomic.zone/coop-cloud/abra/commit/df4e504)
-- Look up local available version from compose files instead of `abra.sh` ([#131](https://git.autonomic.zone/coop-cloud/abra/issues/131))
-- Improve domain polling logging and allow to skip the check altogether with `--no-domain-poll` ([#140](https://git.autonomic.zone/coop-cloud/abra/issues/140), [#141](https://git.autonomic.zone/coop-cloud/abra/issues/141))
-- Support `ABRA_DIR` in the installer script ([4e94a424e94a42](https://git.autonomic.zone/coop-cloud/abra/commit/4e94a424e94a42))
-- Support [abra-hetzner](https://git.autonomic.zone/coop-cloud/abra-hetzner) plugin ([#88](https://git.autonomic.zone/coop-cloud/abra/issues/88))
-
-# abra 0.6.0 (2021-03-17)
-
-- Show version and digest of app if labelled ([98e674b8e8](https://git.autonomic.zone/coop-cloud/abra/commit/98e674b8e83458a83dcbf331e8e34c7188559c4a))
-- Implement basic version checking on deployment ([#82](https://git.autonomic.zone/coop-cloud/abra/issues/82))
-- New `app-catalogue.sh` script to auto-generate app list for documentation ([f163d4b](https://git.autonomic.zone/coop-cloud/abra/commit/f163d4b0fa920232e9d995a22d20fe78b174b3a9))
-- Support app service rollbacks with `abra <app> rollback <service>` ([#76](https://git.autonomic.zone/coop-cloud/abra/issues/76))
-- Detect when latest version is deployed and perform a no-op ([#87](https://git.autonomic.zone/coop-cloud/abra/issues/87))
-- Allow cloning of app repos with different main branches using `-b, --branch=<branch>` ([#80](https://git.autonomic.zone/coop-cloud/abra/issues/80))
-- Protect against lengthy app names which gives Docker trouble later on ([#83](https://git.autonomic.zone/coop-cloud/abra/issues/83))
-- Support removal of secrets and volumes when `rm`'ing apps ([#44](https://git.autonomic.zone/coop-cloud/abra/issues/44))
-- Always choose the default IPv4 address with `abra server <host> init` ([#91](https://git.autonomic.zone/coop-cloud/abra/issues/91))
-- Add `--type=<type>` filtering option to `abra <app> ls` ([0828189](https://git.autonomic.zone/coop-cloud/abra/commit/0828189))
-- Check for bash 4+ ([#96](https://git.autonomic.zone/coop-cloud/abra/commit/0828189))
-- Add `--dev` option to installer using `git clone` ([88d2a75](https://git.autonomic.zone/coop-cloud/abra/commit/88d2a75))
-- Support `--dev` on the `abra upgrade` command also ([bcc15ec](https://git.autonomic.zone/coop-cloud/abra/commit/bcc15ec))
-- Vendor [yq](https://github.com/mikefarah/yq/releases) automatically ([3b59adf](https://git.autonomic.zone/coop-cloud/abra/commit/3b59adf))
-- Extend version handling logic to support all underlying services ([#90](https://git.autonomic.zone/coop-cloud/abra/issues/90))
-- Fix development installation script symlink issue ([#98](https://git.autonomic.zone/coop-cloud/abra/issues/98))
-- Add `app-version.sh` script to help packagers version apps ([28618bd](https://git.autonomic.zone/coop-cloud/abra/commit/28618bd))
-- Add git digest to `abra version` output ([8b41416](https://git.autonomic.zone/coop-cloud/abra/commit/8b41416))
-
-# abra 0.5.0 (2021-03-01)
-
-- `secret auto` merged into `secret generate` and `app new --auto` is now `app new --secrets` ([#64](https://git.autonomic.zone/coop-cloud/abra/pulls/64))
-- Avoid outputting length during secret generation when not in use ([#67](https://git.autonomic.zone/coop-cloud/abra/issues/67))
-- Support graceful failure when missing secret generation commands ([44d3ac3](https://git.autonomic.zone/coop-cloud/abra/commit/44d3ac3a1cb86edc9b9e91eea1a00e70eae14965))
-- Fix secret detection when using new `.env` file format in apps ([5532452](https://git.autonomic.zone/coop-cloud/abra/commit/55324524ca77141666ffe6cc41b62cc71cf89ace))
-- Support choosing an `$EDITOR` when editing configs ([29cc392](https://git.autonomic.zone/coop-cloud/abra/commit/29cc392dff3e93e48e0e2edd3ce11b405c66a95a))
-- "server" shell completion fixed ([8839bd4](https://git.autonomic.zone/coop-cloud/abra/commit/8839bd45951d00dccf4ef81ece445bcc49e13ee6))
-- Drop `multilogs` command ([#56](https://git.autonomic.zone/coop-cloud/abra/pulls/56))
-- Remove `server use` command ([#51](https://git.autonomic.zone/coop-cloud/abra/issues/51))
-- `new <app>` becomes `new <type>` ([#48](https://git.autonomic.zone/coop-cloud/abra/issues/48))
-- `check` is run on `deploy` now and configurable ([77ba565](https://git.autonomic.zone/coop-cloud/abra/commit/77ba5652b2fe15820f5edfa0f642636f7b8eae7e))
-- App configurations are always updated now ([#42](https://git.autonomic.zone/coop-cloud/abra/issues/42))
-- We use docker format `.env` files (no "export" syntax) from now now ([#55](https://git.autonomic.zone/coop-cloud/abra/pulls/55))
-- Rename `<domain>` option to `<app>` and `APP` variable to `TYPE`, see ([#47](https://git.autonomic.zone/coop-cloud/abra/issues/47))
-- Use Docker-in-Docker (dind), and `dind-bats-kcov` Docker image, for `make test` ([1600b62](https://git.autonomic.zone/coop-cloud/abra/commit/1600b6277fbbffc4c6de1e4ba799c7bbe72ec6a0))
-- Add built-in documentation using `abra help <subcommand>...`, see ([#50](https://git.autonomic.zone/coop-cloud/abra/issues/50))
-- `version` subcommand ([e6b24fe](https://git.autonomic.zone/coop-cloud/abra/commit/e6b24fe))
-- Use `# length=x` comments to generate passwords with `pwgen` and drop `KEY`/`PASSWORD` logic ([#68](https://git.autonomic.zone/coop-cloud/abra/issues/68))
-- Global `--skip-update|-U` / `--skip-check|-C` options to make things quicker ([37e8b00](https://git.autonomic.zone/coop-cloud/abra/commit/37e8b00))
-- `app backup` and `app restore` commands; requires per-app definition ([#70](https://git.autonomic.zone/coop-cloud/abra/issues/70))
-- Rename per-type `abra-commands.sh` to `abra.sh`, and include config versions as type-level instead of app-level config ([#43](https://git.autonomic.zone/coop-cloud/abra/issues/43))
-- Show per-subcommand help by adding `-h/--help` to a command line ([#38](https://git.autonomic.zone/coop-cloud/abra/issues/78))
-
-# abra 0.4.1 (2020-12-24)
-
-- Bug-fixes on `app ls --status` & custom commands
-- Add `app ls --server=...` and alias
-
-# abra 0.4.0 (2020-12-24)
-
-- New command-line interface based on docopt
-- `~/.abra` directory instead of expecting local `.env` files
-- Integration tests & code coverage
-
-# abra 0.3.1 (2020-09-27)
-
-- Fix installer version
-
-# abra 0.3.0 (2020-09-27)
-
-- Add multilogs stack logs implementation ([#8](https://git.autonomic.zone/compose-stacks/abra/issues/8))
-- Add beginnings of "monorepo" functionality
-
-# abra 0.2.0 (2020-09-24)
-
-- Prepare for swarm install script using script.d ([#12](https://git.autonomic.zone/compose-stacks/planning/issues/12))
-
-# abra 0.1.2 (2020-09-22)
-
-- Add upgrade command ([#10](https://git.autonomic.zone/autonomic-cooperative/abra/issues/10))
-
-# abra 0.1.1 (2020-09-22)
-
-- Add installer script ([#9](https://git.autonomic.zone/autonomic-cooperative/abra/issues/9))
-
-# abra 0.1.0 (2020-09-22)
-
-- Initial pre-alpha release

Makefile

@@ -0,0 +1,38 @@
+ABRA    := ./cmd/abra
+COMMIT  := $(shell git rev-list -1 HEAD)
+GOPATH  := $(shell go env GOPATH)
+LDFLAGS := "-X 'main.Commit=$(COMMIT)'"
+DIST_LDFLAGS := $(LDFLAGS)" -s -w"
+export GOPRIVATE=coopcloud.tech
+
+all: run test install build clean format check static
+
+run:
+	@go run -ldflags=$(LDFLAGS) $(ABRA)
+
+install:
+	@go install -ldflags=$(LDFLAGS) $(ABRA)
+
+build-dev:
+	@go build -ldflags=$(LDFLAGS) $(ABRA)
+
+build:
+	@go build -ldflags=$(DIST_LDFLAGS) $(ABRA)
+
+clean:
+	@rm '$(GOPATH)/bin/abra'
+
+format:
+	@gofmt -s -w .
+
+check:
+	@test -z $$(gofmt -l .) || (echo "gofmt: formatting issue - run 'make format' to resolve" && exit 1)
+
+static:
+	@staticcheck $(ABRA)
+
+test:
+	@go test ./... -cover
+
+loc:
+	@find . -name "*.go" | xargs wc -l

README.md

@@ -1,91 +1,100 @@
 # abra
 
-[![Build Status](https://drone.autonomic.zone/api/badges/coop-cloud/abra/status.svg)](https://drone.autonomic.zone/coop-cloud/abra)
-[![codecov](https://codecov.io/gh/Autonomic-Cooperative/abra/branch/main/graph/badge.svg?token=aX3I5NMRsj)](undefined)
+> https://coopcloud.tech
 
-> https://cloud.autonomic.zone
+[![Build Status](https://build.coopcloud.tech/api/badges/coop-cloud/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/coop-cloud/abra)
+[![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/coop-cloud/abra)](https://goreportcard.com/report/git.coopcloud.tech/coop-cloud/abra)
 
-The cooperative cloud utility belt 🎩🐇
+The Co-op Cloud utility belt 🎩🐇
 
-`abra` is a command-line tool for managing your own [Co-op Cloud](https://cloud.autonomic.zone). It can provision new servers, create applications, deploy them, run backup and restore operations and a whole lot of other things. It is the go-to tool for day-to-day operations when managing a Co-op Cloud instance.
-
-## Change log
-
-> 🔥 🔥 🔥 Please note, while we are still in [public
-> alpha](https://docs.cloud.autonomic.zone/roadmap/), the `abra` release
-> versioning scheme is not following [semver](https://semver.org/) conventions
-> because we are still in the exploratory phases of building this tool. Please
-> read the changes before upgrading your `abra` installation as there are
-> **most likely** breaking changes coming each release. Sorry for any
-> inconvenience caused, we're working hard to make this tool stable. Semver
-> will be respected when we reach public beta. 🔥 🔥 🔥
-
-See [CHANGELOG.md](./CHANGELOG.md).
-
-## Documentation
-
-> [docs.cloud.autonomic.zone](https://docs.cloud.autonomic.zone/)
+`abra` is a command-line tool for managing your own [Co-op Cloud](https://coopcloud.tech). It can provision new servers, create applications, deploy them, run backup and restore operations and a whole lot of other things. It is the go-to tool for day-to-day operations when managing a Co-op Cloud instance.
 
 ## Install
 
-Install the latest stable release:
+### Arch-based Linux Distros
+
+[abra (coming-soon)](https://aur.archlinux.org/packages/abra/) or for the latest version on git [abra-git](https://aur.archlinux.org/packages/abra-git/)
 
 ```sh
-curl https://install.abra.autonomic.zone | bash
+yay -S abra-git # or abra
 ```
 
-or the bleeding-edge development version:
+### Debian-based Linux Distros
+
+**Coming Soon**
+
+### Homebrew
+
+**Coming Soon**
+
+### Build from source
 
 ```sh
-curl https://install.abra.autonomic.zone | bash -s -- --dev
+git clone https://git.coopcloud.tech/coop-cloud/abra
+cd abra
+go env -w GOPRIVATE=coopcloud.tech
+make install
 ```
 
-The source for this script is [here](./deploy/install.abra.autonomic.zone/installer).
+The abra binary will be in `$GOPATH/bin`.
 
-## Update
+## Autocompletion
 
-Run `abra upgrade` to automatically download and install the latest release
-version.
+**bash**
 
-To update the development version, run `abra upgrade --dev`.
-
-## Hack
-
-It's written in Bash version 4 or greater!
-
-Install it via `curl https://install.abra.autonomic.zone | bash -s -- --dev`, then you can hack on the source in `~/.abra/src`.
-
-The command-line interface is generated via [docopt](http://docopt.org/). If you add arguments then you need to run `make docopt` ro regenerate the parser.
-
-Please remember to update the [CHANGELOG](./CHANGELOG.md) when you make a change.
-
-## Generating a new abra-apps.json
-
-You'll need to install the following requirements:
-
-- [requests](https://docs.python-requests.org/en/master/) (`apt install python3-requests` / `pip install requests`)
-- [skopeo](https://github.com/containers/skopeo) (check [the install docs](https://github.com/containers/skopeo/blob/master/install.md))
-- [jq](https://stedolan.github.io/jq/tutorial/) (`sudo apt-get install jq` or see [the install docs](https://stedolan.github.io/jq/download/))
-- [yq](https://mikefarah.gitbook.io/yq/) (see [the install docs](https://mikefarah.gitbook.io/yq/#install))
-
-Then run `./bin/app-json.py` ([source](./bin/app-json.py)) and it will spit out the JSON file into [deploy/abra-apps.cloud.autonomic.zone/abra-apps.json](./deploy/abra-apps.cloud.autonomic.zone/abra-apps.json).
-
-## Releasing
-
-### `abra`
-
-- Change the `x.x.x` header in [CHANGELOG.md](./CHANGELOG.md) to reflect new version and mark date
-- Update the version in [abra](./abra)
-- Update the version in [deploy/install.abra.autonomic.zone/installer](./deploy/install.abra.autonomic.zone/installer)
-- `git commit` the above changes and then tag it with `git tag <your-new-version>`
-- `git push` and `git push --tags`
-- Deploy a new installer script `make release-installer`
-- Tell the world (CoTech forum, Matrix public channel, Autonomic mastodon, etc.)
-
-### abra-apps.cloud.autonomic.zone
-
-> [abra-apps.cloud.autonomic.zone](https://abra-apps.cloud.autonomic.zone)
-
-```bash
-$ make release-abra-apps
+Copy `autocomplete/bash_autocomplete` into `/etc/bash_completion.d/` and rename
+it to abra.
 ```
+sudo cp autocomplete/bash_autocomplete /etc/bash_completion.d/abra
+source /etc/bash_completion.d/abra
+```
+
+**(fi)zsh**
+
+(fi)zsh doesn't have an autocompletion folder by default but you can create one, then copy `zsh_autocomplete` into it and add a couple lines to your `~/.zshrc` or `~/.fizsh/.fizshrc`
+```
+sudo mkdir /etc/zsh/completion.d/
+sudo cp autocomplete/zsh_autocomplete /etc/zsh/completion.d/abra
+echo "PROG=abra\n_CLI_ZSH_AUTOCOMPLETE_HACK=1\nsource /etc/zsh/completion.d/abra" >> ~/.zshrc
+```
+(replace .zshrc with ~/.fizsh/.fizshrc if you use fizsh)
+
+## Hacking
+
+Install direnv, run `cp .envrc.sample .envrc`, then run `direnv allow` in this directory. This will set coopcloud repos as private due to [this bug.](https://git.coopcloud.tech/coop-cloud/coopcloud.tech/issues/20#issuecomment-8201). Or you can run `go env -w GOPRIVATE=coopcloud.tech` but I'm not sure how persistent this is.
+
+Install [Go >= 1.16](https://golang.org/doc/install) and then:
+
+- `make build` to build
+- `./abra` to run commands
+- `make test` will run tests
+
+Our [Drone CI configuration](.drone.yml) runs a number of sanity on each pushed commit. See the [Makefile](./Makefile) for more handy targets.
+
+Please use the [conventional commit format](https://www.conventionalcommits.org/en/v1.0.0/) for your commits so we can automate our change log.
+
+## Versioning
+
+We use [goreleaser](https://goreleaser.com) to help us automate releases. We use [semver](https://semver.org) for versioning all releases of the tool. While we are still in the public alpha release phase, we will maintain a `0.y.z-alpha` format. Change logs are generated from our commit logs. We are still working this out and aim to refine our release praxis as we go.
+
+For developers, while using this `-alpha` format, the `y` part is the "major" version part. So, if you make breaking changes, you increment that and _not_ the `x` part. So, if you're on `0.1.0-alpha`, then you'd go to `0.1.1-alpha` for a backwards compatible change and `0.2.0-alpha` for a backwards incompatible change.
+
+## Making a new release
+
+- Change `ABRA_VERSION` to match the new tag in [`scripts`](./scripts/installer/installer) (use [semver](https://semver.org))
+- Commit that change (e.g. `git commit -m 'chore: publish next tag 0.3.1-alpha'`)
+- Make a new tag (e.g. `git tag 0.y.z-alpha`)
+- Push the new tag (e.g. `git push && git push --tags`)
+- Wait until the build finishes on [build.coopcloud.tech](https://build.coopcloud.tech/coop-cloud/abra)
+- Check the release worked, (e.g. `abra upgrade; abra version`)
+
+## Fork maintenance
+
+We maintain a fork of [godotenv](https://github.com/Autonomic-Cooperative/godotenv) for two features:
+
+1. multi-line env var support
+2. inline comment parsing
+
+You can upgrade the version here by running `go get github.com/Autonomic-Cooperative/godotenv@<commit>` where `<commit>` is the
+latest commit you want to pin to. We are aiming to migrate to YAML format for the environment configuration, so this should only
+be a temporary thing.

TODO.md

@@ -0,0 +1,67 @@
+# TODO
+
+## Bash feature parity
+
+- [ ] Commands
+  - [x] `abra server`
+    - [x] `ls`
+    - [x] `add`
+    - [x] `new`
+      - [x] `capsul`
+      - [x] `hetzner`
+    - [x] `rm`
+    - [x] `init`
+  - [ ] `abra app`
+    - [x] `ls`
+    - [x] `new`
+    - [x] `backup`
+    - [x] `deploy`
+    - [x] `check`
+    - [x] `version`
+    - [x] `config`
+    - [x] `cp`
+    - [x] `logs`
+    - [x] `ps`
+    - [x] `restore`
+    - [x] `rm`
+    - [x] `run`
+    - [ ] `rollback`
+    - [x] `secret`
+      - [x] `generate`
+      - [x] `insert`
+      - [x] `rm`
+      - [x] `ls`
+    - [x] `undeploy`
+    - [ ] `volume`
+      - [x] `ls` (WIP: knoflook)
+      - [ ] `rm` (WIP: knoflook)
+  - [x] `abra recipe`
+    - [x] `ls`
+    - [x] `create`
+    - [x] `upgrade`
+    - [x] `sync`
+    - [x] `versions`
+    - [x] `lint`
+  - [x] `upgrade`
+  - [x] `version`
+
+## Next phase
+
+- [ ] Polishing UI/UX and testing
+- [ ] Refactoring and code organisation
+- [ ] Automated builds for releasing
+
+## New features
+
+- [ ] Commands
+  - [ ] `abra server`
+    - [ ] `dns`
+      - [ ] `gandi`
+  - [ ] `abra recipe`
+    - [ ] "TBD apps.json generating command" (see [#40](https://git.coopcloud.tech/coop-cloud/go-abra/issues/40))
+- [ ] Package manager integration
+  - [x] AUR
+  - [ ] Debian
+  - [ ] Ubuntu
+  - [ ] Fedora
+  - [ ] Homebrew

autocomplete/bash_autocomplete

@@ -0,0 +1,21 @@
+#! /bin/bash
+
+: ${PROG:=$(basename ${BASH_SOURCE})}
+
+_cli_bash_autocomplete() {
+  if [[ "${COMP_WORDS[0]}" != "source" ]]; then
+    local cur opts base
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    if [[ "$cur" == "-"* ]]; then
+      opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} ${cur} --generate-bash-completion )
+    else
+      opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion )
+    fi
+    COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+    return 0
+  fi
+}
+
+complete -o bashdefault -o default -o nospace -F _cli_bash_autocomplete $PROG
+unset PROG

autocomplete/powershell_autocomplete.ps1

@@ -0,0 +1,9 @@
+$fn = $($MyInvocation.MyCommand.Name)
+$name = $fn -replace "(.*)\.ps1$", '$1'
+Register-ArgumentCompleter -Native -CommandName $name -ScriptBlock {
+     param($commandName, $wordToComplete, $cursorPosition)
+     $other = "$wordToComplete --generate-bash-completion"
+         Invoke-Expression $other | ForEach-Object {
+            [System.Management.Automation.CompletionResult]::new($_, $_, 'ParameterValue', $_)
+         }
+ }

autocomplete/zsh_autocomplete

@@ -0,0 +1,23 @@
+#compdef $PROG
+
+_cli_zsh_autocomplete() {
+
+  local -a opts
+  local cur
+  cur=${words[-1]}
+  if [[ "$cur" == "-"* ]]; then
+    opts=("${(@f)$(_CLI_ZSH_AUTOCOMPLETE_HACK=1 ${words[@]:0:#words[@]-1} ${cur} --generate-bash-completion)}")
+  else
+    opts=("${(@f)$(_CLI_ZSH_AUTOCOMPLETE_HACK=1 ${words[@]:0:#words[@]-1} --generate-bash-completion)}")
+  fi
+
+  if [[ "${opts[1]}" != "" ]]; then
+    _describe 'values' opts
+  else
+    _files
+  fi
+
+  return
+}
+
+compdef _cli_zsh_autocomplete $PROG

bin/app-catalogue.sh

@@ -1,103 +0,0 @@
-#!/bin/bash
-
-# shellcheck disable=SC2119
-
-# Usage: ./app-catalogue.sh
-#
-# Gather metadata from Co-op Cloud apps in $ABRA_DIR/apps (default
-# ~/.abra/apps), and format it as a Markdown table for this page:
-# https://docs.cloud.autonomic.zone/apps/
-
-stack_dir="${ABRA_DIR:-$HOME/.abra}/apps/"
-
-cd "$stack_dir" || exit
-
-# load all README files into ENV_FILES array
-mapfile -t readmes < <(find -L . -name "README.md")
-# FIXME 3wc: requires bash 4, use for loop instead
-
-base_url="https://git.autonomic.zone/coop-cloud"
-
-cat_apps=()
-cat_development=()
-cat_utilities=()
-cat_graveyard=()
-
-get_var() {
-	echo "$1" | grep "$2" | sed 's/^[^:]*: //'
-}
-
-# shellcheck disable=SC2120
-trim() {
-	# accept input as argument or from STDIN, see here:
-	# https://zwbetz.com/passing-input-to-a-bash-function-via-arguments-or-stdin/
-	# shellcheck disable=SC2155
-	local input="$([[ -p /dev/stdin ]] && cat - || echo "$@")"
-	[[ -z "$input" ]] && return 1
-	echo "$input" | tr -d ' '
-}
-
-# shellcheck disable=SC2120
-prettify() {
-	# as above
-	# shellcheck disable=SC2155
-	local input="$([[ -p /dev/stdin ]] && cat - || echo "$@")"
-	[[ -z "$input" ]] && return 1
-
-	echo "$input" | sed -e 's/Yes/✅/' -e 's/No/❌/' -e 's/N\/A/⛔/'
-}
-
-for readme in "${readmes[@]}"; do
-	type="$(basename "${readme%README.md}")"
-	if [ "$type" = "example" ]; then
-		continue
-	fi
-	title="$(grep '^# ' "$type/README.md" | sed 's/^# //' )"
-	# find section between 'metadata' and 'endmetadata' comments
-	metadata="$(awk '/-- metadata --/,/-- endmetadata --/' "$type/README.md")"
-	status="$(get_var "$metadata" "Status")"
-	category="$(get_var "$metadata" "Category" | cut -d',' -f2 | trim)"
-
-	if [ -z "$category" ]; then
-		echo "ERROR: missing category for $type"
-		continue
-	fi
-
-	image="$(get_var "$metadata" "Image" | cut -d',' -f2 | trim)"
-	healthcheck="$(get_var "$metadata" "Healthcheck" | prettify)"
-	backups="$(get_var "$metadata" "Backups" | prettify)"
-	email="$(get_var "$metadata" "Email" | prettify)"
-	tests="$(get_var "$metadata" "Tests" | prettify)"
-	sso="$(get_var "$metadata" "SSO" | prettify)"
-
-	row="| [$title]($base_url/$type) | $status | $image | $healthcheck | $backups | $email | $tests | $sso |"
-
-	category_lower="$(echo "$category" | tr '[:upper:]' '[:lower:]')"
-	eval "cat_$category_lower+=( '$row' )"
-done
-
-headers="
-| **Name** | **Status** | **Image** | **Healtcheck** | **Backups** | **Email** | **CI** | **Single-Sign-On** |
-| --- | --- | --- | --- | --- | --- | --- | --- |"
-
-echo "## Applications"
-echo "$headers"
-printf '%s\n' "${cat_apps[@]}" | sort
-
-echo
-
-echo "## Developer tools"
-echo "$headers"
-printf '%s\n' "${cat_development[@]}" | sort
-
-echo
-
-echo "## Utilities"
-echo "$headers"
-printf '%s\n' "${cat_utilities[@]}" | sort
-
-echo
-
-echo "## Graveyard"
-echo "$headers"
-printf '%s\n' "${cat_graveyard[@]}" | sort

bin/app-json.py

@@ -1,253 +0,0 @@
-#!/usr/bin/env python3
-
-# Usage: ./app-json.py
-#
-# Gather metadata from Co-op Cloud apps in $ABRA_DIR/apps (default
-# ~/.abra/apps), and format it as JSON so that it can be hosted here:
-# https://abra-apps.cloud.autonomic.zone
-
-from json import dump
-from logging import DEBUG, basicConfig, getLogger
-from os import chdir, listdir, mkdir
-from os.path import basename, exists, expanduser
-from pathlib import Path
-from re import findall, search
-from shlex import split
-from subprocess import check_output, DEVNULL
-from sys import exit
-
-from requests import get
-
-HOME_PATH = expanduser("~/")
-CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
-YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq")
-SCRIPT_PATH = Path(__file__).absolute().parent
-REPOS_TO_SKIP = (
-    "abra",
-    "backup-bot",
-    "cloud.autonomic.zone",
-    "docs.cloud.autonomic.zone",
-    "example",
-    "organising",
-    "pyabra",
-    "stack-ssh-deploy",
-    "radicle-seed-node",
-    "coturn"
-)
-
-log = getLogger(__name__)
-basicConfig()
-log.setLevel(DEBUG)
-
-
-def _run_cmd(cmd, shell=False, **kwargs):
-    """Run a shell command."""
-    args = [split(cmd)]
-
-    if shell:
-        args = [cmd]
-        kwargs = {"shell": shell}
-
-    try:
-        return check_output(*args, **kwargs).decode("utf-8").strip()
-    except Exception as exception:
-        log.error(f"Failed to run {cmd}, saw {str(exception)}")
-        exit(1)
-
-
-def get_published_apps_json():
-    """Retrieve already published apps json."""
-    url = "https://abra-apps.cloud.autonomic.zone"
-
-    log.info(f"Retrieving {url}")
-
-    try:
-        return get(url, timeout=5).json()
-    except Exception as exception:
-        log.error(f"Failed to retrieve {url}, saw {str(exception)}")
-        return {}
-
-
-def clone_all_apps():
-    """Clone all Co-op Cloud apps to ~/.abra/apps."""
-    if not exists(CLONES_PATH):
-        mkdir(CLONES_PATH)
-    url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
-
-    log.info(f"Retrieving {url}")
-
-    try:
-        response = get(url, timeout=10)
-    except Exception as exception:
-        log.error(f"Failed to retrieve {url}, saw {str(exception)}")
-        exit(1)
-
-    repos = [[p["name"], p["ssh_url"]] for p in response.json()]
-
-    for name, url in repos:
-        if name in REPOS_TO_SKIP:
-            continue
-
-        if not exists(f"{CLONES_PATH}/{name}"):
-            log.info(f"Retrieving {url}")
-            _run_cmd(f"git clone {url} {CLONES_PATH}/{name}")
-
-            chdir(f"{CLONES_PATH}/{name}")
-            if not int(_run_cmd("git branch --list | wc -l", shell=True)):
-                log.info(f"Guessing main branch is HEAD for {name}")
-                _run_cmd("git checkout main")
-        else:
-            log.info(f"Updating {name}")
-            chdir(f"{CLONES_PATH}/{name}")
-            _run_cmd("git fetch -a")
-
-
-def generate_apps_json():
-    """Generate the abra-apps.json application versions file."""
-    apps_json = {}
-    cached_apps_json = get_published_apps_json()
-
-    for app in listdir(CLONES_PATH):
-        if app in REPOS_TO_SKIP:
-            log.info(f"Skipping {app}")
-            continue
-
-        app_path = f"{CLONES_PATH}/{app}"
-        chdir(app_path)
-
-        log.info(f"Processing {app}")
-        apps_json[app] = {
-            "category": "apps",
-            "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
-            # Note(decentral1se): please note that the app features do not
-            # correspond to version tags. We simply parse the latest features
-            # list from HEAD. This may lead to unexpected situations where
-            # users believe X feature is available under Y version but it is
-            # not.
-            "features": get_app_features(app_path),
-            "versions": get_app_versions(app_path, cached_apps_json),
-        }
-
-    return apps_json
-
-
-def get_app_features(app_path):
-    """Parse features from app repo README files."""
-    features = {}
-
-    chdir(app_path)
-
-    with open(f"{app_path}/README.md", "r") as handle:
-        log.info(f"{app_path}/README.md")
-        contents = handle.read()
-
-    try:
-        for match in findall(r"\*\*.*\s\*", contents):
-            title = search(r"(?<=\*\*).*(?=\*\*)", match).group().lower()
-
-            if title == "image":
-                value = {
-                    "image": search(r"(?<=`).*(?=`)", match).group(),
-                    "url": search(r"(?<=\().*(?=\))", match).group(),
-                    "rating": match.split(",")[1].strip(),
-                    "source": match.split(",")[-1].replace("*", "").strip(),
-                }
-            else:
-                value = match.split(":")[-1].replace("*", "").strip()
-
-            features[title] = value
-    except (IndexError, AttributeError):
-        log.info(f"Can't parse {app_path}/README.md")
-        return {}
-    finally:
-        _run_cmd("git checkout HEAD")
-
-    log.info(f"Parsed {features}")
-
-    return features
-
-
-def get_app_versions(app_path, cached_apps_json):
-    versions = {}
-
-    chdir(app_path)
-
-    tags = _run_cmd("git tag --list").split()
-
-    if not tags:
-        log.info("No tags discovered, moving on")
-        return {}
-
-    initial_branch = _run_cmd("git rev-parse --abbrev-ref HEAD")
-
-    app_name = basename(app_path)
-
-    try:
-        existing_tags = cached_apps_json[app_name]["versions"].keys()
-    except KeyError:
-        existing_tags = []
-
-    for tag in tags:
-        _run_cmd(f"git checkout {tag}",
-                 stderr=DEVNULL)
-
-        services_cmd = f"{YQ_PATH} e '.services | keys | .[]' compose*.yml"
-        services = _run_cmd(services_cmd, shell=True).split()
-
-        parsed_services = []
-        service_versions = {}
-        for service in services:
-            if service in ("null", "---"):
-                continue
-
-            if tag in existing_tags:
-                log.info(f"Skipping {tag} because we've already processed it")
-                existing_versions = cached_apps_json[app_name]["versions"][tag][service]
-                service_versions[service] = existing_versions
-                _run_cmd(f"git checkout {initial_branch}")
-                continue
-
-            if service in parsed_services:
-                log.info(f"Skipped {service}, we've already parsed it locally")
-                continue
-
-            services_cmd = f"{YQ_PATH} e '.services.{service}.image' compose*.yml"
-            images = _run_cmd(services_cmd, shell=True).split()
-
-            for image in images:
-                if image in ("null", "---"):
-                    continue
-
-                images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'"
-                output = _run_cmd(images_cmd, shell=True)
-
-                service_version_info = {
-                    "image": image.split(":")[0],
-                    "tag": image.split(":")[-1],
-                    "digest": output.split(":")[-1][:8],
-                }
-
-                log.info(f"Parsed {service_version_info}")
-                service_versions[service] = service_version_info
-
-            parsed_services.append(service)
-
-        versions[tag] = service_versions
-
-    _run_cmd(f"git checkout {initial_branch}")
-
-    return versions
-
-
-def main():
-    """Run the script."""
-    clone_all_apps()
-
-    target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
-    with open(target, "w", encoding="utf-8") as handle:
-        dump(generate_apps_json(), handle, ensure_ascii=False, indent=4)
-
-    log.info(f"Successfully generated {target}")
-
-
-main()

cli/app/app.go

@@ -0,0 +1,37 @@
+package app
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// AppCommand defines the `abra app` command and ets subcommands
+var AppCommand = &cli.Command{
+	Name:      "app",
+	Usage:     "Manage apps",
+	Aliases:   []string{"a"},
+	ArgsUsage: "<app>",
+	Description: `
+This command provides all the functionality you need to manage the life cycle
+of your apps. From initial deployment, day-2 operations (e.g. backup/restore)
+to scaling apps up and spinning them down.
+`,
+	Subcommands: []*cli.Command{
+		appNewCommand,
+		appConfigCommand,
+		appDeployCommand,
+		appUndeployCommand,
+		appBackupCommand,
+		appRestoreCommand,
+		appRemoveCommand,
+		appCheckCommand,
+		appListCommand,
+		appPsCommand,
+		appLogsCommand,
+		appCpCommand,
+		appRunCommand,
+		appRollbackCommand,
+		appSecretCommand,
+		appVolumeCommand,
+		appVersionCommand,
+	},
+}

cli/app/backup.go

@@ -0,0 +1,78 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var backupAllServices bool
+var backupAllServicesFlag = &cli.BoolFlag{
+	Name:        "all",
+	Value:       false,
+	Destination: &backupAllServices,
+	Aliases:     []string{"a"},
+	Usage:       "Backup all services",
+}
+
+var appBackupCommand = &cli.Command{
+	Name:      "backup",
+	Usage:     "Backup an app",
+	Aliases:   []string{"b"},
+	Flags:     []cli.Flag{backupAllServicesFlag},
+	ArgsUsage: "<service>",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Get(1) != "" && backupAllServices {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<service>' and '--all' together"))
+		}
+
+		abraSh := path.Join(config.ABRA_DIR, "apps", app.Type, "abra.sh")
+		if _, err := os.Stat(abraSh); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("'%s' does not exist?", abraSh)
+			}
+			logrus.Fatal(err)
+		}
+
+		sourceCmd := fmt.Sprintf("source %s", abraSh)
+
+		execCmd := "abra_backup"
+		if !backupAllServices {
+			serviceName := c.Args().Get(1)
+			if serviceName == "" {
+				internal.ShowSubcommandHelpAndError(c, errors.New("no service(s) target provided"))
+			}
+			execCmd = fmt.Sprintf("abra_backup_%s", serviceName)
+		}
+
+		bytes, err := ioutil.ReadFile(abraSh)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !strings.Contains(string(bytes), execCmd) {
+			logrus.Fatalf("%s doesn't have a '%s' function", app.Type, execCmd)
+		}
+
+		sourceAndExec := fmt.Sprintf("%s; %s", sourceCmd, execCmd)
+		cmd := exec.Command("bash", "-c", sourceAndExec)
+		output, err := cmd.Output()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		fmt.Print(string(output))
+
+		return nil
+	},
+}

cli/app/check.go

@@ -0,0 +1,51 @@
+package app
+
+import (
+	"os"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appCheckCommand = &cli.Command{
+	Name:      "check",
+	Usage:     "Check if app is configured correctly",
+	Aliases:   []string{"c"},
+	ArgsUsage: "<service>",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		envSamplePath := path.Join(config.ABRA_DIR, "apps", app.Type, ".env.sample")
+		if _, err := os.Stat(envSamplePath); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("'%s' does not exist?", envSamplePath)
+			}
+			logrus.Fatal(err)
+		}
+
+		envSample, err := config.ReadEnv(envSamplePath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var missing []string
+		for k := range envSample {
+			if _, ok := app.Env[k]; !ok {
+				missing = append(missing, k)
+			}
+		}
+
+		if len(missing) > 0 {
+			missingEnvVars := strings.Join(missing, ", ")
+			logrus.Fatalf("%s is missing %s", app.Path, missingEnvVars)
+		}
+
+		logrus.Info("All necessary environment variables defined")
+
+		return nil
+	},
+}

cli/app/config.go

@@ -0,0 +1,41 @@
+package app
+
+import (
+	"os"
+	"os/exec"
+
+	"coopcloud.tech/abra/cli/internal"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appConfigCommand = &cli.Command{
+	Name:    "config",
+	Aliases: []string{"c"},
+	Usage:   "Edit app config",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		ed, ok := os.LookupEnv("EDITOR")
+		if !ok {
+			edPrompt := &survey.Select{
+				Message: "Which editor do you wish to use?",
+				Options: []string{"vi", "vim", "nvim", "nano", "pico", "emacs"},
+			}
+			if err := survey.AskOne(edPrompt, &ed); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		cmd := exec.Command(ed, app.Path)
+		cmd.Stdin = os.Stdin
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/cp.go

@@ -0,0 +1,121 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appCpCommand = &cli.Command{
+	Name:      "cp",
+	Aliases:   []string{"c"},
+	ArgsUsage: "<src> <dst>",
+	Usage:     "Copy files to/from a running app service",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		src := c.Args().Get(1)
+		dst := c.Args().Get(2)
+		if src == "" {
+			logrus.Fatal("missing <src> argument")
+		} else if dst == "" {
+			logrus.Fatal("missing <dest> argument")
+		}
+
+		parsedSrc := strings.SplitN(src, ":", 2)
+		parsedDst := strings.SplitN(dst, ":", 2)
+		errorMsg := "one of <src>/<dest> arguments must take $SERVICE:$PATH form"
+		if len(parsedSrc) == 2 && len(parsedDst) == 2 {
+			logrus.Fatal(errorMsg)
+		} else if len(parsedSrc) != 2 {
+			if len(parsedDst) != 2 {
+				logrus.Fatal(errorMsg)
+			}
+		} else if len(parsedDst) != 2 {
+			if len(parsedSrc) != 2 {
+				logrus.Fatal(errorMsg)
+			}
+		}
+
+		var service string
+		var srcPath string
+		var dstPath string
+		isToContainer := false // <container:src> <dst>
+		if len(parsedSrc) == 2 {
+			service = parsedSrc[0]
+			srcPath = parsedSrc[1]
+			dstPath = dst
+		} else if len(parsedDst) == 2 {
+			service = parsedDst[0]
+			dstPath = parsedDst[1]
+			srcPath = src
+			isToContainer = true // <src> <container:dst>
+		}
+
+		appFiles, err := config.LoadAppFiles("")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		appEnv, err := config.GetApp(appFiles, app.Name)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		ctx := context.Background()
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters := filters.NewArgs()
+		filters.Add("name", fmt.Sprintf("%s_%s", appEnv.StackName(), service))
+		containers, err := cl.ContainerList(ctx, types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if len(containers) != 1 {
+			logrus.Fatalf("expected 1 container but got %v", len(containers))
+		}
+		container := containers[0]
+
+		if isToContainer {
+			if _, err := os.Stat(srcPath); err != nil {
+				logrus.Fatalf("'%s' does not exist?", srcPath)
+			}
+
+			toTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+			content, err := archive.TarWithOptions(srcPath, toTarOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			copyOpts := types.CopyToContainerOptions{AllowOverwriteDirWithFile: false, CopyUIDGID: false}
+			if err := cl.CopyToContainer(ctx, container.ID, dstPath, content, copyOpts); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			content, _, err := cl.CopyFromContainer(ctx, container.ID, srcPath)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			defer content.Close()
+			fromTarOpts := &archive.TarOptions{NoOverwriteDirNonDir: true, Compression: archive.Gzip}
+			if err := archive.Untar(content, dstPath, fromTarOpts); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+		return nil
+	},
+}

cli/app/deploy.go

@@ -0,0 +1,57 @@
+package app
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	stack "coopcloud.tech/abra/pkg/client/stack"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appDeployCommand = &cli.Command{
+	Name:    "deploy",
+	Aliases: []string{"d"},
+	Usage:   "Deploy an app",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		abraShPath := fmt.Sprintf("%s/%s/%s", config.APPS_DIR, app.Type, "abra.sh")
+		abraShEnv, err := config.ReadAbraShEnvVars(abraShPath)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for k, v := range abraShEnv {
+			app.Env[k] = v
+		}
+		app.Env["STACK_NAME"] = app.StackName()
+
+		composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		deployOpts := stack.Deploy{
+			Composefiles: composeFiles,
+			Namespace:    app.StackName(),
+			Prune:        false,
+			ResolveImage: stack.ResolveImageAlways,
+		}
+		compose, err := config.GetAppComposeConfig(app.Name, deployOpts, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := stack.RunDeploy(cl, deployOpts, compose); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/list.go

@@ -0,0 +1,100 @@
+package app
+
+import (
+	"sort"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var status bool
+var statusFlag = &cli.BoolFlag{
+	Name:        "status",
+	Aliases:     []string{"S"},
+	Value:       false,
+	Usage:       "Show app deployment status",
+	Destination: &status,
+}
+
+var appType string
+var typeFlag = &cli.StringFlag{
+	Name:        "type",
+	Aliases:     []string{"t"},
+	Value:       "",
+	Usage:       "Show apps of a specific type",
+	Destination: &appType,
+}
+
+var listAppServer string
+var listAppServerFlag = &cli.StringFlag{
+	Name:        "server",
+	Aliases:     []string{"s"},
+	Value:       "",
+	Usage:       "Show apps of a specific server",
+	Destination: &listAppServer,
+}
+
+var appListCommand = &cli.Command{
+	Name:  "list",
+	Usage: "List all managed apps",
+	Description: `
+This command looks at your local file system listing of apps and servers (e.g.
+in ~/.abra/) to generate a report of all your apps.
+
+By passing the "--status/-S" flag, you can query all your servers for the
+actual live deployment status. Depending on how many servers you manage, this
+can take some time.
+	`,
+	Aliases: []string{"ls"},
+	Flags: []cli.Flag{
+		statusFlag,
+		listAppServerFlag,
+		typeFlag,
+	},
+	Action: func(c *cli.Context) error {
+		appFiles, err := config.LoadAppFiles(listAppServer)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		apps, err := config.GetApps(appFiles)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		sort.Sort(config.ByServerAndType(apps))
+
+		statuses := map[string]string{}
+		tableCol := []string{"Server", "Type", "Domain"}
+		if status {
+			tableCol = append(tableCol, "Status")
+			statuses, err = config.GetAppStatuses(appFiles)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		table := abraFormatter.CreateTable(tableCol)
+		table.SetAutoMergeCellsByColumnIndex([]int{0})
+
+		for _, app := range apps {
+			var tableRow []string
+			if app.Type == appType || appType == "" {
+				// If type flag is set, check for it, if not, Type == ""
+				tableRow = []string{app.Server, app.Type, app.Domain}
+				if status {
+					if status, ok := statuses[app.StackName()]; ok {
+						tableRow = append(tableRow, status)
+					} else {
+						tableRow = append(tableRow, "unknown")
+					}
+				}
+			}
+			table.Append(tableRow)
+		}
+
+		table.Render()
+		return nil
+	},
+}

cli/app/logs.go

@@ -0,0 +1,112 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	dockerClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// stackLogs lists logs for all stack services
+func stackLogs(stackName string, client *dockerClient.Client) {
+	ctx := context.Background()
+	filters := filters.NewArgs()
+	filters.Add("name", stackName)
+	serviceOpts := types.ServiceListOptions{Filters: filters}
+	services, err := client.ServiceList(ctx, serviceOpts)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	for _, service := range services {
+		wg.Add(1)
+		go func(s string) {
+			logOpts := types.ContainerLogsOptions{
+				Details:    true,
+				Follow:     true,
+				ShowStderr: true,
+				ShowStdout: true,
+				Tail:       "20",
+				Timestamps: true,
+			}
+			logs, err := client.ServiceLogs(ctx, s, logOpts)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			// defer after err check as any err returns a nil io.ReadCloser
+			defer logs.Close()
+
+			_, err = io.Copy(os.Stdout, logs)
+			if err != nil && err != io.EOF {
+				logrus.Fatal(err)
+			}
+		}(service.ID)
+	}
+	wg.Wait()
+	os.Exit(0)
+}
+
+var appLogsCommand = &cli.Command{
+	Name:      "logs",
+	Aliases:   []string{"l"},
+	ArgsUsage: "[<service>]",
+	Usage:     "Tail app logs",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		ctx := context.Background()
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceName := c.Args().Get(1)
+		if serviceName == "" {
+			stackLogs(app.StackName(), cl)
+		}
+
+		service := fmt.Sprintf("%s_%s", app.StackName(), serviceName)
+		filters := filters.NewArgs()
+		filters.Add("name", service)
+		serviceOpts := types.ServiceListOptions{Filters: filters}
+		services, err := cl.ServiceList(ctx, serviceOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if len(services) != 1 {
+			logrus.Fatalf("expected 1 service but got %v", len(services))
+		}
+
+		logOpts := types.ContainerLogsOptions{
+			Details:    true,
+			Follow:     true,
+			ShowStderr: true,
+			ShowStdout: true,
+			Tail:       "20",
+			Timestamps: true,
+		}
+		logs, err := cl.ServiceLogs(ctx, services[0].ID, logOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		// defer after err check as any err returns a nil io.ReadCloser
+		defer logs.Close()
+
+		_, err = io.Copy(os.Stdout, logs)
+		if err != nil && err != io.EOF {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/new.go

@@ -0,0 +1,232 @@
+package app
+
+import (
+	"fmt"
+	"path"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/config"
+	recipePkg "coopcloud.tech/abra/pkg/recipe"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+type secrets map[string]string
+
+var domain string
+var domainFlag = &cli.StringFlag{
+	Name:        "domain",
+	Aliases:     []string{"d"},
+	Value:       "",
+	Usage:       "Choose a domain name",
+	Destination: &domain,
+}
+
+var newAppServer string
+var newAppServerFlag = &cli.StringFlag{
+	Name:        "server",
+	Aliases:     []string{"s"},
+	Value:       "",
+	Usage:       "Show apps of a specific server",
+	Destination: &listAppServer,
+}
+
+var newAppName string
+var newAppNameFlag = &cli.StringFlag{
+	Name:        "app-name",
+	Aliases:     []string{"a"},
+	Value:       "",
+	Usage:       "Choose an app name",
+	Destination: &newAppName,
+}
+
+var appNewDescription = `
+This command takes a recipe and uses it to create a new app. This new app
+configuration is stored in your ~/.abra directory under the appropriate server.
+
+This command does not deploy your app for you. You will need to run "abra app
+deploy <app>" to do so.
+
+You can see what recipes are available (i.e. values for the <recipe> argument)
+by running "abra recipe ls".
+
+Passing the "--secrets/-S" flag will automatically generate secrets for your
+app and store them encrypted at rest on the chosen target server. These
+generated secrets are only visible at generation time, so please take care to
+store them somewhere safe.
+
+You can use the "--pass/-P" to store these generated passwords locally in a
+pass store (see passwordstore.org for more). The pass command must be available
+on your $PATH.
+`
+
+var appNewCommand = &cli.Command{
+	Name:        "new",
+	Usage:       "Create a new app",
+	Aliases:     []string{"n"},
+	Description: appNewDescription,
+	Flags: []cli.Flag{
+		newAppServerFlag,
+		domainFlag,
+		newAppNameFlag,
+		internal.PassFlag,
+		internal.SecretsFlag,
+	},
+	ArgsUsage: "<recipe>",
+	Action:    action,
+}
+
+// getRecipeMeta retrieves the recipe metadata from the recipe catalogue.
+func getRecipeMeta(recipeName string) (catalogue.RecipeMeta, error) {
+	catl, err := catalogue.ReadRecipeCatalogue()
+	if err != nil {
+		return catalogue.RecipeMeta{}, err
+	}
+
+	recipeMeta, ok := catl[recipeName]
+	if !ok {
+		err := fmt.Errorf("recipe '%s' does not exist?", recipeName)
+		return catalogue.RecipeMeta{}, err
+	}
+	if err := recipePkg.EnsureExists(recipeMeta.Name); err != nil {
+		return catalogue.RecipeMeta{}, err
+	}
+
+	return recipeMeta, nil
+}
+
+// ensureDomainFlag checks if the domain flag was used. if not, asks the user for it/
+func ensureDomainFlag() error {
+	if domain == "" {
+		prompt := &survey.Input{
+			Message: "Specify app domain",
+		}
+		if err := survey.AskOne(prompt, &domain); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ensureServerFlag checks if the server flag was used. if not, asks the user for it.
+func ensureServerFlag() error {
+	appFiles, err := config.LoadAppFiles(newAppServer)
+	if err != nil {
+		return err
+	}
+	servers := appFiles.GetServers()
+	if newAppServer == "" {
+		prompt := &survey.Select{
+			Message: "Select app server:",
+			Options: servers,
+		}
+		if err := survey.AskOne(prompt, &newAppServer); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ensureServerFlag checks if the AppName flag was used. if not, asks the user for it.
+func ensureAppNameFlag() error {
+	if newAppName == "" {
+		prompt := &survey.Input{
+			Message: "Specify app name:",
+			Default: config.SanitiseAppName(domain),
+		}
+		if err := survey.AskOne(prompt, &newAppName); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// createSecrets creates all secrets for a new app.
+func createSecrets(sanitisedAppName string) (secrets, error) {
+	appEnvPath := path.Join(config.ABRA_DIR, "servers", newAppServer, fmt.Sprintf("%s.env", sanitisedAppName))
+	appEnv, err := config.ReadEnv(appEnvPath)
+	if err != nil {
+		return nil, err
+	}
+
+	secretEnvVars := secret.ReadSecretEnvVars(appEnv)
+	secrets, err := secret.GenerateSecrets(secretEnvVars, sanitisedAppName, newAppServer)
+	if err != nil {
+		return nil, err
+	}
+
+	if internal.Pass {
+		for secretName := range secrets {
+			secretValue := secrets[secretName]
+			if err := secret.PassInsertSecret(secretValue, secretName, sanitisedAppName, newAppServer); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return secrets, nil
+}
+
+// action is the main command-line action for this package
+func action(c *cli.Context) error {
+	recipe := internal.ValidateRecipe(c)
+
+	if err := config.EnsureAbraDirExists(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	recipeMeta, err := getRecipeMeta(recipe.Name)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	latestVersion := recipeMeta.LatestVersion()
+	if err := recipePkg.EnsureVersion(recipe.Name, latestVersion); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ensureServerFlag(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ensureDomainFlag(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := ensureAppNameFlag(); err != nil {
+		logrus.Fatal(err)
+	}
+
+	sanitisedAppName := config.SanitiseAppName(newAppName)
+	if len(sanitisedAppName) > 45 {
+		logrus.Fatalf("'%s' cannot be longer than 45 characters", sanitisedAppName)
+	}
+
+	if err := config.CopyAppEnvSample(recipe.Name, newAppName, newAppServer); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if internal.Secrets {
+		secrets, err := createSecrets(sanitisedAppName)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretCols := []string{"Name", "Value"}
+		secretTable := abraFormatter.CreateTable(secretCols)
+		for secret := range secrets {
+			secretTable.Append([]string{secret, secrets[secret]})
+		}
+		defer secretTable.Render()
+	}
+
+	tableCol := []string{"Name", "Domain", "Type", "Server"}
+	table := abraFormatter.CreateTable(tableCol)
+	table.Append([]string{sanitisedAppName, domain, recipe.Name, newAppServer})
+	defer table.Render()
+
+	return nil
+}

cli/app/ps.go

@@ -0,0 +1,57 @@
+package app
+
+import (
+	"context"
+	"strings"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"github.com/docker/cli/cli/command/formatter"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appPsCommand = &cli.Command{
+	Name:    "ps",
+	Usage:   "Check app status",
+	Aliases: []string{"p"},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		ctx := context.Background()
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters := filters.NewArgs()
+		filters.Add("name", app.StackName())
+
+		containers, err := cl.ContainerList(ctx, types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableCol := []string{"ID", "Image", "Command", "Created", "Status", "Ports", "Names"}
+		table := abraFormatter.CreateTable(tableCol)
+
+		for _, container := range containers {
+			tableRow := []string{
+				abraFormatter.ShortenID(container.ID),
+				abraFormatter.RemoveSha(container.Image),
+				abraFormatter.Truncate(container.Command),
+				abraFormatter.HumanDuration(container.Created),
+				container.Status,
+				formatter.DisplayablePorts(container.Ports),
+				strings.Join(container.Names, ","),
+			}
+			table.Append(tableRow)
+		}
+
+		table.Render()
+		return nil
+	},
+}

cli/app/remove.go

@@ -0,0 +1,158 @@
+package app
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// Volumes stores the variable from VolumesFlag
+var Volumes bool
+
+// VolumesFlag is used to specify if volumes should be deleted when deleting an app
+var VolumesFlag = &cli.BoolFlag{
+	Name:        "volumes",
+	Value:       false,
+	Destination: &Volumes,
+}
+
+var appRemoveCommand = &cli.Command{
+	Name:    "remove",
+	Usage:   "Remove an already undeployed app",
+	Aliases: []string{"rm"},
+	Flags: []cli.Flag{
+		VolumesFlag,
+		internal.ForceFlag,
+	},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if !internal.Force {
+			response := false
+			prompt := &survey.Confirm{
+				Message: fmt.Sprintf("About to delete %s, are you sure", app.Name),
+			}
+			if err := survey.AskOne(prompt, &response); err != nil {
+				logrus.Fatal(err)
+			}
+			if !response {
+				logrus.Fatal("User aborted app removal")
+			}
+		}
+
+		appFiles, err := config.LoadAppFiles("")
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		ctx := context.Background()
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !internal.Force {
+			// FIXME: only query for app we are interested in, not all of them!
+			statuses, err := config.GetAppStatuses(appFiles)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if statuses[app.Name] == "deployed" {
+				logrus.Fatalf("'%s' is still deployed. Run \"abra app %s undeploy\" or pass --force", app.Name, app.Name)
+			}
+		}
+
+		fs := filters.NewArgs()
+		fs.Add("name", app.Name)
+		secretList, err := cl.SecretList(ctx, types.SecretListOptions{Filters: fs})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secrets := make(map[string]string)
+		var secretNames []string
+
+		for _, cont := range secretList {
+			secrets[cont.Spec.Annotations.Name] = cont.ID // we have to map the names to ID's
+			secretNames = append(secretNames, cont.Spec.Annotations.Name)
+		}
+
+		if len(secrets) > 0 {
+			var secretNamesToRemove []string
+			if !internal.Force {
+				secretsPrompt := &survey.MultiSelect{
+					Message: "Which secrets do you want to remove?",
+					Options: secretNames,
+					Default: secretNames,
+				}
+				if err := survey.AskOne(secretsPrompt, &secretNamesToRemove); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+
+			for _, name := range secretNamesToRemove {
+				err := cl.SecretRemove(ctx, secrets[name])
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				logrus.Info(fmt.Sprintf("Secret: %s removed", name))
+			}
+		} else {
+			logrus.Info("No secrets to remove")
+		}
+
+		volumeListOKBody, err := cl.VolumeList(ctx, fs)
+		volumeList := volumeListOKBody.Volumes
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		var vols []string
+		for _, vol := range volumeList {
+			vols = append(vols, vol.Name)
+		}
+
+		if len(vols) > 0 {
+			if Volumes {
+				var removeVols []string
+				if !internal.Force {
+					volumesPrompt := &survey.MultiSelect{
+						Message: "Which volumes do you want to remove?",
+						Options: vols,
+						Default: vols,
+					}
+					if err := survey.AskOne(volumesPrompt, &removeVols); err != nil {
+						logrus.Fatal(err)
+					}
+				}
+				for _, vol := range removeVols {
+					err := cl.VolumeRemove(ctx, vol, internal.Force) // last argument is for force removing
+					if err != nil {
+						logrus.Fatal(err)
+					}
+					logrus.Info(fmt.Sprintf("Volume %s removed", vol))
+				}
+			} else {
+				logrus.Info("No volumes were removed")
+			}
+		} else {
+			logrus.Info("No volumes to remove")
+		}
+
+		err = os.Remove(app.Path)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		logrus.Info(fmt.Sprintf("File: %s removed", app.Path))
+
+		return nil
+	},
+}

cli/app/restore.go

@@ -0,0 +1,82 @@
+package app
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var restoreAllServices bool
+var restoreAllServicesFlag = &cli.BoolFlag{
+	Name:        "all",
+	Value:       false,
+	Destination: &restoreAllServices,
+	Aliases:     []string{"a"},
+	Usage:       "Restore all services",
+}
+
+var appRestoreCommand = &cli.Command{
+	Name:      "restore",
+	Usage:     "Restore an app from a backup",
+	Aliases:   []string{"r"},
+	Flags:     []cli.Flag{restoreAllServicesFlag},
+	ArgsUsage: "<service> [<backup file>]",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Len() > 1 && restoreAllServices {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use <service>/<backup file> and '--all' together"))
+		}
+
+		abraSh := path.Join(config.ABRA_DIR, "apps", app.Type, "abra.sh")
+		if _, err := os.Stat(abraSh); err != nil {
+			if os.IsNotExist(err) {
+				logrus.Fatalf("'%s' does not exist?", abraSh)
+			}
+			logrus.Fatal(err)
+		}
+
+		sourceCmd := fmt.Sprintf("source %s", abraSh)
+		execCmd := "abra_restore"
+		if !restoreAllServices {
+			serviceName := c.Args().Get(1)
+			if serviceName == "" {
+				internal.ShowSubcommandHelpAndError(c, errors.New("no service(s) target provided"))
+			}
+			execCmd = fmt.Sprintf("abra_restore_%s", serviceName)
+		}
+
+		bytes, err := ioutil.ReadFile(abraSh)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if !strings.Contains(string(bytes), execCmd) {
+			logrus.Fatalf("%s doesn't have a '%s' function", app.Type, execCmd)
+		}
+
+		backupFile := c.Args().Get(2)
+		if backupFile != "" {
+			execCmd = fmt.Sprintf("%s %s", execCmd, backupFile)
+		}
+
+		sourceAndExec := fmt.Sprintf("%s; %s", sourceCmd, execCmd)
+		cmd := exec.Command("bash", "-c", sourceAndExec)
+		output, err := cmd.Output()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		fmt.Print(string(output))
+
+		return nil
+	},
+}

cli/app/rollback.go

@@ -0,0 +1,10 @@
+package app
+
+import "github.com/urfave/cli/v2"
+
+var appRollbackCommand = &cli.Command{
+	Name:      "rollback",
+	Usage:     "Roll an app back to a previous version",
+	Aliases:   []string{"b"},
+	ArgsUsage: "[<version>]",
+}

cli/app/run.go

@@ -0,0 +1,99 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/client/container"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var user string
+var userFlag = &cli.StringFlag{
+	Name:        "user",
+	Value:       "",
+	Destination: &user,
+}
+
+var noTTY bool
+var noTTYFlag = &cli.BoolFlag{
+	Name:        "no-tty",
+	Value:       false,
+	Destination: &noTTY,
+}
+
+var appRunCommand = &cli.Command{
+	Name: "run",
+	Flags: []cli.Flag{
+		noTTYFlag,
+		userFlag,
+	},
+	Aliases:   []string{"r"},
+	ArgsUsage: "<service> <args>...",
+	Usage:     "Run a command in a service container",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Len() < 2 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no <service> provided"))
+		}
+
+		ctx := context.Background()
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceName := c.Args().Get(1)
+		filters := filters.NewArgs()
+		filters.Add("name", fmt.Sprintf("%s_%s", app.StackName(), serviceName))
+
+		containers, err := cl.ContainerList(ctx, types.ContainerListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		if len(containers) > 1 {
+			logrus.Fatalf("expected 1 container but got %d", len(containers))
+		}
+
+		cmd := c.Args().Slice()[2:]
+		execCreateOpts := types.ExecConfig{
+			AttachStderr: true,
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          cmd,
+			Detach:       false,
+			Tty:          true,
+		}
+
+		if user != "" {
+			execCreateOpts.User = user
+		}
+		if noTTY {
+			execCreateOpts.Tty = false
+		}
+
+		// FIXME: an absolutely monumental hack to instantiate another command-line
+		// client withing our command-line client so that we pass something down
+		// the tubes that satisfies the necessary interface requirements. We should
+		// refactor our vendored container code to not require all this cruft.  For
+		// now, It Works.
+		dcli, err := command.NewDockerCli()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if err := container.RunExec(dcli, cl, containers[0].ID, &execCreateOpts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/secret.go

@@ -0,0 +1,250 @@
+package app
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/secret"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var allSecrets bool
+var allSecretsFlag = &cli.BoolFlag{
+	Name:        "all",
+	Aliases:     []string{"A"},
+	Value:       false,
+	Destination: &allSecrets,
+	Usage:       "Generate all secrets",
+}
+
+var appSecretGenerateCommand = &cli.Command{
+	Name:      "generate",
+	Aliases:   []string{"g"},
+	Usage:     "Generate secrets",
+	ArgsUsage: "<secret> <version>",
+	Flags:     []cli.Flag{allSecretsFlag, internal.PassFlag},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Len() == 1 && !allSecrets {
+			err := errors.New("missing arguments <secret>/<version> or '--all'")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		if c.Args().Get(1) != "" && allSecrets {
+			err := errors.New("cannot use '<secret> <version>' and '--all' together")
+			internal.ShowSubcommandHelpAndError(c, err)
+		}
+
+		secretsToCreate := make(map[string]string)
+		secretEnvVars := secret.ReadSecretEnvVars(app.Env)
+		if allSecrets {
+			secretsToCreate = secretEnvVars
+		} else {
+			secretName := c.Args().Get(1)
+			secretVersion := c.Args().Get(2)
+			matches := false
+			for sec := range secretEnvVars {
+				parsed := secret.ParseSecretEnvVarName(sec)
+				if secretName == parsed {
+					secretsToCreate[sec] = secretVersion
+				}
+			}
+			if !matches {
+				logrus.Fatalf("'%s' doesn't exist in the env config?", secretName)
+			}
+		}
+
+		secretVals, err := secret.GenerateSecrets(secretsToCreate, app.StackName(), app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if internal.Pass {
+			for name, data := range secretVals {
+				if err := secret.PassInsertSecret(data, name, app.StackName(), app.Server); err != nil {
+					logrus.Fatal(err)
+				}
+			}
+		}
+
+		if len(secretVals) == 0 {
+			logrus.Warn("no secrets generated")
+			os.Exit(1)
+		}
+
+		tableCol := []string{"Name", "Value"}
+		table := abraFormatter.CreateTable(tableCol)
+		for name, val := range secretVals {
+			table.Append([]string{name, val})
+		}
+		table.Render()
+		logrus.Warn("these secrets are not shown again, please take note of them *now*")
+
+		return nil
+	},
+}
+
+var appSecretInsertCommand = &cli.Command{
+	Name:      "insert",
+	Aliases:   []string{"i"},
+	Usage:     "Insert secret",
+	Flags:     []cli.Flag{internal.PassFlag},
+	ArgsUsage: "<secret> <version> <data>",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Len() != 4 {
+			internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
+		}
+
+		name := c.Args().Get(1)
+		version := c.Args().Get(2)
+		data := c.Args().Get(3)
+
+		secretName := fmt.Sprintf("%s_%s_%s", app.StackName(), name, version)
+		if err := client.StoreSecret(secretName, data, app.Server); err != nil {
+			logrus.Fatal(err)
+		}
+
+		if internal.Pass {
+			if err := secret.PassInsertSecret(data, name, app.StackName(), app.Server); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}
+
+var appSecretRmCommand = &cli.Command{
+	Name:      "remove",
+	Usage:     "Remove a secret",
+	Aliases:   []string{"rm"},
+	Flags:     []cli.Flag{allSecretsFlag, internal.PassFlag},
+	ArgsUsage: "<secret>",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		if c.Args().Get(1) != "" && allSecrets {
+			internal.ShowSubcommandHelpAndError(c, errors.New("cannot use '<secret>' and '--all' together"))
+		}
+
+		if c.Args().Get(1) == "" && !allSecrets {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no secret(s) specified?"))
+		}
+
+		ctx := context.Background()
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters := filters.NewArgs()
+		filters.Add("name", app.StackName())
+		secretList, err := cl.SecretList(ctx, types.SecretListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		secretToRm := c.Args().Get(1)
+		for _, cont := range secretList {
+			secretName := cont.Spec.Annotations.Name
+			parsed := secret.ParseGeneratedSecretName(secretName, app)
+			if allSecrets {
+				if err := cl.SecretRemove(ctx, secretName); err != nil {
+					logrus.Fatal(err)
+				}
+				if internal.Pass {
+					if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
+						logrus.Fatal(err)
+					}
+				}
+			} else {
+				if parsed == secretToRm {
+					if err := cl.SecretRemove(ctx, secretName); err != nil {
+						logrus.Fatal(err)
+					}
+					if internal.Pass {
+						if err := secret.PassRmSecret(parsed, app.StackName(), app.Server); err != nil {
+							logrus.Fatal(err)
+						}
+					}
+				}
+			}
+		}
+
+		return nil
+	},
+}
+
+var appSecretLsCommand = &cli.Command{
+	Name:    "list",
+	Usage:   "List all secrets",
+	Aliases: []string{"ls"},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+		secrets := secret.ReadSecretEnvVars(app.Env)
+
+		tableCol := []string{"Name", "Version", "Generated Name", "Created On Server"}
+		table := abraFormatter.CreateTable(tableCol)
+
+		ctx := context.Background()
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		filters := filters.NewArgs()
+		filters.Add("name", app.StackName())
+		secretList, err := cl.SecretList(ctx, types.SecretListOptions{Filters: filters})
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		remoteSecretNames := make(map[string]bool)
+		for _, cont := range secretList {
+			remoteSecretNames[cont.Spec.Annotations.Name] = true
+		}
+
+		for sec := range secrets {
+			createdRemote := false
+			secretName := secret.ParseSecretEnvVarName(sec)
+			secVal, err := secret.ParseSecretEnvVarValue(secrets[sec])
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, secVal.Version)
+			if _, ok := remoteSecretNames[secretRemoteName]; ok {
+				createdRemote = true
+			}
+			tableRow := []string{secretName, secVal.Version, secretRemoteName, strconv.FormatBool(createdRemote)}
+			table.Append(tableRow)
+		}
+
+		table.Render()
+		return nil
+	},
+}
+
+var appSecretCommand = &cli.Command{
+	Name:      "secret",
+	Aliases:   []string{"s"},
+	Usage:     "Manage app secrets",
+	ArgsUsage: "<command>",
+	Subcommands: []*cli.Command{
+		appSecretGenerateCommand,
+		appSecretInsertCommand,
+		appSecretRmCommand,
+		appSecretLsCommand,
+	},
+}

cli/app/undeploy.go

@@ -0,0 +1,38 @@
+package app
+
+import (
+	"context"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	stack "coopcloud.tech/abra/pkg/client/stack"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appUndeployCommand = &cli.Command{
+	Name:    "undeploy",
+	Aliases: []string{"u"},
+	Usage:   "Undeploy an app",
+	Description: `
+This does not destroy any of the application data. However, you should remain
+vigilant, as your swarm installation will consider any previously attached
+volumes as eligiblef or pruning once undeployed.
+`,
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		ctx := context.Background()
+		cl, err := client.New(app.Server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		rmOpts := stack.Remove{Namespaces: []string{app.StackName()}}
+		if err := stack.RunRemove(ctx, cl, rmOpts); err != nil {
+			logrus.Fatal(err)
+		}
+
+		return nil
+	},
+}

cli/app/version.go

@@ -0,0 +1,107 @@
+package app
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client/stack"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// getImagePath returns the image name
+func getImagePath(image string) (string, error) {
+	img, err := reference.ParseNormalizedNamed(image)
+	if err != nil {
+		return "", err
+	}
+	path := reference.Path(img)
+	if strings.Contains(path, "library") {
+		path = strings.Split(path, "/")[1]
+	}
+	return path, nil
+}
+
+// parseVersionLabel parses a $STACK_NAME_$SERVICE_NAME service label
+func parseServiceName(label string) string {
+	idx := strings.LastIndex(label, "_")
+	return label[idx+1:]
+}
+
+// parseVersionLabel parses a $VERSION-$DIGEST service label
+func parseVersionLabel(label string) (string, string) {
+	// versions may look like v4.2-abcd or v4.2-alpine-abcd
+	idx := strings.LastIndex(label, "-")
+	return label[:idx], label[idx+1:]
+}
+
+var appVersionCommand = &cli.Command{
+	Name:    "version",
+	Aliases: []string{"v"},
+	Usage:   "Show version of all services in app",
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		composeFiles, err := config.GetAppComposeFiles(app.Type, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		opts := stack.Deploy{Composefiles: composeFiles}
+		compose, err := config.GetAppComposeConfig(app.Type, opts, app.Env)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		ch := make(chan stack.StackStatus, len(compose.Services))
+		for _, service := range compose.Services {
+			label := fmt.Sprintf("coop-cloud.%s.%s.version", app.StackName(), service.Name)
+			go func(s string, l string) {
+				ch <- stack.GetDeployedServicesByLabel(s, l)
+			}(app.Server, label)
+		}
+
+		tableCol := []string{"Name", "Image", "Version", "Digest"}
+		table := abraFormatter.CreateTable(tableCol)
+
+		statuses := make(map[string]stack.StackStatus)
+		for range compose.Services {
+			status := <-ch
+			if len(status.Services) > 0 {
+				serviceName := parseServiceName(status.Services[0].Spec.Name)
+				statuses[serviceName] = status
+			}
+		}
+
+		sort.SliceStable(compose.Services, func(i, j int) bool {
+			return compose.Services[i].Name < compose.Services[j].Name
+		})
+
+		for _, service := range compose.Services {
+			if status, ok := statuses[service.Name]; ok {
+				statusService := status.Services[0]
+				label := fmt.Sprintf("coop-cloud.%s.%s.version", app.StackName(), service.Name)
+				version, digest := parseVersionLabel(statusService.Spec.Labels[label])
+				image, err := getImagePath(statusService.Spec.Labels["com.docker.stack.image"])
+				if err != nil {
+					logrus.Fatal(err)
+				}
+				table.Append([]string{service.Name, image, version, digest})
+				continue
+			}
+
+			image, err := getImagePath(service.Image)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			table.Append([]string{service.Name, image, "?", "?"})
+		}
+
+		table.Render()
+		return nil
+	},
+}

cli/app/volume.go

@@ -0,0 +1,95 @@
+package app
+
+import (
+	"context"
+
+	abraFormatter "coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var appVolumeListCommand = &cli.Command{
+	Name:    "list",
+	Usage:   "list volumes associated with an app",
+	Aliases: []string{"ls"},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		ctx := context.Background()
+		volumeList, err := client.GetVolumes(ctx, app.Server, app.Name)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		table := abraFormatter.CreateTable([]string{"DRIVER", "VOLUME NAME"})
+		var volTable [][]string
+		for _, volume := range volumeList {
+			volRow := []string{
+				volume.Driver,
+				volume.Name,
+			}
+			volTable = append(volTable, volRow)
+		}
+
+		table.AppendBulk(volTable)
+		table.Render()
+
+		return nil
+	},
+}
+
+var appVolumeRemoveCommand = &cli.Command{
+	Name:    "remove",
+	Usage:   "remove volume(s) associated with an app",
+	Aliases: []string{"rm"},
+	Flags: []cli.Flag{
+		internal.ForceFlag,
+	},
+	Action: func(c *cli.Context) error {
+		app := internal.ValidateApp(c)
+
+		ctx := context.Background()
+		volumeList, err := client.GetVolumes(ctx, app.Server, app.Name)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		volumeNames := client.GetVolumeNames(volumeList)
+
+		var volumesToRemove []string
+		if !internal.Force {
+			volumesPrompt := &survey.MultiSelect{
+				Message: "Which volumes do you want to remove?",
+				Options: volumeNames,
+				Default: volumeNames,
+			}
+			if err := survey.AskOne(volumesPrompt, &volumesToRemove); err != nil {
+				logrus.Fatal(err)
+			}
+		} else {
+			volumesToRemove = volumeNames
+		}
+
+		err = client.RemoveVolumes(ctx, app.Server, volumesToRemove, internal.Force)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		logrus.Info("Volumes removed successfully.")
+
+		return nil
+	},
+}
+
+var appVolumeCommand = &cli.Command{
+	Name:      "volume",
+	Aliases:   []string{"v"},
+	Usage:     "Manage app volumes",
+	ArgsUsage: "<command>",
+	Subcommands: []*cli.Command{
+		appVolumeListCommand,
+		appVolumeRemoveCommand,
+	},
+}

cli/cli.go

@@ -0,0 +1,76 @@
+// Package cli provides the interface for the command-line.
+package cli
+
+import (
+	"fmt"
+	"os"
+
+	"coopcloud.tech/abra/cli/app"
+	"coopcloud.tech/abra/cli/recipe"
+	"coopcloud.tech/abra/cli/server"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// Verbose stores the variable from VerboseFlag.
+var Verbose bool
+
+// VerboseFlag turns on/off verbose logging down to the INFO level.
+var VerboseFlag = &cli.BoolFlag{
+	Name:        "verbose",
+	Aliases:     []string{"V"},
+	Value:       false,
+	Destination: &Verbose,
+	Usage:       "Show INFO messages",
+}
+
+// Debug stores the variable from DebugFlag.
+var Debug bool
+
+// DebugFlag turns on/off verbose logging down to the DEBUG level.
+var DebugFlag = &cli.BoolFlag{
+	Name:        "debug",
+	Aliases:     []string{"d"},
+	Value:       false,
+	Destination: &Debug,
+	Usage:       "Show DEBUG messages",
+}
+
+// RunApp runs CLI abra app.
+func RunApp(version, commit string) {
+	app := &cli.App{
+		Name: "abra",
+		Usage: `The Co-op Cloud command-line utility belt 🎩🐇
+
+    ____                           ____ _                 _
+   / ___|___         ___  _ __    / ___| | ___  _   _  __| |
+  | |   / _ \ _____ / _ \| '_ \  | |   | |/ _ \| | | |/ _' |
+  | |__| (_) |_____| (_) | |_) | | |___| | (_) | |_| | (_| |
+   \____\___/       \___/| .__/   \____|_|\___/ \__,_|\__,_|
+                         |_|
+		`,
+		Version: fmt.Sprintf("%s-%s", version, commit[:7]),
+		Commands: []*cli.Command{
+			app.AppCommand,
+			server.ServerCommand,
+			recipe.RecipeCommand,
+			VersionCommand,
+			UpgradeCommand,
+		},
+		Flags: []cli.Flag{
+			VerboseFlag,
+			DebugFlag,
+		},
+		Authors: []*cli.Author{
+			&cli.Author{
+				Name:  "Autonomic Co-op",
+				Email: "helo@autonomic.zone",
+			},
+		},
+	}
+	app.EnableBashCompletion = true
+
+	if err := app.Run(os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}

cli/formatter/formatter.go

@@ -0,0 +1,38 @@
+package formatter
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/docker/cli/cli/command/formatter"
+	"github.com/docker/go-units"
+	"github.com/olekukonko/tablewriter"
+)
+
+func ShortenID(str string) string {
+	return str[:12]
+}
+
+func Truncate(str string) string {
+	return fmt.Sprintf(`"%s"`, formatter.Ellipsis(str, 19))
+}
+
+// RemoveSha remove image sha from a string that are added in some docker outputs
+func RemoveSha(str string) string {
+	return strings.Split(str, "@")[0]
+}
+
+// HumanDuration from docker/cli RunningFor() to be accessible outside of the class
+func HumanDuration(timestamp int64) string {
+	date := time.Unix(timestamp, 0)
+	now := time.Now().UTC()
+	return units.HumanDuration(now.Sub(date)) + " ago"
+}
+
+func CreateTable(columns []string) *tablewriter.Table {
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader(columns)
+	return table
+}

cli/internal/command.go

@@ -0,0 +1,38 @@
+package internal
+
+import (
+	"bufio"
+	"fmt"
+	"os/exec"
+)
+
+func RunCmd(cmd *exec.Cmd) error {
+	r, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	cmd.Stderr = cmd.Stdout
+	done := make(chan struct{})
+	scanner := bufio.NewScanner(r)
+
+	go func() {
+		for scanner.Scan() {
+			line := scanner.Text()
+			fmt.Println(line)
+		}
+		done <- struct{}{}
+	}()
+
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+
+	<-done
+
+	if err := cmd.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}

cli/internal/common.go

@@ -0,0 +1,51 @@
+package internal
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// Secrets stores the variable from SecretsFlag
+var Secrets bool
+
+// SecretsFlag turns on/off automatically generating secrets
+var SecretsFlag = &cli.BoolFlag{
+	Name:        "secrets",
+	Aliases:     []string{"S"},
+	Value:       false,
+	Usage:       "Automatically generate secrets",
+	Destination: &Secrets,
+}
+
+// Pass stores the variable from PassFlag
+var Pass bool
+
+// PassFlag turns on/off storing generated secrets in pass
+var PassFlag = &cli.BoolFlag{
+	Name:        "pass",
+	Aliases:     []string{"P"},
+	Value:       false,
+	Usage:       "Store the generated secrets in a local pass store",
+	Destination: &Pass,
+}
+
+// Context is temp
+var Context string
+
+// ContextFlag is temp
+var ContextFlag = &cli.StringFlag{
+	Name:        "context",
+	Value:       "",
+	Aliases:     []string{"c"},
+	Destination: &Context,
+}
+
+// Force force functionality without asking.
+var Force bool
+
+// ForceFlag turns on/off force functionality.
+var ForceFlag = &cli.BoolFlag{
+	Name:        "force",
+	Value:       false,
+	Aliases:     []string{"f"},
+	Destination: &Force,
+}

cli/internal/errors.go

@@ -0,0 +1,18 @@
+package internal
+
+import (
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// ShowSubcommandHelpAndError exits the program on error, logs the error to the
+// terminal, and shows the help command.
+func ShowSubcommandHelpAndError(c *cli.Context, err interface{}) {
+	if err2 := cli.ShowSubcommandHelp(c); err2 != nil {
+		logrus.Error(err2)
+	}
+	logrus.Error(err)
+	os.Exit(1)
+}

cli/internal/validate.go

@@ -0,0 +1,46 @@
+package internal
+
+import (
+	"errors"
+	"os"
+
+	"coopcloud.tech/abra/pkg/app"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/recipe"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// ValidateRecipe ensures the recipe arg is valid.
+func ValidateRecipe(c *cli.Context) recipe.Recipe {
+	recipeName := c.Args().First()
+
+	if recipeName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no recipe provided"))
+	}
+
+	recipe, err := recipe.Get(recipeName)
+	if err != nil {
+		logrus.Fatal(err)
+		os.Exit(1)
+	}
+
+	return recipe
+}
+
+// ValidateApp ensures the app name arg is valid.
+func ValidateApp(c *cli.Context) config.App {
+	appName := c.Args().First()
+
+	if appName == "" {
+		ShowSubcommandHelpAndError(c, errors.New("no app provided"))
+	}
+
+	app, err := app.Get(appName)
+	if err != nil {
+		logrus.Fatal(err)
+		os.Exit(1)
+	}
+
+	return app
+}

cli/recipe/lint.go

@@ -0,0 +1,93 @@
+package recipe
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/tagcmp"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeLintCommand = &cli.Command{
+	Name:      "lint",
+	Usage:     "Lint a recipe",
+	Aliases:   []string{"l"},
+	ArgsUsage: "<recipe>",
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		expectedVersion := false
+		if recipe.Config.Version == "3.8" {
+			expectedVersion = true
+		}
+
+		envSampleProvided := false
+		envSample := fmt.Sprintf("%s/%s/.env.sample", config.APPS_DIR, recipe.Name)
+		if _, err := os.Stat(envSample); !os.IsNotExist(err) {
+			envSampleProvided = true
+		} else if err != nil {
+			logrus.Fatal(err)
+		}
+
+		serviceNamedApp := false
+		traefikEnabled := false
+		healthChecksForAllServices := true
+		allImagesTagged := true
+		noUnstableTags := true
+		semverLikeTags := true
+		for _, service := range recipe.Config.Services {
+			if service.Name == "app" {
+				serviceNamedApp = true
+			}
+
+			for label := range service.Deploy.Labels {
+				if label == "traefik.enable" {
+					if service.Deploy.Labels[label] == "true" {
+						traefikEnabled = true
+					}
+				}
+			}
+
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			if reference.IsNameOnly(img) {
+				allImagesTagged = false
+			}
+
+			tag := img.(reference.NamedTagged).Tag()
+			if tag == "latest" {
+				noUnstableTags = false
+			}
+
+			if !tagcmp.IsParsable(tag) {
+				semverLikeTags = false
+			}
+
+			if service.HealthCheck == nil {
+				healthChecksForAllServices = false
+			}
+		}
+
+		tableCol := []string{"Rule", "Satisfied"}
+		table := formatter.CreateTable(tableCol)
+		table.Append([]string{"Compose files have the expected version", strconv.FormatBool(expectedVersion)})
+		table.Append([]string{"Environment configuration is provided", strconv.FormatBool(envSampleProvided)})
+		table.Append([]string{"Recipe contains a service named 'app'", strconv.FormatBool(serviceNamedApp)})
+		table.Append([]string{"Traefik routing enabled on at least one service", strconv.FormatBool(traefikEnabled)})
+		table.Append([]string{"All services have a healthcheck enabled", strconv.FormatBool(healthChecksForAllServices)})
+		table.Append([]string{"All images are using a tag", strconv.FormatBool(allImagesTagged)})
+		table.Append([]string{"No usage of unstable 'latest' tags", strconv.FormatBool(noUnstableTags)})
+		table.Append([]string{"All tags are using a semver-like format", strconv.FormatBool(semverLikeTags)})
+		table.Render()
+
+		return nil
+	},
+}

cli/recipe/list.go

@@ -0,0 +1,39 @@
+package recipe
+
+import (
+	"fmt"
+	"sort"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeListCommand = &cli.Command{
+	Name:    "list",
+	Usage:   "List available recipes",
+	Aliases: []string{"ls"},
+	Action: func(c *cli.Context) error {
+		catl, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err.Error())
+		}
+
+		recipes := catl.Flatten()
+		sort.Sort(catalogue.ByRecipeName(recipes))
+
+		tableCol := []string{"Name", "Category", "Status"}
+		table := formatter.CreateTable(tableCol)
+
+		for _, recipe := range recipes {
+			status := fmt.Sprintf("%v", recipe.Features.Status)
+			tableRow := []string{recipe.Name, recipe.Category, status}
+			table.Append(tableRow)
+		}
+
+		table.Render()
+
+		return nil
+	},
+}

cli/recipe/new.go

@@ -0,0 +1,80 @@
+package recipe
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"text/template"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/go-git/go-git/v5"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeNewCommand = &cli.Command{
+	Name:      "new",
+	Usage:     "Create a new recipe",
+	Aliases:   []string{"n"},
+	ArgsUsage: "<recipe>",
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		directory := path.Join(config.APPS_DIR, recipe.Name)
+		if _, err := os.Stat(directory); !os.IsNotExist(err) {
+			logrus.Fatalf("'%s' recipe directory already exists?", directory)
+			return nil
+		}
+
+		url := fmt.Sprintf("%s/example.git", config.REPOS_BASE_URL)
+		_, err := git.PlainClone(directory, false, &git.CloneOptions{URL: url, Tags: git.AllTags})
+		if err != nil {
+			logrus.Fatal(err)
+			return nil
+		}
+
+		gitRepo := path.Join(config.APPS_DIR, recipe.Name, ".git")
+		if err := os.RemoveAll(gitRepo); err != nil {
+			logrus.Fatal(err)
+			return nil
+		}
+
+		toParse := []string{
+			path.Join(config.APPS_DIR, recipe.Name, "README.md"),
+			path.Join(config.APPS_DIR, recipe.Name, ".env.sample"),
+			path.Join(config.APPS_DIR, recipe.Name, ".drone.yml"),
+		}
+		for _, path := range toParse {
+			file, err := os.OpenFile(path, os.O_RDWR, 0755)
+			if err != nil {
+				logrus.Fatal(err)
+				return nil
+			}
+
+			tpl, err := template.ParseFiles(path)
+			if err != nil {
+				logrus.Fatal(err)
+				return nil
+			}
+
+			// TODO: ask for description and probably other things so that the
+			// template repository is more "ready" to go than the current best-guess
+			// mode of templating
+			if err := tpl.Execute(file, struct {
+				Name        string
+				Description string
+			}{recipe.Name, "TODO"}); err != nil {
+				logrus.Fatal(err)
+				return nil
+			}
+		}
+
+		logrus.Infof(
+			"New recipe '%s' created in %s, happy hacking!\n",
+			recipe.Name, path.Join(config.APPS_DIR, recipe.Name),
+		)
+
+		return nil
+	},
+}

cli/recipe/recipe.go

@@ -0,0 +1,26 @@
+package recipe
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// RecipeCommand defines all recipe related sub-commands.
+var RecipeCommand = &cli.Command{
+	Name:      "recipe",
+	Usage:     "Manage recipes",
+	ArgsUsage: "<recipe>",
+	Aliases:   []string{"r"},
+	Description: `
+A recipe is a blueprint for an app. It is a bunch of configuration files which
+describe how to deploy and maintain an app. Recipes are maintained by the Co-op
+Cloud community and you can use Abra to read them and create apps for you.
+`,
+	Subcommands: []*cli.Command{
+		recipeListCommand,
+		recipeVersionCommand,
+		recipeNewCommand,
+		recipeUpgradeCommand,
+		recipeSyncCommand,
+		recipeLintCommand,
+	},
+}

cli/recipe/sync.go

@@ -0,0 +1,62 @@
+package recipe
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeSyncCommand = &cli.Command{
+	Name:    "sync",
+	Usage:   "Generate new recipe labels",
+	Aliases: []string{"s"},
+	Description: `
+This command will generate labels for each service which correspond to the
+following format:
+
+    coop-cloud.${STACK_NAME}.${SERVICE_NAME}.version=${IMAGE_TAG}-${IMAGE_DIGEST}
+
+The <recipe> configuration will be updated on the local file system. These
+labels are consumed by abra in other command invocations and used to determine
+the versioning metadata of up-and-running containers are.
+`,
+	ArgsUsage: "<recipe>",
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		hasAppService := false
+		for _, service := range recipe.Config.Services {
+			if service.Name == "app" {
+				hasAppService = true
+			}
+		}
+
+		if !hasAppService {
+			logrus.Fatal(fmt.Sprintf("No 'app' service defined in '%s', cannot proceed", recipe.Name))
+		}
+
+		for _, service := range recipe.Config.Services {
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			digest, err := client.GetTagDigest(img)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			tag := img.(reference.NamedTagged).Tag()
+			label := fmt.Sprintf("coop-cloud.${STACK_NAME}.%s.version=%s-%s", service.Name, tag, digest)
+			if err := recipe.UpdateLabel(service.Name, label); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}

cli/recipe/upgrade.go

@@ -0,0 +1,131 @@
+package recipe
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/tagcmp"
+	"github.com/AlecAivazis/survey/v2"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeUpgradeCommand = &cli.Command{
+	Name:    "upgrade",
+	Usage:   "Upgrade recipe image tags",
+	Aliases: []string{"u"},
+	Description: `
+This command reads and attempts to parse all image tags within the given
+<recipe> configuration and prompt with more recent tags to upgrade to. It will
+update the relevant compose file tags on the local file system.
+
+Some image tags cannot be parsed because they do not follow some sort of
+semver-like convention. In this case, all possible tags will be listed and it
+is up to the end-user to decide.
+
+This is step 1 of upgrading a recipe. Step 2 is running "abra recipe sync
+<recipe>".
+`,
+	ArgsUsage: "<recipe>",
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		for _, service := range recipe.Config.Services {
+			catlVersions, err := catalogue.VersionsOfService(recipe.Name, service.Name)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			img, err := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			image := reference.Path(img)
+			regVersions, err := client.GetRegistryTags(image)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			if strings.Contains(image, "library") {
+				// ParseNormalizedNamed prepends 'library' to images like nginx:<tag>,
+				// postgres:<tag>, i.e. images which do not have a username in the
+				// first position of the string
+				image = strings.Split(image, "/")[1]
+			}
+
+			semverLikeTag := true
+			if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
+				semverLikeTag = false
+			}
+
+			tag, err := tagcmp.Parse(img.(reference.NamedTagged).Tag())
+			if err != nil && semverLikeTag {
+				logrus.Fatal(err)
+			}
+
+			var compatible []tagcmp.Tag
+			for _, regVersion := range regVersions {
+				other, err := tagcmp.Parse(regVersion.Name)
+				if err != nil {
+					continue // skip tags that cannot be parsed
+				}
+
+				if tag.IsCompatible(other) && tag.IsLessThan(other) && !tag.Equals(other) {
+					compatible = append(compatible, other)
+				}
+			}
+
+			sort.Sort(tagcmp.ByTagDesc(compatible))
+
+			if len(compatible) == 0 && semverLikeTag {
+				logrus.Info(fmt.Sprintf("No new versions available for '%s', '%s' is the latest", image, tag))
+				continue // skip on to the next tag and don't update any compose files
+			}
+
+			var compatibleStrings []string
+			for _, compat := range compatible {
+				skip := false
+				for _, catlVersion := range catlVersions {
+					if compat.String() == catlVersion {
+						skip = true
+					}
+				}
+				if !skip {
+					compatibleStrings = append(compatibleStrings, compat.String())
+				}
+			}
+
+			msg := fmt.Sprintf("Which tag would you like to upgrade to? (service: %s, tag: %s)", service.Name, tag)
+			if !tagcmp.IsParsable(img.(reference.NamedTagged).Tag()) {
+				tag := img.(reference.NamedTagged).Tag()
+				logrus.Warning(fmt.Sprintf("Unable to determine versioning semantics of '%s', listing all tags...", tag))
+				msg = fmt.Sprintf("Which tag would you like to upgrade to? (service: %s, tag: %s)", service.Name, tag)
+				compatibleStrings = []string{}
+				for _, regVersion := range regVersions {
+					compatibleStrings = append(compatibleStrings, regVersion.Name)
+				}
+			}
+
+			var upgradeTag string
+			prompt := &survey.Select{
+				Message: msg,
+				Options: compatibleStrings,
+			}
+			if err := survey.AskOne(prompt, &upgradeTag); err != nil {
+				logrus.Fatal(err)
+			}
+
+			if err := recipe.UpdateTag(image, upgradeTag); err != nil {
+				logrus.Fatal(err)
+			}
+		}
+
+		return nil
+	},
+}

cli/recipe/version.go

@@ -0,0 +1,45 @@
+package recipe
+
+import (
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/catalogue"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var recipeVersionCommand = &cli.Command{
+	Name:      "versions",
+	Usage:     "List recipe versions",
+	Aliases:   []string{"v"},
+	ArgsUsage: "<recipe>",
+	Action: func(c *cli.Context) error {
+		recipe := internal.ValidateRecipe(c)
+
+		catalogue, err := catalogue.ReadRecipeCatalogue()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		recipeMeta, ok := catalogue[recipe.Name]
+		if !ok {
+			logrus.Fatalf("'%s' recipe doesn't exist?", recipe.Name)
+		}
+
+		tableCol := []string{"Version", "Service", "Image", "Digest"}
+		table := formatter.CreateTable(tableCol)
+
+		for _, serviceVersion := range recipeMeta.Versions {
+			for tag, meta := range serviceVersion {
+				for service, serviceMeta := range meta {
+					table.Append([]string{tag, service, serviceMeta.Image, serviceMeta.Digest})
+				}
+			}
+		}
+
+		table.SetAutoMergeCells(true)
+		table.Render()
+
+		return nil
+	},
+}

cli/server/add.go

@@ -0,0 +1,29 @@
+package server
+
+import (
+	"fmt"
+
+	"coopcloud.tech/abra/pkg/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var serverAddCommand = &cli.Command{
+	Name:        "add",
+	Usage:       "Add a new server, reachable on <server>.",
+	Aliases:     []string{"a"},
+	ArgsUsage:   "<server> [<user>] [<port>]",
+	Description: "[<user>], [<port>]    SSH connection details",
+	Action: func(c *cli.Context) error {
+		argLen := c.Args().Len()
+		args := c.Args().Slice()
+		if argLen < 3 {
+			args = append(args, make([]string, 3-argLen)...)
+		}
+		if err := client.CreateContext(args[0], args[1], args[2]); err != nil {
+			logrus.Fatal(err)
+		}
+		fmt.Println(args[0])
+		return nil
+	},
+}

cli/server/init.go

@@ -0,0 +1,79 @@
+package server
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"time"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var serverInitCommand = &cli.Command{
+	Name:      "init",
+	Usage:     "Initialise server for deploying apps",
+	Aliases:   []string{"i"},
+	HideHelp:  true,
+	ArgsUsage: "<server>",
+	Description: `
+Initialise swarm mode on the target <server>.
+
+This initialisation explicitly chooses the "single host swarm" mode which uses
+the default IPv4 address as the advertising address. This can be re-configured
+later for more advanced use cases.
+`,
+	Action: func(c *cli.Context) error {
+		server := c.Args().First()
+		if server == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no server provided"))
+		}
+
+		cl, err := client.New(server)
+		if err != nil {
+			return err
+		}
+
+		resolver := &net.Resolver{
+			PreferGo: false,
+			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
+				d := net.Dialer{
+					Timeout: time.Millisecond * time.Duration(10000),
+				}
+				// comrade librehosters DNS resolver https://snopyta.org/service/dns/
+				return d.DialContext(ctx, "udp", "95.216.24.230:53")
+			},
+		}
+
+		ctx := context.Background()
+		ips, err := resolver.LookupIPAddr(ctx, server)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		if len(ips) == 0 {
+			return fmt.Errorf("unable to retrieve ipv4 address for %s", server)
+		}
+		ipv4 := ips[0].IP.To4().String()
+
+		initReq := swarm.InitRequest{
+			ListenAddr:    "0.0.0.0:2377",
+			AdvertiseAddr: ipv4,
+		}
+		if _, err := cl.SwarmInit(ctx, initReq); err != nil {
+			return err
+		}
+
+		netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
+		if _, err := cl.NetworkCreate(ctx, "proxy", netOpts); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}

cli/server/list.go

@@ -0,0 +1,55 @@
+package server
+
+import (
+	"strings"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var serverListCommand = &cli.Command{
+	Name:      "list",
+	Aliases:   []string{"ls"},
+	Usage:     "List locally-defined servers.",
+	ArgsUsage: " ",
+	HideHelp:  true,
+	Action: func(c *cli.Context) error {
+		dockerContextStore := client.NewDefaultDockerContextStore()
+		contexts, err := dockerContextStore.Store.List()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		tableColumns := []string{"Name", "Connection"}
+		table := formatter.CreateTable(tableColumns)
+		defer table.Render()
+
+		serverNames, err := config.ReadServerNames()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		for _, serverName := range serverNames {
+
+			var row []string
+			for _, ctx := range contexts {
+				endpoint, err := client.GetContextEndpoint(ctx)
+				if err != nil && strings.Contains(err.Error(), "does not exist") {
+					// No local context found, we can continue safely
+					continue
+				}
+				if ctx.Name == serverName {
+					row = []string{serverName, endpoint}
+				}
+			}
+			if len(row) == 0 {
+				row = []string{serverName, "UNKNOWN"}
+			}
+			table.Append(row)
+
+		}
+		return nil
+
+	},
+}

cli/server/new.go

@@ -0,0 +1,252 @@
+package server
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"coopcloud.tech/abra/cli/formatter"
+	"coopcloud.tech/abra/cli/internal"
+	"github.com/hetznercloud/hcloud-go/hcloud"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var hetznerCloudType string
+var hetznerCloudImage string
+var hetznerCloudSSHKeys cli.StringSlice
+var hetznerCloudLocation string
+var hetznerCloudAPIToken string
+var serverNewHetznerCloudCommand = &cli.Command{
+	Name:      "hetzner",
+	Usage:     "Create a new Hetzner virtual server",
+	ArgsUsage: "<name>",
+	Description: `
+Create a new Hetzner virtual server.
+
+This command uses the uses the Hetzner Cloud API bindings to send a server
+creation request. You must already have a Hetzner Cloud account and an account
+API token before using this command.
+
+Your token can be loaded from the environment using the HCLOUD_TOKEN
+environment variable or otherwise passing the "--env/-e" flag.
+`,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:        "type",
+			Aliases:     []string{"t"},
+			Usage:       "Server type",
+			Destination: &hetznerCloudType,
+			Value:       "cx11",
+		},
+		&cli.StringFlag{
+			Name:        "image",
+			Aliases:     []string{"i"},
+			Usage:       "Image type",
+			Value:       "debian-10",
+			Destination: &hetznerCloudImage,
+		},
+		&cli.StringSliceFlag{
+			Name:        "ssh-keys",
+			Aliases:     []string{"s"},
+			Usage:       "SSH keys",
+			Destination: &hetznerCloudSSHKeys,
+		},
+		&cli.StringFlag{
+			Name:        "location",
+			Aliases:     []string{"l"},
+			Usage:       "Server location",
+			Value:       "hel1",
+			Destination: &hetznerCloudLocation,
+		},
+		&cli.StringFlag{
+			Name:        "token",
+			Aliases:     []string{"T"},
+			Usage:       "Hetzner Cloud API token",
+			EnvVars:     []string{"HCLOUD_TOKEN"},
+			Destination: &hetznerCloudAPIToken,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		name := c.Args().First()
+		if name == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no name provided"))
+		}
+
+		if hetznerCloudAPIToken == "" {
+			logrus.Fatal("Hetzner Cloud API token is missing, cannot continue")
+		}
+
+		ctx := context.Background()
+		client := hcloud.NewClient(hcloud.WithToken(hetznerCloudAPIToken))
+
+		var sshKeys []*hcloud.SSHKey
+		for _, sshKey := range c.StringSlice("ssh-keys") {
+			sshKey, _, err := client.SSHKey.GetByName(ctx, sshKey)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			sshKeys = append(sshKeys, sshKey)
+		}
+
+		serverOpts := hcloud.ServerCreateOpts{
+			Name:       name,
+			ServerType: &hcloud.ServerType{Name: hetznerCloudType},
+			Image:      &hcloud.Image{Name: hetznerCloudImage},
+			SSHKeys:    sshKeys,
+			Location:   &hcloud.Location{Name: hetznerCloudLocation},
+		}
+		res, _, err := client.Server.Create(ctx, serverOpts)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableColumns := []string{"Name", "IPv4", "Root Password"}
+		table := formatter.CreateTable(tableColumns)
+		if len(sshKeys) > 0 {
+			table.Append([]string{name, res.Server.PublicNet.IPv4.IP.String(), "N/A (using SSH keys)"})
+		} else {
+			table.Append([]string{name, res.Server.PublicNet.IPv4.IP.String(), res.RootPassword})
+		}
+		table.Render()
+
+		return nil
+	},
+}
+
+var capsulInstance string
+var capsulType string
+var capsulImage string
+var capsulSSHKey string
+var capsulAPIToken string
+var serverNewCapsulCommand = &cli.Command{
+	Name:      "capsul",
+	Usage:     "Create a new Capsul virtual server",
+	ArgsUsage: "<name>",
+	Description: `
+Create a new Capsul virtual server.
+
+This command uses the uses the Capsul API bindings of your chosen instance to
+send a server creation request. You must already have an account on your chosen
+Capsul instance before using this command.
+
+Your token can be loaded from the environment using the CAPSUL_TOKEN
+environment variable or otherwise passing the "--env/-e" flag.
+`,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:        "instance",
+			Aliases:     []string{"I"},
+			Usage:       "Capsul instance",
+			Destination: &capsulInstance,
+			Value:       "yolo.servers.coop",
+		},
+		&cli.StringFlag{
+			Name:        "type",
+			Aliases:     []string{"t"},
+			Usage:       "Server type",
+			Value:       "f1-xs",
+			Destination: &capsulType,
+		},
+		&cli.StringFlag{
+			Name:        "image",
+			Aliases:     []string{"i"},
+			Usage:       "Image type",
+			Value:       "debian10",
+			Destination: &capsulImage,
+		},
+		&cli.StringFlag{
+			Name:        "ssh-key",
+			Aliases:     []string{"s"},
+			Usage:       "SSH key",
+			Value:       "",
+			Destination: &capsulSSHKey,
+		},
+		&cli.StringFlag{
+			Name:        "token",
+			Aliases:     []string{"T"},
+			Usage:       "Capsul instance API token",
+			EnvVars:     []string{"CAPSUL_TOKEN"},
+			Destination: &capsulAPIToken,
+		},
+	},
+	Action: func(c *cli.Context) error {
+		name := c.Args().First()
+		if name == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no name provided"))
+		}
+
+		if capsulAPIToken == "" {
+			logrus.Fatal("Capsul API token is missing, cannot continue")
+		}
+
+		// yep, the response time is quite slow, something to fix Capsul side
+		client := &http.Client{Timeout: 20 * time.Second}
+
+		capsulCreateURL := fmt.Sprintf("https://%s/api/capsul/create", capsulInstance)
+		values := map[string]string{
+			"name":      name,
+			"size":      capsulType,
+			"os":        capsulImage,
+			"ssh_key_0": capsulSSHKey,
+		}
+
+		payload, err := json.Marshal(values)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		req, err := http.NewRequest("POST", capsulCreateURL, bytes.NewBuffer(payload))
+		if err != nil {
+			logrus.Fatal(err)
+		}
+
+		req.Header = http.Header{
+			"Content-Type":  []string{"application/json"},
+			"Authorization": []string{capsulAPIToken},
+		}
+
+		res, err := client.Do(req)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+		defer res.Body.Close()
+
+		if res.StatusCode != http.StatusOK {
+			body, err := ioutil.ReadAll(res.Body)
+			if err != nil {
+				panic(err)
+			}
+			logrus.Fatal(string(body))
+		}
+
+		type capsulCreateResponse struct{ ID string }
+		var resp capsulCreateResponse
+		if err := json.NewDecoder(res.Body).Decode(&resp); err != nil {
+			logrus.Fatal(err)
+		}
+
+		tableColumns := []string{"Name", "ID"}
+		table := formatter.CreateTable(tableColumns)
+		table.Append([]string{name, resp.ID})
+		table.Render()
+
+		return nil
+	},
+}
+
+var serverNewCommand = &cli.Command{
+	Name:        "new",
+	Usage:       "Create a new server using a 3rd party provider",
+	Description: "Use a provider plugin to create a new server which can then be used to house a new Co-op Cloud installation.",
+	ArgsUsage:   "<provider>",
+	Subcommands: []*cli.Command{
+		serverNewHetznerCloudCommand,
+		serverNewCapsulCommand,
+	},
+}

cli/server/remove.go

@@ -0,0 +1,31 @@
+package server
+
+import (
+	"errors"
+
+	"coopcloud.tech/abra/cli/internal"
+	"coopcloud.tech/abra/pkg/client"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var serverRemoveCommand = &cli.Command{
+	Name:    "remove",
+	Aliases: []string{"rm"},
+	Usage:   "Remove a server",
+	Description: `
+This does not destroy the actual server. It simply removes it from Abra
+internal bookkeeping so that it is not managed any more.
+`,
+	HideHelp: true,
+	Action: func(c *cli.Context) error {
+		server := c.Args().First()
+		if server == "" {
+			internal.ShowSubcommandHelpAndError(c, errors.New("no server provided"))
+		}
+		if err := client.DeleteContext(server); err != nil {
+			logrus.Fatal(err)
+		}
+		return nil
+	},
+}

cli/server/server.go

@@ -0,0 +1,26 @@
+package server
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// ServerCommand defines the `abra server` command and its subcommands
+var ServerCommand = &cli.Command{
+	Name:    "server",
+	Aliases: []string{"s"},
+	Usage:   "Manage servers",
+	Description: `
+Manage the lifecycle of a server.
+
+These commands support creating new servers using 3rd party integrations,
+initialising existing servers to support Co-op Cloud deployments and managing
+the connections to those servers.
+`,
+	Subcommands: []*cli.Command{
+		serverNewCommand,
+		serverInitCommand,
+		serverAddCommand,
+		serverListCommand,
+		serverRemoveCommand,
+	},
+}

cli/upgrade.go

@@ -0,0 +1,22 @@
+package cli
+
+import (
+	"os/exec"
+
+	"coopcloud.tech/abra/cli/internal"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+// UpgradeCommand upgrades abra in-place.
+var UpgradeCommand = &cli.Command{
+	Name:  "upgrade",
+	Usage: "Upgrade abra",
+	Action: func(c *cli.Context) error {
+		cmd := exec.Command("bash", "-c", "curl -s https://install.abra.coopcloud.tech | bash")
+		if err := internal.RunCmd(cmd); err != nil {
+			logrus.Fatal(err)
+		}
+		return nil
+	},
+}

cli/version.go

@@ -0,0 +1,15 @@
+package cli
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+// VersionCommand prints the version of abra.
+var VersionCommand = &cli.Command{
+	Name:  "version",
+	Usage: "Print version",
+	Action: func(c *cli.Context) error {
+		cli.VersionPrinter(c)
+		return nil
+	},
+}

cmd/abra/main.go

@@ -0,0 +1,24 @@
+// Package main provides the command-line entrypoint.
+package main
+
+import (
+	"coopcloud.tech/abra/cli"
+)
+
+// Version is the current version of abra.
+var Version string
+
+// Commit is the current commit of abra.
+var Commit string
+
+func main() {
+	// If not set in the ld-flags
+	if Version == "" {
+		Version = "dev"
+	}
+	if Commit == "" {
+		Commit = "       "
+	}
+
+	cli.RunApp(Version, Commit)
+}

completion/_abra

@@ -1,52 +0,0 @@
-#compdef abra
-
-_abra () {
-	local context state line curcontext="$curcontext" ret=1
-	_arguments -n : \
-		{-h,--help}'[Help message]' \
-		'1:commands:(app server)' \
-		'*::arguments:->arguments' \
-		&& ret=0
-
-	case $state in
-		(arguments)
-			curcontext="${curcontext%:*:*}:abra-arguments-$words[1]:"
-			case $words[1] in
-				(app)
-					_arguments \
-						'1: :_abra_apps' \
-					&& ret=0
-				;;
-				(server)
-					_arguments \
-						'1:servers:_abra_servers' \
-					&& ret=0
-				;;
-			esac
-		;;
-	esac
-
-	return ret
-}
-
-_abra_servers() {
-	_path_files -/W $HOME/.abra/servers
-}
-
-_abra_apps()
-{
-	local newapps apps=($HOME/.abra/servers/*/*.env)
-	typeset -a apps
-	newapps=()
-	for app in $apps; do
-		newapps+=($(_abra_basename "${app}"))
-	done
-	_describe -t apps 'app' newapps
-}
-
-_abra_basename()
-{
-    printf -- "${1##*/}"
-}
-
-_abra "$@"

completion/abra.bash

@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-
-_abra_basename()
-{
-    echo "${1##*/}"
-}
-
-_abra_servers()
-{
-    # FIXME 3wc: copied from abra/get_servers()
-    shopt -s nullglob dotglob
-    local SERVERS=(~/.abra/servers/*)
-    shopt -u nullglob dotglob
-
-    for SERVER in "${SERVERS[@]}"; do
-        _abra_basename "${SERVER}"
-    done
-}
-
-_abra_complete_servers()
-{
-    mapfile -t COMPREPLY < <(compgen -W "$(_abra_servers)" -- "$1")
-}
-
-_abra_apps()
-{
-    shopt -s nullglob dotglob
-    local APPS=(~/.abra/servers/*/*.env)
-    shopt -u nullglob dotglob
-
-    for APP in "${APPS[@]}"; do
-        _abra_basename "${APP%.env}"
-    done
-}
-
-_abra_complete_apps()
-{
-    mapfile -t COMPREPLY < <(compgen -W "$(_abra_apps)" -- "$1")
-}
-
-_abra_complete()
-{
-    compopt +o default +o nospace
-    COMPREPLY=()
-
-    local -r cmds='
-        app
-        server
-    '
-    local -r short_opts='-e      -h     -s      -v'
-    local -r long_opts='--env --help --stack --version'
-
-    # Scan through the command line and find the abra command
-    # (if present), as well as its expected position.
-    local cmd
-    local cmd_index=1 # Expected index of the command token.
-    local i
-    for (( i = 1; i < ${#COMP_WORDS[@]}; i++ )); do
-        local word="${COMP_WORDS[i]}"
-        case "$word" in
-            -*)
-                ((cmd_index++))
-                ;;
-            *)
-                cmd="$word"
-                break
-                ;;
-        esac
-    done
-
-    local cur="${COMP_WORDS[COMP_CWORD]}"
-
-    if (( COMP_CWORD < cmd_index )); then
-        # Offer option completions.
-        case "$cur" in
-            --*)
-                mapfile -t COMPREPLY < <(compgen -W "$long_opts" -- "$cur")
-                ;;
-            -*)
-                mapfile -t COMPREPLY < <(compgen -W "$short_opts" -- "$cur")
-                ;;
-            *)
-                # Skip completion; we should never get here.
-                ;;
-        esac
-    elif (( COMP_CWORD == cmd_index )); then
-        # Offer command name completions.
-        mapfile -t COMPREPLY < <(compgen -W "$cmds" -- "$cur")
-    else
-        # Offer command argument completions.
-        case "$cmd" in
-            server)
-                # Offer exactly one server name completion.
-                if (( COMP_CWORD == cmd_index + 1 )); then
-                    _abra_complete_servers "$cur"
-                fi
-                ;;
-            app)
-                # Offer exactly one app completion.
-                if (( COMP_CWORD == cmd_index + 1 )); then
-                    _abra_complete_apps "$cur"
-                fi
-                ;;
-            #help)
-            #    # Offer exactly one command name completion.
-            #    if (( COMP_CWORD == cmd_index + 1 )); then
-            #        COMPREPLY=($(compgen -W "$cmds" -- "$cur"))
-            #    fi
-            #    ;;
-            *)
-                # Unknown command or unknowable argument.
-                ;;
-        esac
-    fi
-}
-
-complete -o default -F _abra_complete abra

deploy/abra-apps.cloud.autonomic.zone/compose.yml

@@ -1,38 +0,0 @@
----
-version: "3.8"
-
-services:
-  app:
-    image: "nginx:stable"
-    configs:
-      - source: abra_conf
-        target: /etc/nginx/conf.d/abra.conf
-      - source: abra_apps_json
-        target: /var/www/abra-apps/abra-apps.json
-    volumes:
-      - "public:/var/www/abra-apps"
-    networks:
-      - proxy
-    deploy:
-      update_config:
-        failure_action: rollback
-        order: start-first
-      labels:
-        - "traefik.enable=true"
-        - "traefik.http.services.abra-apps.loadbalancer.server.port=80"
-        - "traefik.http.routers.abra-apps.rule=Host(`abra-apps.cloud.autonomic.zone`)"
-        - "traefik.http.routers.abra-apps.entrypoints=web-secure"
-        - "traefik.http.routers.abra-apps.tls.certresolver=production"
-
-configs:
-  abra_apps_json:
-    file: abra-apps.json
-  abra_conf:
-    file: nginx.conf
-
-networks:
-  proxy:
-    external: true
-
-volumes:
-  public:

deploy/abra-apps.cloud.autonomic.zone/nginx.conf

@@ -1,10 +0,0 @@
-server {
-  listen         80 default_server;
-  server_name    abra-apps.cloud.autonomic.zone;
-
-  location / {
-    root /var/www/abra-apps;
-    add_header Content-Type application/json;
-    index abra-apps.json;
-  }
-}

deploy/install.abra.autonomic.zone/installer

@@ -1,35 +0,0 @@
-#!/bin/bash
-
-ABRA_VERSION="0.7.2"
-GIT_URL="https://git.autonomic.zone/coop-cloud/abra"
-ABRA_SRC="$GIT_URL/raw/tag/$ABRA_VERSION/abra"
-ABRA_DIR="${ABRA_DIR:-$HOME/.abra/}"
-
-function install_abra_release {
-  mkdir -p "$HOME/.local/bin"
-  curl "$ABRA_SRC" > "$HOME/.local/bin/abra"
-  chmod +x "$HOME/.local/bin/abra"
-  echo "abra installed to $HOME/.local/bin/abra"
-}
-
-function install_abra_dev {
-  mkdir -p "$ABRA_DIR/"
-  if [[ ! -d "$ABRA_DIR/src" ]]; then
-    git clone "$GIT_URL" "$ABRA_DIR/src"
-  fi
-  (cd "$ABRA_DIR/src" && git pull origin main && cd - || exit)
-  mkdir -p "$HOME/.local/bin"
-  ln -sf "$ABRA_DIR/src/abra" "$HOME/.local/bin/abra"
-  echo "abra installed to $HOME/.local/bin/abra (development bleeding edge)"
-}
-
-function run_installation {
-  if [ "$1" = "--dev" ]; then
-    install_abra_dev
-  else
-    install_abra_release
-  fi
-}
-
-run_installation "$@"
-exit 0

go.mod

@@ -0,0 +1,94 @@
+module coopcloud.tech/abra
+
+go 1.17
+
+require (
+	coopcloud.tech/tagcmp v0.0.0-20210906102006-2a8edd82d75d
+	github.com/AlecAivazis/survey/v2 v2.3.1
+	github.com/Autonomic-Cooperative/godotenv v1.3.1
+	github.com/docker/cli v20.10.8+incompatible
+	github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/docker v20.10.8+incompatible
+	github.com/docker/go-units v0.4.0
+	github.com/go-git/go-git/v5 v5.4.2
+	github.com/hetznercloud/hcloud-go v1.32.0
+	github.com/moby/sys/signal v0.5.0
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
+	github.com/olekukonko/tablewriter v0.0.5
+	github.com/pkg/errors v0.9.1
+	github.com/schultz-is/passgen v1.0.1
+	github.com/sirupsen/logrus v1.8.1
+	github.com/urfave/cli/v2 v2.3.0
+	gotest.tools/v3 v3.0.3
+)
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.4.17 // indirect
+	github.com/Microsoft/hcsshim v0.8.21 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/containerd/cgroups v1.0.1 // indirect
+	github.com/containerd/containerd v1.5.5 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/docker/docker-credential-helpers v0.6.4 // indirect
+	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-metrics v0.0.1 // indirect
+	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/fvbommel/sortorder v1.0.2 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.0 // indirect
+	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
+	github.com/mattn/go-colorable v0.1.2 // indirect
+	github.com/mattn/go-isatty v0.0.8 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
+	github.com/miekg/pkcs11 v1.0.3 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/moby/sys/mountinfo v0.4.1 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/runc v1.0.2 // indirect
+	github.com/prometheus/client_golang v1.11.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.26.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/spf13/cobra v1.0.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/theupdateframework/notary v0.7.0 // indirect
+	github.com/xanzy/ssh-agent v0.3.0 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	go.opencensus.io v0.22.3 // indirect
+	golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect
+	golang.org/x/net v0.0.0-20210326060303-6b1517762897 // indirect
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
+	golang.org/x/term v0.0.0-20210503060354-a79de5458b56 // indirect
+	golang.org/x/text v0.3.4 // indirect
+	google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a // indirect
+	google.golang.org/grpc v1.33.2 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+)

makefile

@@ -1,58 +0,0 @@
-.PHONY: test shellcheck docopt kcov codecov release-installer
-
-test:
-	@sudo DOCKER_CONTEXT=default docker run \
-		-v $$(pwd):/workdir \
-		--privileged \
-		-d \
-		--name=abra-test-dind \
-		-e DOCKER_TLS_CERTDIR="" \
-		decentral1se/docker-dind-bats-kcov
-	@DOCKER_CONTEXT=default docker exec \
-		-it \
-		abra-test-dind \
-		sh -c "cd /workdir && bats /workdir/tests"
-	@DOCKER_CONTEXT=default docker stop abra-test-dind
-	@DOCKER_CONTEXT=default docker rm abra-test-dind
-
-shellcheck:
-	@docker run \
-		-it \
-		--rm \
-		-v $$(pwd):/workdir \
-		koalaman/shellcheck-alpine \
-		shellcheck /workdir/abra && \
-		shellcheck /workdir/bin/*.sh
-
-docopt:
-	@if [ ! -d ".venv" ]; then \
-		python3 -m venv .venv && \
-		.venv/bin/pip install -U pip setuptools wheel && \
-		.venv/bin/pip install docopt-sh; \
-	fi
-	.venv/bin/docopt.sh abra
-
-kcov:
-	@docker run \
-		-it \
-		--rm \
-		-v $$(pwd):/workdir \
-		kcov/kcov:latest \
-		sh -c "kcov /workdir/coverage /workdir/abra || true"
-
-codecov: SHELL:=/bin/bash
-codecov:
-	@bash <(curl -s https://codecov.io/bash) \
-		-s coverage -t $$(pass show hosts/swarm.autonomic.zone/drone/codecov/token)
-
-release-installer:
-	@DOCKER_CONTEXT=swarm.autonomic.zone \
-		docker stack rm abra-installer-script && \
-		cd deploy/install.abra.autonomic.zone && \
-		DOCKER_CONTEXT=swarm.autonomic.zone docker stack deploy -c compose.yml abra-installer-script
-
-release-abra-apps:
-	@DOCKER_CONTEXT=swarm.autonomic.zone \
-		docker stack rm abra-apps-json && \
-		cd deploy/abra-apps.cloud.autonomic.zone && \
-		DOCKER_CONTEXT=swarm.autonomic.zone docker stack deploy -c compose.yml abra-apps-json

pkg/app/app.go

@@ -0,0 +1,20 @@
+package app
+
+import (
+	"coopcloud.tech/abra/pkg/config"
+)
+
+// Get retrieves an app
+func Get(appName string) (config.App, error) {
+	files, err := config.LoadAppFiles("")
+	if err != nil {
+		return config.App{}, err
+	}
+
+	app, err := config.GetApp(files, appName)
+	if err != nil {
+		return config.App{}, err
+	}
+
+	return app, nil
+}

pkg/catalogue/catalogue.go

@@ -0,0 +1,214 @@
+// Package catalogue provides ways of interacting with recipe catalogues which
+// are JSON data structures which contain meta information about recipes (e.g.
+// what versions of the Nextcloud recipe are available?).
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"coopcloud.tech/abra/pkg/config"
+	"coopcloud.tech/abra/pkg/web"
+)
+
+// RecipeCatalogueURL is the only current recipe catalogue available.
+const RecipeCatalogueURL = "https://apps.coopcloud.tech"
+
+// image represents a recipe container image.
+type image struct {
+	Image  string `json:"image"`
+	Rating string `json:"rating"`
+	Source string `json:"source"`
+	URL    string `json:"url"`
+}
+
+// features represent what top-level features a recipe supports (e.g. does this
+// recipe support backups?).
+type features struct {
+	Backups     string `json:"backups"`
+	Email       string `json:"email"`
+	Healthcheck string `json:"healthcheck"`
+	Image       image  `json:"image"`
+	Status      int    `json:"status"`
+	Tests       string `json:"tests"`
+}
+
+// tag represents a git tag.
+type tag = string
+
+// service represents a service within a recipe.
+type service = string
+
+// serviceMeta represents meta info associated with a service.
+type serviceMeta struct {
+	Digest string `json:"digest"`
+	Image  string `json:"image"`
+	Tag    string `json:"tag"`
+}
+
+// RecipeMeta represents metadata for a recipe in the abra catalogue.
+type RecipeMeta struct {
+	Category      string                            `json:"category"`
+	DefaultBranch string                            `json:"default_branch"`
+	Description   string                            `json:"description"`
+	Features      features                          `json:"features"`
+	Icon          string                            `json:"icon"`
+	Name          string                            `json:"name"`
+	Repository    string                            `json:"repository"`
+	Versions      []map[tag]map[service]serviceMeta `json:"versions"`
+	Website       string                            `json:"website"`
+}
+
+// LatestVersion returns the latest version of a recipe.
+func (r RecipeMeta) LatestVersion() string {
+	var version string
+
+	// apps.json versions are sorted so the last key is latest
+	latest := r.Versions[len(r.Versions)-1]
+
+	for tag := range latest {
+		version = tag
+	}
+
+	return version
+}
+
+// Name represents a recipe name.
+type Name = string
+
+// RecipeCatalogue represents the entire recipe catalogue.
+type RecipeCatalogue map[Name]RecipeMeta
+
+// Flatten converts AppCatalogue to slice
+func (r RecipeCatalogue) Flatten() []RecipeMeta {
+	recipes := make([]RecipeMeta, 0, len(r))
+	for name := range r {
+		recipes = append(recipes, r[name])
+	}
+	return recipes
+}
+
+// ByRecipeName sorts recipes by name.
+type ByRecipeName []RecipeMeta
+
+func (r ByRecipeName) Len() int      { return len(r) }
+func (r ByRecipeName) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
+func (r ByRecipeName) Less(i, j int) bool {
+	return strings.ToLower(r[i].Name) < strings.ToLower(r[j].Name)
+}
+
+// recipeCatalogueFSIsLatest checks whether the recipe catalogue stored locally
+// is up to date.
+func recipeCatalogueFSIsLatest() (bool, error) {
+	httpClient := &http.Client{Timeout: web.Timeout}
+	res, err := httpClient.Head(RecipeCatalogueURL)
+	if err != nil {
+		return false, err
+	}
+
+	lastModified := res.Header["Last-Modified"][0]
+	parsed, err := time.Parse(time.RFC1123, lastModified)
+	if err != nil {
+		return false, err
+	}
+	info, err := os.Stat(config.APPS_JSON)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+
+	localModifiedTime := info.ModTime().Unix()
+	remoteModifiedTime := parsed.Unix()
+
+	if localModifiedTime < remoteModifiedTime {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// ReadRecipeCatalogue reads the recipe catalogue.
+func ReadRecipeCatalogue() (RecipeCatalogue, error) {
+	recipes := make(RecipeCatalogue)
+
+	recipeFSIsLatest, err := recipeCatalogueFSIsLatest()
+	if err != nil {
+		return nil, err
+	}
+
+	if !recipeFSIsLatest {
+		if err := readRecipeCatalogueWeb(&recipes); err != nil {
+			return nil, err
+		}
+		return recipes, nil
+	}
+
+	if err := readRecipeCatalogueFS(&recipes); err != nil {
+		return nil, err
+	}
+
+	return recipes, nil
+}
+
+// readRecipeCatalogueFS reads the catalogue from the file system.
+func readRecipeCatalogueFS(target interface{}) error {
+	recipesJSONFS, err := ioutil.ReadFile(config.APPS_JSON)
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
+		return err
+	}
+	return nil
+}
+
+// readRecipeCatalogueWeb reads the catalogue from the web.
+func readRecipeCatalogueWeb(target interface{}) error {
+	if err := web.ReadJSON(RecipeCatalogueURL, &target); err != nil {
+		return err
+	}
+
+	recipesJSON, err := json.MarshalIndent(target, "", "    ")
+	if err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(config.APPS_JSON, recipesJSON, 0644); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// VersionsOfService lists the version of a service.
+func VersionsOfService(recipe, serviceName string) ([]string, error) {
+	catalogue, err := ReadRecipeCatalogue()
+	if err != nil {
+		return nil, err
+	}
+
+	rec, ok := catalogue[recipe]
+	if !ok {
+		return nil, fmt.Errorf("recipe '%s' does not exist?", recipe)
+	}
+
+	versions := []string{}
+	alreadySeen := make(map[string]bool)
+	for _, serviceVersion := range rec.Versions {
+		for tag := range serviceVersion {
+			if _, ok := alreadySeen[tag]; !ok {
+				alreadySeen[tag] = true
+				versions = append(versions, tag)
+			}
+		}
+	}
+
+	return versions, nil
+}

pkg/client/client.go

@@ -0,0 +1,52 @@
+// Package client provides Docker client initiatialisation functions.
+package client
+
+import (
+	"net/http"
+	"os"
+
+	"github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+// New initiates a new Docker client.
+func New(contextName string) (*client.Client, error) {
+	context, err := GetContext(contextName)
+	if err != nil {
+		return nil, err
+	}
+
+	ctxEndpoint, err := GetContextEndpoint(context)
+	if err != nil {
+		return nil, err
+	}
+
+	helper := newConnectionHelper(ctxEndpoint)
+	httpClient := &http.Client{
+		// No tls, no proxy
+		Transport: &http.Transport{
+			DialContext: helper.Dialer,
+		},
+	}
+
+	var clientOpts []client.Opt
+	clientOpts = append(clientOpts,
+		client.WithHTTPClient(httpClient),
+		client.WithHost(helper.Host),
+		client.WithDialContext(helper.Dialer),
+	)
+
+	version := os.Getenv("DOCKER_API_VERSION")
+	if version != "" {
+		clientOpts = append(clientOpts, client.WithVersion(version))
+	} else {
+		clientOpts = append(clientOpts, client.WithAPIVersionNegotiation())
+	}
+
+	cl, err := client.NewClientWithOpts(clientOpts...)
+	if err != nil {
+		logrus.Fatalf("unable to create Docker client: %s", err)
+	}
+
+	return cl, nil
+}

pkg/client/connection.go

@@ -0,0 +1,45 @@
+package client
+
+import (
+	"github.com/docker/cli/cli/connhelper"
+	"github.com/docker/cli/cli/context/docker"
+	dCliContextStore "github.com/docker/cli/cli/context/store"
+	dClient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+func newConnectionHelper(daemonURL string) *connhelper.ConnectionHelper {
+	helper, err := connhelper.GetConnectionHelper(daemonURL)
+
+	if err != nil {
+		logrus.Fatal(err)
+	}
+	return helper
+}
+
+func getDockerEndpoint(host string) (docker.Endpoint, error) {
+	skipTLSVerify := false
+	ep := docker.Endpoint{
+		EndpointMeta: docker.EndpointMeta{
+			Host:          host,
+			SkipTLSVerify: skipTLSVerify,
+		},
+	}
+	// try to resolve a docker client, validating the configuration
+	opts, err := ep.ClientOpts()
+	if err != nil {
+		return docker.Endpoint{}, err
+	}
+	if _, err := dClient.NewClientWithOpts(opts...); err != nil {
+		return docker.Endpoint{}, err
+	}
+	return ep, nil
+}
+
+func getDockerEndpointMetadataAndTLS(host string) (docker.EndpointMeta, *dCliContextStore.EndpointTLSData, error) {
+	ep, err := getDockerEndpoint(host)
+	if err != nil {
+		return docker.EndpointMeta{}, nil, err
+	}
+	return ep.EndpointMeta, ep.TLSData.ToStoreTLSData(), nil
+}

pkg/client/container/LICENSE

@@ -0,0 +1,191 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2013-2017 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pkg/client/container/README.md

@@ -0,0 +1,7 @@
+# github.com/docker/cli/cli/command/container
+
+Due to this literally just being copy-pasted from the lib, the Apache license
+will be posted in this folder. Small edits to the source code have been to
+function names and parts we don't need deleted.
+
+Same vibe as [../convert](../convert).

pkg/client/container/exec.go

@@ -0,0 +1,130 @@
+package container
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/docker/cli/cli"
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	apiclient "github.com/docker/docker/client"
+	"github.com/sirupsen/logrus"
+)
+
+func RunExec(dockerCli command.Cli, client *apiclient.Client, containerID string, execConfig *types.ExecConfig) error {
+	ctx := context.Background()
+
+	// We need to check the tty _before_ we do the ContainerExecCreate, because
+	// otherwise if we error out we will leak execIDs on the server (and
+	// there's no easy way to clean those up). But also in order to make "not
+	// exist" errors take precedence we do a dummy inspect first.
+	if _, err := client.ContainerInspect(ctx, containerID); err != nil {
+		return err
+	}
+	if !execConfig.Detach {
+		if err := dockerCli.In().CheckTty(execConfig.AttachStdin, execConfig.Tty); err != nil {
+			return err
+		}
+	}
+
+	response, err := client.ContainerExecCreate(ctx, containerID, *execConfig)
+	if err != nil {
+		return err
+	}
+
+	execID := response.ID
+	if execID == "" {
+		return errors.New("exec ID empty")
+	}
+
+	if execConfig.Detach {
+		execStartCheck := types.ExecStartCheck{
+			Detach: execConfig.Detach,
+			Tty:    execConfig.Tty,
+		}
+		return client.ContainerExecStart(ctx, execID, execStartCheck)
+	}
+	return interactiveExec(ctx, dockerCli, client, execConfig, execID)
+}
+
+func interactiveExec(ctx context.Context, dockerCli command.Cli, client *apiclient.Client,
+	execConfig *types.ExecConfig, execID string) error {
+	// Interactive exec requested.
+	var (
+		out, stderr io.Writer
+		in          io.ReadCloser
+	)
+
+	if execConfig.AttachStdin {
+		in = dockerCli.In()
+	}
+	if execConfig.AttachStdout {
+		out = dockerCli.Out()
+	}
+	if execConfig.AttachStderr {
+		if execConfig.Tty {
+			stderr = dockerCli.Out()
+		} else {
+			stderr = dockerCli.Err()
+		}
+	}
+
+	execStartCheck := types.ExecStartCheck{
+		Tty: execConfig.Tty,
+	}
+	resp, err := client.ContainerExecAttach(ctx, execID, execStartCheck)
+	if err != nil {
+		return err
+	}
+	defer resp.Close()
+
+	errCh := make(chan error, 1)
+
+	go func() {
+		defer close(errCh)
+		errCh <- func() error {
+			streamer := hijackedIOStreamer{
+				streams:      dockerCli,
+				inputStream:  in,
+				outputStream: out,
+				errorStream:  stderr,
+				resp:         resp,
+				tty:          execConfig.Tty,
+				detachKeys:   execConfig.DetachKeys,
+			}
+
+			return streamer.stream(ctx)
+		}()
+	}()
+
+	if execConfig.Tty && dockerCli.In().IsTerminal() {
+		if err := MonitorTtySize(ctx, client, dockerCli, execID, true); err != nil {
+			fmt.Fprintln(dockerCli.Err(), "Error monitoring TTY size:", err)
+		}
+	}
+
+	if err := <-errCh; err != nil {
+		logrus.Debugf("Error hijack: %s", err)
+		return err
+	}
+
+	return getExecExitStatus(ctx, client, execID)
+}
+
+func getExecExitStatus(ctx context.Context, client apiclient.ContainerAPIClient, execID string) error {
+	resp, err := client.ContainerExecInspect(ctx, execID)
+	if err != nil {
+		// If we can't connect, then the daemon probably died.
+		if !apiclient.IsErrConnectionFailed(err) {
+			return err
+		}
+		return cli.StatusError{StatusCode: -1}
+	}
+	status := resp.ExitCode
+	if status != 0 {
+		return cli.StatusError{StatusCode: status}
+	}
+	return nil
+}

pkg/client/container/hijack.go

@@ -0,0 +1,208 @@
+package container
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"runtime"
+	"sync"
+
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/moby/term"
+	"github.com/sirupsen/logrus"
+)
+
+// The default escape key sequence: ctrl-p, ctrl-q
+// TODO: This could be moved to `pkg/term`.
+var defaultEscapeKeys = []byte{16, 17}
+
+// A hijackedIOStreamer handles copying input to and output from streams to the
+// connection.
+type hijackedIOStreamer struct {
+	streams      command.Streams
+	inputStream  io.ReadCloser
+	outputStream io.Writer
+	errorStream  io.Writer
+
+	resp types.HijackedResponse
+
+	tty        bool
+	detachKeys string
+}
+
+// stream handles setting up the IO and then begins streaming stdin/stdout
+// to/from the hijacked connection, blocking until it is either done reading
+// output, the user inputs the detach key sequence when in TTY mode, or when
+// the given context is cancelled.
+func (h *hijackedIOStreamer) stream(ctx context.Context) error {
+	restoreInput, err := h.setupInput()
+	if err != nil {
+		return fmt.Errorf("unable to setup input stream: %s", err)
+	}
+
+	defer restoreInput()
+
+	outputDone := h.beginOutputStream(restoreInput)
+	inputDone, detached := h.beginInputStream(restoreInput)
+
+	select {
+	case err := <-outputDone:
+		return err
+	case <-inputDone:
+		// Input stream has closed.
+		if h.outputStream != nil || h.errorStream != nil {
+			// Wait for output to complete streaming.
+			select {
+			case err := <-outputDone:
+				return err
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		}
+		return nil
+	case err := <-detached:
+		// Got a detach key sequence.
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (h *hijackedIOStreamer) setupInput() (restore func(), err error) {
+	if h.inputStream == nil || !h.tty {
+		// No need to setup input TTY.
+		// The restore func is a nop.
+		return func() {}, nil
+	}
+
+	if err := setRawTerminal(h.streams); err != nil {
+		return nil, fmt.Errorf("unable to set IO streams as raw terminal: %s", err)
+	}
+
+	// Use sync.Once so we may call restore multiple times but ensure we
+	// only restore the terminal once.
+	var restoreOnce sync.Once
+	restore = func() {
+		restoreOnce.Do(func() {
+			restoreTerminal(h.streams, h.inputStream)
+		})
+	}
+
+	// Wrap the input to detect detach escape sequence.
+	// Use default escape keys if an invalid sequence is given.
+	escapeKeys := defaultEscapeKeys
+	if h.detachKeys != "" {
+		customEscapeKeys, err := term.ToBytes(h.detachKeys)
+		if err != nil {
+			logrus.Warnf("invalid detach escape keys, using default: %s", err)
+		} else {
+			escapeKeys = customEscapeKeys
+		}
+	}
+
+	h.inputStream = ioutils.NewReadCloserWrapper(term.NewEscapeProxy(h.inputStream, escapeKeys), h.inputStream.Close)
+
+	return restore, nil
+}
+
+func (h *hijackedIOStreamer) beginOutputStream(restoreInput func()) <-chan error {
+	if h.outputStream == nil && h.errorStream == nil {
+		// There is no need to copy output.
+		return nil
+	}
+
+	outputDone := make(chan error)
+	go func() {
+		var err error
+
+		// When TTY is ON, use regular copy
+		if h.outputStream != nil && h.tty {
+			_, err = io.Copy(h.outputStream, h.resp.Reader)
+			// We should restore the terminal as soon as possible
+			// once the connection ends so any following print
+			// messages will be in normal type.
+			restoreInput()
+		} else {
+			_, err = stdcopy.StdCopy(h.outputStream, h.errorStream, h.resp.Reader)
+		}
+
+		logrus.Debug("[hijack] End of stdout")
+
+		if err != nil {
+			logrus.Debugf("Error receiveStdout: %s", err)
+		}
+
+		outputDone <- err
+	}()
+
+	return outputDone
+}
+
+func (h *hijackedIOStreamer) beginInputStream(restoreInput func()) (doneC <-chan struct{}, detachedC <-chan error) {
+	inputDone := make(chan struct{})
+	detached := make(chan error)
+
+	go func() {
+		if h.inputStream != nil {
+			_, err := io.Copy(h.resp.Conn, h.inputStream)
+			// We should restore the terminal as soon as possible
+			// once the connection ends so any following print
+			// messages will be in normal type.
+			restoreInput()
+
+			logrus.Debug("[hijack] End of stdin")
+
+			if _, ok := err.(term.EscapeError); ok {
+				detached <- err
+				return
+			}
+
+			if err != nil {
+				// This error will also occur on the receive
+				// side (from stdout) where it will be
+				// propagated back to the caller.
+				logrus.Debugf("Error sendStdin: %s", err)
+			}
+		}
+
+		if err := h.resp.CloseWrite(); err != nil {
+			logrus.Debugf("Couldn't send EOF: %s", err)
+		}
+
+		close(inputDone)
+	}()
+
+	return inputDone, detached
+}
+
+func setRawTerminal(streams command.Streams) error {
+	if err := streams.In().SetRawTerminal(); err != nil {
+		return err
+	}
+	return streams.Out().SetRawTerminal()
+}
+
+// nolint: unparam
+func restoreTerminal(streams command.Streams, in io.Closer) error {
+	streams.In().RestoreTerminal()
+	streams.Out().RestoreTerminal()
+	// WARNING: DO NOT REMOVE THE OS CHECKS !!!
+	// For some reason this Close call blocks on darwin..
+	// As the client exits right after, simply discard the close
+	// until we find a better solution.
+	//
+	// This can also cause the client on Windows to get stuck in Win32 CloseHandle()
+	// in some cases. See https://github.com/docker/docker/issues/28267#issuecomment-288237442
+	// Tracked internally at Microsoft by VSO #11352156. In the
+	// Windows case, you hit this if you are using the native/v2 console,
+	// not the "legacy" console, and you start the client in a new window. eg
+	// `start docker run --rm -it microsoft/nanoserver cmd /s /c echo foobar`
+	// will hang. Remove start, and it won't repro.
+	if in != nil && runtime.GOOS != "darwin" && runtime.GOOS != "windows" {
+		return in.Close()
+	}
+	return nil
+}

pkg/client/container/tty.go

@@ -0,0 +1,98 @@
+package container
+
+import (
+	"context"
+	"fmt"
+	"os"
+	gosignal "os/signal"
+	"runtime"
+	"time"
+
+	"github.com/docker/cli/cli/command"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+	apiclient "github.com/docker/docker/client"
+	"github.com/moby/sys/signal"
+	"github.com/sirupsen/logrus"
+)
+
+// resizeTtyTo resizes tty to specific height and width
+func resizeTtyTo(ctx context.Context, client client.ContainerAPIClient, id string, height, width uint, isExec bool) error {
+	if height == 0 && width == 0 {
+		return nil
+	}
+
+	options := types.ResizeOptions{
+		Height: height,
+		Width:  width,
+	}
+
+	var err error
+	if isExec {
+		err = client.ContainerExecResize(ctx, id, options)
+	} else {
+		err = client.ContainerResize(ctx, id, options)
+	}
+
+	if err != nil {
+		logrus.Debugf("Error resize: %s\r", err)
+	}
+	return err
+}
+
+// resizeTty is to resize the tty with cli out's tty size
+func resizeTty(ctx context.Context, client *apiclient.Client, cli command.Cli, id string, isExec bool) error {
+	height, width := cli.Out().GetTtySize()
+	return resizeTtyTo(ctx, client, id, height, width, isExec)
+}
+
+// initTtySize is to init the tty's size to the same as the window, if there is an error, it will retry 5 times.
+func initTtySize(ctx context.Context, client *apiclient.Client, cli command.Cli, id string, isExec bool, resizeTtyFunc func(ctx context.Context, client *apiclient.Client, cli command.Cli, id string, isExec bool) error) {
+	rttyFunc := resizeTtyFunc
+	if rttyFunc == nil {
+		rttyFunc = resizeTty
+	}
+	if err := rttyFunc(ctx, client, cli, id, isExec); err != nil {
+		go func() {
+			var err error
+			for retry := 0; retry < 5; retry++ {
+				time.Sleep(10 * time.Millisecond)
+				if err = rttyFunc(ctx, client, cli, id, isExec); err == nil {
+					break
+				}
+			}
+			if err != nil {
+				fmt.Fprintln(cli.Err(), "failed to resize tty, using default size")
+			}
+		}()
+	}
+}
+
+// MonitorTtySize updates the container tty size when the terminal tty changes size
+func MonitorTtySize(ctx context.Context, client *apiclient.Client, cli command.Cli, id string, isExec bool) error {
+	initTtySize(ctx, client, cli, id, isExec, resizeTty)
+	if runtime.GOOS == "windows" {
+		go func() {
+			prevH, prevW := cli.Out().GetTtySize()
+			for {
+				time.Sleep(time.Millisecond * 250)
+				h, w := cli.Out().GetTtySize()
+
+				if prevW != w || prevH != h {
+					resizeTty(ctx, client, cli, id, isExec)
+				}
+				prevH = h
+				prevW = w
+			}
+		}()
+	} else {
+		sigchan := make(chan os.Signal, 1)
+		gosignal.Notify(sigchan, signal.SIGWINCH)
+		go func() {
+			for range sigchan {
+				resizeTty(ctx, client, cli, id, isExec)
+			}
+		}()
+	}
+	return nil
+}

pkg/client/context.go

@@ -0,0 +1,119 @@
+package client
+
+import (
+	"errors"
+	"fmt"
+
+	command "github.com/docker/cli/cli/command"
+	dConfig "github.com/docker/cli/cli/config"
+	context "github.com/docker/cli/cli/context"
+	"github.com/docker/cli/cli/context/docker"
+	contextStore "github.com/docker/cli/cli/context/store"
+	"github.com/moby/term"
+)
+
+type Context = contextStore.Metadata
+
+func CreateContext(contextName string, user string, port string) error {
+	host := contextName
+	if user != "" {
+		host = fmt.Sprintf("%s@%s", user, host)
+	}
+	if port != "" {
+		host = fmt.Sprintf("%s:%s", host, port)
+	}
+	host = fmt.Sprintf("ssh://%s", host)
+	if err := createContext(contextName, host); err != nil {
+		return err
+	}
+	return nil
+}
+
+// createContext interacts with Docker Context to create a Docker context config
+func createContext(name string, host string) error {
+	s := NewDefaultDockerContextStore()
+	contextMetadata := contextStore.Metadata{
+		Endpoints: make(map[string]interface{}),
+		Name:      name,
+	}
+	contextTLSData := contextStore.ContextTLSData{
+		Endpoints: make(map[string]contextStore.EndpointTLSData),
+	}
+	dockerEP, dockerTLS, err := getDockerEndpointMetadataAndTLS(host)
+	if err != nil {
+		return err
+	}
+	contextMetadata.Endpoints[docker.DockerEndpoint] = dockerEP
+	if dockerTLS != nil {
+		contextTLSData.Endpoints[docker.DockerEndpoint] = *dockerTLS
+	}
+
+	if err := s.CreateOrUpdate(contextMetadata); err != nil {
+		return err
+	}
+	if err := s.ResetTLSMaterial(name, &contextTLSData); err != nil {
+		return err
+	}
+	return nil
+}
+
+func DeleteContext(name string) error {
+	if name == "default" {
+		return errors.New("context 'default' cannot be removed")
+	}
+	if _, err := GetContext(name); err != nil {
+		return err
+	}
+
+	// remove any context that might be loaded
+	// TODO: Check if the context we are removing is the active one rather than doing it all the time
+	cfg := dConfig.LoadDefaultConfigFile(nil)
+	cfg.CurrentContext = ""
+	if err := cfg.Save(); err != nil {
+		return err
+	}
+
+	return NewDefaultDockerContextStore().Remove(name)
+}
+
+func GetContext(contextName string) (contextStore.Metadata, error) {
+	ctx, err := NewDefaultDockerContextStore().GetMetadata(contextName)
+	if err != nil {
+		return contextStore.Metadata{}, err
+	}
+	return ctx, nil
+}
+
+func GetContextEndpoint(ctx contextStore.Metadata) (string, error) {
+	// safe to use docker key hardcoded since abra doesn't use k8s... yet...
+	endpointmeta, ok := ctx.Endpoints["docker"].(context.EndpointMetaBase)
+	if !ok {
+		err := errors.New("context lacks Docker endpoint")
+		return "", err
+	}
+	return endpointmeta.Host, nil
+}
+
+func newContextStore(dir string, config contextStore.Config) contextStore.Store {
+	return contextStore.New(dir, config)
+}
+
+func NewDefaultDockerContextStore() *command.ContextStoreWithDefault {
+	// Grabbing the stderr from Docker commands
+	// Much easier to fit this into the code we are using to replicate docker cli commands
+	_, _, stderr := term.StdStreams()
+	// TODO: Look into custom docker configs in case users want that
+	dockerConfig := dConfig.LoadDefaultConfigFile(stderr)
+	contextDir := dConfig.ContextStoreDir()
+	storeConfig := command.DefaultContextStoreConfig()
+	store := newContextStore(contextDir, storeConfig)
+
+	dockerContextStore := &command.ContextStoreWithDefault{
+		Store: store,
+		Resolver: func() (*command.DefaultContext, error) {
+			// nil for the Opts because it works without it and its a cli thing
+			return command.ResolveDefaultContext(nil, dockerConfig, storeConfig, stderr)
+		},
+	}
+	return dockerContextStore
+}

pkg/client/context_test.go

@@ -0,0 +1,52 @@
+package client_test
+
+import (
+	"testing"
+
+	"coopcloud.tech/abra/pkg/client"
+	dContext "github.com/docker/cli/cli/context"
+	dCliContextStore "github.com/docker/cli/cli/context/store"
+)
+
+type TestContext struct {
+	context           dCliContextStore.Metadata
+	expected_endpoint string
+}
+
+func dockerContext(host, key string) TestContext {
+	dockerContext := dCliContextStore.Metadata{
+		Name:     "foo",
+		Metadata: nil,
+		Endpoints: map[string]interface{}{
+			key: dContext.EndpointMetaBase{
+				Host:          host,
+				SkipTLSVerify: false,
+			},
+		},
+	}
+	return TestContext{
+		context:           dockerContext,
+		expected_endpoint: host,
+	}
+}
+
+func TestGetContextEndpoint(t *testing.T) {
+	var testDockerContexts = []TestContext{
+		dockerContext("ssh://foobar", "docker"),
+		dockerContext("ssh://foobar", "k8"),
+	}
+	for _, context := range testDockerContexts {
+		endpoint, err := client.GetContextEndpoint(context.context)
+		if err != nil {
+			if err.Error() != "context lacks Docker endpoint" {
+				t.Error(err)
+			}
+		} else {
+			if endpoint != context.expected_endpoint {
+				t.Errorf("did not get correct context endpoint. Expected: %s, received: %s", context.expected_endpoint, endpoint)
+			}
+		}
+
+	}
+
+}

pkg/client/convert/LICENSE

@@ -0,0 +1,191 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2013-2017 Docker, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pkg/client/convert/README.md

@@ -0,0 +1,10 @@
+# github.com/docker/cli/cli/compose/convert
+
+DISCLAIMER: This is like the entire `github.com/docker/cli/cli/compose/convert`
+package. This should be an easy import but importing it creates DEPENDENCY
+HELL. I tried for an hour to fix it but it would work. TRY TO FIX AT YOUR OWN
+RISK!!!
+
+Due to this literally just being copy-pasted from the lib, the Apache license
+will be posted in this folder. Small edits to the source code have been to
+function names and parts we don't need deleted.

pkg/client/convert/compose.go

@@ -0,0 +1,199 @@
+package convert
+
+import (
+	"io/ioutil"
+	"strings"
+
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types"
+	networktypes "github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/swarm"
+)
+
+const (
+	// LabelNamespace is the label used to track stack resources
+	LabelNamespace = "com.docker.stack.namespace"
+)
+
+// Namespace mangles names by prepending the name
+type Namespace struct {
+	name string
+}
+
+// Scope prepends the namespace to a name
+func (n Namespace) Scope(name string) string {
+	return n.name + "_" + name
+}
+
+// Descope returns the name without the namespace prefix
+func (n Namespace) Descope(name string) string {
+	return strings.TrimPrefix(name, n.name+"_")
+}
+
+// Name returns the name of the namespace
+func (n Namespace) Name() string {
+	return n.name
+}
+
+// NewNamespace returns a new Namespace for scoping of names
+func NewNamespace(name string) Namespace {
+	return Namespace{name: name}
+}
+
+// AddStackLabel returns labels with the namespace label added
+func AddStackLabel(namespace Namespace, labels map[string]string) map[string]string {
+	if labels == nil {
+		labels = make(map[string]string)
+	}
+	labels[LabelNamespace] = namespace.name
+	return labels
+}
+
+type networkMap map[string]composetypes.NetworkConfig
+
+// Networks from the compose-file type to the engine API type
+func Networks(namespace Namespace, networks networkMap, servicesNetworks map[string]struct{}) (map[string]types.NetworkCreate, []string) {
+	if networks == nil {
+		networks = make(map[string]composetypes.NetworkConfig)
+	}
+
+	externalNetworks := []string{}
+	result := make(map[string]types.NetworkCreate)
+	for internalName := range servicesNetworks {
+		network := networks[internalName]
+		if network.External.External {
+			externalNetworks = append(externalNetworks, network.Name)
+			continue
+		}
+
+		createOpts := types.NetworkCreate{
+			Labels:     AddStackLabel(namespace, network.Labels),
+			Driver:     network.Driver,
+			Options:    network.DriverOpts,
+			Internal:   network.Internal,
+			Attachable: network.Attachable,
+		}
+
+		if network.Ipam.Driver != "" || len(network.Ipam.Config) > 0 {
+			createOpts.IPAM = &networktypes.IPAM{}
+		}
+
+		if network.Ipam.Driver != "" {
+			createOpts.IPAM.Driver = network.Ipam.Driver
+		}
+		for _, ipamConfig := range network.Ipam.Config {
+			config := networktypes.IPAMConfig{
+				Subnet: ipamConfig.Subnet,
+			}
+			createOpts.IPAM.Config = append(createOpts.IPAM.Config, config)
+		}
+
+		networkName := namespace.Scope(internalName)
+		if network.Name != "" {
+			networkName = network.Name
+		}
+		result[networkName] = createOpts
+	}
+
+	return result, externalNetworks
+}
+
+// Secrets converts secrets from the Compose type to the engine API type
+func Secrets(namespace Namespace, secrets map[string]composetypes.SecretConfig) ([]swarm.SecretSpec, error) {
+	result := []swarm.SecretSpec{}
+	for name, secret := range secrets {
+		if secret.External.External {
+			continue
+		}
+
+		var obj swarmFileObject
+		var err error
+		if secret.Driver != "" {
+			obj = driverObjectConfig(namespace, name, composetypes.FileObjectConfig(secret))
+		} else {
+			obj, err = fileObjectConfig(namespace, name, composetypes.FileObjectConfig(secret))
+		}
+		if err != nil {
+			return nil, err
+		}
+		spec := swarm.SecretSpec{Annotations: obj.Annotations, Data: obj.Data}
+		if secret.Driver != "" {
+			spec.Driver = &swarm.Driver{
+				Name:    secret.Driver,
+				Options: secret.DriverOpts,
+			}
+		}
+		if secret.TemplateDriver != "" {
+			spec.Templating = &swarm.Driver{
+				Name: secret.TemplateDriver,
+			}
+		}
+		result = append(result, spec)
+	}
+	return result, nil
+}
+
+// Configs converts config objects from the Compose type to the engine API type
+func Configs(namespace Namespace, configs map[string]composetypes.ConfigObjConfig) ([]swarm.ConfigSpec, error) {
+	result := []swarm.ConfigSpec{}
+	for name, config := range configs {
+		if config.External.External {
+			continue
+		}
+
+		obj, err := fileObjectConfig(namespace, name, composetypes.FileObjectConfig(config))
+		if err != nil {
+			return nil, err
+		}
+		spec := swarm.ConfigSpec{Annotations: obj.Annotations, Data: obj.Data}
+		if config.TemplateDriver != "" {
+			spec.Templating = &swarm.Driver{
+				Name: config.TemplateDriver,
+			}
+		}
+		result = append(result, spec)
+	}
+	return result, nil
+}
+
+type swarmFileObject struct {
+	Annotations swarm.Annotations
+	Data        []byte
+}
+
+func driverObjectConfig(namespace Namespace, name string, obj composetypes.FileObjectConfig) swarmFileObject {
+	if obj.Name != "" {
+		name = obj.Name
+	} else {
+		name = namespace.Scope(name)
+	}
+
+	return swarmFileObject{
+		Annotations: swarm.Annotations{
+			Name:   name,
+			Labels: AddStackLabel(namespace, obj.Labels),
+		},
+		Data: []byte{},
+	}
+}
+
+func fileObjectConfig(namespace Namespace, name string, obj composetypes.FileObjectConfig) (swarmFileObject, error) {
+	data, err := ioutil.ReadFile(obj.File)
+	if err != nil {
+		return swarmFileObject{}, err
+	}
+
+	if obj.Name != "" {
+		name = obj.Name
+	} else {
+		name = namespace.Scope(name)
+	}
+
+	return swarmFileObject{
+		Annotations: swarm.Annotations{
+			Name:   name,
+			Labels: AddStackLabel(namespace, obj.Labels),
+		},
+		Data: data,
+	}, nil
+}

pkg/client/convert/compose_test.go

@@ -0,0 +1,171 @@
+package convert
+
+import (
+	"testing"
+
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/network"
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+	"gotest.tools/v3/fs"
+)
+
+func TestNamespaceScope(t *testing.T) {
+	scoped := Namespace{name: "foo"}.Scope("bar")
+	assert.Check(t, is.Equal("foo_bar", scoped))
+}
+
+func TestAddStackLabel(t *testing.T) {
+	labels := map[string]string{
+		"something": "labeled",
+	}
+	actual := AddStackLabel(Namespace{name: "foo"}, labels)
+	expected := map[string]string{
+		"something":    "labeled",
+		LabelNamespace: "foo",
+	}
+	assert.Check(t, is.DeepEqual(expected, actual))
+}
+
+func TestNetworks(t *testing.T) {
+	namespace := Namespace{name: "foo"}
+	serviceNetworks := map[string]struct{}{
+		"normal":        {},
+		"outside":       {},
+		"default":       {},
+		"attachablenet": {},
+		"named":         {},
+	}
+	source := networkMap{
+		"normal": composetypes.NetworkConfig{
+			Driver: "overlay",
+			DriverOpts: map[string]string{
+				"opt": "value",
+			},
+			Ipam: composetypes.IPAMConfig{
+				Driver: "driver",
+				Config: []*composetypes.IPAMPool{
+					{
+						Subnet: "10.0.0.0",
+					},
+				},
+			},
+			Labels: map[string]string{
+				"something": "labeled",
+			},
+		},
+		"outside": composetypes.NetworkConfig{
+			External: composetypes.External{External: true},
+			Name:     "special",
+		},
+		"attachablenet": composetypes.NetworkConfig{
+			Driver:     "overlay",
+			Attachable: true,
+		},
+		"named": composetypes.NetworkConfig{
+			Name: "othername",
+		},
+	}
+	expected := map[string]types.NetworkCreate{
+		"foo_default": {
+			Labels: map[string]string{
+				LabelNamespace: "foo",
+			},
+		},
+		"foo_normal": {
+			Driver: "overlay",
+			IPAM: &network.IPAM{
+				Driver: "driver",
+				Config: []network.IPAMConfig{
+					{
+						Subnet: "10.0.0.0",
+					},
+				},
+			},
+			Options: map[string]string{
+				"opt": "value",
+			},
+			Labels: map[string]string{
+				LabelNamespace: "foo",
+				"something":    "labeled",
+			},
+		},
+		"foo_attachablenet": {
+			Driver:     "overlay",
+			Attachable: true,
+			Labels: map[string]string{
+				LabelNamespace: "foo",
+			},
+		},
+		"othername": {
+			Labels: map[string]string{LabelNamespace: "foo"},
+		},
+	}
+
+	networks, externals := Networks(namespace, source, serviceNetworks)
+	assert.DeepEqual(t, expected, networks)
+	assert.DeepEqual(t, []string{"special"}, externals)
+}
+
+func TestSecrets(t *testing.T) {
+	namespace := Namespace{name: "foo"}
+
+	secretText := "this is the first secret"
+	secretFile := fs.NewFile(t, "convert-secrets", fs.WithContent(secretText))
+	defer secretFile.Remove()
+
+	source := map[string]composetypes.SecretConfig{
+		"one": {
+			File:   secretFile.Path(),
+			Labels: map[string]string{"monster": "mash"},
+		},
+		"ext": {
+			External: composetypes.External{
+				External: true,
+			},
+		},
+	}
+
+	specs, err := Secrets(namespace, source)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Len(specs, 1))
+	secret := specs[0]
+	assert.Check(t, is.Equal("foo_one", secret.Name))
+	assert.Check(t, is.DeepEqual(map[string]string{
+		"monster":      "mash",
+		LabelNamespace: "foo",
+	}, secret.Labels))
+	assert.Check(t, is.DeepEqual([]byte(secretText), secret.Data))
+}
+
+func TestConfigs(t *testing.T) {
+	namespace := Namespace{name: "foo"}
+
+	configText := "this is the first config"
+	configFile := fs.NewFile(t, "convert-configs", fs.WithContent(configText))
+	defer configFile.Remove()
+
+	source := map[string]composetypes.ConfigObjConfig{
+		"one": {
+			File:   configFile.Path(),
+			Labels: map[string]string{"monster": "mash"},
+		},
+		"ext": {
+			External: composetypes.External{
+				External: true,
+			},
+		},
+	}
+
+	specs, err := Configs(namespace, source)
+	assert.NilError(t, err)
+	assert.Assert(t, is.Len(specs, 1))
+	config := specs[0]
+	assert.Check(t, is.Equal("foo_one", config.Name))
+	assert.Check(t, is.DeepEqual(map[string]string{
+		"monster":      "mash",
+		LabelNamespace: "foo",
+	}, config.Labels))
+	assert.Check(t, is.DeepEqual([]byte(configText), config.Data))
+}

pkg/client/convert/service.go

@@ -0,0 +1,861 @@
+package convert
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/cli/opts"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/swarm"
+	swarmtypes "github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/versions"
+	"github.com/docker/docker/client"
+	"github.com/docker/go-units"
+	"github.com/pkg/errors"
+)
+
+const (
+	defaultNetwork = "default"
+	// LabelImage is the label used to store image name provided in the compose file
+	LabelImage = "com.docker.stack.image"
+)
+
+// ParseSecrets retrieves the secrets with the requested names and fills
+// secret IDs into the secret references.
+func ParseSecrets(client client.SecretAPIClient, requestedSecrets []*swarmtypes.SecretReference) ([]*swarmtypes.SecretReference, error) {
+	if len(requestedSecrets) == 0 {
+		return []*swarmtypes.SecretReference{}, nil
+	}
+
+	secretRefs := make(map[string]*swarmtypes.SecretReference)
+	ctx := context.Background()
+
+	for _, secret := range requestedSecrets {
+		if _, exists := secretRefs[secret.File.Name]; exists {
+			return nil, errors.Errorf("duplicate secret target for %s not allowed", secret.SecretName)
+		}
+		secretRef := new(swarmtypes.SecretReference)
+		*secretRef = *secret
+		secretRefs[secret.File.Name] = secretRef
+	}
+
+	args := filters.NewArgs()
+	for _, s := range secretRefs {
+		args.Add("name", s.SecretName)
+	}
+
+	secrets, err := client.SecretList(ctx, types.SecretListOptions{
+		Filters: args,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	foundSecrets := make(map[string]string)
+	for _, secret := range secrets {
+		foundSecrets[secret.Spec.Annotations.Name] = secret.ID
+	}
+
+	addedSecrets := []*swarmtypes.SecretReference{}
+
+	for _, ref := range secretRefs {
+		id, ok := foundSecrets[ref.SecretName]
+		if !ok {
+			return nil, errors.Errorf("secret not found: %s", ref.SecretName)
+		}
+
+		// set the id for the ref to properly assign in swarm
+		// since swarm needs the ID instead of the name
+		ref.SecretID = id
+		addedSecrets = append(addedSecrets, ref)
+	}
+
+	return addedSecrets, nil
+}
+
+// ParseConfigs retrieves the configs from the requested names and converts
+// them to config references to use with the spec
+func ParseConfigs(client client.ConfigAPIClient, requestedConfigs []*swarmtypes.ConfigReference) ([]*swarmtypes.ConfigReference, error) {
+	if len(requestedConfigs) == 0 {
+		return []*swarmtypes.ConfigReference{}, nil
+	}
+
+	// the configRefs map has two purposes: it prevents duplication of config
+	// target filenames, and it it used to get all configs so we can resolve
+	// their IDs. unfortunately, there are other targets for ConfigReferences,
+	// besides just a File; specifically, the Runtime target, which is used for
+	// CredentialSpecs. Therefore, we need to have a list of ConfigReferences
+	// that are not File targets as well. at this time of writing, the only use
+	// for Runtime targets is CredentialSpecs. However, to future-proof this
+	// functionality, we should handle the case where multiple Runtime targets
+	// are in use for the same Config, and we should deduplicate
+	// such ConfigReferences, as no matter how many times the Config is used,
+	// it is only needed to be referenced once.
+	configRefs := make(map[string]*swarmtypes.ConfigReference)
+	runtimeRefs := make(map[string]*swarmtypes.ConfigReference)
+	ctx := context.Background()
+
+	for _, config := range requestedConfigs {
+		// copy the config, so we don't mutate the args
+		configRef := new(swarmtypes.ConfigReference)
+		*configRef = *config
+
+		if config.Runtime != nil {
+			// by assigning to a map based on ConfigName, if the same Config
+			// is required as a Runtime target for multiple purposes, we only
+			// include it once in the final set of configs.
+			runtimeRefs[config.ConfigName] = config
+			// continue, so we skip the logic below for handling file-type
+			// configs
+			continue
+		}
+
+		if _, exists := configRefs[config.File.Name]; exists {
+			return nil, errors.Errorf("duplicate config target for %s not allowed", config.ConfigName)
+		}
+
+		configRefs[config.File.Name] = configRef
+	}
+
+	args := filters.NewArgs()
+	for _, s := range configRefs {
+		args.Add("name", s.ConfigName)
+	}
+	for _, s := range runtimeRefs {
+		args.Add("name", s.ConfigName)
+	}
+
+	configs, err := client.ConfigList(ctx, types.ConfigListOptions{
+		Filters: args,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	foundConfigs := make(map[string]string)
+	for _, config := range configs {
+		foundConfigs[config.Spec.Annotations.Name] = config.ID
+	}
+
+	addedConfigs := []*swarmtypes.ConfigReference{}
+
+	for _, ref := range configRefs {
+		id, ok := foundConfigs[ref.ConfigName]
+		if !ok {
+			return nil, errors.Errorf("config not found: %s", ref.ConfigName)
+		}
+
+		// set the id for the ref to properly assign in swarm
+		// since swarm needs the ID instead of the name
+		ref.ConfigID = id
+		addedConfigs = append(addedConfigs, ref)
+	}
+
+	// unfortunately, because the key of configRefs and runtimeRefs is different
+	// values that may collide, we can't just do some fancy trickery to
+	// concat maps, we need to do two separate loops
+	for _, ref := range runtimeRefs {
+		id, ok := foundConfigs[ref.ConfigName]
+		if !ok {
+			return nil, errors.Errorf("config not found: %s", ref.ConfigName)
+		}
+
+		ref.ConfigID = id
+		addedConfigs = append(addedConfigs, ref)
+	}
+
+	return addedConfigs, nil
+}
+
+// Services from compose-file types to engine API types
+func Services(
+	namespace Namespace,
+	config *composetypes.Config,
+	client client.CommonAPIClient,
+) (map[string]swarm.ServiceSpec, error) {
+	result := make(map[string]swarm.ServiceSpec)
+
+	services := config.Services
+	volumes := config.Volumes
+	networks := config.Networks
+
+	for _, service := range services {
+		secrets, err := convertServiceSecrets(client, namespace, service.Secrets, config.Secrets)
+		if err != nil {
+			return nil, errors.Wrapf(err, "service %s", service.Name)
+		}
+		configs, err := convertServiceConfigObjs(client, namespace, service, config.Configs)
+		if err != nil {
+			return nil, errors.Wrapf(err, "service %s", service.Name)
+		}
+
+		serviceSpec, err := Service(client.ClientVersion(), namespace, service, networks, volumes, secrets, configs)
+		if err != nil {
+			return nil, errors.Wrapf(err, "service %s", service.Name)
+		}
+		result[service.Name] = serviceSpec
+	}
+
+	return result, nil
+}
+
+// Service converts a ServiceConfig into a swarm ServiceSpec
+func Service(
+	apiVersion string,
+	namespace Namespace,
+	service composetypes.ServiceConfig,
+	networkConfigs map[string]composetypes.NetworkConfig,
+	volumes map[string]composetypes.VolumeConfig,
+	secrets []*swarm.SecretReference,
+	configs []*swarm.ConfigReference,
+) (swarm.ServiceSpec, error) {
+	name := namespace.Scope(service.Name)
+	endpoint := convertEndpointSpec(service.Deploy.EndpointMode, service.Ports)
+
+	mode, err := convertDeployMode(service.Deploy.Mode, service.Deploy.Replicas)
+	if err != nil {
+		return swarm.ServiceSpec{}, err
+	}
+
+	mounts, err := Volumes(service.Volumes, volumes, namespace)
+	if err != nil {
+		return swarm.ServiceSpec{}, err
+	}
+
+	resources, err := convertResources(service.Deploy.Resources)
+	if err != nil {
+		return swarm.ServiceSpec{}, err
+	}
+
+	restartPolicy, err := convertRestartPolicy(
+		service.Restart, service.Deploy.RestartPolicy)
+	if err != nil {
+		return swarm.ServiceSpec{}, err
+	}
+
+	healthcheck, err := convertHealthcheck(service.HealthCheck)
+	if err != nil {
+		return swarm.ServiceSpec{}, err
+	}
+
+	networks, err := convertServiceNetworks(service.Networks, networkConfigs, namespace, service.Name)
+	if err != nil {
+		return swarm.ServiceSpec{}, err
+	}
+
+	dnsConfig := convertDNSConfig(service.DNS, service.DNSSearch)
+
+	var privileges swarm.Privileges
+	privileges.CredentialSpec, err = convertCredentialSpec(
+		namespace, service.CredentialSpec, configs,
+	)
+	if err != nil {
+		return swarm.ServiceSpec{}, err
+	}
+
+	var logDriver *swarm.Driver
+	if service.Logging != nil {
+		logDriver = &swarm.Driver{
+			Name:    service.Logging.Driver,
+			Options: service.Logging.Options,
+		}
+	}
+
+	capAdd, capDrop := opts.EffectiveCapAddCapDrop(service.CapAdd, service.CapDrop)
+
+	serviceSpec := swarm.ServiceSpec{
+		Annotations: swarm.Annotations{
+			Name:   name,
+			Labels: AddStackLabel(namespace, service.Deploy.Labels),
+		},
+		TaskTemplate: swarm.TaskSpec{
+			ContainerSpec: &swarm.ContainerSpec{
+				Image:           service.Image,
+				Command:         service.Entrypoint,
+				Args:            service.Command,
+				Hostname:        service.Hostname,
+				Hosts:           convertExtraHosts(service.ExtraHosts),
+				DNSConfig:       dnsConfig,
+				Healthcheck:     healthcheck,
+				Env:             sortStrings(convertEnvironment(service.Environment)),
+				Labels:          AddStackLabel(namespace, service.Labels),
+				Dir:             service.WorkingDir,
+				User:            service.User,
+				Mounts:          mounts,
+				StopGracePeriod: composetypes.ConvertDurationPtr(service.StopGracePeriod),
+				StopSignal:      service.StopSignal,
+				TTY:             service.Tty,
+				OpenStdin:       service.StdinOpen,
+				Secrets:         secrets,
+				Configs:         configs,
+				ReadOnly:        service.ReadOnly,
+				Privileges:      &privileges,
+				Isolation:       container.Isolation(service.Isolation),
+				Init:            service.Init,
+				Sysctls:         service.Sysctls,
+				CapabilityAdd:   capAdd,
+				CapabilityDrop:  capDrop,
+				Ulimits:         convertUlimits(service.Ulimits),
+			},
+			LogDriver:     logDriver,
+			Resources:     resources,
+			RestartPolicy: restartPolicy,
+			Placement: &swarm.Placement{
+				Constraints: service.Deploy.Placement.Constraints,
+				Preferences: getPlacementPreference(service.Deploy.Placement.Preferences),
+				MaxReplicas: service.Deploy.Placement.MaxReplicas,
+			},
+		},
+		EndpointSpec:   endpoint,
+		Mode:           mode,
+		UpdateConfig:   convertUpdateConfig(service.Deploy.UpdateConfig),
+		RollbackConfig: convertUpdateConfig(service.Deploy.RollbackConfig),
+	}
+
+	// add an image label to serviceSpec
+	serviceSpec.Labels[LabelImage] = service.Image
+
+	// ServiceSpec.Networks is deprecated and should not have been used by
+	// this package. It is possible to update TaskTemplate.Networks, but it
+	// is not possible to update ServiceSpec.Networks. Unfortunately, we
+	// can't unconditionally start using TaskTemplate.Networks, because that
+	// will break with older daemons that don't support migrating from
+	// ServiceSpec.Networks to TaskTemplate.Networks. So which field to use
+	// is conditional on daemon version.
+	if versions.LessThan(apiVersion, "1.29") {
+		serviceSpec.Networks = networks
+	} else {
+		serviceSpec.TaskTemplate.Networks = networks
+	}
+	return serviceSpec, nil
+}
+
+func getPlacementPreference(preferences []composetypes.PlacementPreferences) []swarm.PlacementPreference {
+	result := []swarm.PlacementPreference{}
+	for _, preference := range preferences {
+		spreadDescriptor := preference.Spread
+		result = append(result, swarm.PlacementPreference{
+			Spread: &swarm.SpreadOver{
+				SpreadDescriptor: spreadDescriptor,
+			},
+		})
+	}
+	return result
+}
+
+func sortStrings(strs []string) []string {
+	sort.Strings(strs)
+	return strs
+}
+
+func convertServiceNetworks(
+	networks map[string]*composetypes.ServiceNetworkConfig,
+	networkConfigs networkMap,
+	namespace Namespace,
+	name string,
+) ([]swarm.NetworkAttachmentConfig, error) {
+	if len(networks) == 0 {
+		networks = map[string]*composetypes.ServiceNetworkConfig{
+			defaultNetwork: {},
+		}
+	}
+
+	nets := []swarm.NetworkAttachmentConfig{}
+	for networkName, network := range networks {
+		networkConfig, ok := networkConfigs[networkName]
+		if !ok && networkName != defaultNetwork {
+			return nil, errors.Errorf("undefined network %q", networkName)
+		}
+		var aliases []string
+		if network != nil {
+			aliases = network.Aliases
+		}
+		target := namespace.Scope(networkName)
+		if networkConfig.Name != "" {
+			target = networkConfig.Name
+		}
+		netAttachConfig := swarm.NetworkAttachmentConfig{
+			Target:  target,
+			Aliases: aliases,
+		}
+		// Only add default aliases to user defined networks. Other networks do
+		// not support aliases.
+		if container.NetworkMode(target).IsUserDefined() {
+			netAttachConfig.Aliases = append(netAttachConfig.Aliases, name)
+		}
+		nets = append(nets, netAttachConfig)
+	}
+
+	sort.Slice(nets, func(i, j int) bool {
+		return nets[i].Target < nets[j].Target
+	})
+	return nets, nil
+}
+
+// TODO: fix secrets API so that SecretAPIClient is not required here
+func convertServiceSecrets(
+	client client.SecretAPIClient,
+	namespace Namespace,
+	secrets []composetypes.ServiceSecretConfig,
+	secretSpecs map[string]composetypes.SecretConfig,
+) ([]*swarm.SecretReference, error) {
+	refs := []*swarm.SecretReference{}
+
+	lookup := func(key string) (composetypes.FileObjectConfig, error) {
+		secretSpec, exists := secretSpecs[key]
+		if !exists {
+			return composetypes.FileObjectConfig{}, errors.Errorf("undefined secret %q", key)
+		}
+		return composetypes.FileObjectConfig(secretSpec), nil
+	}
+	for _, secret := range secrets {
+		obj, err := convertFileObject(namespace, composetypes.FileReferenceConfig(secret), lookup)
+		if err != nil {
+			return nil, err
+		}
+
+		file := swarm.SecretReferenceFileTarget(obj.File)
+		refs = append(refs, &swarm.SecretReference{
+			File:       &file,
+			SecretName: obj.Name,
+		})
+	}
+
+	secrs, err := ParseSecrets(client, refs)
+	if err != nil {
+		return nil, err
+	}
+	// sort to ensure idempotence (don't restart services just because the entries are in different order)
+	sort.SliceStable(secrs, func(i, j int) bool { return secrs[i].SecretName < secrs[j].SecretName })
+	return secrs, err
+}
+
+// convertServiceConfigObjs takes an API client, a namespace, a ServiceConfig,
+// and a set of compose Config specs, and creates the swarm ConfigReferences
+// required by the serivce. Unlike convertServiceSecrets, this takes the whole
+// ServiceConfig, because some Configs may be needed as a result of other
+// fields (like CredentialSpecs).
+//
+// TODO: fix configs API so that ConfigsAPIClient is not required here
+func convertServiceConfigObjs(
+	client client.ConfigAPIClient,
+	namespace Namespace,
+	service composetypes.ServiceConfig,
+	configSpecs map[string]composetypes.ConfigObjConfig,
+) ([]*swarm.ConfigReference, error) {
+	refs := []*swarm.ConfigReference{}
+
+	lookup := func(key string) (composetypes.FileObjectConfig, error) {
+		configSpec, exists := configSpecs[key]
+		if !exists {
+			return composetypes.FileObjectConfig{}, errors.Errorf("undefined config %q", key)
+		}
+		return composetypes.FileObjectConfig(configSpec), nil
+	}
+	for _, config := range service.Configs {
+		obj, err := convertFileObject(namespace, composetypes.FileReferenceConfig(config), lookup)
+		if err != nil {
+			return nil, err
+		}
+
+		file := swarm.ConfigReferenceFileTarget(obj.File)
+		refs = append(refs, &swarm.ConfigReference{
+			File:       &file,
+			ConfigName: obj.Name,
+		})
+	}
+
+	// finally, after converting all of the file objects, create any
+	// Runtime-type configs that are needed. these are configs that are not
+	// mounted into the container, but are used in some other way by the
+	// container runtime. Currently, this only means CredentialSpecs, but in
+	// the future it may be used for other fields
+
+	// grab the CredentialSpec out of the Service
+	credSpec := service.CredentialSpec
+	// if the credSpec uses a config, then we should grab the config name, and
+	// create a config reference for it. A File or Registry-type CredentialSpec
+	// does not need this operation.
+	if credSpec.Config != "" {
+		// look up the config in the configSpecs.
+		obj, err := lookup(credSpec.Config)
+		if err != nil {
+			return nil, err
+		}
+
+		// get the actual correct name.
+		name := namespace.Scope(credSpec.Config)
+		if obj.Name != "" {
+			name = obj.Name
+		}
+
+		// now append a Runtime-type config.
+		refs = append(refs, &swarm.ConfigReference{
+			ConfigName: name,
+			Runtime:    &swarm.ConfigReferenceRuntimeTarget{},
+		})
+
+	}
+
+	confs, err := ParseConfigs(client, refs)
+	if err != nil {
+		return nil, err
+	}
+	// sort to ensure idempotence (don't restart services just because the entries are in different order)
+	sort.SliceStable(confs, func(i, j int) bool { return confs[i].ConfigName < confs[j].ConfigName })
+	return confs, err
+}
+
+type swarmReferenceTarget struct {
+	Name string
+	UID  string
+	GID  string
+	Mode os.FileMode
+}
+
+type swarmReferenceObject struct {
+	File swarmReferenceTarget
+	ID   string
+	Name string
+}
+
+func convertFileObject(
+	namespace Namespace,
+	config composetypes.FileReferenceConfig,
+	lookup func(key string) (composetypes.FileObjectConfig, error),
+) (swarmReferenceObject, error) {
+	obj, err := lookup(config.Source)
+	if err != nil {
+		return swarmReferenceObject{}, err
+	}
+
+	source := namespace.Scope(config.Source)
+	if obj.Name != "" {
+		source = obj.Name
+	}
+
+	target := config.Target
+	if target == "" {
+		target = config.Source
+	}
+
+	uid := config.UID
+	gid := config.GID
+	if uid == "" {
+		uid = "0"
+	}
+	if gid == "" {
+		gid = "0"
+	}
+	mode := config.Mode
+	if mode == nil {
+		mode = uint32Ptr(0444)
+	}
+
+	return swarmReferenceObject{
+		File: swarmReferenceTarget{
+			Name: target,
+			UID:  uid,
+			GID:  gid,
+			Mode: os.FileMode(*mode),
+		},
+		Name: source,
+	}, nil
+}
+
+func uint32Ptr(value uint32) *uint32 {
+	return &value
+}
+
+// convertExtraHosts converts <host>:<ip> mappings to SwarmKit notation:
+// "IP-address hostname(s)". The original order of mappings is preserved.
+func convertExtraHosts(extraHosts composetypes.HostsList) []string {
+	hosts := []string{}
+	for _, hostIP := range extraHosts {
+		if v := strings.SplitN(hostIP, ":", 2); len(v) == 2 {
+			// Convert to SwarmKit notation: IP-address hostname(s)
+			hosts = append(hosts, fmt.Sprintf("%s %s", v[1], v[0]))
+		}
+	}
+	return hosts
+}
+
+func convertHealthcheck(healthcheck *composetypes.HealthCheckConfig) (*container.HealthConfig, error) {
+	if healthcheck == nil {
+		return nil, nil
+	}
+	var (
+		timeout, interval, startPeriod time.Duration
+		retries                        int
+	)
+	if healthcheck.Disable {
+		if len(healthcheck.Test) != 0 {
+			return nil, errors.Errorf("test and disable can't be set at the same time")
+		}
+		return &container.HealthConfig{
+			Test: []string{"NONE"},
+		}, nil
+
+	}
+	if healthcheck.Timeout != nil {
+		timeout = time.Duration(*healthcheck.Timeout)
+	}
+	if healthcheck.Interval != nil {
+		interval = time.Duration(*healthcheck.Interval)
+	}
+	if healthcheck.StartPeriod != nil {
+		startPeriod = time.Duration(*healthcheck.StartPeriod)
+	}
+	if healthcheck.Retries != nil {
+		retries = int(*healthcheck.Retries)
+	}
+	return &container.HealthConfig{
+		Test:        healthcheck.Test,
+		Timeout:     timeout,
+		Interval:    interval,
+		Retries:     retries,
+		StartPeriod: startPeriod,
+	}, nil
+}
+
+func convertRestartPolicy(restart string, source *composetypes.RestartPolicy) (*swarm.RestartPolicy, error) {
+	// TODO: log if restart is being ignored
+	if source == nil {
+		policy, err := opts.ParseRestartPolicy(restart)
+		if err != nil {
+			return nil, err
+		}
+		switch {
+		case policy.IsNone():
+			return nil, nil
+		case policy.IsAlways(), policy.IsUnlessStopped():
+			return &swarm.RestartPolicy{
+				Condition: swarm.RestartPolicyConditionAny,
+			}, nil
+		case policy.IsOnFailure():
+			attempts := uint64(policy.MaximumRetryCount)
+			return &swarm.RestartPolicy{
+				Condition:   swarm.RestartPolicyConditionOnFailure,
+				MaxAttempts: &attempts,
+			}, nil
+		default:
+			return nil, errors.Errorf("unknown restart policy: %s", restart)
+		}
+	}
+
+	return &swarm.RestartPolicy{
+		Condition:   swarm.RestartPolicyCondition(source.Condition),
+		Delay:       composetypes.ConvertDurationPtr(source.Delay),
+		MaxAttempts: source.MaxAttempts,
+		Window:      composetypes.ConvertDurationPtr(source.Window),
+	}, nil
+}
+
+func convertUpdateConfig(source *composetypes.UpdateConfig) *swarm.UpdateConfig {
+	if source == nil {
+		return nil
+	}
+	parallel := uint64(1)
+	if source.Parallelism != nil {
+		parallel = *source.Parallelism
+	}
+	return &swarm.UpdateConfig{
+		Parallelism:     parallel,
+		Delay:           time.Duration(source.Delay),
+		FailureAction:   source.FailureAction,
+		Monitor:         time.Duration(source.Monitor),
+		MaxFailureRatio: source.MaxFailureRatio,
+		Order:           source.Order,
+	}
+}
+
+func convertResources(source composetypes.Resources) (*swarm.ResourceRequirements, error) {
+	resources := &swarm.ResourceRequirements{}
+	var err error
+	if source.Limits != nil {
+		var cpus int64
+		if source.Limits.NanoCPUs != "" {
+			cpus, err = opts.ParseCPUs(source.Limits.NanoCPUs)
+			if err != nil {
+				return nil, err
+			}
+		}
+		resources.Limits = &swarm.Limit{
+			NanoCPUs:    cpus,
+			MemoryBytes: int64(source.Limits.MemoryBytes),
+			Pids:        source.Limits.Pids,
+		}
+	}
+	if source.Reservations != nil {
+		var cpus int64
+		if source.Reservations.NanoCPUs != "" {
+			cpus, err = opts.ParseCPUs(source.Reservations.NanoCPUs)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		var generic []swarm.GenericResource
+		for _, res := range source.Reservations.GenericResources {
+			var r swarm.GenericResource
+
+			if res.DiscreteResourceSpec != nil {
+				r.DiscreteResourceSpec = &swarm.DiscreteGenericResource{
+					Kind:  res.DiscreteResourceSpec.Kind,
+					Value: res.DiscreteResourceSpec.Value,
+				}
+			}
+
+			generic = append(generic, r)
+		}
+
+		resources.Reservations = &swarm.Resources{
+			NanoCPUs:         cpus,
+			MemoryBytes:      int64(source.Reservations.MemoryBytes),
+			GenericResources: generic,
+		}
+	}
+	return resources, nil
+}
+
+func convertEndpointSpec(endpointMode string, source []composetypes.ServicePortConfig) *swarm.EndpointSpec {
+	portConfigs := []swarm.PortConfig{}
+	for _, port := range source {
+		portConfig := swarm.PortConfig{
+			Protocol:      swarm.PortConfigProtocol(port.Protocol),
+			TargetPort:    port.Target,
+			PublishedPort: port.Published,
+			PublishMode:   swarm.PortConfigPublishMode(port.Mode),
+		}
+		portConfigs = append(portConfigs, portConfig)
+	}
+
+	sort.Slice(portConfigs, func(i, j int) bool {
+		return portConfigs[i].PublishedPort < portConfigs[j].PublishedPort
+	})
+
+	return &swarm.EndpointSpec{
+		Mode:  swarm.ResolutionMode(strings.ToLower(endpointMode)),
+		Ports: portConfigs,
+	}
+}
+
+func convertEnvironment(source map[string]*string) []string {
+	var output []string
+
+	for name, value := range source {
+		switch value {
+		case nil:
+			output = append(output, name)
+		default:
+			output = append(output, fmt.Sprintf("%s=%s", name, *value))
+		}
+	}
+
+	return output
+}
+
+func convertDeployMode(mode string, replicas *uint64) (swarm.ServiceMode, error) {
+	serviceMode := swarm.ServiceMode{}
+
+	switch mode {
+	case "global":
+		if replicas != nil {
+			return serviceMode, errors.Errorf("replicas can only be used with replicated mode")
+		}
+		serviceMode.Global = &swarm.GlobalService{}
+	case "replicated", "":
+		serviceMode.Replicated = &swarm.ReplicatedService{Replicas: replicas}
+	default:
+		return serviceMode, errors.Errorf("Unknown mode: %s", mode)
+	}
+	return serviceMode, nil
+}
+
+func convertDNSConfig(DNS []string, DNSSearch []string) *swarm.DNSConfig {
+	if DNS != nil || DNSSearch != nil {
+		return &swarm.DNSConfig{
+			Nameservers: DNS,
+			Search:      DNSSearch,
+		}
+	}
+	return nil
+}
+
+func convertCredentialSpec(namespace Namespace, spec composetypes.CredentialSpecConfig, refs []*swarm.ConfigReference) (*swarm.CredentialSpec, error) {
+	var o []string
+
+	// Config was added in API v1.40
+	if spec.Config != "" {
+		o = append(o, `"Config"`)
+	}
+	if spec.File != "" {
+		o = append(o, `"File"`)
+	}
+	if spec.Registry != "" {
+		o = append(o, `"Registry"`)
+	}
+	l := len(o)
+	switch {
+	case l == 0:
+		return nil, nil
+	case l == 2:
+		return nil, errors.Errorf("invalid credential spec: cannot specify both %s and %s", o[0], o[1])
+	case l > 2:
+		return nil, errors.Errorf("invalid credential spec: cannot specify both %s, and %s", strings.Join(o[:l-1], ", "), o[l-1])
+	}
+	swarmCredSpec := swarm.CredentialSpec(spec)
+	// if we're using a swarm Config for the credential spec, over-write it
+	// here with the config ID
+	if swarmCredSpec.Config != "" {
+		for _, config := range refs {
+			if swarmCredSpec.Config == config.ConfigName {
+				swarmCredSpec.Config = config.ConfigID
+				return &swarmCredSpec, nil
+			}
+		}
+		// if none of the configs match, try namespacing
+		for _, config := range refs {
+			if namespace.Scope(swarmCredSpec.Config) == config.ConfigName {
+				swarmCredSpec.Config = config.ConfigID
+				return &swarmCredSpec, nil
+			}
+		}
+		return nil, errors.Errorf("invalid credential spec: spec specifies config %v, but no such config can be found", swarmCredSpec.Config)
+	}
+	return &swarmCredSpec, nil
+}
+
+func convertUlimits(origUlimits map[string]*composetypes.UlimitsConfig) []*units.Ulimit {
+	newUlimits := make(map[string]*units.Ulimit)
+	for name, u := range origUlimits {
+		if u.Single != 0 {
+			newUlimits[name] = &units.Ulimit{
+				Name: name,
+				Soft: int64(u.Single),
+				Hard: int64(u.Single),
+			}
+		} else {
+			newUlimits[name] = &units.Ulimit{
+				Name: name,
+				Soft: int64(u.Soft),
+				Hard: int64(u.Hard),
+			}
+		}
+	}
+	var ulimits []*units.Ulimit
+	for _, ulimit := range newUlimits {
+		ulimits = append(ulimits, ulimit)
+	}
+	sort.SliceStable(ulimits, func(i, j int) bool {
+		return ulimits[i].Name < ulimits[j].Name
+	})
+	return ulimits
+}

pkg/client/convert/service_test.go

@@ -0,0 +1,678 @@
+package convert
+
+import (
+	"context"
+	"os"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+)
+
+func TestConvertRestartPolicyFromNone(t *testing.T) {
+	policy, err := convertRestartPolicy("no", nil)
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual((*swarm.RestartPolicy)(nil), policy))
+}
+
+func TestConvertRestartPolicyFromUnknown(t *testing.T) {
+	_, err := convertRestartPolicy("unknown", nil)
+	assert.Error(t, err, "unknown restart policy: unknown")
+}
+
+func TestConvertRestartPolicyFromAlways(t *testing.T) {
+	policy, err := convertRestartPolicy("always", nil)
+	expected := &swarm.RestartPolicy{
+		Condition: swarm.RestartPolicyConditionAny,
+	}
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, policy))
+}
+
+func TestConvertRestartPolicyFromFailure(t *testing.T) {
+	policy, err := convertRestartPolicy("on-failure:4", nil)
+	attempts := uint64(4)
+	expected := &swarm.RestartPolicy{
+		Condition:   swarm.RestartPolicyConditionOnFailure,
+		MaxAttempts: &attempts,
+	}
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, policy))
+}
+
+func strPtr(val string) *string {
+	return &val
+}
+
+func TestConvertEnvironment(t *testing.T) {
+	source := map[string]*string{
+		"foo": strPtr("bar"),
+		"key": strPtr("value"),
+	}
+	env := convertEnvironment(source)
+	sort.Strings(env)
+	assert.Check(t, is.DeepEqual([]string{"foo=bar", "key=value"}, env))
+}
+
+func TestConvertExtraHosts(t *testing.T) {
+	source := composetypes.HostsList{
+		"zulu:127.0.0.2",
+		"alpha:127.0.0.1",
+		"zulu:ff02::1",
+	}
+	assert.Check(t, is.DeepEqual([]string{"127.0.0.2 zulu", "127.0.0.1 alpha", "ff02::1 zulu"}, convertExtraHosts(source)))
+}
+
+func TestConvertResourcesFull(t *testing.T) {
+	source := composetypes.Resources{
+		Limits: &composetypes.ResourceLimit{
+			NanoCPUs:    "0.003",
+			MemoryBytes: composetypes.UnitBytes(300000000),
+		},
+		Reservations: &composetypes.Resource{
+			NanoCPUs:    "0.002",
+			MemoryBytes: composetypes.UnitBytes(200000000),
+		},
+	}
+	resources, err := convertResources(source)
+	assert.NilError(t, err)
+
+	expected := &swarm.ResourceRequirements{
+		Limits: &swarm.Limit{
+			NanoCPUs:    3000000,
+			MemoryBytes: 300000000,
+		},
+		Reservations: &swarm.Resources{
+			NanoCPUs:    2000000,
+			MemoryBytes: 200000000,
+		},
+	}
+	assert.Check(t, is.DeepEqual(expected, resources))
+}
+
+func TestConvertResourcesOnlyMemory(t *testing.T) {
+	source := composetypes.Resources{
+		Limits: &composetypes.ResourceLimit{
+			MemoryBytes: composetypes.UnitBytes(300000000),
+		},
+		Reservations: &composetypes.Resource{
+			MemoryBytes: composetypes.UnitBytes(200000000),
+		},
+	}
+	resources, err := convertResources(source)
+	assert.NilError(t, err)
+
+	expected := &swarm.ResourceRequirements{
+		Limits: &swarm.Limit{
+			MemoryBytes: 300000000,
+		},
+		Reservations: &swarm.Resources{
+			MemoryBytes: 200000000,
+		},
+	}
+	assert.Check(t, is.DeepEqual(expected, resources))
+}
+
+func TestConvertHealthcheck(t *testing.T) {
+	retries := uint64(10)
+	timeout := composetypes.Duration(30 * time.Second)
+	interval := composetypes.Duration(2 * time.Millisecond)
+	source := &composetypes.HealthCheckConfig{
+		Test:     []string{"EXEC", "touch", "/foo"},
+		Timeout:  &timeout,
+		Interval: &interval,
+		Retries:  &retries,
+	}
+	expected := &container.HealthConfig{
+		Test:     source.Test,
+		Timeout:  time.Duration(timeout),
+		Interval: time.Duration(interval),
+		Retries:  10,
+	}
+
+	healthcheck, err := convertHealthcheck(source)
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, healthcheck))
+}
+
+func TestConvertHealthcheckDisable(t *testing.T) {
+	source := &composetypes.HealthCheckConfig{Disable: true}
+	expected := &container.HealthConfig{
+		Test: []string{"NONE"},
+	}
+
+	healthcheck, err := convertHealthcheck(source)
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, healthcheck))
+}
+
+func TestConvertHealthcheckDisableWithTest(t *testing.T) {
+	source := &composetypes.HealthCheckConfig{
+		Disable: true,
+		Test:    []string{"EXEC", "touch"},
+	}
+	_, err := convertHealthcheck(source)
+	assert.Error(t, err, "test and disable can't be set at the same time")
+}
+
+func TestConvertEndpointSpec(t *testing.T) {
+	source := []composetypes.ServicePortConfig{
+		{
+			Protocol:  "udp",
+			Target:    53,
+			Published: 1053,
+			Mode:      "host",
+		},
+		{
+			Target:    8080,
+			Published: 80,
+		},
+	}
+	endpoint := convertEndpointSpec("vip", source)
+
+	expected := swarm.EndpointSpec{
+		Mode: swarm.ResolutionMode(strings.ToLower("vip")),
+		Ports: []swarm.PortConfig{
+			{
+				TargetPort:    8080,
+				PublishedPort: 80,
+			},
+			{
+				Protocol:      "udp",
+				TargetPort:    53,
+				PublishedPort: 1053,
+				PublishMode:   "host",
+			},
+		},
+	}
+
+	assert.Check(t, is.DeepEqual(expected, *endpoint))
+}
+
+func TestConvertServiceNetworksOnlyDefault(t *testing.T) {
+	networkConfigs := networkMap{}
+
+	configs, err := convertServiceNetworks(
+		nil, networkConfigs, NewNamespace("foo"), "service")
+
+	expected := []swarm.NetworkAttachmentConfig{
+		{
+			Target:  "foo_default",
+			Aliases: []string{"service"},
+		},
+	}
+
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, configs))
+}
+
+func TestConvertServiceNetworks(t *testing.T) {
+	networkConfigs := networkMap{
+		"front": composetypes.NetworkConfig{
+			External: composetypes.External{External: true},
+			Name:     "fronttier",
+		},
+		"back": composetypes.NetworkConfig{},
+	}
+	networks := map[string]*composetypes.ServiceNetworkConfig{
+		"front": {
+			Aliases: []string{"something"},
+		},
+		"back": {
+			Aliases: []string{"other"},
+		},
+	}
+
+	configs, err := convertServiceNetworks(
+		networks, networkConfigs, NewNamespace("foo"), "service")
+
+	expected := []swarm.NetworkAttachmentConfig{
+		{
+			Target:  "foo_back",
+			Aliases: []string{"other", "service"},
+		},
+		{
+			Target:  "fronttier",
+			Aliases: []string{"something", "service"},
+		},
+	}
+
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, configs))
+}
+
+func TestConvertServiceNetworksCustomDefault(t *testing.T) {
+	networkConfigs := networkMap{
+		"default": composetypes.NetworkConfig{
+			External: composetypes.External{External: true},
+			Name:     "custom",
+		},
+	}
+	networks := map[string]*composetypes.ServiceNetworkConfig{}
+
+	configs, err := convertServiceNetworks(
+		networks, networkConfigs, NewNamespace("foo"), "service")
+
+	expected := []swarm.NetworkAttachmentConfig{
+		{
+			Target:  "custom",
+			Aliases: []string{"service"},
+		},
+	}
+
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, configs))
+}
+
+func TestConvertDNSConfigEmpty(t *testing.T) {
+	dnsConfig := convertDNSConfig(nil, nil)
+	assert.Check(t, is.DeepEqual((*swarm.DNSConfig)(nil), dnsConfig))
+}
+
+var (
+	nameservers = []string{"8.8.8.8", "9.9.9.9"}
+	search      = []string{"dc1.example.com", "dc2.example.com"}
+)
+
+func TestConvertDNSConfigAll(t *testing.T) {
+	dnsConfig := convertDNSConfig(nameservers, search)
+	assert.Check(t, is.DeepEqual(&swarm.DNSConfig{
+		Nameservers: nameservers,
+		Search:      search,
+	}, dnsConfig))
+}
+
+func TestConvertDNSConfigNameservers(t *testing.T) {
+	dnsConfig := convertDNSConfig(nameservers, nil)
+	assert.Check(t, is.DeepEqual(&swarm.DNSConfig{
+		Nameservers: nameservers,
+		Search:      nil,
+	}, dnsConfig))
+}
+
+func TestConvertDNSConfigSearch(t *testing.T) {
+	dnsConfig := convertDNSConfig(nil, search)
+	assert.Check(t, is.DeepEqual(&swarm.DNSConfig{
+		Nameservers: nil,
+		Search:      search,
+	}, dnsConfig))
+}
+
+func TestConvertCredentialSpec(t *testing.T) {
+	tests := []struct {
+		name        string
+		in          composetypes.CredentialSpecConfig
+		out         *swarm.CredentialSpec
+		configs     []*swarm.ConfigReference
+		expectedErr string
+	}{
+		{
+			name: "empty",
+		},
+		{
+			name:        "config-and-file",
+			in:          composetypes.CredentialSpecConfig{Config: "0bt9dmxjvjiqermk6xrop3ekq", File: "somefile.json"},
+			expectedErr: `invalid credential spec: cannot specify both "Config" and "File"`,
+		},
+		{
+			name:        "config-and-registry",
+			in:          composetypes.CredentialSpecConfig{Config: "0bt9dmxjvjiqermk6xrop3ekq", Registry: "testing"},
+			expectedErr: `invalid credential spec: cannot specify both "Config" and "Registry"`,
+		},
+		{
+			name:        "file-and-registry",
+			in:          composetypes.CredentialSpecConfig{File: "somefile.json", Registry: "testing"},
+			expectedErr: `invalid credential spec: cannot specify both "File" and "Registry"`,
+		},
+		{
+			name:        "config-and-file-and-registry",
+			in:          composetypes.CredentialSpecConfig{Config: "0bt9dmxjvjiqermk6xrop3ekq", File: "somefile.json", Registry: "testing"},
+			expectedErr: `invalid credential spec: cannot specify both "Config", "File", and "Registry"`,
+		},
+		{
+			name:        "missing-config-reference",
+			in:          composetypes.CredentialSpecConfig{Config: "missing"},
+			expectedErr: "invalid credential spec: spec specifies config missing, but no such config can be found",
+			configs: []*swarm.ConfigReference{
+				{
+					ConfigName: "someName",
+					ConfigID:   "missing",
+				},
+			},
+		},
+		{
+			name: "namespaced-config",
+			in:   composetypes.CredentialSpecConfig{Config: "name"},
+			configs: []*swarm.ConfigReference{
+				{
+					ConfigName: "namespaced-config_name",
+					ConfigID:   "someID",
+				},
+			},
+			out: &swarm.CredentialSpec{Config: "someID"},
+		},
+		{
+			name: "config",
+			in:   composetypes.CredentialSpecConfig{Config: "someName"},
+			configs: []*swarm.ConfigReference{
+				{
+					ConfigName: "someOtherName",
+					ConfigID:   "someOtherID",
+				}, {
+					ConfigName: "someName",
+					ConfigID:   "someID",
+				},
+			},
+			out: &swarm.CredentialSpec{Config: "someID"},
+		},
+		{
+			name: "file",
+			in:   composetypes.CredentialSpecConfig{File: "somefile.json"},
+			out:  &swarm.CredentialSpec{File: "somefile.json"},
+		},
+		{
+			name: "registry",
+			in:   composetypes.CredentialSpecConfig{Registry: "testing"},
+			out:  &swarm.CredentialSpec{Registry: "testing"},
+		},
+	}
+
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			namespace := NewNamespace(tc.name)
+			swarmSpec, err := convertCredentialSpec(namespace, tc.in, tc.configs)
+
+			if tc.expectedErr != "" {
+				assert.Error(t, err, tc.expectedErr)
+			} else {
+				assert.NilError(t, err)
+			}
+			assert.DeepEqual(t, swarmSpec, tc.out)
+		})
+	}
+}
+
+func TestConvertUpdateConfigOrder(t *testing.T) {
+	// test default behavior
+	updateConfig := convertUpdateConfig(&composetypes.UpdateConfig{})
+	assert.Check(t, is.Equal("", updateConfig.Order))
+
+	// test start-first
+	updateConfig = convertUpdateConfig(&composetypes.UpdateConfig{
+		Order: "start-first",
+	})
+	assert.Check(t, is.Equal(updateConfig.Order, "start-first"))
+
+	// test stop-first
+	updateConfig = convertUpdateConfig(&composetypes.UpdateConfig{
+		Order: "stop-first",
+	})
+	assert.Check(t, is.Equal(updateConfig.Order, "stop-first"))
+}
+
+func TestConvertFileObject(t *testing.T) {
+	namespace := NewNamespace("testing")
+	config := composetypes.FileReferenceConfig{
+		Source: "source",
+		Target: "target",
+		UID:    "user",
+		GID:    "group",
+		Mode:   uint32Ptr(0644),
+	}
+	swarmRef, err := convertFileObject(namespace, config, lookupConfig)
+	assert.NilError(t, err)
+
+	expected := swarmReferenceObject{
+		Name: "testing_source",
+		File: swarmReferenceTarget{
+			Name: config.Target,
+			UID:  config.UID,
+			GID:  config.GID,
+			Mode: os.FileMode(0644),
+		},
+	}
+	assert.Check(t, is.DeepEqual(expected, swarmRef))
+}
+
+func lookupConfig(key string) (composetypes.FileObjectConfig, error) {
+	if key != "source" {
+		return composetypes.FileObjectConfig{}, errors.New("bad key")
+	}
+	return composetypes.FileObjectConfig{}, nil
+}
+
+func TestConvertFileObjectDefaults(t *testing.T) {
+	namespace := NewNamespace("testing")
+	config := composetypes.FileReferenceConfig{Source: "source"}
+	swarmRef, err := convertFileObject(namespace, config, lookupConfig)
+	assert.NilError(t, err)
+
+	expected := swarmReferenceObject{
+		Name: "testing_source",
+		File: swarmReferenceTarget{
+			Name: config.Source,
+			UID:  "0",
+			GID:  "0",
+			Mode: os.FileMode(0444),
+		},
+	}
+	assert.Check(t, is.DeepEqual(expected, swarmRef))
+}
+
+func TestServiceConvertsIsolation(t *testing.T) {
+	src := composetypes.ServiceConfig{
+		Isolation: "hyperv",
+	}
+	result, err := Service("1.35", Namespace{name: "foo"}, src, nil, nil, nil, nil)
+	assert.NilError(t, err)
+	assert.Check(t, is.Equal(container.IsolationHyperV, result.TaskTemplate.ContainerSpec.Isolation))
+}
+
+func TestConvertServiceSecrets(t *testing.T) {
+	namespace := Namespace{name: "foo"}
+	secrets := []composetypes.ServiceSecretConfig{
+		{Source: "foo_secret"},
+		{Source: "bar_secret"},
+	}
+	secretSpecs := map[string]composetypes.SecretConfig{
+		"foo_secret": {
+			Name: "foo_secret",
+		},
+		"bar_secret": {
+			Name: "bar_secret",
+		},
+	}
+	client := &fakeClient{
+		secretListFunc: func(opts types.SecretListOptions) ([]swarm.Secret, error) {
+			assert.Check(t, is.Contains(opts.Filters.Get("name"), "foo_secret"))
+			assert.Check(t, is.Contains(opts.Filters.Get("name"), "bar_secret"))
+			return []swarm.Secret{
+				{Spec: swarm.SecretSpec{Annotations: swarm.Annotations{Name: "foo_secret"}}},
+				{Spec: swarm.SecretSpec{Annotations: swarm.Annotations{Name: "bar_secret"}}},
+			}, nil
+		},
+	}
+	refs, err := convertServiceSecrets(client, namespace, secrets, secretSpecs)
+	assert.NilError(t, err)
+	expected := []*swarm.SecretReference{
+		{
+			SecretName: "bar_secret",
+			File: &swarm.SecretReferenceFileTarget{
+				Name: "bar_secret",
+				UID:  "0",
+				GID:  "0",
+				Mode: 0444,
+			},
+		},
+		{
+			SecretName: "foo_secret",
+			File: &swarm.SecretReferenceFileTarget{
+				Name: "foo_secret",
+				UID:  "0",
+				GID:  "0",
+				Mode: 0444,
+			},
+		},
+	}
+	assert.DeepEqual(t, expected, refs)
+}
+
+func TestConvertServiceConfigs(t *testing.T) {
+	namespace := Namespace{name: "foo"}
+	service := composetypes.ServiceConfig{
+		Configs: []composetypes.ServiceConfigObjConfig{
+			{Source: "foo_config"},
+			{Source: "bar_config"},
+		},
+		CredentialSpec: composetypes.CredentialSpecConfig{
+			Config: "baz_config",
+		},
+	}
+	configSpecs := map[string]composetypes.ConfigObjConfig{
+		"foo_config": {
+			Name: "foo_config",
+		},
+		"bar_config": {
+			Name: "bar_config",
+		},
+		"baz_config": {
+			Name: "baz_config",
+		},
+	}
+	client := &fakeClient{
+		configListFunc: func(opts types.ConfigListOptions) ([]swarm.Config, error) {
+			assert.Check(t, is.Contains(opts.Filters.Get("name"), "foo_config"))
+			assert.Check(t, is.Contains(opts.Filters.Get("name"), "bar_config"))
+			assert.Check(t, is.Contains(opts.Filters.Get("name"), "baz_config"))
+			return []swarm.Config{
+				{Spec: swarm.ConfigSpec{Annotations: swarm.Annotations{Name: "foo_config"}}},
+				{Spec: swarm.ConfigSpec{Annotations: swarm.Annotations{Name: "bar_config"}}},
+				{Spec: swarm.ConfigSpec{Annotations: swarm.Annotations{Name: "baz_config"}}},
+			}, nil
+		},
+	}
+	refs, err := convertServiceConfigObjs(client, namespace, service, configSpecs)
+	assert.NilError(t, err)
+	expected := []*swarm.ConfigReference{
+		{
+			ConfigName: "bar_config",
+			File: &swarm.ConfigReferenceFileTarget{
+				Name: "bar_config",
+				UID:  "0",
+				GID:  "0",
+				Mode: 0444,
+			},
+		},
+		{
+			ConfigName: "baz_config",
+			Runtime:    &swarm.ConfigReferenceRuntimeTarget{},
+		},
+		{
+			ConfigName: "foo_config",
+			File: &swarm.ConfigReferenceFileTarget{
+				Name: "foo_config",
+				UID:  "0",
+				GID:  "0",
+				Mode: 0444,
+			},
+		},
+	}
+	assert.DeepEqual(t, expected, refs)
+}
+
+type fakeClient struct {
+	client.Client
+	secretListFunc func(types.SecretListOptions) ([]swarm.Secret, error)
+	configListFunc func(types.ConfigListOptions) ([]swarm.Config, error)
+}
+
+func (c *fakeClient) SecretList(ctx context.Context, options types.SecretListOptions) ([]swarm.Secret, error) {
+	if c.secretListFunc != nil {
+		return c.secretListFunc(options)
+	}
+	return []swarm.Secret{}, nil
+}
+
+func (c *fakeClient) ConfigList(ctx context.Context, options types.ConfigListOptions) ([]swarm.Config, error) {
+	if c.configListFunc != nil {
+		return c.configListFunc(options)
+	}
+	return []swarm.Config{}, nil
+}
+
+func TestConvertUpdateConfigParallelism(t *testing.T) {
+	parallel := uint64(4)
+
+	// test default behavior
+	updateConfig := convertUpdateConfig(&composetypes.UpdateConfig{})
+	assert.Check(t, is.Equal(uint64(1), updateConfig.Parallelism))
+
+	// Non default value
+	updateConfig = convertUpdateConfig(&composetypes.UpdateConfig{
+		Parallelism: &parallel,
+	})
+	assert.Check(t, is.Equal(parallel, updateConfig.Parallelism))
+}
+
+func TestConvertServiceCapAddAndCapDrop(t *testing.T) {
+	tests := []struct {
+		title   string
+		in, out composetypes.ServiceConfig
+	}{
+		{
+			title: "default behavior",
+		},
+		{
+			title: "some values",
+			in: composetypes.ServiceConfig{
+				CapAdd:  []string{"SYS_NICE", "CAP_NET_ADMIN"},
+				CapDrop: []string{"CHOWN", "CAP_NET_ADMIN", "DAC_OVERRIDE", "CAP_FSETID", "CAP_FOWNER"},
+			},
+			out: composetypes.ServiceConfig{
+				CapAdd:  []string{"CAP_NET_ADMIN", "CAP_SYS_NICE"},
+				CapDrop: []string{"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_FOWNER", "CAP_FSETID"},
+			},
+		},
+		{
+			title: "adding ALL capabilities",
+			in: composetypes.ServiceConfig{
+				CapAdd:  []string{"ALL", "CAP_NET_ADMIN"},
+				CapDrop: []string{"CHOWN", "CAP_NET_ADMIN", "DAC_OVERRIDE", "CAP_FSETID", "CAP_FOWNER"},
+			},
+			out: composetypes.ServiceConfig{
+				CapAdd:  []string{"ALL"},
+				CapDrop: []string{"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_FOWNER", "CAP_FSETID", "CAP_NET_ADMIN"},
+			},
+		},
+		{
+			title: "dropping ALL capabilities",
+			in: composetypes.ServiceConfig{
+				CapAdd:  []string{"CHOWN", "CAP_NET_ADMIN", "DAC_OVERRIDE", "CAP_FSETID", "CAP_FOWNER"},
+				CapDrop: []string{"ALL", "CAP_NET_ADMIN", "CAP_FOO"},
+			},
+			out: composetypes.ServiceConfig{
+				CapAdd:  []string{"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_FOWNER", "CAP_FSETID", "CAP_NET_ADMIN"},
+				CapDrop: []string{"ALL"},
+			},
+		},
+	}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.title, func(t *testing.T) {
+			result, err := Service("1.41", Namespace{name: "foo"}, tc.in, nil, nil, nil, nil)
+			assert.NilError(t, err)
+			assert.Check(t, is.DeepEqual(result.TaskTemplate.ContainerSpec.CapabilityAdd, tc.out.CapAdd))
+			assert.Check(t, is.DeepEqual(result.TaskTemplate.ContainerSpec.CapabilityDrop, tc.out.CapDrop))
+		})
+	}
+}

pkg/client/convert/volume.go

@@ -0,0 +1,162 @@
+package convert
+
+import (
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types/mount"
+	"github.com/pkg/errors"
+)
+
+type volumes map[string]composetypes.VolumeConfig
+
+// Volumes from compose-file types to engine api types
+func Volumes(serviceVolumes []composetypes.ServiceVolumeConfig, stackVolumes volumes, namespace Namespace) ([]mount.Mount, error) {
+	var mounts []mount.Mount
+
+	for _, volumeConfig := range serviceVolumes {
+		mount, err := convertVolumeToMount(volumeConfig, stackVolumes, namespace)
+		if err != nil {
+			return nil, err
+		}
+		mounts = append(mounts, mount)
+	}
+	return mounts, nil
+}
+
+func createMountFromVolume(volume composetypes.ServiceVolumeConfig) mount.Mount {
+	return mount.Mount{
+		Type:        mount.Type(volume.Type),
+		Target:      volume.Target,
+		ReadOnly:    volume.ReadOnly,
+		Source:      volume.Source,
+		Consistency: mount.Consistency(volume.Consistency),
+	}
+}
+
+func handleVolumeToMount(
+	volume composetypes.ServiceVolumeConfig,
+	stackVolumes volumes,
+	namespace Namespace,
+) (mount.Mount, error) {
+	result := createMountFromVolume(volume)
+
+	if volume.Tmpfs != nil {
+		return mount.Mount{}, errors.New("tmpfs options are incompatible with type volume")
+	}
+	if volume.Bind != nil {
+		return mount.Mount{}, errors.New("bind options are incompatible with type volume")
+	}
+	// Anonymous volumes
+	if volume.Source == "" {
+		return result, nil
+	}
+
+	stackVolume, exists := stackVolumes[volume.Source]
+	if !exists {
+		return mount.Mount{}, errors.Errorf("undefined volume %q", volume.Source)
+	}
+
+	result.Source = namespace.Scope(volume.Source)
+	result.VolumeOptions = &mount.VolumeOptions{}
+
+	if volume.Volume != nil {
+		result.VolumeOptions.NoCopy = volume.Volume.NoCopy
+	}
+
+	if stackVolume.Name != "" {
+		result.Source = stackVolume.Name
+	}
+
+	// External named volumes
+	if stackVolume.External.External {
+		return result, nil
+	}
+
+	result.VolumeOptions.Labels = AddStackLabel(namespace, stackVolume.Labels)
+	if stackVolume.Driver != "" || stackVolume.DriverOpts != nil {
+		result.VolumeOptions.DriverConfig = &mount.Driver{
+			Name:    stackVolume.Driver,
+			Options: stackVolume.DriverOpts,
+		}
+	}
+
+	return result, nil
+}
+
+func handleBindToMount(volume composetypes.ServiceVolumeConfig) (mount.Mount, error) {
+	result := createMountFromVolume(volume)
+
+	if volume.Source == "" {
+		return mount.Mount{}, errors.New("invalid bind source, source cannot be empty")
+	}
+	if volume.Volume != nil {
+		return mount.Mount{}, errors.New("volume options are incompatible with type bind")
+	}
+	if volume.Tmpfs != nil {
+		return mount.Mount{}, errors.New("tmpfs options are incompatible with type bind")
+	}
+	if volume.Bind != nil {
+		result.BindOptions = &mount.BindOptions{
+			Propagation: mount.Propagation(volume.Bind.Propagation),
+		}
+	}
+	return result, nil
+}
+
+func handleTmpfsToMount(volume composetypes.ServiceVolumeConfig) (mount.Mount, error) {
+	result := createMountFromVolume(volume)
+
+	if volume.Source != "" {
+		return mount.Mount{}, errors.New("invalid tmpfs source, source must be empty")
+	}
+	if volume.Bind != nil {
+		return mount.Mount{}, errors.New("bind options are incompatible with type tmpfs")
+	}
+	if volume.Volume != nil {
+		return mount.Mount{}, errors.New("volume options are incompatible with type tmpfs")
+	}
+	if volume.Tmpfs != nil {
+		result.TmpfsOptions = &mount.TmpfsOptions{
+			SizeBytes: volume.Tmpfs.Size,
+		}
+	}
+	return result, nil
+}
+
+func handleNpipeToMount(volume composetypes.ServiceVolumeConfig) (mount.Mount, error) {
+	result := createMountFromVolume(volume)
+
+	if volume.Source == "" {
+		return mount.Mount{}, errors.New("invalid npipe source, source cannot be empty")
+	}
+	if volume.Volume != nil {
+		return mount.Mount{}, errors.New("volume options are incompatible with type npipe")
+	}
+	if volume.Tmpfs != nil {
+		return mount.Mount{}, errors.New("tmpfs options are incompatible with type npipe")
+	}
+	if volume.Bind != nil {
+		result.BindOptions = &mount.BindOptions{
+			Propagation: mount.Propagation(volume.Bind.Propagation),
+		}
+	}
+	return result, nil
+}
+
+func convertVolumeToMount(
+	volume composetypes.ServiceVolumeConfig,
+	stackVolumes volumes,
+	namespace Namespace,
+) (mount.Mount, error) {
+
+	switch volume.Type {
+	case "volume", "":
+		return handleVolumeToMount(volume, stackVolumes, namespace)
+	case "bind":
+		return handleBindToMount(volume)
+	case "tmpfs":
+		return handleTmpfsToMount(volume)
+	case "npipe":
+		return handleNpipeToMount(volume)
+	}
+	return mount.Mount{}, errors.New("volume type must be volume, bind, tmpfs or npipe")
+}

pkg/client/convert/volume_test.go

@@ -0,0 +1,361 @@
+package convert
+
+import (
+	"testing"
+
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types/mount"
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+)
+
+func TestConvertVolumeToMountAnonymousVolume(t *testing.T) {
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "volume",
+		Target: "/foo/bar",
+	}
+	expected := mount.Mount{
+		Type:   mount.TypeVolume,
+		Target: "/foo/bar",
+	}
+	mount, err := convertVolumeToMount(config, volumes{}, NewNamespace("foo"))
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, mount))
+}
+
+func TestConvertVolumeToMountAnonymousBind(t *testing.T) {
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "bind",
+		Target: "/foo/bar",
+		Bind: &composetypes.ServiceVolumeBind{
+			Propagation: "slave",
+		},
+	}
+	_, err := convertVolumeToMount(config, volumes{}, NewNamespace("foo"))
+	assert.Error(t, err, "invalid bind source, source cannot be empty")
+}
+
+func TestConvertVolumeToMountUnapprovedType(t *testing.T) {
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "foo",
+		Target: "/foo/bar",
+	}
+	_, err := convertVolumeToMount(config, volumes{}, NewNamespace("foo"))
+	assert.Error(t, err, "volume type must be volume, bind, tmpfs or npipe")
+}
+
+func TestConvertVolumeToMountConflictingOptionsBindInVolume(t *testing.T) {
+	namespace := NewNamespace("foo")
+
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "volume",
+		Source: "foo",
+		Target: "/target",
+		Bind: &composetypes.ServiceVolumeBind{
+			Propagation: "slave",
+		},
+	}
+	_, err := convertVolumeToMount(config, volumes{}, namespace)
+	assert.Error(t, err, "bind options are incompatible with type volume")
+}
+
+func TestConvertVolumeToMountConflictingOptionsTmpfsInVolume(t *testing.T) {
+	namespace := NewNamespace("foo")
+
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "volume",
+		Source: "foo",
+		Target: "/target",
+		Tmpfs: &composetypes.ServiceVolumeTmpfs{
+			Size: 1000,
+		},
+	}
+	_, err := convertVolumeToMount(config, volumes{}, namespace)
+	assert.Error(t, err, "tmpfs options are incompatible with type volume")
+}
+
+func TestConvertVolumeToMountConflictingOptionsVolumeInBind(t *testing.T) {
+	namespace := NewNamespace("foo")
+
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "bind",
+		Source: "/foo",
+		Target: "/target",
+		Volume: &composetypes.ServiceVolumeVolume{
+			NoCopy: true,
+		},
+	}
+	_, err := convertVolumeToMount(config, volumes{}, namespace)
+	assert.Error(t, err, "volume options are incompatible with type bind")
+}
+
+func TestConvertVolumeToMountConflictingOptionsTmpfsInBind(t *testing.T) {
+	namespace := NewNamespace("foo")
+
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "bind",
+		Source: "/foo",
+		Target: "/target",
+		Tmpfs: &composetypes.ServiceVolumeTmpfs{
+			Size: 1000,
+		},
+	}
+	_, err := convertVolumeToMount(config, volumes{}, namespace)
+	assert.Error(t, err, "tmpfs options are incompatible with type bind")
+}
+
+func TestConvertVolumeToMountConflictingOptionsBindInTmpfs(t *testing.T) {
+	namespace := NewNamespace("foo")
+
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "tmpfs",
+		Target: "/target",
+		Bind: &composetypes.ServiceVolumeBind{
+			Propagation: "slave",
+		},
+	}
+	_, err := convertVolumeToMount(config, volumes{}, namespace)
+	assert.Error(t, err, "bind options are incompatible with type tmpfs")
+}
+
+func TestConvertVolumeToMountConflictingOptionsVolumeInTmpfs(t *testing.T) {
+	namespace := NewNamespace("foo")
+
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "tmpfs",
+		Target: "/target",
+		Volume: &composetypes.ServiceVolumeVolume{
+			NoCopy: true,
+		},
+	}
+	_, err := convertVolumeToMount(config, volumes{}, namespace)
+	assert.Error(t, err, "volume options are incompatible with type tmpfs")
+}
+
+func TestConvertVolumeToMountNamedVolume(t *testing.T) {
+	stackVolumes := volumes{
+		"normal": composetypes.VolumeConfig{
+			Driver: "glusterfs",
+			DriverOpts: map[string]string{
+				"opt": "value",
+			},
+			Labels: map[string]string{
+				"something": "labeled",
+			},
+		},
+	}
+	namespace := NewNamespace("foo")
+	expected := mount.Mount{
+		Type:     mount.TypeVolume,
+		Source:   "foo_normal",
+		Target:   "/foo",
+		ReadOnly: true,
+		VolumeOptions: &mount.VolumeOptions{
+			Labels: map[string]string{
+				LabelNamespace: "foo",
+				"something":    "labeled",
+			},
+			DriverConfig: &mount.Driver{
+				Name: "glusterfs",
+				Options: map[string]string{
+					"opt": "value",
+				},
+			},
+			NoCopy: true,
+		},
+	}
+	config := composetypes.ServiceVolumeConfig{
+		Type:     "volume",
+		Source:   "normal",
+		Target:   "/foo",
+		ReadOnly: true,
+		Volume: &composetypes.ServiceVolumeVolume{
+			NoCopy: true,
+		},
+	}
+	mount, err := convertVolumeToMount(config, stackVolumes, namespace)
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, mount))
+}
+
+func TestConvertVolumeToMountNamedVolumeWithNameCustomizd(t *testing.T) {
+	stackVolumes := volumes{
+		"normal": composetypes.VolumeConfig{
+			Name:   "user_specified_name",
+			Driver: "vsphere",
+			DriverOpts: map[string]string{
+				"opt": "value",
+			},
+			Labels: map[string]string{
+				"something": "labeled",
+			},
+		},
+	}
+	namespace := NewNamespace("foo")
+	expected := mount.Mount{
+		Type:     mount.TypeVolume,
+		Source:   "user_specified_name",
+		Target:   "/foo",
+		ReadOnly: true,
+		VolumeOptions: &mount.VolumeOptions{
+			Labels: map[string]string{
+				LabelNamespace: "foo",
+				"something":    "labeled",
+			},
+			DriverConfig: &mount.Driver{
+				Name: "vsphere",
+				Options: map[string]string{
+					"opt": "value",
+				},
+			},
+			NoCopy: true,
+		},
+	}
+	config := composetypes.ServiceVolumeConfig{
+		Type:     "volume",
+		Source:   "normal",
+		Target:   "/foo",
+		ReadOnly: true,
+		Volume: &composetypes.ServiceVolumeVolume{
+			NoCopy: true,
+		},
+	}
+	mount, err := convertVolumeToMount(config, stackVolumes, namespace)
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, mount))
+}
+
+func TestConvertVolumeToMountNamedVolumeExternal(t *testing.T) {
+	stackVolumes := volumes{
+		"outside": composetypes.VolumeConfig{
+			Name:     "special",
+			External: composetypes.External{External: true},
+		},
+	}
+	namespace := NewNamespace("foo")
+	expected := mount.Mount{
+		Type:          mount.TypeVolume,
+		Source:        "special",
+		Target:        "/foo",
+		VolumeOptions: &mount.VolumeOptions{NoCopy: false},
+	}
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "volume",
+		Source: "outside",
+		Target: "/foo",
+	}
+	mount, err := convertVolumeToMount(config, stackVolumes, namespace)
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, mount))
+}
+
+func TestConvertVolumeToMountNamedVolumeExternalNoCopy(t *testing.T) {
+	stackVolumes := volumes{
+		"outside": composetypes.VolumeConfig{
+			Name:     "special",
+			External: composetypes.External{External: true},
+		},
+	}
+	namespace := NewNamespace("foo")
+	expected := mount.Mount{
+		Type:   mount.TypeVolume,
+		Source: "special",
+		Target: "/foo",
+		VolumeOptions: &mount.VolumeOptions{
+			NoCopy: true,
+		},
+	}
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "volume",
+		Source: "outside",
+		Target: "/foo",
+		Volume: &composetypes.ServiceVolumeVolume{
+			NoCopy: true,
+		},
+	}
+	mount, err := convertVolumeToMount(config, stackVolumes, namespace)
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, mount))
+}
+
+func TestConvertVolumeToMountBind(t *testing.T) {
+	stackVolumes := volumes{}
+	namespace := NewNamespace("foo")
+	expected := mount.Mount{
+		Type:        mount.TypeBind,
+		Source:      "/bar",
+		Target:      "/foo",
+		ReadOnly:    true,
+		BindOptions: &mount.BindOptions{Propagation: mount.PropagationShared},
+	}
+	config := composetypes.ServiceVolumeConfig{
+		Type:     "bind",
+		Source:   "/bar",
+		Target:   "/foo",
+		ReadOnly: true,
+		Bind:     &composetypes.ServiceVolumeBind{Propagation: "shared"},
+	}
+	mount, err := convertVolumeToMount(config, stackVolumes, namespace)
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, mount))
+}
+
+func TestConvertVolumeToMountVolumeDoesNotExist(t *testing.T) {
+	namespace := NewNamespace("foo")
+	config := composetypes.ServiceVolumeConfig{
+		Type:     "volume",
+		Source:   "unknown",
+		Target:   "/foo",
+		ReadOnly: true,
+	}
+	_, err := convertVolumeToMount(config, volumes{}, namespace)
+	assert.Error(t, err, "undefined volume \"unknown\"")
+}
+
+func TestConvertTmpfsToMountVolume(t *testing.T) {
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "tmpfs",
+		Target: "/foo/bar",
+		Tmpfs: &composetypes.ServiceVolumeTmpfs{
+			Size: 1000,
+		},
+	}
+	expected := mount.Mount{
+		Type:         mount.TypeTmpfs,
+		Target:       "/foo/bar",
+		TmpfsOptions: &mount.TmpfsOptions{SizeBytes: 1000},
+	}
+	mount, err := convertVolumeToMount(config, volumes{}, NewNamespace("foo"))
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, mount))
+}
+
+func TestConvertTmpfsToMountVolumeWithSource(t *testing.T) {
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "tmpfs",
+		Source: "/bar",
+		Target: "/foo/bar",
+		Tmpfs: &composetypes.ServiceVolumeTmpfs{
+			Size: 1000,
+		},
+	}
+
+	_, err := convertVolumeToMount(config, volumes{}, NewNamespace("foo"))
+	assert.Error(t, err, "invalid tmpfs source, source must be empty")
+}
+
+func TestConvertVolumeToMountAnonymousNpipe(t *testing.T) {
+	config := composetypes.ServiceVolumeConfig{
+		Type:   "npipe",
+		Source: `\\.\pipe\foo`,
+		Target: `\\.\pipe\foo`,
+	}
+	expected := mount.Mount{
+		Type:   mount.TypeNamedPipe,
+		Source: `\\.\pipe\foo`,
+		Target: `\\.\pipe\foo`,
+	}
+	mount, err := convertVolumeToMount(config, volumes{}, NewNamespace("foo"))
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(expected, mount))
+}

pkg/client/registry.go

@@ -0,0 +1,170 @@
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/web"
+	"github.com/docker/distribution/reference"
+)
+
+type RawTag struct {
+	Layer string
+	Name  string
+}
+
+type RawTags []RawTag
+
+var registryURL = "https://registry.hub.docker.com/v1/repositories/%s/tags"
+
+func GetRegistryTags(image string) (RawTags, error) {
+	var tags RawTags
+
+	tagsUrl := fmt.Sprintf(registryURL, image)
+	if err := web.ReadJSON(tagsUrl, &tags); err != nil {
+		return tags, err
+	}
+
+	return tags, nil
+}
+
+// getRegv2Token retrieves a registry v2 authentication token.
+func getRegv2Token(image reference.Named) (string, error) {
+	img := reference.Path(image)
+	authTokenURL := fmt.Sprintf("https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull", img)
+	req, err := http.NewRequest("GET", authTokenURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	client := &http.Client{Timeout: web.Timeout}
+	res, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		_, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", nil
+	}
+
+	tokenRes := struct {
+		Token  string
+		Expiry string
+		Issued string
+	}{}
+
+	if err := json.Unmarshal(body, &tokenRes); err != nil {
+		return "", err
+	}
+
+	return tokenRes.Token, nil
+}
+
+// GetTagDigest retrieves an image digest from a v2 registry
+func GetTagDigest(image reference.Named) (string, error) {
+	img := reference.Path(image)
+	tag := image.(reference.NamedTagged).Tag()
+	manifestURL := fmt.Sprintf("https://index.docker.io/v2/%s/manifests/%s", img, tag)
+
+	req, err := http.NewRequest("GET", manifestURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	token, err := getRegv2Token(image)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header = http.Header{
+		"Accept": []string{
+			"application/vnd.docker.distribution.manifest.v2+json",
+			"application/vnd.docker.distribution.manifest.list.v2+json",
+		},
+		"Authorization": []string{fmt.Sprintf("Bearer %s", token)},
+	}
+
+	client := &http.Client{Timeout: web.Timeout}
+	res, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		_, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+
+	registryResT1 := struct {
+		SchemaVersion int
+		MediaType     string
+		Manifests     []struct {
+			MediaType string
+			Size      int
+			Digest    string
+			Platform  struct {
+				Architecture string
+				Os           string
+			}
+		}
+	}{}
+
+	registryResT2 := struct {
+		SchemaVersion int
+		MediaType     string
+		Config        struct {
+			MediaType string
+			Size      int
+			Digest    string
+		}
+		Layers []struct {
+			MediaType string
+			Size      int
+			Digest    string
+		}
+	}{}
+
+	if err := json.Unmarshal(body, &registryResT1); err != nil {
+		return "", err
+	}
+
+	var digest string
+	for _, manifest := range registryResT1.Manifests {
+		if string(manifest.Platform.Architecture) == "amd64" {
+			digest = strings.Split(manifest.Digest, ":")[1][:7]
+		}
+	}
+
+	if digest == "" {
+		if err := json.Unmarshal(body, &registryResT2); err != nil {
+			return "", err
+		}
+		digest = strings.Split(registryResT2.Config.Digest, ":")[1][:7]
+	}
+
+	if digest == "" {
+		return "", fmt.Errorf("Unable to retrieve amd64 digest for '%s'", image)
+	}
+
+	return digest, nil
+}

pkg/client/secret.go

@@ -0,0 +1,25 @@
+package client
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types/swarm"
+)
+
+func StoreSecret(secretName, secretValue, server string) error {
+	cl, err := New(server)
+	if err != nil {
+		return err
+	}
+
+	ctx := context.Background()
+	ann := swarm.Annotations{Name: secretName}
+	spec := swarm.SecretSpec{Annotations: ann, Data: []byte(secretValue)}
+
+	// We don't bother with the secret IDs for now
+	if _, err := cl.SecretCreate(ctx, spec); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/client/stack/loader.go

@@ -0,0 +1,131 @@
+package stack
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/docker/cli/cli/compose/loader"
+	"github.com/docker/cli/cli/compose/schema"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/sirupsen/logrus"
+)
+
+// LoadComposefile parse the composefile specified in the cli and returns its Config and version.
+func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
+	configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
+	if err != nil {
+		return nil, err
+	}
+
+	dicts := getDictsFrom(configDetails.ConfigFiles)
+	config, err := loader.Load(configDetails)
+	if err != nil {
+		if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
+			return nil, fmt.Errorf("compose file contains unsupported options:\n\n%s",
+				propertyWarnings(fpe.Properties))
+		}
+
+		return nil, err
+	}
+
+	unsupportedProperties := loader.GetUnsupportedProperties(dicts...)
+	if len(unsupportedProperties) > 0 {
+		logrus.Warnf("Ignoring unsupported options: %s\n\n",
+			strings.Join(unsupportedProperties, ", "))
+	}
+
+	deprecatedProperties := loader.GetDeprecatedProperties(dicts...)
+	if len(deprecatedProperties) > 0 {
+		logrus.Warnf("Ignoring deprecated options:\n\n%s\n\n",
+			propertyWarnings(deprecatedProperties))
+	}
+	return config, nil
+}
+
+func getDictsFrom(configFiles []composetypes.ConfigFile) []map[string]interface{} {
+	dicts := []map[string]interface{}{}
+
+	for _, configFile := range configFiles {
+		dicts = append(dicts, configFile.Config)
+	}
+
+	return dicts
+}
+
+func propertyWarnings(properties map[string]string) string {
+	var msgs []string
+	for name, description := range properties {
+		msgs = append(msgs, fmt.Sprintf("%s: %s", name, description))
+	}
+	sort.Strings(msgs)
+	return strings.Join(msgs, "\n\n")
+}
+
+func getConfigDetails(composefiles []string, appEnv map[string]string) (composetypes.ConfigDetails, error) {
+	var details composetypes.ConfigDetails
+
+	absPath, err := filepath.Abs(composefiles[0])
+	if err != nil {
+		return details, err
+	}
+	details.WorkingDir = filepath.Dir(absPath)
+
+	details.ConfigFiles, err = loadConfigFiles(composefiles)
+	if err != nil {
+		return details, err
+	}
+	// Take the first file version (2 files can't have different version)
+	details.Version = schema.Version(details.ConfigFiles[0].Config)
+	details.Environment = appEnv
+	return details, err
+}
+
+func buildEnvironment(env []string) (map[string]string, error) {
+	result := make(map[string]string, len(env))
+	for _, s := range env {
+		// if value is empty, s is like "K=", not "K".
+		if !strings.Contains(s, "=") {
+			return result, fmt.Errorf("unexpected environment %q", s)
+		}
+		kv := strings.SplitN(s, "=", 2)
+		result[kv[0]] = kv[1]
+	}
+	return result, nil
+}
+
+func loadConfigFiles(filenames []string) ([]composetypes.ConfigFile, error) {
+	var configFiles []composetypes.ConfigFile
+
+	for _, filename := range filenames {
+		configFile, err := loadConfigFile(filename)
+		if err != nil {
+			return configFiles, err
+		}
+		configFiles = append(configFiles, *configFile)
+	}
+
+	return configFiles, nil
+}
+
+func loadConfigFile(filename string) (*composetypes.ConfigFile, error) {
+	var bytes []byte
+	var err error
+
+	bytes, err = ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	config, err := loader.ParseYAML(bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	return &composetypes.ConfigFile{
+		Filename: filename,
+		Config:   config,
+	}, nil
+}

pkg/client/stack/options.go

@@ -0,0 +1,15 @@
+package stack
+
+// Deploy holds docker stack deploy options
+type Deploy struct {
+	Composefiles     []string
+	Namespace        string
+	ResolveImage     string
+	SendRegistryAuth bool
+	Prune            bool
+}
+
+// Remove holds docker stack remove options
+type Remove struct {
+	Namespaces []string
+}

pkg/client/stack/remove.go

@@ -0,0 +1,138 @@
+package stack
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/versions"
+	apiclient "github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// RunRemove is the swarm implementation of docker stack remove
+func RunRemove(ctx context.Context, client *apiclient.Client, opts Remove) error {
+	var errs []string
+	for _, namespace := range opts.Namespaces {
+		services, err := getStackServices(ctx, client, namespace)
+		if err != nil {
+			return err
+		}
+
+		networks, err := getStackNetworks(ctx, client, namespace)
+		if err != nil {
+			return err
+		}
+
+		var secrets []swarm.Secret
+		if versions.GreaterThanOrEqualTo(client.ClientVersion(), "1.25") {
+			secrets, err = getStackSecrets(ctx, client, namespace)
+			if err != nil {
+				return err
+			}
+		}
+
+		var configs []swarm.Config
+		if versions.GreaterThanOrEqualTo(client.ClientVersion(), "1.30") {
+			configs, err = getStackConfigs(ctx, client, namespace)
+			if err != nil {
+				return err
+			}
+		}
+
+		if len(services)+len(networks)+len(secrets)+len(configs) == 0 {
+			logrus.Warning(fmt.Errorf("nothing found in stack: %s", namespace))
+			continue
+		}
+
+		hasError := removeServices(ctx, client, services)
+		hasError = removeSecrets(ctx, client, secrets) || hasError
+		hasError = removeConfigs(ctx, client, configs) || hasError
+		hasError = removeNetworks(ctx, client, networks) || hasError
+
+		if hasError {
+			errs = append(errs, fmt.Sprintf("failed to remove some resources from stack: %s", namespace))
+		}
+	}
+
+	if len(errs) > 0 {
+		return errors.Errorf(strings.Join(errs, "\n"))
+	}
+
+	return nil
+}
+
+func sortServiceByName(services []swarm.Service) func(i, j int) bool {
+	return func(i, j int) bool {
+		return services[i].Spec.Name < services[j].Spec.Name
+	}
+}
+
+func removeServices(
+	ctx context.Context,
+	client *apiclient.Client,
+	services []swarm.Service,
+) bool {
+	var hasError bool
+	sort.Slice(services, sortServiceByName(services))
+	for _, service := range services {
+		logrus.Infof("removing service %s\n", service.Spec.Name)
+		if err := client.ServiceRemove(ctx, service.ID); err != nil {
+			hasError = true
+			logrus.Fatalf("failed to remove service %s: %s", service.ID, err)
+		}
+	}
+	return hasError
+}
+
+func removeNetworks(
+	ctx context.Context,
+	client *apiclient.Client,
+	networks []types.NetworkResource,
+) bool {
+	var hasError bool
+	for _, network := range networks {
+		logrus.Infof("removing network %s\n", network.Name)
+		if err := client.NetworkRemove(ctx, network.ID); err != nil {
+			hasError = true
+			logrus.Fatalf("failed to remove network %s: %s", network.ID, err)
+		}
+	}
+	return hasError
+}
+
+func removeSecrets(
+	ctx context.Context,
+	client *apiclient.Client,
+	secrets []swarm.Secret,
+) bool {
+	var hasError bool
+	for _, secret := range secrets {
+		logrus.Infof("Removing secret %s\n", secret.Spec.Name)
+		if err := client.SecretRemove(ctx, secret.ID); err != nil {
+			hasError = true
+			logrus.Fatalf("Failed to remove secret %s: %s", secret.ID, err)
+		}
+	}
+	return hasError
+}
+
+func removeConfigs(
+	ctx context.Context,
+	client *apiclient.Client,
+	configs []swarm.Config,
+) bool {
+	var hasError bool
+	for _, config := range configs {
+		logrus.Infof("removing config %s\n", config.Spec.Name)
+		if err := client.ConfigRemove(ctx, config.ID); err != nil {
+			hasError = true
+			logrus.Fatalf("failed to remove config %s: %s", config.ID, err)
+		}
+	}
+	return hasError
+}

pkg/client/stack/stack.go

@@ -0,0 +1,393 @@
+package stack
+
+import (
+	"context"
+	"strings"
+
+	abraClient "coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/client/convert"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/versions"
+	"github.com/docker/docker/client"
+	dockerclient "github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Resolve image constants
+const (
+	defaultNetworkDriver = "overlay"
+	ResolveImageAlways   = "always"
+	ResolveImageChanged  = "changed"
+	ResolveImageNever    = "never"
+)
+
+type StackStatus struct {
+	Services []swarm.Service
+	Err      error
+}
+
+func getStackFilter(namespace string) filters.Args {
+	filter := filters.NewArgs()
+	filter.Add("label", convert.LabelNamespace+"="+namespace)
+	return filter
+}
+
+func getStackServiceFilter(namespace string) filters.Args {
+	return getStackFilter(namespace)
+}
+
+func getAllStacksFilter() filters.Args {
+	filter := filters.NewArgs()
+	filter.Add("label", convert.LabelNamespace)
+	return filter
+}
+
+func getStackServices(ctx context.Context, dockerclient client.APIClient, namespace string) ([]swarm.Service, error) {
+	return dockerclient.ServiceList(ctx, types.ServiceListOptions{Filters: getStackServiceFilter(namespace)})
+}
+
+// GetDeployedServicesByLabel filters services by label
+func GetDeployedServicesByLabel(contextName string, label string) StackStatus {
+	cl, err := abraClient.New(contextName)
+	if err != nil {
+		if strings.Contains(err.Error(), "does not exist") {
+			// No local context found, bail out gracefully
+			return StackStatus{[]swarm.Service{}, nil}
+		}
+		return StackStatus{[]swarm.Service{}, err}
+	}
+
+	ctx := context.Background()
+	filters := filters.NewArgs()
+	filters.Add("label", label)
+	services, err := cl.ServiceList(ctx, types.ServiceListOptions{Filters: filters})
+	if err != nil {
+		return StackStatus{[]swarm.Service{}, err}
+	}
+
+	return StackStatus{services, nil}
+}
+
+func GetAllDeployedServices(contextName string) StackStatus {
+	cl, err := abraClient.New(contextName)
+	if err != nil {
+		if strings.Contains(err.Error(), "does not exist") {
+			// No local context found, bail out gracefully
+			return StackStatus{[]swarm.Service{}, nil}
+		}
+		return StackStatus{[]swarm.Service{}, err}
+	}
+
+	ctx := context.Background()
+	services, err := cl.ServiceList(ctx, types.ServiceListOptions{Filters: getAllStacksFilter()})
+	if err != nil {
+		return StackStatus{[]swarm.Service{}, err}
+	}
+
+	return StackStatus{services, nil}
+}
+
+// pruneServices removes services that are no longer referenced in the source
+func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace convert.Namespace, services map[string]struct{}) {
+	oldServices, err := getStackServices(ctx, cl, namespace.Name())
+	if err != nil {
+		logrus.Infof("Failed to list services: %s\n", err)
+	}
+
+	pruneServices := []swarm.Service{}
+	for _, service := range oldServices {
+		if _, exists := services[namespace.Descope(service.Spec.Name)]; !exists {
+			pruneServices = append(pruneServices, service)
+		}
+	}
+	removeServices(ctx, cl, pruneServices)
+}
+
+// RunDeploy is the swarm implementation of docker stack deploy
+func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config) error {
+	ctx := context.Background()
+
+	if err := validateResolveImageFlag(&opts); err != nil {
+		return err
+	}
+	// client side image resolution should not be done when the supported
+	// server version is older than 1.30
+	if versions.LessThan(cl.ClientVersion(), "1.30") {
+		opts.ResolveImage = ResolveImageNever
+	}
+
+	return deployCompose(ctx, cl, opts, cfg)
+}
+
+// validateResolveImageFlag validates the opts.resolveImage command line option
+func validateResolveImageFlag(opts *Deploy) error {
+	switch opts.ResolveImage {
+	case ResolveImageAlways, ResolveImageChanged, ResolveImageNever:
+		return nil
+	default:
+		return errors.Errorf("Invalid option %s for flag --resolve-image", opts.ResolveImage)
+	}
+}
+
+func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config) error {
+	namespace := convert.NewNamespace(opts.Namespace)
+
+	if opts.Prune {
+		services := map[string]struct{}{}
+		for _, service := range config.Services {
+			services[service.Name] = struct{}{}
+		}
+		pruneServices(ctx, cl, namespace, services)
+	}
+
+	serviceNetworks := getServicesDeclaredNetworks(config.Services)
+	networks, externalNetworks := convert.Networks(namespace, config.Networks, serviceNetworks)
+	if err := validateExternalNetworks(ctx, cl, externalNetworks); err != nil {
+		return err
+	}
+	if err := createNetworks(ctx, cl, namespace, networks); err != nil {
+		return err
+	}
+
+	secrets, err := convert.Secrets(namespace, config.Secrets)
+	if err != nil {
+		return err
+	}
+	if err := createSecrets(ctx, cl, secrets); err != nil {
+		return err
+	}
+
+	configs, err := convert.Configs(namespace, config.Configs)
+	if err != nil {
+		return err
+	}
+	if err := createConfigs(ctx, cl, configs); err != nil {
+		return err
+	}
+
+	services, err := convert.Services(namespace, config, cl)
+	if err != nil {
+		return err
+	}
+	return deployServices(ctx, cl, services, namespace, opts.SendRegistryAuth, opts.ResolveImage)
+}
+
+func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) map[string]struct{} {
+	serviceNetworks := map[string]struct{}{}
+	for _, serviceConfig := range serviceConfigs {
+		if len(serviceConfig.Networks) == 0 {
+			serviceNetworks["default"] = struct{}{}
+			continue
+		}
+		for network := range serviceConfig.Networks {
+			serviceNetworks[network] = struct{}{}
+		}
+	}
+	return serviceNetworks
+}
+
+func validateExternalNetworks(ctx context.Context, client dockerclient.NetworkAPIClient, externalNetworks []string) error {
+	for _, networkName := range externalNetworks {
+		if !container.NetworkMode(networkName).IsUserDefined() {
+			// Networks that are not user defined always exist on all nodes as
+			// local-scoped networks, so there's no need to inspect them.
+			continue
+		}
+		network, err := client.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{})
+		switch {
+		case dockerclient.IsErrNotFound(err):
+			return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed", networkName)
+		case err != nil:
+			return err
+		case network.Scope != "swarm":
+			return errors.Errorf("network %q is declared as external, but it is not in the right scope: %q instead of \"swarm\"", networkName, network.Scope)
+		}
+	}
+	return nil
+}
+
+func createSecrets(ctx context.Context, cl *dockerclient.Client, secrets []swarm.SecretSpec) error {
+	for _, secretSpec := range secrets {
+		secret, _, err := cl.SecretInspectWithRaw(ctx, secretSpec.Name)
+		switch {
+		case err == nil:
+			// secret already exists, then we update that
+			if err := cl.SecretUpdate(ctx, secret.ID, secret.Meta.Version, secretSpec); err != nil {
+				return errors.Wrapf(err, "failed to update secret %s", secretSpec.Name)
+			}
+		case dockerclient.IsErrNotFound(err):
+			// secret does not exist, then we create a new one.
+			logrus.Infof("Creating secret %s\n", secretSpec.Name)
+			if _, err := cl.SecretCreate(ctx, secretSpec); err != nil {
+				return errors.Wrapf(err, "failed to create secret %s", secretSpec.Name)
+			}
+		default:
+			return err
+		}
+	}
+	return nil
+}
+
+func createConfigs(ctx context.Context, cl *dockerclient.Client, configs []swarm.ConfigSpec) error {
+	for _, configSpec := range configs {
+		config, _, err := cl.ConfigInspectWithRaw(ctx, configSpec.Name)
+		switch {
+		case err == nil:
+			// config already exists, then we update that
+			if err := cl.ConfigUpdate(ctx, config.ID, config.Meta.Version, configSpec); err != nil {
+				return errors.Wrapf(err, "failed to update config %s", configSpec.Name)
+			}
+		case dockerclient.IsErrNotFound(err):
+			// config does not exist, then we create a new one.
+			logrus.Infof("Creating config %s\n", configSpec.Name)
+			if _, err := cl.ConfigCreate(ctx, configSpec); err != nil {
+				return errors.Wrapf(err, "failed to create config %s", configSpec.Name)
+			}
+		default:
+			return err
+		}
+	}
+	return nil
+}
+
+func createNetworks(ctx context.Context, cl *dockerclient.Client, namespace convert.Namespace, networks map[string]types.NetworkCreate) error {
+	existingNetworks, err := getStackNetworks(ctx, cl, namespace.Name())
+	if err != nil {
+		return err
+	}
+
+	existingNetworkMap := make(map[string]types.NetworkResource)
+	for _, network := range existingNetworks {
+		existingNetworkMap[network.Name] = network
+	}
+
+	for name, createOpts := range networks {
+		if _, exists := existingNetworkMap[name]; exists {
+			continue
+		}
+
+		if createOpts.Driver == "" {
+			createOpts.Driver = defaultNetworkDriver
+		}
+
+		logrus.Infof("Creating network %s\n", name)
+		if _, err := cl.NetworkCreate(ctx, name, createOpts); err != nil {
+			return errors.Wrapf(err, "failed to create network %s", name)
+		}
+	}
+	return nil
+}
+
+func deployServices(
+	ctx context.Context,
+	cl *dockerclient.Client,
+	services map[string]swarm.ServiceSpec,
+	namespace convert.Namespace,
+	sendAuth bool,
+	resolveImage string) error {
+	existingServices, err := getStackServices(ctx, cl, namespace.Name())
+	if err != nil {
+		return err
+	}
+
+	existingServiceMap := make(map[string]swarm.Service)
+	for _, service := range existingServices {
+		existingServiceMap[service.Spec.Name] = service
+	}
+
+	for internalName, serviceSpec := range services {
+		var (
+			name        = namespace.Scope(internalName)
+			image       = serviceSpec.TaskTemplate.ContainerSpec.Image
+			encodedAuth string
+		)
+
+		// FIXME: disable for now as not sure how to avoid having a `dockerCli`
+		// instance here and would rather not copy/pasta that entire module in
+		// right now for something that we don't even support right now. Will skip
+		// this for now.
+		if sendAuth {
+			// Retrieve encoded auth token from the image reference
+			// encodedAuth, err = command.RetrieveAuthTokenFromImage(ctx, dockerCli, image)
+			// if err != nil {
+			// 	return err
+			// }
+		}
+
+		if service, exists := existingServiceMap[name]; exists {
+			logrus.Infof("Updating service %s (id: %s)\n", name, service.ID)
+
+			updateOpts := types.ServiceUpdateOptions{EncodedRegistryAuth: encodedAuth}
+
+			switch resolveImage {
+			case ResolveImageAlways:
+				// image should be updated by the server using QueryRegistry
+				updateOpts.QueryRegistry = true
+			case ResolveImageChanged:
+				if image != service.Spec.Labels[convert.LabelImage] {
+					// Query the registry to resolve digest for the updated image
+					updateOpts.QueryRegistry = true
+				} else {
+					// image has not changed; update the serviceSpec with the
+					// existing information that was set by QueryRegistry on the
+					// previous deploy. Otherwise this will trigger an incorrect
+					// service update.
+					serviceSpec.TaskTemplate.ContainerSpec.Image = service.Spec.TaskTemplate.ContainerSpec.Image
+				}
+			default:
+				if image == service.Spec.Labels[convert.LabelImage] {
+					// image has not changed; update the serviceSpec with the
+					// existing information that was set by QueryRegistry on the
+					// previous deploy. Otherwise this will trigger an incorrect
+					// service update.
+					serviceSpec.TaskTemplate.ContainerSpec.Image = service.Spec.TaskTemplate.ContainerSpec.Image
+				}
+			}
+
+			// Stack deploy does not have a `--force` option. Preserve existing
+			// ForceUpdate value so that tasks are not re-deployed if not updated.
+			// TODO move this to API client?
+			serviceSpec.TaskTemplate.ForceUpdate = service.Spec.TaskTemplate.ForceUpdate
+
+			response, err := cl.ServiceUpdate(ctx, service.ID, service.Version, serviceSpec, updateOpts)
+			if err != nil {
+				return errors.Wrapf(err, "failed to update service %s", name)
+			}
+
+			for _, warning := range response.Warnings {
+				logrus.Warn(warning)
+			}
+		} else {
+			logrus.Infof("Creating service %s\n", name)
+
+			createOpts := types.ServiceCreateOptions{EncodedRegistryAuth: encodedAuth}
+
+			// query registry if flag disabling it was not set
+			if resolveImage == ResolveImageAlways || resolveImage == ResolveImageChanged {
+				createOpts.QueryRegistry = true
+			}
+
+			if _, err := cl.ServiceCreate(ctx, serviceSpec, createOpts); err != nil {
+				return errors.Wrapf(err, "failed to create service %s", name)
+			}
+		}
+	}
+	return nil
+}
+
+func getStackNetworks(ctx context.Context, dockerclient client.APIClient, namespace string) ([]types.NetworkResource, error) {
+	return dockerclient.NetworkList(ctx, types.NetworkListOptions{Filters: getStackFilter(namespace)})
+}
+
+func getStackSecrets(ctx context.Context, dockerclient client.APIClient, namespace string) ([]swarm.Secret, error) {
+	return dockerclient.SecretList(ctx, types.SecretListOptions{Filters: getStackFilter(namespace)})
+}
+
+func getStackConfigs(ctx context.Context, dockerclient client.APIClient, namespace string) ([]swarm.Config, error) {
+	return dockerclient.ConfigList(ctx, types.ConfigListOptions{Filters: getStackFilter(namespace)})
+}

pkg/client/volumes.go

@@ -0,0 +1,51 @@
+package client
+
+import (
+	"context"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/sirupsen/logrus"
+)
+
+func GetVolumes(ctx context.Context, server string, appName string) ([]*types.Volume, error) {
+
+	cl, err := New(server)
+	if err != nil {
+		return nil, err
+	}
+
+	fs := filters.NewArgs()
+	fs.Add("name", appName)
+
+	volumeListOKBody, err := cl.VolumeList(ctx, fs)
+	volumeList := volumeListOKBody.Volumes
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	return volumeList, nil
+}
+
+func GetVolumeNames(volumes []*types.Volume) []string {
+	var volumeNames []string
+	for _, vol := range volumes {
+		volumeNames = append(volumeNames, vol.Name)
+	}
+	return volumeNames
+}
+
+func RemoveVolumes(ctx context.Context, server string, volumeNames []string, force bool) error {
+	cl, err := New(server)
+	if err != nil {
+		return err
+	}
+	for _, volName := range volumeNames {
+		err := cl.VolumeRemove(ctx, volName, force)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+
+}

pkg/compose/compose.go

@@ -0,0 +1,115 @@
+package compose
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/client/stack"
+	loader "coopcloud.tech/abra/pkg/client/stack"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/docker/distribution/reference"
+	"github.com/sirupsen/logrus"
+)
+
+// UpdateTag updates an image tag in-place on file system local compose files.
+func UpdateTag(pattern, image, tag string) error {
+	composeFiles, err := filepath.Glob(pattern)
+	if err != nil {
+		return err
+	}
+
+	for _, composeFile := range composeFiles {
+		opts := stack.Deploy{Composefiles: []string{composeFile}}
+		emptyEnv := make(map[string]string)
+		compose, err := loader.LoadComposefile(opts, emptyEnv)
+		if err != nil {
+			return err
+		}
+
+		for _, service := range compose.Services {
+			if service.Image == "" {
+				continue // may be a compose.$optional.yml file
+			}
+
+			img, _ := reference.ParseNormalizedNamed(service.Image)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			composeImage := reference.Path(img)
+			if strings.Contains(composeImage, "library") {
+				// ParseNormalizedNamed prepends 'library' to images like nginx:<tag>,
+				// postgres:<tag>, i.e. images which do not have a username in the
+				// first position of the string
+				composeImage = strings.Split(composeImage, "/")[1]
+			}
+			composeTag := img.(reference.NamedTagged).Tag()
+
+			if image == composeImage {
+				bytes, err := ioutil.ReadFile(composeFile)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+
+				old := fmt.Sprintf("%s:%s", composeImage, composeTag)
+				new := fmt.Sprintf("%s:%s", composeImage, tag)
+				replacedBytes := strings.Replace(string(bytes), old, new, -1)
+
+				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// UpdateLabel updates a label in-place on file system local compose files.
+func UpdateLabel(pattern, serviceName, label string) error {
+	composeFiles, err := filepath.Glob(pattern)
+	if err != nil {
+		return err
+	}
+
+	for _, composeFile := range composeFiles {
+		opts := stack.Deploy{Composefiles: []string{composeFile}}
+		emptyEnv := make(map[string]string)
+		compose, err := loader.LoadComposefile(opts, emptyEnv)
+		if err != nil {
+			return err
+		}
+
+		serviceExists := false
+		var service composetypes.ServiceConfig
+		for _, s := range compose.Services {
+			if s.Name == serviceName {
+				service = s
+				serviceExists = true
+			}
+		}
+
+		if !serviceExists {
+			continue
+		}
+
+		for oldLabel, value := range service.Deploy.Labels {
+			if strings.HasPrefix(oldLabel, "coop-cloud") {
+				bytes, err := ioutil.ReadFile(composeFile)
+				if err != nil {
+					return err
+				}
+
+				old := fmt.Sprintf("coop-cloud.${STACK_NAME}.%s.version=%s", service.Name, value)
+				replacedBytes := strings.Replace(string(bytes), old, label, -1)
+				if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}

pkg/config/app.go

@@ -0,0 +1,258 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/client/convert"
+	loader "coopcloud.tech/abra/pkg/client/stack"
+	stack "coopcloud.tech/abra/pkg/client/stack"
+	composetypes "github.com/docker/cli/cli/compose/types"
+)
+
+// Type aliases to make code hints easier to understand
+
+// AppEnv is a map of the values in an apps env config
+type AppEnv = map[string]string
+
+// AppName is AppName
+type AppName = string
+
+// AppFile represents app env files on disk without reading the contents
+type AppFile struct {
+	Path   string
+	Server string
+}
+
+// AppFiles is a slice of appfiles
+type AppFiles map[AppName]AppFile
+
+// App reprents an app with its env file read into memory
+type App struct {
+	Name   AppName
+	Type   string
+	Domain string
+	Env    AppEnv
+	Server string
+	Path   string
+}
+
+// StackName gets what the docker safe stack name is for the app
+func (a App) StackName() string {
+	return SanitiseAppName(a.Name)
+}
+
+// SORTING TYPES
+
+// ByServer sort a slice of Apps
+type ByServer []App
+
+func (a ByServer) Len() int      { return len(a) }
+func (a ByServer) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByServer) Less(i, j int) bool {
+	return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server)
+}
+
+// ByServerAndType sort a slice of Apps
+type ByServerAndType []App
+
+func (a ByServerAndType) Len() int      { return len(a) }
+func (a ByServerAndType) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByServerAndType) Less(i, j int) bool {
+	if a[i].Server == a[j].Server {
+		return strings.ToLower(a[i].Type) < strings.ToLower(a[j].Type)
+	}
+	return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server)
+}
+
+// ByType sort a slice of Apps
+type ByType []App
+
+func (a ByType) Len() int      { return len(a) }
+func (a ByType) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByType) Less(i, j int) bool {
+	return strings.ToLower(a[i].Type) < strings.ToLower(a[j].Type)
+}
+
+// ByName sort a slice of Apps
+type ByName []App
+
+func (a ByName) Len() int      { return len(a) }
+func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByName) Less(i, j int) bool {
+	return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name)
+}
+
+func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
+	env, err := ReadEnv(appFile.Path)
+	if err != nil {
+		return App{}, fmt.Errorf("env file for '%s' couldn't be read: %s", name, err.Error())
+	}
+	app, err := newApp(env, name, appFile)
+	if err != nil {
+		return App{}, fmt.Errorf("env file for '%s' has issues: %s", name, err.Error())
+	}
+	return app, nil
+}
+
+// newApp creates new App object
+func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
+	// Checking for type as it is required - apps wont work without it
+	domain := env["DOMAIN"]
+	apptype, ok := env["TYPE"]
+	if !ok {
+		return App{}, errors.New("missing TYPE variable")
+	}
+	return App{
+		Name:   name,
+		Domain: domain,
+		Type:   apptype,
+		Env:    env,
+		Server: appFile.Server,
+		Path:   appFile.Path,
+	}, nil
+}
+
+// LoadAppFiles gets all app files for a given set of servers or all servers
+func LoadAppFiles(servers ...string) (AppFiles, error) {
+	appFiles := make(AppFiles)
+	if len(servers) == 1 {
+		if servers[0] == "" {
+			// Empty servers flag, one string will always be passed
+			var err error
+			servers, err = getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	for _, server := range servers {
+		serverDir := path.Join(ABRA_SERVER_FOLDER, server)
+		files, err := getAllFilesInDirectory(serverDir)
+		if err != nil {
+			return nil, err
+		}
+		for _, file := range files {
+			appName := strings.TrimSuffix(file.Name(), ".env")
+			appFilePath := path.Join(ABRA_SERVER_FOLDER, server, file.Name())
+			appFiles[appName] = AppFile{
+				Path:   appFilePath,
+				Server: server,
+			}
+		}
+	}
+	return appFiles, nil
+}
+
+// GetApp loads an apps settings, reading it from file, in preparation to use it
+//
+// ONLY use when ready to use the env file to keep IO down
+func GetApp(apps AppFiles, name AppName) (App, error) {
+	appFile, exists := apps[name]
+	if !exists {
+		return App{}, fmt.Errorf("cannot find app with name '%s'", name)
+	}
+	app, err := readAppEnvFile(appFile, name)
+	if err != nil {
+		return App{}, err
+	}
+	return app, nil
+}
+
+// GetApps returns a slice of Apps with their env files read from a given slice of AppFiles
+func GetApps(appFiles AppFiles) ([]App, error) {
+	var apps []App
+	for name := range appFiles {
+		app, err := GetApp(appFiles, name)
+		if err != nil {
+			return nil, err
+		}
+		apps = append(apps, app)
+	}
+	return apps, nil
+}
+
+// CopyAppEnvSample copies the example env file for the app into the users env files
+func CopyAppEnvSample(appType, appName, server string) error {
+	envSamplePath := path.Join(ABRA_DIR, "apps", appType, ".env.sample")
+	envSample, err := ioutil.ReadFile(envSamplePath)
+	if err != nil {
+		return err
+	}
+
+	appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
+	if _, err := os.Stat(appEnvPath); err == nil {
+		return fmt.Errorf("%s already exists?", appEnvPath)
+	}
+
+	err = ioutil.WriteFile(appEnvPath, envSample, 0755)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// SanitiseAppName makes a app name usable with Docker by replacing illegal characters
+func SanitiseAppName(name string) string {
+	return strings.ReplaceAll(name, ".", "_")
+}
+
+// GetAppStatuses queries servers to check the deployment status of given apps
+func GetAppStatuses(appFiles AppFiles) (map[string]string, error) {
+	servers := appFiles.GetServers()
+	ch := make(chan stack.StackStatus, len(servers))
+	for _, server := range servers {
+		go func(s string) { ch <- stack.GetAllDeployedServices(s) }(server)
+	}
+
+	statuses := map[string]string{}
+	for range servers {
+		status := <-ch
+		for _, service := range status.Services {
+			name := service.Spec.Labels[convert.LabelNamespace]
+			if _, ok := statuses[name]; !ok {
+				statuses[name] = "deployed"
+			}
+		}
+	}
+
+	return statuses, nil
+}
+
+// GetAppComposeFiles gets the list of compose files for an app which should be
+// merged into a composetypes.Config while respecting the COMPOSE_FILE env var.
+func GetAppComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
+	if _, ok := appEnv["COMPOSE_FILE"]; !ok {
+		pattern := fmt.Sprintf("%s/%s/compose**yml", APPS_DIR, recipe)
+		composeFiles, err := filepath.Glob(pattern)
+		if err != nil {
+			return composeFiles, err
+		}
+		return composeFiles, nil
+	}
+
+	var composeFiles []string
+	composeFileEnvVar := appEnv["COMPOSE_FILE"]
+	for _, file := range strings.Split(composeFileEnvVar, ":") {
+		path := fmt.Sprintf("%s/%s/%s", APPS_DIR, recipe, file)
+		composeFiles = append(composeFiles, path)
+	}
+	return composeFiles, nil
+}
+
+// GetAppComposeConfig retrieves a compose specification for a recipe. This
+// specification is the result of a merge of all the compose.**.yml files in
+// the recipe repository.
+func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*composetypes.Config, error) {
+	compose, err := loader.LoadComposefile(opts, appEnv)
+	if err != nil {
+		return &composetypes.Config{}, err
+	}
+	return compose, nil
+}

pkg/config/app_test.go

@@ -0,0 +1,37 @@
+package config
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestNewApp(t *testing.T) {
+	app, err := newApp(expectedAppEnv, appName, expectedAppFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, expectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	}
+}
+
+func TestReadAppEnvFile(t *testing.T) {
+	app, err := readAppEnvFile(expectedAppFile, appName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, expectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	}
+}
+
+func TestGetApp(t *testing.T) {
+	// TODO: Test failures as well as successes
+	app, err := GetApp(expectedAppFiles, appName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(app, expectedApp) {
+		t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
+	}
+}

pkg/config/env.go

@@ -0,0 +1,141 @@
+package config
+
+import (
+	"bufio"
+	"fmt"
+	"io/fs"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/Autonomic-Cooperative/godotenv"
+	"github.com/sirupsen/logrus"
+)
+
+var ABRA_DIR = os.ExpandEnv("$HOME/.abra")
+var ABRA_SERVER_FOLDER = path.Join(ABRA_DIR, "servers")
+var APPS_JSON = path.Join(ABRA_DIR, "apps.json")
+var APPS_DIR = path.Join(ABRA_DIR, "apps")
+var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
+
+func (a AppFiles) GetServers() []string {
+	var unique []string
+	servers := make(map[string]struct{})
+	for _, appFile := range a {
+		if _, ok := servers[appFile.Server]; !ok {
+			servers[appFile.Server] = struct{}{}
+			unique = append(unique, appFile.Server)
+		}
+	}
+	return unique
+}
+
+func ReadEnv(filePath string) (AppEnv, error) {
+	var envFile AppEnv
+	envFile, err := godotenv.Read(filePath)
+	if err != nil {
+		return nil, err
+	}
+	return envFile, nil
+}
+
+func ReadServerNames() ([]string, error) {
+	serverNames, err := getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
+	if err != nil {
+		return nil, err
+	}
+	return serverNames, nil
+}
+
+// getAllFilesInDirectory returns filenames of all files in directory
+func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
+	var realFiles []fs.FileInfo
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return nil, err
+	}
+	for _, file := range files {
+		// Follow any symlinks
+		filePath := path.Join(directory, file.Name())
+		realPath, err := filepath.EvalSymlinks(filePath)
+		if err != nil {
+			logrus.Warningf("broken symlink in your abra config folders: '%s'", filePath)
+		} else {
+			realFile, err := os.Stat(realPath)
+			if err != nil {
+				return nil, err
+			}
+			if !realFile.IsDir() {
+				realFiles = append(realFiles, file)
+			}
+		}
+
+	}
+	return realFiles, nil
+}
+
+// getAllFoldersInDirectory returns both folder and symlink paths
+func getAllFoldersInDirectory(directory string) ([]string, error) {
+	var folders []string
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return nil, err
+	}
+	if len(files) == 0 {
+		return nil, fmt.Errorf("directory is empty: '%s'", directory)
+	}
+	for _, file := range files {
+		// Check if file is directory or symlink
+		if file.IsDir() || file.Mode()&fs.ModeSymlink != 0 {
+			filePath := path.Join(directory, file.Name())
+			realDir, err := filepath.EvalSymlinks(filePath)
+			if err != nil {
+				logrus.Warningf("broken symlink in your abra config folders: '%s'", filePath)
+			} else if stat, err := os.Stat(realDir); err == nil && stat.IsDir() {
+				// path is a directory
+				folders = append(folders, file.Name())
+			}
+		}
+	}
+	return folders, nil
+}
+
+// EnsureAbraDirExists checks for the abra config folder and throws error if not
+func EnsureAbraDirExists() error {
+	if _, err := os.Stat(ABRA_DIR); os.IsNotExist(err) {
+		if err := os.Mkdir(ABRA_DIR, 0777); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
+	envVars := make(map[string]string)
+
+	file, err := os.Open(abraSh)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return envVars, fmt.Errorf("'%s' does not exist?", abraSh)
+		}
+		return envVars, err
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.Contains(line, "export") {
+			splitVals := strings.Split(line, "export ")
+			envVarDef := splitVals[len(splitVals)-1]
+			keyVal := strings.Split(envVarDef, "=")
+			if len(keyVal) != 2 {
+				return envVars, fmt.Errorf("couldn't parse %s", line)
+			}
+			envVars[keyVal[0]] = keyVal[1]
+		}
+	}
+
+	return envVars, nil
+}

pkg/config/env_test.go

@@ -0,0 +1,84 @@
+package config
+
+import (
+	"os"
+	"path"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+var testFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
+var validAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
+
+// make sure these are in alphabetical order
+var tFolders = []string{"folder1", "folder2"}
+var tFiles = []string{"bar", "foo"}
+
+var appName = "ecloud"
+var serverName = "evil.corp"
+
+var expectedAppEnv = AppEnv{
+	"DOMAIN": "ecloud.evil.corp",
+	"TYPE":   "ecloud",
+}
+
+var expectedApp = App{
+	Name:   appName,
+	Type:   expectedAppEnv["TYPE"],
+	Domain: expectedAppEnv["DOMAIN"],
+	Env:    expectedAppEnv,
+	Path:   expectedAppFile.Path,
+	Server: expectedAppFile.Server,
+}
+
+var expectedAppFile = AppFile{
+	Path:   path.Join(validAbraConf, "servers", serverName, appName+".env"),
+	Server: serverName,
+}
+
+var expectedAppFiles = map[string]AppFile{
+	appName: expectedAppFile,
+}
+
+// var expectedServerNames = []string{"evil.corp"}
+
+func TestGetAllFoldersInDirectory(t *testing.T) {
+	folders, err := getAllFoldersInDirectory(testFolder)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(folders, tFolders) {
+		t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(tFolders, ","), strings.Join(folders, ","))
+	}
+}
+
+func TestGetAllFilesInDirectory(t *testing.T) {
+	files, err := getAllFilesInDirectory(testFolder)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var fileNames []string
+	for _, file := range files {
+		fileNames = append(fileNames, file.Name())
+	}
+	if !reflect.DeepEqual(fileNames, tFiles) {
+		t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(tFiles, ","), strings.Join(fileNames, ","))
+	}
+}
+
+func TestReadEnv(t *testing.T) {
+	env, err := ReadEnv(expectedAppFile.Path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(env, expectedAppEnv) {
+		t.Fatalf(
+			"did not get expected application settings. Expected: DOMAIN=%s TYPE=%s; Got: DOMAIN=%s TYPE=%s",
+			expectedAppEnv["DOMAIN"],
+			expectedAppEnv["TYPE"],
+			env["DOMAIN"],
+			env["TYPE"],
+		)
+	}
+}

pkg/recipe/recipe.go

@@ -0,0 +1,127 @@
+package recipe
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/client/stack"
+	loader "coopcloud.tech/abra/pkg/client/stack"
+	"coopcloud.tech/abra/pkg/compose"
+	"coopcloud.tech/abra/pkg/config"
+	composetypes "github.com/docker/cli/cli/compose/types"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+)
+
+// Recipe represents a recipe.
+type Recipe struct {
+	Name   string
+	Config *composetypes.Config
+}
+
+// UpdateLabel updates a recipe label
+func (r Recipe) UpdateLabel(serviceName, label string) error {
+	pattern := fmt.Sprintf("%s/%s/compose**yml", config.APPS_DIR, r.Name)
+	if err := compose.UpdateLabel(pattern, serviceName, label); err != nil {
+		return err
+	}
+	return nil
+}
+
+// UpdateTag updates a recipe tag
+func (r Recipe) UpdateTag(image, tag string) error {
+	pattern := fmt.Sprintf("%s/%s/compose**yml", config.APPS_DIR, r.Name)
+	if err := compose.UpdateTag(pattern, image, tag); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Get retrieves a recipe.
+func Get(recipeName string) (Recipe, error) {
+	if err := EnsureExists(recipeName); err != nil {
+		return Recipe{}, err
+	}
+
+	pattern := fmt.Sprintf("%s/%s/compose**yml", config.APPS_DIR, recipeName)
+	composeFiles, err := filepath.Glob(pattern)
+	if err != nil {
+		return Recipe{}, err
+	}
+
+	opts := stack.Deploy{Composefiles: composeFiles}
+	emptyEnv := make(map[string]string)
+	config, err := loader.LoadComposefile(opts, emptyEnv)
+	if err != nil {
+		return Recipe{}, err
+	}
+
+	return Recipe{Name: recipeName, Config: config}, nil
+}
+
+// EnsureExists checks whether a recipe has been cloned locally or not.
+func EnsureExists(recipe string) error {
+	recipeDir := path.Join(config.ABRA_DIR, "apps", strings.ToLower(recipe))
+
+	if _, err := os.Stat(recipeDir); os.IsNotExist(err) {
+		url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipe)
+		_, err := git.PlainClone(recipeDir, false, &git.CloneOptions{URL: url, Tags: git.AllTags})
+		if err != nil {
+			// try with main branch because Git is being a Git
+			_, err := git.PlainClone(recipeDir, false, &git.CloneOptions{
+				URL:           url,
+				Tags:          git.AllTags,
+				ReferenceName: plumbing.ReferenceName("refs/heads/main"),
+			})
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// EnsureVersion checks whether a specific version exists for a recipe.
+func EnsureVersion(recipeName, version string) error {
+	recipeDir := path.Join(config.ABRA_DIR, "apps", recipeName)
+
+	repo, err := git.PlainOpen(recipeDir)
+	if err != nil {
+		return err
+	}
+
+	tags, err := repo.Tags()
+	if err != nil {
+		return nil
+	}
+
+	var tagRef plumbing.ReferenceName
+	if err := tags.ForEach(func(ref *plumbing.Reference) (err error) {
+		if ref.Name().Short() == version {
+			tagRef = ref.Name()
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	if tagRef.String() == "" {
+		return fmt.Errorf("%s is not available?", version)
+	}
+
+	worktree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	opts := &git.CheckoutOptions{Branch: tagRef, Keep: true}
+	if err := worktree.Checkout(opts); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/secret/pass.go

@@ -0,0 +1,49 @@
+package secret
+
+import (
+	"errors"
+	"fmt"
+	"os/exec"
+
+	"github.com/sirupsen/logrus"
+)
+
+// PassInsertSecret inserts a secret into a pass store.
+func PassInsertSecret(secretValue, secretName, appName, server string) error {
+	if _, err := exec.LookPath("pass"); err != nil {
+		return errors.New("pass command not found on $PATH, is it installed?")
+	}
+
+	cmd := fmt.Sprintf(
+		"echo %s | pass insert hosts/%s/%s/%s -m",
+		secretValue, server, appName, secretName,
+	)
+
+	if err := exec.Command("bash", "-c", cmd).Run(); err != nil {
+		return err
+	}
+
+	logrus.Infof("'%s' inserted into pass store", secretName)
+
+	return nil
+}
+
+// PassRmSecret deletes a secret from a pass store.
+func PassRmSecret(secretName, appName, server string) error {
+	if _, err := exec.LookPath("pass"); err != nil {
+		return errors.New("pass command not found on $PATH, is it installed?")
+	}
+
+	cmd := fmt.Sprintf(
+		"pass rm --force hosts/%s/%s/%s",
+		server, appName, secretName,
+	)
+
+	if err := exec.Command("bash", "-c", cmd).Run(); err != nil {
+		return err
+	}
+
+	logrus.Infof("'%s' removed from pass store", secretName)
+
+	return nil
+}

pkg/secret/secret.go

@@ -0,0 +1,151 @@
+// Package secret provides functionality for generating and storing secrets
+// both in a remote swarm and locally within supported storage such as pass
+// stores.
+package secret
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"coopcloud.tech/abra/pkg/client"
+	"coopcloud.tech/abra/pkg/config"
+	"github.com/schultz-is/passgen"
+)
+
+// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
+// secret definition.
+type secretValue struct {
+	Version string
+	Length  int
+}
+
+// GeneratePasswords generates passwords.
+func GeneratePasswords(count, length uint) ([]string, error) {
+	passwords, err := passgen.GeneratePasswords(
+		count,
+		length,
+		passgen.AlphabetDefault,
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return passwords, nil
+}
+
+// GeneratePassphrases generates human readable and rememberable passphrases.
+func GeneratePassphrases(count uint) ([]string, error) {
+	passphrases, err := passgen.GeneratePassphrases(
+		count,
+		passgen.PassphraseWordCountDefault,
+		rune('-'),
+		passgen.PassphraseCasingDefault,
+		passgen.WordListDefault,
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return passphrases, nil
+}
+
+func ReadSecretEnvVars(appEnv config.AppEnv) map[string]string {
+	secretEnvVars := make(map[string]string)
+	for envVar := range appEnv {
+		regex := regexp.MustCompile(`^SECRET.*VERSION.*`)
+		if string(regex.Find([]byte(envVar))) != "" {
+			secretEnvVars[envVar] = appEnv[envVar]
+		}
+	}
+	return secretEnvVars
+}
+
+// TODO: should probably go in the config/app package?
+func ParseSecretEnvVarName(secretEnvVar string) string {
+	withoutPrefix := strings.TrimPrefix(secretEnvVar, "SECRET_")
+	withoutSuffix := strings.TrimSuffix(withoutPrefix, "_VERSION")
+	return strings.ToLower(withoutSuffix)
+}
+
+// TODO: should probably go in the config/app package?
+func ParseGeneratedSecretName(secret string, appEnv config.App) string {
+	name := fmt.Sprintf("%s_", appEnv.StackName())
+	withoutAppName := strings.TrimPrefix(secret, name)
+	idx := strings.LastIndex(withoutAppName, "_")
+	return withoutAppName[:idx]
+}
+
+// TODO: should probably go in the config/app package?
+func ParseSecretEnvVarValue(secret string) (secretValue, error) {
+	values := strings.Split(secret, "#")
+	if len(values) == 0 {
+		return secretValue{}, fmt.Errorf("unable to parse '%s'", secret)
+	}
+	if len(values) == 1 {
+		return secretValue{Version: values[0], Length: 0}, nil
+	} else {
+		split := strings.Split(values[1], "=")
+		parsed := split[len(split)-1]
+		stripped := strings.ReplaceAll(parsed, " ", "")
+		length, err := strconv.Atoi(stripped)
+		if err != nil {
+			return secretValue{}, err
+		}
+		version := strings.ReplaceAll(values[0], " ", "")
+		return secretValue{Version: version, Length: length}, nil
+	}
+}
+
+// GenerateSecrets generates secrets locally and sends them to a remote server for storage.
+func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (map[string]string, error) {
+	secrets := make(map[string]string)
+
+	ch := make(chan error, len(secretEnvVars))
+	for secretEnvVar := range secretEnvVars {
+		go func(s string) {
+			secretName := ParseSecretEnvVarName(s)
+			secretValue, err := ParseSecretEnvVarValue(secretEnvVars[s])
+			if err != nil {
+				ch <- err
+				return
+			}
+			secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secretValue.Version)
+			if secretValue.Length > 0 {
+				passwords, err := GeneratePasswords(1, uint(secretValue.Length))
+				if err != nil {
+					ch <- err
+					return
+				}
+				if err := client.StoreSecret(secretRemoteName, passwords[0], server); err != nil {
+					ch <- err
+					return
+				}
+				secrets[secretName] = passwords[0]
+			} else {
+				passphrases, err := GeneratePassphrases(1)
+				if err != nil {
+					ch <- err
+					return
+				}
+				if err := client.StoreSecret(secretRemoteName, passphrases[0], server); err != nil {
+					ch <- err
+				}
+				secrets[secretName] = passphrases[0]
+			}
+			ch <- nil
+		}(secretEnvVar)
+	}
+
+	for range secretEnvVars {
+		err := <-ch
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return secrets, nil
+}

pkg/web/web.go

@@ -0,0 +1,23 @@
+// Package web provides functionality for dealing with web operations.
+package web
+
+import (
+	"encoding/json"
+	"net/http"
+	"time"
+)
+
+// Timeout is the time it takes before a web request bails out waiting for a
+// request.
+const Timeout = 10 * time.Second
+
+// ReadJSON reads JSON and parses it into your chosen interface pointer
+func ReadJSON(url string, target interface{}) error {
+	httpClient := &http.Client{Timeout: Timeout}
+	res, err := httpClient.Get(url)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	return json.NewDecoder(res.Body).Decode(target)
+}