Prefer --fast for skipping all checks

Closes https://git.autonomic.zone/coop-cloud/abra/issues/169

.drone.yml

@@ -18,32 +18,61 @@ steps:
     commands:
       - bats tests
 
-  - name: collect code coverage
+  - name: publish image
-    failure: ignore # until we fix this
+    image: plugins/docker
-    image: decentral1se/docker-dind-bats-kcov
-    commands:
-      - kcov . bats tests || true
-
-  - name: send code coverage report to codecov
-    failure: ignore # until we fix this
-    image: plugins/codecov
     settings:
+      auto_tag: true
+      username:
+        from_secret: docker_reg_username
+      password:
+        from_secret: docker_reg_passwd
+      repo: decentral1se/abra
+      tags: latest
+    depends_on:
+      - run shellcheck
+      - run flake8
+      - run unit tests
+    when:
+      event:
+        exclude:
+          - pull_request
+
+  - name: trigger downstream builds
+    image: plugins/downstream
+    settings:
+      server: https://drone.autonomic.zone
       token:
-        from_secret: codecov_token
+        from_secret: decentral1se_token
-        required: true
+      fork: true
+      repositories:
+        - coop-cloud/drone-abra
+    depends_on:
+      - run shellcheck
+      - run flake8
+      - run unit tests
+      - publish image
+    when:
+      event:
+        exclude:
+          - pull_request
 
-  - name: notify rocket chat
+  - name: notify on failure
-    image: plugins/slack
+    image: plugins/matrix
     settings:
-      webhook:
+      homeserver: https://matrix.autonomic.zone
-        from_secret: rc_builds_url
+      roomid: "IFazIpLtxiScqbHqoa:autonomic.zone"
-      username: comradebritney
+      userid: "@autono-bot:autonomic.zone"
-      channel: "internal.builds"
+      accesstoken:
-      template: "{{repo.owner}}/{{repo.name}} build failed: {{build.link}}"
+        from_secret: autono_bot_access_token
+    depends_on:
+      - run shellcheck
+      - run flake8
+      - run unit tests
+      - publish image
+      - trigger downstream builds
     when:
       status:
         - failure
-
 trigger:
   branch:
     - main

.gitignore

@@ -1,2 +1,5 @@
+*.json
+*.pyc
 /.venv
+__pycache__
 coverage/

CHANGELOG.md

@@ -1,5 +1,5 @@
-> 🔥 🔥 🔥 Please note, while we are still in [public
+> 🔥 🔥 🔥 Please note, while we are still in
-> alpha](https://docs.cloud.autonomic.zone/roadmap/), the `abra` release
+> [public alpha](https://docs.cloud.autonomic.zone/roadmap/), the `abra` release
 > versioning scheme is not following [semver](https://semver.org/) conventions
 > because we are still in the exploratory phases of building this tool. Please
 > read the changes before upgrading your `abra` installation as there are
@@ -9,6 +9,57 @@
 
 # abra x.x.x (UNRELEASED)
 
+- Add `--bump` to `deploy` command to allow packagers to make minor package related releases ([#173](https://git.autonomic.zone/coop-cloud/abra/issues/173))
+- Drop `--skip-version-check`/`--no-domain-poll`/`--no-state-poll` in favour of `--fast` ([#169](https://git.autonomic.zone/coop-cloud/abra/issues/169))
+
+# abra 9.0.0 (2021-06-10)
+
+- Add Docker image for `abra` ([64d578cf91](https://git.autonomic.zone/coop-cloud/abra/commit/64d578cf914bd2bad378ea4ef375747d10b33191))
+- Support unattended mode for recipe releasing ([3759bcd641](https://git.autonomic.zone/coop-cloud/abra/commit/3759bcd641cf60611c13927e83425e773d2bb629))
+- Add Renovate bot configuraiton script ([9fadc430a7](https://git.autonomic.zone/coop-cloud/abra/commit/9fadc430a7bb2d554c0ee26c0f9b6c51dc5b0475))
+- Add release automation via [drone-abra](https://git.autonomic.zone/coop-cloud/drone-abra) ([#56](https://git.autonomic.zone/coop-cloud/organising/issues/56))
+- Move `apps.json` generation to [auto-apps-json](https://git.autonomic.zone/coop-cloud/auto-apps-json) ([#125](https://git.autonomic.zone/coop-cloud/abra/issues/125))
+- Add Github mirroring script ([4ef433312d](https://git.autonomic.zone/coop-cloud/abra/commit/4ef433312dd0b0ace91b3c285f82f3973093d92d))
+- Add `--chaos` flag to deploy (always choose latest Git commit) ([#178](https://git.autonomic.zone/coop-cloud/abra/issues/178))
+
+# abra 8.0.1 (2021-05-31)
+
+- Fix help for `... app ... volume ls` ([efad71c470](https://git.autonomic.zone/coop-cloud/abra/commits/branch/main))
+- Only output secrets warnings once ([#143](https://git.autonomic.zone/coop-cloud/abra/issues/143))
+- Migrate `abra` installation script to `coopcloud.tech` domain ([#150](https://git.autonomic.zone/coop-cloud/abra/issues/150))
+- Add `--no-state-poll` to avoid success/failure forecasting on deployment ([#165](https://git.autonomic.zone/coop-cloud/abra/issues/165))
+
+# abra 8.0.0 (2021-05-30)
+
+- Fix secret length generation ([f537417](https://git.autonomic.zone/coop-cloud/abra/commit/1b85bf3d37280e9632c315d759c0f2d09c039fef))
+- Fix checking out new apps ([#164](https://git.autonomic.zone/coop-cloud/abra/issues/164)
+- Give up if YAML is invalid ([#154](https://git.autonomic.zone/coop-cloud/abra/issues/154))
+- Switch from wget to cURL ([fc0caaa](https://git.autonomic.zone/coop-cloud/abra/commit/fc0caaa))
+- Add Bash completion for `recipe ..` ([8c93d1a](https://git.autonomic.zone/coop-cloud/abra/commit/8c93d1a))
+- Tweak README parsing in `app-json.py` ([b14219b](https://git.autonomic.zone/coop-cloud/abra/commit/b14219b))
+- Add fallback names to `app.json` ([#157](https://git.autonomic.zone/coop-cloud/abra/issues/157))
+- Remove duplicate message ([#155](https://git.autonomic.zone/coop-cloud/abra/issues/155))
+- Add `deploy --fast` ([a7f7c96](https://git.autonomic.zone/coop-cloud/abra/commit/a7f7c96))
+- Add `app .. volume` commands, fix volume deletion with `app .. delete --volumes` ([#161](https://git.autonomic.zone/coop-cloud/abra/issues/161))
+
+# abra 0.7.4 (2021-05-10)
+
+- Sort `apps.json` when publishing ([39a7fc0](https://git.autonomic.zone/coop-cloud/abra/commit/39a7fc04fb5df1a6d78b84f51838530ab3eb76db))
+- Fix publishing of rating for new apps ([0e28af9](https://git.autonomic.zone/coop-cloud/abra/commit/0e28af9eb1af6c6da705b4614ddd173c60576629))
+- Detect compose filenames in `n+1` release generation ([ffc569e](https://git.autonomic.zone/coop-cloud/abra/commit/ffc569e275df7ca784a4db1a3331e17975fd8c87))
+- Fix secret generation when specifying length ([3a353f4](https://git.autonomic.zone/coop-cloud/abra/commit/3a353f4062baccde2c9f175b03afb2db6d462ae4))
+
+# abra 0.7.3 (2021-04-28)
+
+- Only check for pw(q)gen if we're actually trying to use them ([#147](https://git.autonomic.zone/coop-cloud/abra/issues/147))
+- Use apps.coopcloud.tech for app data hosting & download ([75bd599](https://git.autonomic.zone/coop-cloud/abra/commit/75bd599))
+- Choose latest commit messages for new tags ([#144](https://git.autonomic.zone/coop-cloud/abra/issues/144))
+- Handle recipes without an `app` service in `recipe .. release` ([#151](https://git.autonomic.zone/coop-cloud/abra/issues/151))
+
+# abra 0.7.2 (2021-04-07)
+
+- Fix installation script development installs (again! Thanks Bash!) ([4747d9b7](https://git.autonomic.zone/coop-cloud/abra/commit/4747d9b7fb5fba914f210b6570bfe2db0b53da23))
+
 # abra 0.7.1 (2021-04-07)
 
 - Fix installation script development installs ([8f2fadb3c](https://git.autonomic.zone/coop-cloud/abra/commit/8f2fadb3c43c5915520f5ea531ea3815c2ba8531))
@@ -33,7 +84,7 @@
 - Recognise when apps are already undeployed with `abra app <app> undeploy` ([#123](https://git.autonomic.zone/coop-cloud/abra/issues/123))
 - Add `abra doctor` command to help diagnose setup issues ([#119](https://git.autonomic.zone/coop-cloud/abra/issues/119))
 - Add apps version and feature catalogue generation script ([#121](https://git.autonomic.zone/coop-cloud/abra/issues/121))
-- New `--skip-version-check` option to `deploy` ([df4e504](https://git.autonomic.zone/coop-cloud/abra/commit/df4e504)
+- New `--skip-version-check` option to `deploy` ([df4e504](https://git.autonomic.zone/coop-cloud/abra/commit/df4e504))
 - Look up local available version from compose files instead of `abra.sh` ([#131](https://git.autonomic.zone/coop-cloud/abra/issues/131))
 - Improve domain polling logging and allow to skip the check altogether with `--no-domain-poll` ([#140](https://git.autonomic.zone/coop-cloud/abra/issues/140), [#141](https://git.autonomic.zone/coop-cloud/abra/issues/141))
 - Support `ABRA_DIR` in the installer script ([4e94a424e94a42](https://git.autonomic.zone/coop-cloud/abra/commit/4e94a424e94a42))

Dockerfile

@@ -0,0 +1,33 @@
+FROM alpine:latest
+
+RUN apk add --upgrade --no-cache \
+  bash \
+  curl \
+  git \
+  grep \
+  openssh-client \
+  py3-requests \
+  skopeo \
+  util-linux
+
+RUN mkdir -p ~./local/bin
+RUN mkdir -p ~/.abra/apps
+RUN mkdir -p ~/.abra/vendor
+RUN mkdir -p ~/.ssh/
+
+RUN ssh-keyscan -p 2222 git.autonomic.zone > ~/.ssh/known_hosts
+
+RUN curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 --output ~/.abra/vendor/jq
+RUN chmod +x ~/.abra/vendor/jq
+
+RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.9.3/yq_linux_amd64 --output ~/.abra/vendor/yq
+RUN chmod +x ~/.abra/vendor/yq
+
+# Note(decentral1se): it is fine to always use the development branch because
+# our Drone CI docker auto-tagger will publish official release tags and
+# otherwise give us the latest abra on the latest tag
+RUN curl https://install.abra.coopcloud.tech | bash -s -- --dev
+
+COPY bin/* /root/.local/bin/
+
+ENTRYPOINT ["/root/.local/bin/abra"]

README.md

@@ -1,18 +1,17 @@
 # abra
 
 [![Build Status](https://drone.autonomic.zone/api/badges/coop-cloud/abra/status.svg)](https://drone.autonomic.zone/coop-cloud/abra)
-[![codecov](https://codecov.io/gh/Autonomic-Cooperative/abra/branch/main/graph/badge.svg?token=aX3I5NMRsj)](undefined)
 
-> https://cloud.autonomic.zone
+> https://coopcloud.tech
 
-The cooperative cloud utility belt 🎩🐇
+The Co-op Cloud utility belt 🎩🐇
 
-`abra` is a command-line tool for managing your own [Co-op Cloud](https://cloud.autonomic.zone). It can provision new servers, create applications, deploy them, run backup and restore operations and a whole lot of other things. It is the go-to tool for day-to-day operations when managing a Co-op Cloud instance.
+`abra` is a command-line tool for managing your own [Co-op Cloud](https://coopcloud.tech). It can provision new servers, create applications, deploy them, run backup and restore operations and a whole lot of other things. It is the go-to tool for day-to-day operations when managing a Co-op Cloud instance.
 
 ## Change log
 
 > 🔥 🔥 🔥 Please note, while we are still in [public
-> alpha](https://docs.cloud.autonomic.zone/roadmap/), the `abra` release
+> alpha](https://docs.coopcloud.tech/roadmap/), the `abra` release
 > versioning scheme is not following [semver](https://semver.org/) conventions
 > because we are still in the exploratory phases of building this tool. Please
 > read the changes before upgrading your `abra` installation as there are
@@ -24,23 +23,39 @@ See [CHANGELOG.md](./CHANGELOG.md).
 
 ## Documentation
 
-> [docs.cloud.autonomic.zone](https://docs.cloud.autonomic.zone/)
+> [docs.coopcloud.tech](https://docs.coopcloud.tech)
 
 ## Install
 
+Requirements:
+
+- `pwqgen` (optional)
+- `pwgen` (optional)
+- `curl`
+- `docker`
+- `bash` >= 4
+
 Install the latest stable release:
 
 ```sh
-curl https://install.abra.autonomic.zone | bash
+curl https://install.abra.coopcloud.tech | bash
 ```
 
 or the bleeding-edge development version:
 
 ```sh
-curl https://install.abra.autonomic.zone | bash -s -- --dev
+curl https://install.abra.coopcloud.tech | bash -s -- --dev
 ```
 
-The source for this script is [here](./deploy/install.abra.autonomic.zone/installer).
+The source for this script is [here](./deploy/install.abra.coopcloud.tech/installer).
+
+## Container
+
+An [image](https://hub.docker.com/r/decentral1se/abra) is also provided.
+
+```
+docker run decentral1se/abra app ls
+```
 
 ## Update
 
@@ -53,39 +68,22 @@ To update the development version, run `abra upgrade --dev`.
 
 It's written in Bash version 4 or greater!
 
-Install it via `curl https://install.abra.autonomic.zone | bash -s -- --dev`, then you can hack on the source in `~/.abra/src`.
+Install it via `curl https://install.abra.coopcloud.tech | bash -s -- --dev`, then you can hack on the source in `~/.abra/src`.
 
 The command-line interface is generated via [docopt](http://docopt.org/). If you add arguments then you need to run `make docopt` ro regenerate the parser.
 
 Please remember to update the [CHANGELOG](./CHANGELOG.md) when you make a change.
 
-## Generating a new abra-apps.json
-
-You'll need to install the following requirements:
-
-- [requests](https://docs.python-requests.org/en/master/) (`apt install python3-requests` / `pip install requests`)
-- [skopeo](https://github.com/containers/skopeo) (check [the install docs](https://github.com/containers/skopeo/blob/master/install.md))
-- [jq](https://stedolan.github.io/jq/tutorial/) (`sudo apt-get install jq` or see [the install docs](https://stedolan.github.io/jq/download/))
-- [yq](https://mikefarah.gitbook.io/yq/) (see [the install docs](https://mikefarah.gitbook.io/yq/#install))
-
-Then run `./bin/app-json.py` ([source](./bin/app-json.py)) and it will spit out the JSON file into [deploy/abra-apps.cloud.autonomic.zone/abra-apps.json](./deploy/abra-apps.cloud.autonomic.zone/abra-apps.json).
-
 ## Releasing
 
 ### `abra`
 
+> [install.abra.coopcloud.tech](https://install.abra.coopcloud.tech)
+
 - Change the `x.x.x` header in [CHANGELOG.md](./CHANGELOG.md) to reflect new version and mark date
 - Update the version in [abra](./abra)
-- Update the version in [deploy/install.abra.autonomic.zone/installer](./deploy/install.abra.autonomic.zone/installer)
+- Update the version in [deploy/install.abra.coopcloud.tech/installer](./deploy/install.abra.coopcloud.tech/installer)
 - `git commit` the above changes and then tag it with `git tag <your-new-version>`
 - `git push` and `git push --tags`
 - Deploy a new installer script `make release-installer`
 - Tell the world (CoTech forum, Matrix public channel, Autonomic mastodon, etc.)
-
-### abra-apps.cloud.autonomic.zone
-
-> [abra-apps.cloud.autonomic.zone](https://abra-apps.cloud.autonomic.zone)
-
-```bash
-$ make release-abra-apps
-```

bin/abralib.py

@@ -0,0 +1,108 @@
+"""Shared utilities for bin/*.py scripts."""
+
+from logging import DEBUG, basicConfig, getLogger
+from os import chdir, mkdir
+from os.path import exists, expanduser
+from pathlib import Path
+from shlex import split
+from subprocess import check_output
+from sys import exit
+
+from requests import get
+
+HOME_PATH = expanduser("~/")
+CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
+REPOS_TO_SKIP = (
+    "abra",
+    "abra-apps",
+    "abra-gandi",
+    "abra-hetzner",
+    "auto-apps-json",
+    "auto-mirror",
+    "backup-bot",
+    "coopcloud.tech",
+    "coturn",
+    "docker-cp-deploy",
+    "docker-dind-bats-kcov",
+    "docs.coopcloud.tech",
+    "example",
+    "gardening",
+    "organising",
+    "pyabra",
+    "radicle-seed-node",
+    "stack-ssh-deploy",
+    "swarm-cronjob",
+)
+YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq")
+JQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/jq")
+
+log = getLogger(__name__)
+basicConfig()
+log.setLevel(DEBUG)
+
+
+def _run_cmd(cmd, shell=False, **kwargs):
+    """Run a shell command."""
+    args = [split(cmd)]
+
+    if shell:
+        args = [cmd]
+        kwargs = {"shell": shell}
+
+    try:
+        return check_output(*args, **kwargs).decode("utf-8").strip()
+    except Exception as exception:
+        log.error(f"Failed to run {cmd}, saw {str(exception)}")
+        exit(1)
+
+
+def get_repos_json():
+    """ Retrieve repo list from Gitea """
+
+    url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
+
+    log.info(f"Retrieving {url}")
+
+    repos = []
+    response = True
+    page = 1
+
+    try:
+        while response:
+            log.info(f"Trying to fetch page {page}")
+            response = get(url + f"?page={page}", timeout=10).json()
+            repos.extend(response)
+            page += 1
+
+        return repos
+    except Exception as exception:
+        log.error(f"Failed to retrieve {url}, saw {str(exception)}")
+        exit(1)
+
+
+def clone_all_apps(repos_json, ssh=False):
+    """Clone all Co-op Cloud apps to ~/.abra/apps."""
+    if not exists(CLONES_PATH):
+        mkdir(CLONES_PATH)
+
+    if ssh:
+        repos = [[p["name"], p["ssh_url"]] for p in repos_json]
+    else:
+        repos = [[p["name"], p["clone_url"]] for p in repos_json]
+
+    for name, url in repos:
+        if name in REPOS_TO_SKIP:
+            continue
+
+        if not exists(f"{CLONES_PATH}/{name}"):
+            log.info(f"Retrieving {url}")
+            _run_cmd(f"git clone {url} {CLONES_PATH}/{name}")
+
+            chdir(f"{CLONES_PATH}/{name}")
+            if not int(_run_cmd("git branch --list | wc -l", shell=True)):
+                log.info(f"Guessing main branch is HEAD for {name}")
+                _run_cmd("git checkout main")
+        else:
+            log.info(f"Updating {name}")
+            chdir(f"{CLONES_PATH}/{name}")
+            _run_cmd("git fetch -a")

bin/app-json.py

@@ -4,60 +4,31 @@
 #
 # Gather metadata from Co-op Cloud apps in $ABRA_DIR/apps (default
 # ~/.abra/apps), and format it as JSON so that it can be hosted here:
-# https://abra-apps.cloud.autonomic.zone
+# https://apps.coopcloud.tech
 
 from json import dump
-from logging import DEBUG, basicConfig, getLogger
+from os import chdir, getcwd, listdir
-from os import chdir, listdir, mkdir
+from os.path import basename
-from os.path import basename, exists, expanduser
-from pathlib import Path
 from re import findall, search
-from shlex import split
+from subprocess import DEVNULL
-from subprocess import check_output, DEVNULL
-from sys import exit
 
 from requests import get
 
-HOME_PATH = expanduser("~/")
+from abralib import (
-CLONES_PATH = Path(f"{HOME_PATH}/.abra/apps").absolute()
+    CLONES_PATH,
-YQ_PATH = Path(f"{HOME_PATH}/.abra/vendor/yq")
+    JQ_PATH,
-SCRIPT_PATH = Path(__file__).absolute().parent
+    REPOS_TO_SKIP,
-REPOS_TO_SKIP = (
+    YQ_PATH,
-    "abra",
+    _run_cmd,
-    "backup-bot",
+    clone_all_apps,
-    "cloud.autonomic.zone",
+    get_repos_json,
-    "docs.cloud.autonomic.zone",
+    log,
-    "example",
-    "organising",
-    "pyabra",
-    "stack-ssh-deploy",
-    "radicle-seed-node",
-    "coturn"
 )
 
-log = getLogger(__name__)
-basicConfig()
-log.setLevel(DEBUG)
-
-
-def _run_cmd(cmd, shell=False, **kwargs):
-    """Run a shell command."""
-    args = [split(cmd)]
-
-    if shell:
-        args = [cmd]
-        kwargs = {"shell": shell}
-
-    try:
-        return check_output(*args, **kwargs).decode("utf-8").strip()
-    except Exception as exception:
-        log.error(f"Failed to run {cmd}, saw {str(exception)}")
-        exit(1)
-
 
 def get_published_apps_json():
     """Retrieve already published apps json."""
-    url = "https://abra-apps.cloud.autonomic.zone"
+    url = "https://apps.coopcloud.tech"
 
     log.info(f"Retrieving {url}")
 
@@ -68,41 +39,7 @@ def get_published_apps_json():
         return {}
 
 
-def clone_all_apps():
+def generate_apps_json(repos_json):
-    """Clone all Co-op Cloud apps to ~/.abra/apps."""
-    if not exists(CLONES_PATH):
-        mkdir(CLONES_PATH)
-    url = "https://git.autonomic.zone/api/v1/orgs/coop-cloud/repos"
-
-    log.info(f"Retrieving {url}")
-
-    try:
-        response = get(url, timeout=10)
-    except Exception as exception:
-        log.error(f"Failed to retrieve {url}, saw {str(exception)}")
-        exit(1)
-
-    repos = [[p["name"], p["ssh_url"]] for p in response.json()]
-
-    for name, url in repos:
-        if name in REPOS_TO_SKIP:
-            continue
-
-        if not exists(f"{CLONES_PATH}/{name}"):
-            log.info(f"Retrieving {url}")
-            _run_cmd(f"git clone {url} {CLONES_PATH}/{name}")
-
-            chdir(f"{CLONES_PATH}/{name}")
-            if not int(_run_cmd("git branch --list | wc -l", shell=True)):
-                log.info(f"Guessing main branch is HEAD for {name}")
-                _run_cmd("git checkout main")
-        else:
-            log.info(f"Updating {name}")
-            chdir(f"{CLONES_PATH}/{name}")
-            _run_cmd("git fetch -a")
-
-
-def generate_apps_json():
     """Generate the abra-apps.json application versions file."""
     apps_json = {}
     cached_apps_json = get_published_apps_json()
@@ -112,37 +49,47 @@ def generate_apps_json():
             log.info(f"Skipping {app}")
             continue
 
+        repo_details = next(filter(lambda x: x["name"] == app, repos_json), {})
+
         app_path = f"{CLONES_PATH}/{app}"
         chdir(app_path)
 
+        metadata = get_app_metadata(app_path)
+
+        name = metadata.pop("name", app)
+
         log.info(f"Processing {app}")
         apps_json[app] = {
-            "category": "apps",
+            "name": name,
-            "repository": f"https://git.autonomic.zone/coop-cloud/{app}.git",
+            "category": metadata.get("category", ""),
-            # Note(decentral1se): please note that the app features do not
+            "repository": repo_details.get("clone_url", ""),
-            # correspond to version tags. We simply parse the latest features
+            "default_branch": repo_details.get("default_branch", ""),
-            # list from HEAD. This may lead to unexpected situations where
+            "description": repo_details.get("description", ""),
-            # users believe X feature is available under Y version but it is
+            "website": repo_details.get("website", ""),
-            # not.
+            "features": metadata,
-            "features": get_app_features(app_path),
             "versions": get_app_versions(app_path, cached_apps_json),
+            "icon": repo_details.get("avatar_url", ""),
         }
 
     return apps_json
 
 
-def get_app_features(app_path):
+def get_app_metadata(app_path):
-    """Parse features from app repo README files."""
+    """Parse metadata from app repo README files."""
-    features = {}
+    metadata = {}
 
     chdir(app_path)
 
-    with open(f"{app_path}/README.md", "r") as handle:
+    try:
-        log.info(f"{app_path}/README.md")
+        with open(f"{app_path}/README.md", "r") as handle:
-        contents = handle.read()
+            log.info(f"{app_path}/README.md")
+            contents = handle.read()
+    except Exception:
+        log.info(f"No {app_path}/README.md discovered, moving on")
+        return {}
 
     try:
-        for match in findall(r"\*\*.*\s\*", contents):
+        for match in findall(r"\*\*.*", contents):
             title = search(r"(?<=\*\*).*(?=\*\*)", match).group().lower()
 
             if title == "image":
@@ -152,19 +99,24 @@ def get_app_features(app_path):
                     "rating": match.split(",")[1].strip(),
                     "source": match.split(",")[-1].replace("*", "").strip(),
                 }
+            elif title == "status":
+                value = {"❶💚": 1, "❷💛": 2, "❸🍎": 3, "❹💣": 4, "?": 5, "": 5}[
+                    match.split(":")[-1].replace("*", "").strip()
+                ]
             else:
                 value = match.split(":")[-1].replace("*", "").strip()
 
-            features[title] = value
+            metadata[title] = value
+        metadata["name"] = findall(r"^# (.*)", contents)[0]
     except (IndexError, AttributeError):
         log.info(f"Can't parse {app_path}/README.md")
         return {}
     finally:
         _run_cmd("git checkout HEAD")
 
-    log.info(f"Parsed {features}")
+    log.info(f"Parsed {metadata}")
 
-    return features
+    return metadata
 
 
 def get_app_versions(app_path, cached_apps_json):
@@ -188,8 +140,7 @@ def get_app_versions(app_path, cached_apps_json):
         existing_tags = []
 
     for tag in tags:
-        _run_cmd(f"git checkout {tag}",
+        _run_cmd(f"git checkout {tag}", stderr=DEVNULL)
-                 stderr=DEVNULL)
 
         services_cmd = f"{YQ_PATH} e '.services | keys | .[]' compose*.yml"
         services = _run_cmd(services_cmd, shell=True).split()
@@ -200,7 +151,10 @@ def get_app_versions(app_path, cached_apps_json):
             if service in ("null", "---"):
                 continue
 
-            if tag in existing_tags:
+            if (
+                tag in existing_tags
+                and service in cached_apps_json[app_name]["versions"][tag]
+            ):
                 log.info(f"Skipping {tag} because we've already processed it")
                 existing_versions = cached_apps_json[app_name]["versions"][tag][service]
                 service_versions[service] = existing_versions
@@ -218,7 +172,7 @@ def get_app_versions(app_path, cached_apps_json):
                 if image in ("null", "---"):
                     continue
 
-                images_cmd = f"skopeo inspect docker://{image} | jq '.Digest'"
+                images_cmd = f"skopeo inspect docker://{image} | {JQ_PATH} '.Digest'"
                 output = _run_cmd(images_cmd, shell=True)
 
                 service_version_info = {
@@ -241,11 +195,18 @@ def get_app_versions(app_path, cached_apps_json):
 
 def main():
     """Run the script."""
-    clone_all_apps()
+    repos_json = get_repos_json()
+    clone_all_apps(repos_json)
 
-    target = f"{SCRIPT_PATH}/../deploy/abra-apps.cloud.autonomic.zone/abra-apps.json"
+    target = f"{getcwd()}/apps.json"
     with open(target, "w", encoding="utf-8") as handle:
-        dump(generate_apps_json(), handle, ensure_ascii=False, indent=4)
+        dump(
+            generate_apps_json(repos_json),
+            handle,
+            ensure_ascii=False,
+            indent=4,
+            sort_keys=True,
+        )
 
     log.info(f"Successfully generated {target}")
 

bin/clone-all-apps.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+# Usage: ./clone-all-apps.py
+#
+# Clone all available apps into ~/.abra/apps using ssh:// URLs
+
+from abralib import clone_all_apps, get_repos_json
+
+
+def main():
+    """Run the script."""
+    repos_json = get_repos_json()
+    clone_all_apps(repos_json, ssh=True)
+
+
+main()

bin/github-sync.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+# Usage: ./github-sync.py
+#
+# Mirror repositories to Github (Fuck M$, get it straight)
+
+from os import chdir, environ, listdir
+
+from abralib import (
+    CLONES_PATH,
+    REPOS_TO_SKIP,
+    _run_cmd,
+    clone_all_apps,
+    get_repos_json,
+    log,
+)
+
+
+def main():
+    """Run the script."""
+    repos_json = get_repos_json()
+    clone_all_apps(repos_json)
+
+    for app in listdir(CLONES_PATH):
+        if app in REPOS_TO_SKIP:
+            log.info(f"Skipping {app}")
+            continue
+
+        app_path = f"{CLONES_PATH}/{app}"
+        chdir(app_path)
+
+        log.info(f"Mirroring {app}...")
+
+        token = environ.get("GITHUB_ACCESS_TOKEN")
+        remote = (
+            f"https://decentral1se:{token}@github.com/Autonomic-Cooperative/{app}.git"
+        )
+
+        _run_cmd(
+            f"git remote add github {remote} || true",
+            shell=True,
+        )
+
+        _run_cmd("git push github --all")
+
+
+main()

bin/renovate-ls-apps.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+
+# Usage: ./renovate-ls-apps.py
+#
+# Output list of apps for Renovate bot configuration
+
+from abralib import REPOS_TO_SKIP, get_repos_json
+
+
+def main():
+    """Run the script."""
+    repos = [p["full_name"] for p in get_repos_json()]
+    repos.sort()
+    for repo in repos:
+        if repo.split("/")[-1] in REPOS_TO_SKIP:
+            continue
+        print(f'"{repo}",')
+
+
+main()

completion/abra.bash

@@ -38,6 +38,22 @@ _abra_complete_apps()
     mapfile -t COMPREPLY < <(compgen -W "$(_abra_apps)" -- "$1")
 }
 
+_abra_recipes()
+{
+    shopt -s nullglob dotglob
+    local RECIPES=(~/.abra/apps/*)
+    shopt -u nullglob dotglob
+
+    for RECIPE in "${RECIPES[@]}"; do
+        _abra_basename "${RECIPE%.env}"
+    done
+}
+
+_abra_complete_recipes()
+{
+    mapfile -t COMPREPLY < <(compgen -W "$(_abra_recipes)" -- "$1")
+}
+
 _abra_complete()
 {
     compopt +o default +o nospace
@@ -46,6 +62,7 @@ _abra_complete()
     local -r cmds='
         app
         server
+        recipe
     '
     local -r short_opts='-e      -h     -s      -v'
     local -r long_opts='--env --help --stack --version'
@@ -101,6 +118,12 @@ _abra_complete()
                     _abra_complete_apps "$cur"
                 fi
                 ;;
+            recipe)
+                # Offer exactly one app completion.
+                if (( COMP_CWORD == cmd_index + 1 )); then
+                    _abra_complete_recipes "$cur"
+                fi
+                ;;
             #help)
             #    # Offer exactly one command name completion.
             #    if (( COMP_CWORD == cmd_index + 1 )); then

deploy/abra-apps.cloud.autonomic.zone/compose.yml

@@ -1,38 +0,0 @@
----
-version: "3.8"
-
-services:
-  app:
-    image: "nginx:stable"
-    configs:
-      - source: abra_conf
-        target: /etc/nginx/conf.d/abra.conf
-      - source: abra_apps_json
-        target: /var/www/abra-apps/abra-apps.json
-    volumes:
-      - "public:/var/www/abra-apps"
-    networks:
-      - proxy
-    deploy:
-      update_config:
-        failure_action: rollback
-        order: start-first
-      labels:
-        - "traefik.enable=true"
-        - "traefik.http.services.abra-apps.loadbalancer.server.port=80"
-        - "traefik.http.routers.abra-apps.rule=Host(`abra-apps.cloud.autonomic.zone`)"
-        - "traefik.http.routers.abra-apps.entrypoints=web-secure"
-        - "traefik.http.routers.abra-apps.tls.certresolver=production"
-
-configs:
-  abra_apps_json:
-    file: abra-apps.json
-  abra_conf:
-    file: nginx.conf
-
-networks:
-  proxy:
-    external: true
-
-volumes:
-  public:

deploy/abra-apps.cloud.autonomic.zone/nginx.conf

@@ -1,10 +0,0 @@
-server {
-  listen         80 default_server;
-  server_name    abra-apps.cloud.autonomic.zone;
-
-  location / {
-    root /var/www/abra-apps;
-    add_header Content-Type application/json;
-    index abra-apps.json;
-  }
-}

deploy/install.abra.coopcloud.tech/compose.yml

@@ -20,7 +20,7 @@ services:
       labels:
         - "traefik.enable=true"
         - "traefik.http.services.abra-installer.loadbalancer.server.port=80"
-        - "traefik.http.routers.abra-installer.rule=Host(`install.abra.autonomic.zone`)"
+        - "traefik.http.routers.abra-installer.rule=Host(`install.abra.autonomic.zone`,`install.abra.coopcloud.tech`)"
         - "traefik.http.routers.abra-installer.entrypoints=web-secure"
         - "traefik.http.routers.abra-installer.tls.certresolver=production"
 

deploy/install.abra.coopcloud.tech/installer

@@ -1,9 +1,9 @@
 #!/bin/bash
 
-ABRA_VERSION="0.7.1"
+ABRA_VERSION="9.0.0"
 GIT_URL="https://git.autonomic.zone/coop-cloud/abra"
 ABRA_SRC="$GIT_URL/raw/tag/$ABRA_VERSION/abra"
-ABRA_DIR="${ABRA_DIR:+$HOME/.abra/}"
+ABRA_DIR="${ABRA_DIR:-$HOME/.abra}"
 
 function install_abra_release {
   mkdir -p "$HOME/.local/bin"

deploy/install.abra.coopcloud.tech/nginx.conf

@@ -1,6 +1,6 @@
 server {
   listen         80 default_server;
-  server_name    install.abra.autonomic.zone;
+  server_name    install.abra.autonomic.zone install.abra.coopcloud.tech;
 
   location / {
     root /var/www/abra-installer;

makefile

@@ -1,4 +1,4 @@
-.PHONY: test shellcheck docopt kcov codecov release-installer
+.PHONY: test shellcheck docopt release-installer build push
 
 test:
 	@sudo DOCKER_CONTEXT=default docker run \
@@ -7,13 +7,13 @@ test:
 		-d \
 		--name=abra-test-dind \
 		-e DOCKER_TLS_CERTDIR="" \
-		decentral1se/docker-dind-bats-kcov
+		decentral1se/docker-dind-bats-kcov \
-	@DOCKER_CONTEXT=default docker exec \
+	@DOCKER_CONTEXT=default sudo docker exec \
 		-it \
 		abra-test-dind \
 		sh -c "cd /workdir && bats /workdir/tests"
-	@DOCKER_CONTEXT=default docker stop abra-test-dind
+	@DOCKER_CONTEXT=default sudo docker stop abra-test-dind
-	@DOCKER_CONTEXT=default docker rm abra-test-dind
+	@DOCKER_CONTEXT=default sudo docker rm abra-test-dind
 
 shellcheck:
 	@docker run \
@@ -32,27 +32,14 @@ docopt:
 	fi
 	.venv/bin/docopt.sh abra
 
-kcov:
-	@docker run \
-		-it \
-		--rm \
-		-v $$(pwd):/workdir \
-		kcov/kcov:latest \
-		sh -c "kcov /workdir/coverage /workdir/abra || true"
-
-codecov: SHELL:=/bin/bash
-codecov:
-	@bash <(curl -s https://codecov.io/bash) \
-		-s coverage -t $$(pass show hosts/swarm.autonomic.zone/drone/codecov/token)
-
 release-installer:
 	@DOCKER_CONTEXT=swarm.autonomic.zone \
 		docker stack rm abra-installer-script && \
-		cd deploy/install.abra.autonomic.zone && \
+		cd deploy/install.abra.coopcloud.tech && \
 		DOCKER_CONTEXT=swarm.autonomic.zone docker stack deploy -c compose.yml abra-installer-script
 
-release-abra-apps:
+build:
-	@DOCKER_CONTEXT=swarm.autonomic.zone \
+	@docker build -t decentral1se/abra .
-		docker stack rm abra-apps-json && \
+
-		cd deploy/abra-apps.cloud.autonomic.zone && \
+push: build
-		DOCKER_CONTEXT=swarm.autonomic.zone docker stack deploy -c compose.yml abra-apps-json
+	@docker push decentral1se/abra