Add disco build-abra-in-docker advice

Thanks @V

Dockerfile

@@ -1,4 +1,4 @@
-FROM squidfunk/mkdocs-material:8.3.9
+FROM squidfunk/mkdocs-material:9.1.21
 
 EXPOSE 8000
 
@@ -8,4 +8,6 @@ WORKDIR /docs
 
 RUN apk add --no-cache curl
 
-RUN pip install mkdocs-awesome-pages-plugin mkdocs-material-extensions
+RUN pip install \
+  mkdocs-awesome-pages-plugin==2.9.1 \
+  mkdocs-material-extensions==1.1.1

custom_theme/partials/header.html

@@ -1,19 +1,61 @@
 {#-
   This file was copied from the Material theme
-  You can find the file in .venv/lib/python3.10/site-packages/material/partials/header.html
+  You can find the file in https://raw.githubusercontent.com/squidfunk/mkdocs-material/master/src/partials/header.html
 -#}
+<!--
+  Copyright (c) 2016-2023 Martin Donath <martin.donath@squidfunk.com>
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to
+  deal in the Software without restriction, including without limitation the
+  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+  sell copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+  IN THE SOFTWARE.
+-->
+
+<!-- Determine base classes -->
 {% set class = "md-header" %}
 {% if "navigation.tabs.sticky" in features %}
-  {% set class = class ~ " md-header--lifted" %}
+  {% set class = class ~ " md-header--shadow md-header--lifted" %}
+{% elif "navigation.tabs" not in features %}
+  {% set class = class ~ " md-header--shadow" %}
 {% endif %}
+
+<!-- Header -->
 <header class="{{ class }}" data-md-component="header">
-  <nav class="md-header__inner md-grid" aria-label="{{ lang.t('header.title') }}">
-    <a href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}" title="{{ config.site_name | e }}" class="md-header__button md-logo" aria-label="{{ config.site_name }}" data-md-component="logo">
+  <nav
+    class="md-header__inner md-grid"
+    aria-label="{{ lang.t('header') }}"
+  >
+
+    <!-- Link to home -->
+    <a
+      href="{{ config.extra.homepage | d(nav.homepage.url, true) | url }}"
+      title="{{ config.site_name | e }}"
+      class="md-header__button md-logo"
+      aria-label="{{ config.site_name }}"
+      data-md-component="logo"
+    >
       {% include "partials/logo.html" %}
     </a>
+
+    <!-- Button to open drawer -->
     <label class="md-header__button md-icon" for="__drawer">
       {% include ".icons/material/menu" ~ ".svg" %}
     </label>
+
+    <!-- Header title -->
     <div class="md-header__title" data-md-component="header-title">
       <div class="md-header__ellipsis">
         <div class="md-header__topic">
@@ -23,7 +65,7 @@
         </div>
         <div class="md-header__topic" data-md-component="header-topic">
           <span class="md-ellipsis">
-            {% if page and page.meta and page.meta.title %}
+            {% if page.meta and page.meta.title %}
               {{ page.meta.title }}
             {% else %}
               {{ page.title }}
@@ -32,53 +74,94 @@
         </div>
       </div>
     </div>
-    {% if not config.theme.palette is mapping %}
-      <form class="md-header__option" data-md-component="palette">
-        {% for option in config.theme.palette %}
-          {% set primary = option.primary | replace(" ", "-") | lower %}
-          {% set accent  = option.accent  | replace(" ", "-") | lower %}
-          <input class="md-option" data-md-color-media="{{ option.media }}" data-md-color-scheme="{{ option.scheme }}" data-md-color-primary="{{ primary }}" data-md-color-accent="{{ accent }}" {% if option.toggle %} aria-label="{{ option.toggle.name }}" {% else %} aria-hidden="true" {% endif %} type="radio" name="__palette" id="__palette_{{ loop.index }}">
-          {% if option.toggle %}
-            <label class="md-header__button md-icon" title="{{ option.toggle.name }}" for="__palette_{{ loop.index0 or loop.length }}" hidden>
-              {% include ".icons/" ~ option.toggle.icon ~ ".svg" %}
-            </label>
-          {% endif %}
-        {% endfor %}
-      </form>
+
+    <!-- Color palette -->
+    {% if config.theme.palette %}
+      {% if not config.theme.palette is mapping %}
+        <form class="md-header__option" data-md-component="palette">
+          {% for option in config.theme.palette %}
+            {% set scheme  = option.scheme  | d("default", true) %}
+            {% set primary = option.primary | d("indigo", true) %}
+            {% set accent  = option.accent  | d("indigo", true) %}
+            <input
+              class="md-option"
+              data-md-color-media="{{ option.media }}"
+              data-md-color-scheme="{{ scheme | replace(' ', '-') }}"
+              data-md-color-primary="{{ primary | replace(' ', '-') }}"
+              data-md-color-accent="{{ accent | replace(' ', '-') }}"
+              {% if option.toggle %}
+                aria-label="{{ option.toggle.name }}"
+              {% else  %}
+                aria-hidden="true"
+              {% endif %}
+              type="radio"
+              name="__palette"
+              id="__palette_{{ loop.index }}"
+            />
+            {% if option.toggle %}
+              <label
+                class="md-header__button md-icon"
+                title="{{ option.toggle.name }}"
+                for="__palette_{{ loop.index0 or loop.length }}"
+                hidden
+              >
+                {% include ".icons/" ~ option.toggle.icon ~ ".svg" %}
+              </label>
+            {% endif %}
+          {% endfor %}
+        </form>
+      {% endif %}
     {% endif %}
+
+    <!-- Site language selector -->
     {% if config.extra.alternate %}
       <div class="md-header__option">
         <div class="md-select">
           {% set icon = config.theme.icon.alternate or "material/translate" %}
-          <button class="md-header__button md-icon" aria-label="{{ lang.t('select.language.title') }}">
+          <button
+            class="md-header__button md-icon"
+            aria-label="{{ lang.t('select.language') }}"
+          >
             {% include ".icons/" ~ icon ~ ".svg" %}
           </button>
           <div class="md-select__inner">
             <ul class="md-select__list">
               {% for alt in config.extra.alternate %}
                 <li class="md-select__item">
-                  <a href="{{ alt.link | url }}" hreflang="{{ alt.lang }}" class="md-select__link">
+                  <a
+                    href="{{ alt.link | url }}"
+                    hreflang="{{ alt.lang }}"
+                    class="md-select__link"
+                  >
                     {{ alt.name }}
                   </a>
                 </li>
-                {% endfor %}
+              {% endfor %}
             </ul>
           </div>
         </div>
       </div>
     {% endif %}
+
+    <!-- Button to open search modal -->
+    {% if "material/search" in config.plugins %}
+      <label class="md-header__button md-icon" for="__search">
+        {% include ".icons/material/magnify.svg" %}
+      </label>
+
+      <!-- Search interface -->
+      {% include "partials/search.html" %}
+    {% endif %}
+
+    <!-- Repository information -->
     {% if config.repo_url %}
       <div class="md-header__source">
         {% include "partials/source.html" %}
       </div>
     {% endif %}
-    {% if "search" in config["plugins"] %}
-      <label class="md-header__button md-icon" for="__search">
-        {% include ".icons/material/magnify.svg" %}
-      </label>
-      {% include "partials/search.html" %}
-    {% endif %}
   </nav>
+
+  <!-- Navigation tabs (sticky) -->
   {% if "navigation.tabs.sticky" in features %}
     {% if "navigation.tabs" in features %}
       {% include "partials/tabs.html" %}

docs/abra/cheat-sheet.md

@@ -28,8 +28,7 @@ flags: `-f/--force`, `-C/--chaos`
 flags: `-f/--force`, `-V/--volumes`
 
 ### add/remove server
-- `abra server add $SERVER $USERNAME $SSH_PORT`
-flags: `-p/--provision`, `-l/--local`
+- `abra server add $SERVER`
 - `abra server remove $SERVER`
 flags: `-s/--server`
 
@@ -50,3 +49,31 @@ flags: `-p/--publish`, `-r/--dry-run`, `-x,y,z`
 - deploy the changed version to your test instance
 - determine how serious your change is (semver.org for reference)
 - `abra recipe release $RECIPE [$VERSION]`
+
+### Advanced Listing using `jq`
+
+Several `abra` commands can output JSON formatted tables, and can thus be queried and filtered with the tool [jq](https://stedolan.github.io/jq/ "jq JSON Query tool"). We can also format these outputs with [tv](https://github.com/uzimaru0000/tv "tv Table Viewer") into a pretty table. 
+
+
+Currently, `abra recipe ls`, `abra server ls`, and `abra app ls` support the `-m` machine readable output flag which outputs JSON.
+
+#### Filter recipes by "category"
+
+`abra recipe ls -m | jq '[.[] | select(.category == "Utilities") ]' | tv`
+
+As you can see we, we're selecting all recipes where category is "Utilities".
+
+#### Filter apps by state `deployed`
+
+!!! info 
+    `abra app ls -S` queries each server in your added server list, where as without the `-S` it only lists from your local copy of the sever files (thus providing no information about actual state of the apps)
+
+!!! info 
+    `abra app ls` lists apps grouped into a server object, with statistics about the server. In `jq` we can select the entire apps list with `.[].apps[]`.
+
+`abra app ls -m -S |jq '[.[].apps[] | select(.status == "deployed") | del(.upgrade)]' |tv`
+
+The `del(.upgrade)` filter filters out available versions for the recipe in question for that row. It could be useful to leave in if you want a list of deployed apps that need an upgrade.
+
+
+

docs/abra/hack.md

@@ -16,10 +16,51 @@ Install [Go >= 1.16](https://golang.org/doc/install) and then:
 - `make install` will install it to `$GOPATH/bin`
 - `go get <package>` and `go mod tidy` to add a new dependency
 
-Our [Drone CI configuration](https://git.coopcloud.tech/coop-cloud/abra/src/branch/main/.drone.yml) runs a number of sanity on each pushed commit. See the [Makefile](./Makefile) for more handy targets.
+Our [Drone CI configuration](https://git.coopcloud.tech/coop-cloud/abra/src/branch/main/.drone.yml) runs a number of checks on each pushed commit. See the [Makefile](https://git.coopcloud.tech/coop-cloud/abra/src/branch/main/Makefile) for more handy targets.
 
 Please use the [conventional commit format](https://www.conventionalcommits.org/en/v1.0.0/) for your commits so we can automate our change log.
 
+### Using the `abra` public API
+
+Warning, there is currently no stability promise for the `abra` public API! Most of the internals are exposed in order to allow a free hand for developers to try build stuff. If people start to build things then we can start the discussion on what is useful to have open/closed and keep stable etc. Please let us know if you depend on the APIs!
+
+The `pkg.go.dev` documentation is [here](https://pkg.go.dev/coopcloud.tech/abra). Here's a brief example to get you going:
+
+```go
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	abraClient "coopcloud.tech/abra/pkg/client"
+	dockerClient "github.com/docker/docker/client"
+)
+
+func getClient(serverName string) (*dockerClient.Client, error) {
+	cl, err := abraClient.New(serverName)
+	if err != nil {
+		return nil, fmt.Errorf("getClient: %s", err)
+	}
+	return cl, nil
+}
+
+func main() {
+	cl, err := getClient("foo.example.com")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+    // do stuff with the client...
+    // https://pkg.go.dev/github.com/docker/docker/client
+}
+```
+
+Some tools that are making use of the API so far are:
+
+* [`kadabra`](https://git.coopcloud.tech/coop-cloud/abra/src/branch/main/cmd/kadabra/main.go)
+
 ### Cross-compiling
 
 If there's no official release for the architecture you use, you can cross-compile `abra` very easily. Clone the source code from [here](https://git.coopcloud.tech/coop-cloud/abra) and then:
@@ -29,6 +70,18 @@ If there's no official release for the architecture you use, you can cross-compi
 - run `git checkout <tag>`, where `<tag>` is the latest version
 - run `GOOS=<os> GOARCH=<arch> [GOARM=<arm>] make build`. You only have to use `GOARM` if you're building for ARM, this specifies the ARM version (5,6,7 etc). See [this](https://go.dev/doc/install/source#environment) for a list of all supported OS'es and architectures.
 
+### Building in Docker
+
+If you are living under a curse of constant Go environment problems, it might be easier to build `abra` using Docker:
+
+```
+sudo setenforce 0  # SELinux probably won't allow Docker to access files
+docker run -it -v $PWD:/abra golang:1.19.6 bash
+cd /abra
+. .envrc
+git config --global --add safe.directory /abra  # work around funky file permissions
+make build
+```
 
 ## Release management
 
@@ -38,7 +91,7 @@ For developers, while using this `-beta` format, the `y` part is the "major" ver
 
 ### Making a new release
 
-- Change `ABRA_VERSION` to match the new tag in [`scripts`](./scripts/installer/installer) (use [semver](https://semver.org))
+- Change `ABRA_VERSION` in [`scripts/installer/installer`](https://git.coopcloud.tech/coop-cloud/abra/src/branch/main/scripts/installer/installer) to match the new tag (use [semver](https://semver.org))
 - Commit that change (e.g. `git commit -m 'chore: publish next tag x.y.z-beta'`)
 - Make a new tag (e.g. `git tag -a x.y.z-beta`)
 - Push the new tag (e.g. `git push && git push --tags`)
@@ -54,4 +107,8 @@ We maintain a fork of [godotenv](https://github.com/Autonomic-Cooperative/godote
 
 ### `docker/client`
 
-A number of modules in [pkg/upstream](./pkg/upstream) are copy/pasta'd from the upstream [docker/docker/client](https://pkg.go.dev/github.com/docker/docker/client). We had to do this because upstream are not exposing their API as public.
+A number of modules in [pkg/upstream](https://git.coopcloud.tech/coop-cloud/abra/src/branch/main/pkg/upstream) are copy/pasta'd from the upstream [docker/docker/client](https://pkg.go.dev/github.com/docker/docker/client). We had to do this because upstream are not exposing their API as public.
+
+### `github.com/schultz-is/passgen`
+
+Due to [`coop-cloud/organising#358`](https://git.coopcloud.tech/coop-cloud/organising/issues/358).

docs/abra/index.md

@@ -4,7 +4,11 @@ title: Abra
 
 <a href="https://github.com/egonelbre/gophers"><img align="right" width="250" src="https://github.com/egonelbre/gophers/raw/master/.thumb/sketch/adventure/poking-fire.png"/></a>
 
-`abra` is our flagship client & command-line tool which has been developed specifically in the context of the Co-op Cloud project for the purpose of making the day-to-day operations of operators and maintainers pleasant & convenient. It is libre software, written in Go and maintained and extended by the community :heart:
+[![Build Status](https://build.coopcloud.tech/api/badges/coop-cloud/abra/status.svg?ref=refs/heads/main)](https://build.coopcloud.tech/coop-cloud/abra)
+[![Go Report Card](https://goreportcard.com/badge/git.coopcloud.tech/coop-cloud/abra)](https://goreportcard.com/report/git.coopcloud.tech/coop-cloud/abra)
+[![Go Reference](https://pkg.go.dev/badge/coopcloud.tech/abra.svg)](https://pkg.go.dev/coopcloud.tech/abra)
+
+`abra` is the flagship client & command-line for Co-op Cloud. It has been developed specifically for the purpose of making the day-to-day operations of operators and maintainers pleasant & convenient. It is libre software, written in Go and maintained and extended by the community :heart:
 
 Once you've got `abra` installed, you can start your own Co-op Cloud deployment. `abra` allows you to create, deploy and maintain libre software apps. It supports working with existing servers or can create new servers (supported providers: [Servers.coop](https://servers.coop/) & [Hetzner](https://hetzner.com)). It can also help you manage your DNS configuration (supported providers: [Gandi](https://gandi.net)).
 

docs/abra/install.md

@@ -2,6 +2,10 @@
 title: Install
 ---
 
+!!! warning
+
+    We've seen reports that `abra` under [WSL](https://learn.microsoft.com/en-us/windows/wsl/about) doesn't work due to an underlying bug in Docker context handling. See [`coop-cloud/organising#406`](https://git.coopcloud.tech/coop-cloud/organising/issues/406) and [`docker/for-win#13180`](https://github.com/docker/for-win/issues/13180) for more.
+
 ## Stable release
 
 ```
@@ -17,3 +21,17 @@ curl https://install.abra.coopcloud.tech | bash -s -- --rc
 ## Installer script source
 
 You can view that [here](https://git.coopcloud.tech/coop-cloud/abra/src/branch/main/scripts/installer/installer).
+
+## Using Docker
+
+```
+docker run \
+	-v $HOME/.abra:/.abra \
+	git.coopcloud.tech/coop-cloud/abra app ls
+```
+
+!!! note
+	If you're using symlinks, e.g. for [sharing
+	`~/.abra`](/operators/handbook/#sharing-abra), add more `-v` options for each
+	directory you're symlinking to, e.g. `-v
+	$HOME/Projects/CoopCloud/apps:/home/user/Projects/CoopCloud/apps`

docs/abra/trouble.md

@@ -4,55 +4,25 @@ title: Troubleshoot
 
 ## Where do I report `abra` bugs / feature requests?
 
-You can use [this issue tracker](https://git.coopcloud.tech/coop-cloud/abra/issues/new).
+You can use [this issue tracker](https://git.coopcloud.tech/coop-cloud/organising/issues/new/choose).
 
 ## SSH connection issues?
 
-`abra` tries its best to learn from your system configuration or command-line input what the correct SSH connection details are for a given server. This doesn't always work out. Here are some things to try to fix it.
+When you run `abra server add <host>`, `abra` will read from your `~/.ssh/config` and try to match a `Host <host>` entry. If you can `ssh <host>` then you should be able to `abra server add <host>`.
 
-First, ensure that you can `ssh <my-server>` and things work. If you can't SSH to your server then neither can `abra`. If you have a password protected SSH key, then you'll need to make sure your `ssh-agent` is running and you've added your SSH key part:
+For example, if you do `abra server add example.com`, you should have a matching entry that looks like this:
 
 ```
-eval $(ssh-agent -k)
-ssh-add ~/.ssh/<my-secret-key-part>
-ssh-add -L # validate loaded keys
+Host example.com
+  Hostname example.com
+  User exampleUser
+  Port 12345
+  IdentityFile ~/.ssh/example@somewhere
 ```
 
-The first thing `abra` will check for is the connection details listed in `abra server ls`. Check those details are correct. If you haven't managed to `abra server add` your server yet, then no details will appear in that list. You may need to take a look at [this entry](/abra/trouble/#abra-server-ls-shows-the-wrong-details) to clean up old values depending on your situation.
-
-`abra` will then try to read your `~/.ssh/config` entries and match the server domain against a `Host` entry. So, if you do `ssh myserver.com` and you have:
-
-```
-Host myserver.com
-  Hostname myserver.com
-  User myuser
-  Port 222
-  IdentityFile ~/.ssh/my@myserver.com
-```
-
-Then `abra` should have all it needs to build a working SSH connection. You can validate this by passing `-d/--debug` to your commands.
-
-However, sometimes, you use an alias in your SSH configuration, say:
-
-```
-Host mys
-...
-```
-
-So that you can simply type `ssh mys`. `abra` won't be able to match against those entries to discover connection details. You can use aliases to remedy this:
-
-```
-Host mys, myserver.com
-...
-```
-
-`abra` will try to read the relevant `IdentityFile` entry from your `~/.ssh/config` but if it can't make a match, it will rely on your key being added to the `ssh-agent`.
-
-Due to a limitation in our implementation, `abra` uses 2 methods of making SSH connections, the main `abra` -> `remote docker` connection using `/usr/bin/ssh` which can seamlessly pick up loaded SSH keys. However, for SSH host key checking, `abra` uses an SSH library & Golang SSH internals. We're working on resolving this to a single implementation but it is tricky work.
-
 ## "abra server ls" shows the wrong details?
 
-You can use `abra server rm` to remove the incorrect details. Make sure to take a backup of your `~/.abra/servers/<domain>` first. You can then try to re-create by using `abra server add ...` again, making sure to take care if you need to use `<user> <port>`, see `abra server add -h` for more help on this.
+You can use `abra server rm` to remove the incorrect details. Make sure to take a backup of your `~/.abra/servers/<domain>` first. You can then try to re-create by using `abra server add ...` again.
 
 However, if you have Docker installed on the same machine you have `abra`, then there might be some confusion. If you run `docker context ls` you'll see that Docker uses context connection strings also. `abra` simply uses this approach. Sometimes, your Docker defined context details & your `abra` context details can get out of sync. You can use `docker context rm` to resolve this.
 
@@ -62,7 +32,7 @@ If you need to create a new context from Docker, you can do:
 docker context create <domain> --docker "host=ssh://<user>@<domain>:<port>"
 ```
 
-(This is what we used to before we wrote `abra` to make it more convenient.)
+This is what we used to before we wrote `abra` to make it more convenient.
 
 ## Command-line flag handling is weird?
 
@@ -89,7 +59,7 @@ We're still waiting for upstream patch which resovles this.
 
 We're sorry, it's an issue with an upstream dependency. See [`#291`](https://git.coopcloud.tech/coop-cloud/organising/issues/291) for more.
 
-## I need some feature from the old depreciated bash abra?
+## I need some feature from the old deprecated bash abra?
 
 There is an archive of the [old code here](https://git.coopcloud.tech/coop-cloud/abra-bash).
 
@@ -100,6 +70,10 @@ git clone https://git.coopcloud.tech/coop-cloud/abra-bash ~/.abra/bash-src
 ln -s ~/.abra/bash-src/abra ~/.local/bin/babra
 ```
 
-## I am seeing very weird `lookup <domain> on <ip>: write udp <ip>: write: operation not permitted` errors
+## "Network not found" when deploying?
 
-You should turn off your VPN. `abra` has trouble dealing with it right now. We welcome change sets to make it work though!
+This appears to be an upstream issue for which we can't do much in `abra` to solve. See [`coop-cloud/organising#420`](https://git.coopcloud.tech/coop-cloud/organising/issues/420) for more info. The work-around is to leave more time in between undeploy/deploy operations so the runtime can catch up.
+
+## Caller path in debug stacktrace doesn't exist
+
+Debug stacktrace currently begins with `/drone/` due to CI. Remove the initial `/drone/` and the path is relative to the abra project root.

docs/abra/upgrade.md

@@ -18,6 +18,54 @@ abra upgrade --rc
 
 ## Migration guides
 
+> General release notes are [here](https://git.coopcloud.tech/coop-cloud/abra/releases/)
+
+### `0.7.x-beta` -> `0.8.x-beta`
+
+> Currently still a release candidate! Please help test 😌
+
+- We now have an `--offline` flag instead of relying on internal logic to try
+  to decide when offline/online works best. It's up to you! A lot of `abra`
+  operations require network access, so it is not really truly "offline". The
+  logic prefers local filesystem access when this flag is passed. E.g. if there
+  is a local copy of the catalogue, then don't `git pull`.
+
+- There is a new linting rule for catching invalid tags in recipe versions.
+  This is an seemingly unavoidable issue that requires some maintenance work.
+  If you run into the error, here's some
+  [docs](https://docs.coopcloud.tech/maintainers/handbook/#r014-invalid-lightweight-tag)
+  to help work through it.
+
+- `~/.abra/catalogue` is now *only* updated via `git pull`. You may need to
+  `cd ~/.abra/catalogue && git checkout .` to get `abra` to stop complaining about
+  unstaged changes.
+
+### `0.6.x-beta` -> `0.7.x-beta`
+
+- **ALERTA, ALERTA**, security related issue: all `$domain.env` env vars are now exposed to the deployment via the `app` service container. Each `FOO=BAR` is exported within the context of the container. If you have any privately committed secrets in your `.env` files, please migrate them to the `secrets: ...` configuration in the recipe. This change was made to facilitate tooling which can support auto-upgrading of apps in a deployment.
+
+- `abra` can no longer install Docker, initialise swarm mode and the proxy network. It will check if a Docker install exists and is in swarm mode or not and error out accordingly. We leave the provisioning to tools that are designed for that and reduce the command-line surface that we have to maintain going forward.
+
+- `abra server add <host> <args>` 👉 `abra server add <host>`. We have finally removed the custom SSH handling code and now solely rely on invoking `/usr/bin/ssh` directly and reading from the `~/.ssh/config`. The `<host>` argument should correspond to a `Host <host>` entry in your `~/.ssh/config` or in an `Include <file>` statement (hosts are retrieved via `ssh -G <host>`). This means "how does `abra` interact with SSH is 1) do you have an `~/.ssh/config` entry for `<host>` 2) can you `ssh <host>` successfully? 3) there is no 3. It's an easier mental model and also the way `abra-bash` works, hence, less weird obscure errors. `<host>` being public a domain name is still required.
+
+- `abra` no longer tries to do the TOFU host key verification prompt. We follow the praxis of the Docker CLI and just give up when host keys are not validated. We leave it to folks to SSH in and verify themselves.
+
+- Digests have been removed from the catalogue generation. They are not being used elsewhere and were significantly slowing down generation.
+
+### `0.5.x-beta` -> `0.6.x-beta`
+
+- Using `{{ .Domain }}` in recipe `.envrc.sample` files went away because it
+  was portable enough. We revert to replacing e.g `gitea.example.com` with the
+  domain. See
+  [`8fad34e`](https://git.coopcloud.tech/coop-cloud/abra/commit/8fad34e) for
+  more.
+
+- If your `abra.sh` scripts depend on `/bin/sh` and `/bin/bash` is available in
+  the container then `/bin/bash` will be used from now on. `/bin/sh` is only
+  now used if `/bin/bash` is not available. See
+  [`7f745ff`](https://git.coopcloud.tech/coop-cloud/abra/commit/7f745ff) for
+  more.
+
 ### `v0.4.x` -> `v0.5.x`
 
 - The only breaking change was making `abra` understand that the JSON dump for the recipes listing lives on [recipes.coopcloud.tech/recipes.json](https://recipes.coopcloud.tech) instead of [recipes.coopcloud.tech](https://recipes.coopcloud.tech).

docs/democracy/decisions.md

@@ -1,5 +0,0 @@
----
-title: Decisions
----
-
-Placeholder for all the wonderful decisions we will make together.

docs/democracy/index.md

@@ -1,5 +0,0 @@
----
-title: Democracy
----
-
-Placeholder for all the wonderful things we will do together.

docs/federation/faq.md

@@ -0,0 +1,28 @@
+---
+title: FAQ
+---
+
+## What is the Co-op Cloud Federation?
+
+> We're still working things out, here's what know so far!
+
+* It's membership based
+* It's operates democratically
+* It's about mutualising resources and sharing
+* We want to do nice things together
+
+## How many votes makes quorom for a Large Decision?
+
+According to [Resolution 001](/federation/resolutions/passed/001), large decisions can pass when:
+
+> more than 50% of total number of federation members :+1: votes
+
+Please see [the membership docs](/federation/membership) to get the up-to-date membership listing and find the final count for quorom.
+
+## How do I join the federation?
+
+According to [Resolution 002](/federation/resolutions/passed/002):
+
+> To join the federation an existing member must create a large decision to approve of the new member (paid or solidarity).
+
+So, please [get in touch](/intro/contact) if you'd like to join!

docs/federation/finance.md

@@ -0,0 +1,75 @@
+---
+title: Finance
+---
+
+> If you have any questions or run into problems, please chat with us on
+> `#coopcloud-finance:autonomic.zone`
+
+## Agreeing to spend money (Budgets)
+
+
+
+## Sending and receiving money
+
+It's slightly complicated, because money is complicated, but here's how it works. There are two moving parts:
+
+* The Co-op Cloud Open Collective
+* The Autonomic Wise account
+
+Autonomic is [the fiscal host](https://docs.opencollective.com/help/fiscal-hosts/fiscal-hosts) for the Co-op Cloud Open Collective (OC).
+
+OC helps us make all expenses and transfers transparent to the Federation. No actual money is handled via the OC interface. All payments are done via the Autonomic [Wise](https://wise.com) account. The total sum of the available funds shows on the OC page is the actual amount that is held in the Autonomic Wise account.
+
+Autonomic Co-op members commit to support the federation by doing the financial adminstration work for the time being. Autonomic is publicy registered, has a bank account, files taxes etc. All financial comings/goings are kept on the books internally at Autonomic. This could be further mutualised or another collective could pick this up in the future.
+
+Autonomic does not eat the transfer costs from the Wise account when paying out expense for members. That is charged to the Federation common fund.
+
+### How to get paid via Open Collective
+
+* [Create an account on Open Collective](https://opencollective.com/create-account)
+* Go to the [Co-op Cloud Open Collective](https://opencollective.com/coop-cloud)
+* Click [SUBMIT EXPENSE](https://opencollective.com/coop-cloud/expenses/new)
+
+**Important** Please include bank details in your expense so that we can make a bank transfer. We do not currently support payments via Paypal and other platforms.
+
+If you urgently need the money, please let us know on the Co-op Cloud Finance channel.
+
+Finally, please let us know what your username/email is for your Open Collective account so we can add you to the team. This helps us build up the view of our community from the perspective of our Open Collective page.
+
+### How to pay someone via Wise
+
+> **Note**: only Autonomic Co-op members can do this
+
+* First off, be wary of two things: 1) the currency conversion 2) the transaction fees of Wise. For 1) we have the complicating factor that the OC represents the common fund in GBP but our internal Wise jar is EUR. Then you're getting deeper into trouble if someone wants to get paid in e.g. USD.
+
+* In order to cover the transaction fee, you need to fake do the transfer to see what you'll be charged and then add that to what you withdraw from the jar. This is because Autonomic does not eat the cost of the transfer from Wise, that is charged to the Federation.
+
+* First step is to withdraw cash from the Co-op Cloud jar. It will automatically be transferred to the general EUR jar because the Co-op Cloud jar is also in EUR.
+
+* To transfer to USD, you don't have to use USD, you can use GBP/EUR directly. It's easier to make the direct payment from the jar you transferred it to. This is purely because it is easier to follow it in the accounting bookkeeping later on.
+
+* When making the payment, do the following:
+    * Select international transfer, choose your requird `$currency`
+    * Put correct amount in "recipient gets exactly" to get Wise to figure out the correct amount
+    * Open the invoice in Open Collective and look for the expense number, e.g. "Expense #132373" and put this in the reference number of the payment
+    * Note how long the transfer will take (Wise should tell you)
+
+* Mark the expense as paid in Open Collective. Use the "manual" method.
+
+* Let the member know the payment is on the way and how long it will take (if you have time).
+
+#### FAQ
+
+##### Where are the bank details of federation members?
+
+Please see [`Finance.md` in the internal Federation Wiki](https://git.coopcloud.tech/Federation/organising/wiki/Finance)
+
+##### What transfer type do we use for USD?
+
+`ACH`. If you see `Abartn`, that is the `ACH routing number`.
+
+### Tiers on Open Collective
+
+* Infrastructure Sustainability: Folks who are making use of Co-op Cloud digital infrastructure (e.g. [git.coopcloud.tech](https://git.coopcloud.tech)) and want to help out with maintenance costs. All recurring donations are spent directly on running costs and system adminstration labour. Thanks for considering!
+
+* Federation Membership: Dues paid by members of the Co-op Cloud Federation. Please see "Resolution 002: Membership/Dues 2023-03-22" for more information. There may be further decisions made around dues, please refer to the Federation documentation on [docs.coopcloud.tech/federation](https://docs.coopcloud.tech/federation).

docs/federation/index.md

@@ -0,0 +1,14 @@
+---
+title: Federation
+---
+
+Welcome to the Co-op Cloud Federation documentation!
+
+This is the public facing page where we publish all things federation in the open.
+
+- [FAQ](/federation/faq): Take a look if you're curious about the Federation is about 🤓
+- [Resolutions](/federation/resolutions): All draft, in-progress and passed resolutions ✊
+- [Finance](/federation/finance): How we deal with money 💸
+- [Membership](/federation/membership): See who's already joined in 🥰
+- [Minutes](/federation/minutes): All minutes from our meetings 📒
+- [Digital tools](/federation/tools): Tools we use to organise online 🔌

docs/federation/membership.md

@@ -0,0 +1,17 @@
+---
+title: Membership
+---
+
+> Are you also interested in joining the federation? Please see [Resolution 002](/federation/resolutions/passed/002/) for our process on how to join. If you have any questions, [drop us a line](/intro/contact/) with us for a chat
+
+| Name | Dues paid up? | Notes | Contact |
+| -------- | -------- | -------- |-------- |
+| Agaric | - | - | `@wolcen:matrix.org` |
+| Flancia | - | - | `@vera:fairydust.space` |
+| Autonomic | - | - | `@3wc` `@cas` `@decentral1se` `@knoflook` `@travvy` |
+| Bonfire | - | - | `@mayel:matrix.org` + Ivan (`@cambriale:matrix.org`) |
+| Doop.coop | - | - | `@yusf:gottsnack.net` |
+| Local IT | - | - | Philipp (`@yksflip:matrix.kaputt.cloud`) + `@moritz:matrix.local-it.org` |
+| ruangrupa | - | - | Henry `@babystepper:matrix.org` |
+| UTAW | - | - | `@javielico:matrix.org` |
+| ??? | - | - | `@mirsal:1312.media` |

docs/federation/minutes/2022-03-03.md

@@ -0,0 +1,319 @@
+---
+title: 2022-03-03
+---
+
+## Co-op Cloud Federation Bootstrapping
+
+_Please add any suggested agenda items here. We'll add meeting notes to this page after the meeting has happened_
+
+## Metadata
+
+* Time / date: March 3 @ 1500-1630 UTC https://time.is/0300PM_3_Mar_2023_in_UTC
+* Location: https://meet.jit.si/coop-cloud-federation-bootstrap-meeting
+* Real-time note taking will happen at: https://pad.autonomic.zone/XVhRKvAaRHmaEIR14KBxLA# (and be migrated here after the meeting)
+
+## Agenda
+
+(All times UTC, as sharp as possible)
+
+| Start | End | Topic | Time |
+| -------- | -------- | -------- | -------- |
+| 1500 | - | Meeting opens | - |
+| 1500 | 1510 | introductions | 10m |
+| 1510 | 1520 | confirming the agenda | 10m |
+| 1520 | 1540 | decision-making process | 20m |
+| 1540 | 1450 | break | 10m |
+| 1450 | 1610 | small-group discussions | 20m |
+| 1610 | 1630 | report-back / next steps | 20m |
+
+Suggested topics for small-group discussions:
+
+1. What software tools do we want to use for our organising?
+2. How should the finances of the federation work?
+3. Where else can we promote Co-op Cloud?
+4. Development priorities
+
+## Meeting notes
+
+### Agenda
+
+(All times UTC, as sharp as possible)
+
+| Start | End | Topic | Time |
+| -------- | -------- | -------- | -------- |
+| 1500 | - | Meeting opens | - |
+| 1500 | 1510 | introductions | 10m |
+| 1510 | 1520 | confirming the agenda | 10m |
+| 1520 | 1540 | decision-making process | 20m |
+| 1540 | 1450 | break | 10m |
+| 1450 | 1610 | small-group discussions | 20m |
+| 1610 | 1630 | report-back / next steps | 20m |
+
+Suggested topics for small-group discussions:
+
+1. What software tools do we want to use for our organising?
+2. How should the finances of the federation work?
+3. Where else can we promote Co-op Cloud?
+4. Development priorities
+
+### Introductions
+
+- name
+- pronouns
+- co-op you're part of
+- favorite natural place
+
+Attending:
+
+* Trav (Autonomic) [facilitation]
+* dc1 (Autonomic)
+* Phillip (Local-IT)
+* kawaiipunk (Autonomic)
+* V (Flancia [coop] - https://anagora.org [software platform])
+* Cas (Autonomic) [main notes]
+* Josef (Doop-Coop)
+* Wolcen (Agaric)
+* Ivan (Bonfire)
+* Mayel (Bonfire)
+* Calix (Autonomic) [facilitation]
+* Jamie (FarmOS)
+* Mirsal
+
+### Confirming Agenda
+
+- V: Question about overall objective.
+- Calix: Any suggested answers?
+- dc1: To see who wants to come with on setting up the fed. Getting going making descisions together. Also how people see themselves participating on an ongoing basis.
+- Calix: Any other intentions?
+
+[Mirsal joins]
+
+Recap:
+- How to make Decisions
+
+- Wolcen: Is there an existing org or is this the start?
+- Trav: This is the kick off. We start here.
+- dc1: Points to proposal. Mostly from Autonomic's viewpoint, which we hope to build on.
+
+- V: Asking about transcription bots.
+- Calix: Suggests that we table for now for later discussion.
+
+### decision-making process
+
+proposal: https://git.coopcloud.tech/Federation/Federation/wiki/Proposals
+
+- Trav: We adapted an old proposal for descision making process.
+- https://pad.autonomic.zone/s/MLafJE2jC#Overview
+- https://coopcloud.tech/blog/federation-proposal/
+
+- Trav: We consider this an important step in group formation. Summary: Proposal is written up and posted on channel, voting occurs via emoji reactions and after a time period it is passed.
+- dc1: Context: Autonomic has been initiating the project heretofor and has made all descisions, but we want the community to have that power rather than Autonomic.
+
+- Calix: We are in bootstraps as far as descision making so we have hope we can do it ad hoc this time.
+
+- V: Question about using Loomio.
+
+- kawaiipunk: Loomio while it is good seems to have a lot of bloat to it and the complexity of the forum functionality. Proposes we try to the most minimal mechanics possible. Autonomic uses this process.
+Calix: We don't have SSO for Coopcloud that we all have access. Gitea is a good platform for bootstrapping since it has its own accounts and almost everyone already has accounts on it for functional reasons.
+Wolcen: keeping things in as few places as possible seems better.
+- dc1: +1 for minimal. We are at the start, and all things are vague so we perhaps just do the minimal possible and keep going with what we have
+
+- V: As a member of not autonomic lacking context. tnx to dc1 for clarifying.
+
+- Wolcen: Asks about technology for automating the gitea+wiki->matrix crossover.
+- dc1: No automation current we can always change that later
+
+- kawaiipunk: Would the voting proposal come into effect after this meeting. Propose that it should. - Explanation that we operate more on a consent basis than exactly a consensus and this proposal continues that.
+
+- Wolcen: Asks about how to determine the magnitude of a descision and when the proposal can be acted on based on votes.
+- dc1: General descions are made as more than one person. This informs how to do the calculation for the magnitude.
+- kawaiipunk: Medium descisions can pass without everyone interacting with them. Good descisions are transparent and reversable. Controversial things generally have more than one blocker.
+
+### break time
+
+### Roundup about breakouts
+
+### Checking on qualified yes answers on poll about descision making
+
+1 qualified yes. Giving space for the qualifications. No takers.
+
+- V: Suggests that Autonomic lead that choice.
+- kawaiipunk: Agrees. If we're following consensus, unless anyone blocks it passes.
+
+**We pass the descision making process.**
+
+### Breakouts
+
+* Calix: Suggestions for other breakout topics?
+
+* Wolcen: 1 & 4 seem closely tied and should be merged.
+
+* dc1: Membership should be a topic.
+
+* We vote on poll now.
+
+#### Breakout 1 + 3 (technology and technology)
+
+> (See notes below)
+
+We summarized where we're at, what technologies are being used for organizing, what are the dependencies for coop cloud, and what is being developed.
+
+A couple of little things that are interesting from the perspective of development priorities:
+
+- debug / entrypoint tools
+- catalog normalization
+
+#### Breakout 2 (finances)
+
+> (See notes below)
+
+Summary: Federation model has tried and failed several times. Priority should be getting some money in. Give what you can should be the main thing and we can see if we need to tweak that.
+
+#### Breakout 4 (membership)
+
+> (See notes below)
+
+Broke down what membership means. Three classes of membership and what their powers are.
+
+- Community member (individual level)
+- Recipe maintainer
+- Federation members - which is the main descion making member.
+
+Discussed the processes involved in creating members, as in notes.
+
+MVP is protection from capital interest. Existing members vote on new members? Minimal and lean.
+
+### We vote on next meeting time
+
+### Checkout process
+
+> How do you feel? What about future?
+
+* Calix: Feeling inspired. Topics would love to talk about: Working groups? What comms channels are needed?
+
+* V: Feeling amazing. In one month?
+
+* dc1: Feeling great, very productive. Looking forward to future.
+
+* Trav: Great! THink this works well. Good that we can actually make descisions. Excited and optimistic for future.
+
+* Mayel: Feel good about meeting.  Really good to see energies. Great awesome keep going.
+
+* Wolcen: Feels great. Good to see it making progress. Missed meeting pad - having official links for future meetings would be nice to have.
+
+* Cas: Great. Looking forward to nitty gritty stuff.
+
+* kawaiipunk: Tired, but good. Good discussions. Excited to have it open up to other people. Maybe we could hack on technical stuff. Interested in recipes and doing more formal organization.
+
+* Phillipp: Excited this was really great. Language barrier made it a little challenging. Next meeting has a lot of questions and things to do.
+
+### Poll
+
+* 1 month wins for next meeting.
+
+### Breakout room minutes
+
+#### Breakout room #1
+
+We have a rambling discussion about things related to software.
+
+List of technologies currently in use:
+
+- gitea [wiki, source repo, kanban?]
+- matrix [chat, community organization]
+
+Technologies under development/coop cloud
+
+- Abra
+- Recipes
+- Recipe catalog
+
+Technology dependencies
+
+- Docker + swarm
+
+Technology decision considerations
+
+- What about using other 'containers', 'virtual instances', etc?
+    - Part of the value proposition is that it operates on current standards.
+
+Development priorities
+
+- Standardizing recipe catalog acquisition
+- Debug/other hooks/entrypoints
+
+#### Breakout room #2
+
+Breakout Room #2: How should the finances of the federation work?
+
+* Present: Trav, Mayel, yksflip, d1, V
+* trav taking notes
+
+* d1: want to map what we're already doing. At first unpaid then back-paid. Grant money is gone. Now asking clients to contribute 50% of fee towards Co-op Cloud development. Have not dumped money into OC yet.
+
+* V: why tell clients? Transparency?
+
+* d1: yeah transparency, helps get paid out and helps groups understand what they're contributing to.
+
+* m: some part of revenue goes back to maintain commons. we know theres more than the config, we have admin, meetings, more services/tools/etc. besides membership dues, I would draft some suggestions/templates for how hosters can split revenue between infra/config/upstream project development...
+
+* yksflip: contribute time to co-op cloud atm. but happy to shuffle money directly to open collective. main funding is public funding, funders need to see who is working on what, transparency/overview stuff. they also have money to give to freelancers. could we have ways to say we give % of the project to the open collective of the fedi. still trying to make local-it sustainable. funding until july and then open question how to go on sustainably.
+
+* t: concern on finances is, having enough. paying for meetings is noble but we'll lose money fast. having a prioritisation of where money goes would be great. then as we have cash we put it where we want.
+
+* d1: contribute comensurate to number of members in co-op? what would be a practical model? do we attach it to membership (dues?).
+
+* v: dues and per-x contribution? are these two differen things?
+
+* d1: 2 approaches, end goal is the same, have members contribute to the project
+
+* V: we are amalgum, hard to quantify member #s
+
+* yksflip: best to start with something easy. Co-ops of this size/# of instance, guidelines. Try it and then discuss again in a few months. Compensating meetings is great but maybe some things are more urgent.
+
+* V: how does Autonomic do this currently?
+
+* d1: simpler for us as 1 org vs fed. All funds into 1 pot. Tricky to get right. Did call with Co-op Cycle, bike delivery. Start with financial contribution from the start. Money makes things happen.
+
+#### Breakout #3 did not happen
+
+#### Breakout #4: Membership
+
+* Jamie (facilitating)
+* Kawaiipunk
+* Calix (notes)
+
+What are we hoping to learn / decide?
+
+* kp: 3 levels of membership:
+  * community member
+  * maintainers of recipes
+  * federation members
+
+* question: boundaries around co-operativeness. open to organisations / individuals? what org. structures permitted for members. co-ops only? worker co-ops only? allied orgs? capitalist orgs? do we allow co-ops
+
+* calix: Q: what is the process for deciding any of above questions? new members, dues, etc.
+
+* j: is there a way for e.g. corporate members to be community members? any previous steps towards decision-making with co-op cloud?
+
+* Calix: Cooperative Technologists [...] when someone wants to joing it's a network-wide process, case-by-case, open to anyone to in the group; then w/ Autonomic, it's a 2-step process where someone joins provisionally, then as full member after a period of time.
+
+* j: B-Corp, somewhat controversial category at this point as to whether it means anything or not. Consideration to take network approach to avoid "legalistic" approach, strict "by the books".
+
+* kp: might also be a problem of assessing what counts as a cooperative for international members, different legal formations around the world.
+
+* c: Proposal: the 3 cats, fed-members should be cooperatively run, and aligned with values of the federation. Large decision among federation members to add a new one.
+
+* J: Like it. Open it as a question.
+
+* c: Could use our new decision making process? Ask for comments before it goes live.
+
+* kp: Seems quite fundamental. Best to keep simple and iterate.
+
+* calix: should there be a federation members only matrix channel to discuss federation decisions?
+
+* kp: guess there'd have to be under current plans. or does it set up too much of an "in group"? proposals could be posted publicly, voting could be in secret, results public? emoji voting in a private channel. transparency = positive.
+
+* jamie: public "who voted how"
+
+* kp: maybe literally just collecting of voting

docs/federation/minutes/index.md

@@ -0,0 +1,3 @@
+---
+title: Minutes
+---

docs/federation/resolutions/drafts/index.md

@@ -0,0 +1,3 @@
+---
+title: Drafts
+---

docs/federation/resolutions/in-progress/011.md

@@ -0,0 +1,22 @@
+---
+title: "Resolution 011: Budget 005: Backup improvements - 2023-07-23"
+---
+
+- Deadline: 2022-08-06
+- Size: Large
+
+### Summary
+
+Agree Budget 005, for up to 10 hours (€200) to implement changes to the Co-op Cloud backup system, including `abra`, and `backup-bot-two`.
+
+More information, including background to the changes, and breakdown of the work, can be found here: https://pad.local-it.org/4chDYXkBQZywXKj0zs_Ymg#
+
+### Details (Budget 005)
+
+**Budget amount**: Up to EUR €200
+
+**Who will implement this**: Calix/3wc, decentral1se, Philipp and Moritz
+
+**When will the money be spent**: Before the end of August 2023.
+
+**What is the money for**: Improvements to Co-op Cloud backup tools

docs/federation/resolutions/in-progress/index.md

@@ -0,0 +1,3 @@
+---
+title: In progress
+---

docs/federation/resolutions/index.md

@@ -0,0 +1,18 @@
+---
+title: Resolutions
+---
+
+### Resolution Template
+
+```javascript
+## Resolution <number>: <title> - <date>
+
+- Deadline: Date
+- Size: large or medium
+
+### Summary
+Who this affects, and what it does
+
+### Details
+A narrative with details
+```

docs/federation/resolutions/passed/001.md

@@ -0,0 +1,59 @@
+---
+title: "Proposal 001: Decision Making Process - 2023-03-03"
+---
+
+- Deadline: 2023-03-03 (live voting)
+- Size: large
+
+### Summary
+
+Institute descision making process as per below. Special consensus voting in organization meeting rather than the below process.
+
+### Decision Making Process
+
+* Write up a proposal using the below template, and add to the [Proposals wiki page](https://git.coopcloud.tech/Federation/Federation/wiki/Proposals).
+* Specify if they are a large or medium proposal
+* Votes are done via emoji-reaction in the Community Organising Matrix channel (<https://matrix.to/#/#coopcloud-comm-org:autonomic.zone>)
+* List the decision on the [decisions page](https://docs.coopcloud.tech/federation/resolutions) on our documentation
+* Decisions can be split intro three categories: Small, Medium and Large.
+* Votes can be in favour :+1:, against :-1: (block),  or abstain :shrug:
+* Announce the result in the [Federation chat (#coop-cloud-fedi:autonomic.zone)](https://docs.coopcloud.tech/intro/contact/#matrix) and record it on the [decisions page](https://docs.coopcloud.tech/federation/resolutions) of the documentation
+
+### Types of Proposals
+
+#### Small - “Get on and do a thing”
+
+* Up to individual members to decide if they should just make the decision, or share it with the rest of the members to seek consensus.
+
+#### Medium - “consensus pending objections”
+
+* Potentially about shared tools, recipes, abra, etc.
+* Doesn’t have an effect on the direction or operation of Co-op Cloud as a whole.
+* If any member of Co-op Cloud thinks it’s a Large decision, achieve Maximum Consensus™ (see [below](https://pad.autonomic.zone/PtNbWo-7Tt-CKXvC6kxvZQ?view#Large---Maximum-Consensus-%E2%84%A2))
+* proposals must have a minimum deadline of 2 weeks from when they are proposed
+* Pass requirements:
+  * at least one :+1: vote
+  * no :-1: votes
+
+#### Large - “Maximum Consensus ™”
+
+* Important decisions affecting the operation, direction, working conditions and finances of Co-op Cloud.
+* proposals must have a minimum deadline of 2 weeks from when they are proposed
+* Pass requirements:
+  * more than 50% of total number of federation members :+1: votes
+  * no :-1: votes
+
+### Proposal Template
+
+```javascript
+## Resolution <number>: <title> - <date>
+
+- Deadline: Date
+- Size: large or medium
+
+### Summary
+Who this affects, and what it does
+
+### Details
+A narritive with details
+```

docs/federation/resolutions/passed/002.md

@@ -0,0 +1,25 @@
+---
+title: "Resolution 002: Membership/Dues - 2023-03-22"
+---
+
+* Deadline: 2023-04-11
+* Passed on 2023-04-13
+* Size: Large
+
+### Summary
+
+1. Set membership dues for the Co-op Cloud Federation at EUR 10/month, to be reviewed in 6 months’ time, in October 2023.
+2. Approval of new members requires a Large decision
+3. Groups who had a member attend the first federation meeting are all founding members (are already in the federation, do not require a decision to be added)
+
+### Details
+
+Obviously this plan is not long-term financially sustainable. The idea is to use Autonomic’s remaining funds for the federation to collectivise the process of working this out over the next few months.
+
+#### Dues
+
+Members are required to make a minimum monthly EUR 10 (or EUR 60/year) donation through Open Collective. Members who are able are encouraged to donate more. Individuals/groups wanting to join Co-op Cloud who aren’t able to make a financial contribution may request a solidarity free membership.
+
+#### Membership
+
+To join the federation an existing member must create a large decision to approve of the new member (paid or solidarity). All collectives who attended the first federation meeting are already granted membership and are asked to setup recurring donations as soon as possible.

docs/federation/resolutions/passed/003.md

@@ -0,0 +1,17 @@
+---
+title: "Resolution 003: Paid work - 2023-03-22"
+---
+
+* Deadline: 2023-04-11
+* Passed on 2023-04-13
+* Size: Large
+
+### Summary
+
+1. Set the wage for Co-op Cloud Federation work at €20/h. Review these numbers in 6 months’ time, in October 2023.
+
+### Details
+
+Work is paid at EUR 20/hour. This may be increased via future decisions if so desired by the collective.
+
+Members must do their own taxes for wages earned. To get paid, worker members must invoice via the Co-op Cloud Open Collective. Invoices need to include times and descriptions.

docs/federation/resolutions/passed/004.md

@@ -0,0 +1,38 @@
+---
+title: "Resolution 004: Budgeting - 2023-03-22"
+---
+
+* Deadline: 2023-04-11
+* Passed on 2023-04-13
+* Size: Large
+
+### Summary
+
+1. All paid work must be within a Budget
+2. The first Budget as 8 hours / month, for member groups’ participation in organising meetings
+
+### Details
+
+All paid work must be within a Budget, agreed using a Large Decision. Budgets are ideally clumped as much as possible to reduce decision fatigue. Budgets include “who will do the work”. The agreed person can give the task to another Federation member.
+
+Participation in organising meetings is paid for up to one person per member organisation. Additional people are welcome to attend; we encourage member organisations to pay for their additional attendees’ time themselves, if possible.
+
+#### Budget: Monthly meetings
+
+> **Budget amount:** EUR 960
+>
+> **Who will implement this:** Up to 1 person from each member organisation
+>
+> **When will the money be spent:** Over the next 6 months, until the meeting in October 2023.
+>
+> **What is the money for:** Paying attendees of monthly organising meetings
+
+#### Budget template:
+
+```
+**Budget name:** Buying ponies
+**Budget amount:** EUR 100,000
+**Who will implement this:** Ade from Ponies.coop
+**When will the work happen:** Tomorrow
+**What is the money for:: Buying one pony for each member organisation
+```

docs/federation/resolutions/passed/005.md

@@ -0,0 +1,19 @@
+---
+title: "Resolution 005: Public federation membership, notes and decisions - 2023-04-14"
+---
+
+* Deadline: 2023-04-17
+* Passed: 2023-04-18
+* Size: medium
+
+### Summary
+
+The following federation info will be made public on [`docs.coopcloud.tech/federation`](https://docs.coopcloud.tech/federation/):
+
+- Federation membership
+- Meeting minutes
+- Decisions which have passed
+
+### Details
+
+This will make the process of documenting easier to mutualise and increase transparency for those interested in joining. The [`git.coopcloud.tech/Federation`](https://git.coopcloud.tech/Federation/Federation/wiki/) wiki can still be used for storing private details such as bank account information. If members do not want to be listed, they can do so even when this decision passes.

docs/federation/resolutions/passed/006.md

@@ -0,0 +1,25 @@
+---
+title: "Resolution 006: Budget 002: Resolution Writing-up - 2023-05-29"
+---
+
+# Resolution 006: Budget 002: Resolution Writing-up - 2023-05-29
+
+- Deadline: 2022-06-12
+- Size: Large
+
+### Summary
+
+Agree Budget 002, for €100 for @decentral1se to write up 2 resolutions.
+
+### Details (Budget YYY)
+
+**Budget amount**: EUR 100
+
+**Who will implement this**: @decentral1se
+
+**When will the money be spent**: By 2023-07-03
+
+**What is the money for**: Writing up two Resolutions:
+
+1. A buffer for federation common fund.
+2. To set up a standing critical fixes budget each month until the buffer in (1) is hit

docs/federation/resolutions/passed/007.md

@@ -0,0 +1,17 @@
+---
+title: "Resolution 007: 1 year dues waiver for Doop.coop - 2023-06-19"
+---
+
+- Deadline: 2023-07-03
+- Size: Medium
+
+### Summary
+
+Waive membership dues for Doop.coop for the first year of their membership, from July 2023 to June 2024.
+
+### Details
+
+Yusf said:
+
+> Hai! As our coop is a side gig, we've had a very low turnaround in the coop last fiscal. As such we've hadn't had the time to raise our revenue yet in this cycle so my question for the federation is:
+> Is it possible for us (already joined), Doop Coop, to apply for this solidarity free membership the first year after which we'll be able to put in the fee?

docs/federation/resolutions/passed/008.md

@@ -0,0 +1,24 @@
+---
+title: "Resolution 008: Budget 003: Paying invoices - 2023-06-19"
+---
+
+- Deadline: 2022-07-03
+- Size: Large
+
+### Summary
+
+Agree Budget 003, for up to €20/month for an Autonomic member to pay invoices submitted for Co-op Cloud Federation work.
+
+### Details (Budget 003)
+
+**Budget amount**: EUR €20/month
+
+**Who will implement this**: Autonomic
+
+**When will the money be spent**: Monthly
+
+**What is the money for**: Paying for the work involved in paying invoices submitted via OpenCollective.
+
+## A note about decentralisation
+
+Unfortunately, there doesn't seem to be a good way to open this task to other Co-op Cloud Federation members, until we get a dedicated bank account for the Federation, but if anyone has ideas about this, please let us know!

docs/federation/resolutions/passed/009.md

@@ -0,0 +1,22 @@
+---
+title: "Resolution 009: Federation common fund buffer - 2023-07-03"
+---
+
+## Resolution 009: Federation common fund buffer - 2023-07-03
+
+- Deadline: 2023-07-17
+- Size: Large
+
+### Summary
+
+The Co-op Cloud Federation has [a common fund](https://opencollective.com/coop-cloud#category-BUDGET) which can be used by federation members by proposing [budgets](https://docs.coopcloud.tech/federation/resolutions/passed/004/).
+
+With Resolution 009, we want to introduce the idea of a common fund buffer.
+
+This buffer is proposed to be set at 1,500 EUR. When the buffer is reached, no more budgets can be proposed. When the fund increases once more, budgets can resume.
+
+The buffer is in place to reserve funds for critical collective spending when we have limited income.
+
+### Details
+
+1,500 EUR is fairly arbitrarily chosen but seems to be a reasonable amount of money to cover situations where, e.g. we need to pay for servers, domains, mail accounts or other common critical infrastructure.

docs/federation/resolutions/passed/010.md

@@ -0,0 +1,32 @@
+---
+title: "Resolution 010: Budget 004: Critical fixes - 2023-07-03"
+---
+
+## Resolution 010: Budget 004: Critical fixes - 2023-07-03
+
+- Deadline: 2023-07-17
+- Size: Large
+
+### Summary
+
+We propose to have a standing budget of 10 hrs / month available for fixes in Abra, Co-op Cloud recipes and other critical tools (e.g. recipes.coopcloud.tech) in the Co-op Cloud ecosystem.
+
+A fix is deemed critial when it is listed on this coop-cloud/organising board:
+
+> https://git.coopcloud.tech/coop-cloud/organising/projects/24
+
+This board is collectively gardened by Co-op Cloud participants (both federation members and not). The process for adding a ticket to the board requires getting confirmation from at least one other member of the federation.
+
+This budget can be claimed by any volunteer who would like to develop the fix. If the volunteer is not a Co-op Cloud federation member, they must first be "vouched for" by a federation member. This is an informal process which can be arranged via the Matrix chat. This aims to assure agreement on timing and what the fix should contain beforehand.
+
+Fixes can be claimed by assiging yourself to the ticket. If within 1 week there is no updates on the ticket, another volunteer can propose to take over. This process is also informal: please @ the original volunteer and give some reasonable time for them to reply (suggested: 1 day).
+
+If the fix is urgent and things need to move faster, please state so on the ticket. Please consult with at least one other member of the federation to confirm that there is indeed agreement on the urgency of the fix.
+
+### Details (Budget 004)
+
+This budget stands open and available for use until the common fund buffer is reached. Please see [Resolution 009](https://docs.coopcloud.tech/federation/resolutions/passed/009/) for further details on the buffer.
+
+Please see [Resolution 003](https://docs.coopcloud.tech/federation/resolutions/passed/003/) for details on the hourly rate for this work.
+
+Invoices can be submitted via the Co-op Cloud Open Collective. Please see [these docs](https://docs.coopcloud.tech/federation/finance/#how-to-get-paid-via-open-collective) for how to do that.

docs/federation/tools.md

@@ -0,0 +1,11 @@
+---
+title: Digital tools
+---
+
+- [Public documentation](https://docs.coopcloud.tech/federation)
+- [Organising repository (private)](https://git.coopcloud.tech/Federation/organising)
+- [Wiki (private)](https://git.coopcloud.tech/Federation/organising/wiki)
+- [Git hosting](https://git.coopcloud.tech/)
+- [Matrix Space](https://matrix.to/#/#coop-cloud-space:autonomic.zone)
+- [Website](https://coopcloud.tech/)
+- [Drone CI/CD](https://build.coopcloud.tech)

docs/get-involved/index.md

@@ -12,15 +12,11 @@ We are happy to have designers, critical thinkers, artists, hackers, documenters
 
 There are a number of "roles" such as "operator", "maintainer", "organiser" which we've tried to come up with to make it more clear how you can relate to the project and how you can find ways to be involved which suit your interests. If you don't fit one of these roles, that is fine.
 
-We have [a weekly check-in](/get-involved/#kite-flying-hours) for contributors of this project to let each other know what we're working on, how much time we've spent on it and how to coordinate further work.
+We have [an irregular online check-in](/organisers/handbook/#kite-flying-hours) for contributors of this project to let each other know what we're working on, how much time we've spent on it and how to coordinate further work.
 
 We have a [status page](/intro/bikemap) showing what we are aiming to achieve in the near future. That gives a good overview of where we're going together.
 
-From this status page, we use an [issue tracker](https://git.coopcloud.tech/coop-cloud/organising/issues) where we hold discussions about what we want to do. We categorise these issues according to the bike map using these [milestones](https://git.coopcloud.tech/coop-cloud/organising/milestones). Finally, use this [board](https://git.coopcloud.tech/coop-cloud/organising/projects/8) to keep track of what we're working on right now. We collectively review these things on a weekly/monthly basis to keep track of our time spent vs. budget available.
-
-Once you've found something to work on and are introduced, we'll give you an account on our [time tracking infrastructure](https://kimai.autonomic.zone) where you can log your times. This helps us reduce the burden of financial and time keeping admin falling on one person.
-
-We have received funding via [the ECF](https://culturalfoundation.eu/initiatives/culture-of-solidarity-fund) and can offer £16 hourly rate for your work. We've written more on why we think it is important to compensate all contributions for this project below.
+We use [issue trackers](https://git.coopcloud.tech/coop-cloud/organising/issues) and [project boards](https://git.coopcloud.tech/coop-cloud/organising/projects) to keep track of what we're working on right now. We collectively review these, to keep track of our time spent vs. budget available.
 
 ## Compensation
 

docs/glossary/index.md

@@ -12,11 +12,11 @@ An app is a libre software that you use, e.g. Wordpress, Gitea, Jitsi, Nextcloud
 
 ## Container
 
-A [Docker](/glossary#docker) term: a running instance of an [image](/glossary#image), running processes that are isolated from the host system.
+A [Docker](#docker) term: a running instance of an [image](#image), running processes that are isolated from the host system.
 
 ## Deployment
 
-When you run `abra app deploy <domain>`, `abra` reads a [recipe](/glossary#recipe) configuration and creates an [app](/glossary#app).
+When you run `abra app deploy <domain>`, `abra` reads a [recipe](#recipe) configuration and creates an [app](#app).
 
 ## Docker
 
@@ -24,36 +24,36 @@ When you run `abra app deploy <domain>`, `abra` reads a [recipe](/glossary#recip
 
 ## Environment variables
 
-Variables passed from the shell to processes invoked by it. They are used for configuring [services](/glossary#service).
+Variables passed from the shell to processes invoked by it. They are used for configuring [services](#service).
 
 ## Environment file
 
-A file contained in a [recipe](/glossary#recipe) describing the contents of [environmental variables](/glossary#environment-variables).
+A file contained in a [recipe](#recipe) describing the contents of [environment variables](#environment-variables).
 
 ## Image
 
-A [Docker](/glossary#docker) term: a template for creating [containers](/glossary#container), describing their file structure and installed binaries.
+A [Docker](#docker) term: a template for creating [containers](#container), describing their file structure and installed binaries.
 
 ## Proxy network
 
-A [Docker](glossary#docker) related concept: a virtual network created on the server machine used for communicating between [services](/glossary#service). Any [service](/glossary#service) can be plugged into more than one [network](/glossary#network), allowing for control over data sharing between them.
+A [Docker](#docker) related concept: a virtual network created on the server machine used for communicating between [services](#service). Any [service](#service) can be plugged into more than one [network](#network), allowing for control over data sharing between them.
 
 ## Recipe
 
-A recipe is what we call the configuration files that are used to deploy an [app](/glossary#app). When you run `abra app deploy <domain>`, `abra` is reading a recipe configuration, such as [the gitea recipe](https://git.coopcloud.tech/coop-cloud/gitea), in order to know how to deploy a new Gitea instance. When we speak of a "digital configuraiton commons", we're primarily referring to the [growing collection of recipes](https://git.coopcloud.tech/coop-cloud).
+A recipe is what we call the configuration files that are used to deploy an [app](#app). When you run `abra app deploy <domain>`, `abra` is reading a recipe configuration, such as [the gitea recipe](https://git.coopcloud.tech/coop-cloud/gitea), in order to know how to deploy a new Gitea instance. When we speak of a "digital configuration commons", we're primarily referring to the [growing collection of recipes](https://recipes.coopcloud.tech).
 
 ## Secret
 
-A [Docker](/glossary#docker) related concept: A way to store passwords encrypted on disk and mounted inside the [containers](/glossary#container) as files that can be read that contain the secret. See the [Docker secrets documentation for more](https://docs.docker.com/engine/swarm/secrets/). `abra` makes use of this approach to store secrets for deployed [apps](/glossary#app).
+A [Docker](#docker) related concept: A way to store passwords encrypted on disk and mounted inside the [containers](#container) as files that can be read that contain the secret. See the [Docker secrets documentation for more](https://docs.docker.com/engine/swarm/secrets/). `abra` makes use of this approach to store secrets for deployed [apps](#app).
 
 ## Service
 
-A [Docker](glossary#docker) term: a single [container](/glossary#container) that is a part of a [stack](glossary#stack).
+A [Docker](#docker) term: a single [container](#container) that is a part of a [stack](#stack).
 
 ## Stack
 
-A [Docker](glossary#docker) term: one or more [services](/glossary#service) running together to provide a functionality.
+A [Docker](#docker) term: one or more [services](#service) running together to provide a functionality.
 
 ## Volume
 
-A [Docker](/glossary#docker) term: a directory that can be mounted inside a [container](/glossary#container) to store data. Because [containers](/glossary#container) are meant to be non-changeable and disposable, any data that is supposed to not be lost between updates or restarts is stored in volumes.
+A [Docker](#docker) term: a directory that can be mounted inside a [container](#container) to store data. Because [containers](#container) are meant to be non-changeable and disposable, any data that is supposed to not be lost between updates or restarts is stored in volumes.

docs/intro/bikemap.md

@@ -6,4 +6,4 @@ title: Bike map
 
 - We are working towards a stable `1.0.0` release.
 
-- What we're currently working on is listed on this issue tracker: [`coop-cloud/organising`](https://git.autonomic.zone/coop-cloud/organising/issues).
+- What we're currently working on is listed on this issue tracker: [`coop-cloud/organising`](https://git.coopcloud.tech/coop-cloud/organising/issues).

docs/intro/contact.md

@@ -12,14 +12,6 @@ title: Get in touch
 
 Here is a link to the [Matrix space](https://matrix.to/#/!xSMwGbdVehScXcIFwS:autonomic.zone?via=autonomic.zone&via=matrix.org&via=1312.media) to see all channels.
 
-- [`#coopcloud:autonomic.zone`](https://matrix.to/#/!JSVYWCRXSVMrAzgeKB:autonomic.zone?via=autonomic.zone) General chat and announcements (low traffic)
-- [`#coopcloud-tech:autonomic.zone`](https://matrix.to/#/!DfXPgKLoYCvjHithgS:autonomic.zone?via=autonomic.zone) Technical discussions (some techno babble)
-- [`#coopcloud-dev:autonomic.zone`](https://matrix.to/#/!IFazIpLtxiScqbHqoa:autonomic.zone?via=autonomic.zone) Intense developer chat (a lot of techno babble)
-
-### XMPP
-
-> Coming Soon :tm:
-
 ## Forum
 
 [`community.coops.tech`](https://community.coops.tech/)

docs/intro/credits.md

@@ -6,7 +6,7 @@ title: Credits & thanks
 
 Special thanks to:
 
-- [Doop Coop](mailto:cluck@doop.coop), for making a transparent version of the Co-op Cloud logo, and helping with OSX alpha testing.
+- [Doop Coop](mailto:cluck@doop.coop), for making a transparent version of the Co-op Cloud logo, Matrix room avatars and helping with OSX alpha testing.
 - [Social.coop](https://social.coop), for warmly welcoming us onto [`social.coop/@coopcloud`](https://social.coop/@coopcloud).
 - [Servers.coop](https://servers.coop), for hosting our digital infrastructure (website, builds, git hosting, etc.).
 - Every single last one of you heroic & patient beta testers, you are all comrades of the highest order of kropotkin :heart:

docs/intro/faq.md

@@ -34,7 +34,7 @@ The project was started by workers at [Autonomic](https://autonomic.zone/) which
 
 Please read our [initial project announcement post](https://autonomic.zone/blog/co-op-cloud/) for more on this.
 
-Also see our [strategy page](/strategy/).
+Also see our [strategy page](../strategy/).
 
 ## How do I make a recipe for (package) an app?
 
@@ -176,13 +176,13 @@ This means that we can patch our app containers directly in conversation with up
 
 We definitely recommend using best-in-class security auditing tools like [docker-bench-security](https://github.com/docker/docker-bench-security), IDS systems like [OSSEC](https://www.ossec.net/), security profiles like [Apparmor](https://docs.docker.com/engine/security/apparmor/) and hooking these into your existing monitoring, alert and update maintenance flows.
 
-It's up to how you want to arrange your system. For example, Co-op Cloud also allows you to compartmentalise different apps onto different servers. You could stack a bunch of apps on one big server or you could deploy one app per server.
+It's up to you how you want to arrange your system. For example, Co-op Cloud also allows you to compartmentalise different apps onto different servers. You could stack a bunch of apps on one big server or you could deploy one app per server.
 
 These are organisational concerns that Co-op Cloud can't solve for you which any software system will require. See this [additional question](/intro/faq/#what-is-important-to-consider-when-running-containers-in-production) for further information.
 
 ## What is important to consider when running containers in production?
 
-The Co-op Cloud uses [containers](/faq/#why-containers) as a fundamental building block. Therefore it is important to be aware of some general principles for container management in production environments. These are typically things that you will want to discuss within your co-op or democratic collective about how to prioritise and build up process for.
+The Co-op Cloud uses [containers](#why-containers) as a fundamental building block. Therefore it is important to be aware of some general principles for container management in production environments. These are typically things that you will want to discuss within your co-op or democratic collective about how to prioritise and build up process for.
 
 However, as the Co-op Cloud project is still very young, we're also still thinking about how we can make the platform itself mitigate problematic issues and make the maintenance of containers a more stable experience.
 
@@ -216,7 +216,7 @@ While the industry is bordering on a [k8s](https://kubernetes.io/) obsession and
 
 We hope to see a container orchestrator tool that is not directly linked to a for-profit company emerge soon but for now, this is what we have.
 
-If you want to learn more, see [dockerswarm.rocks](https://dockerswarm.rocks/) for a nice guide.
+If you want to learn more, see [dockerswarm.rocks](https://dockerswarm.rocks/) for a nice guide. See also [`BretFisher/awesome-swarm`](https://github.com/BretFisher/awesome-swarm).
 
 ## What licensing model do you use?
 
@@ -226,7 +226,7 @@ The Co-op Cloud is and will always be available under [copyleft licenses](https:
 
 It is true that if you install 3 apps and each one requires a MySQL database, then you will have 3 installations of MySQL on your system, running in containers.
 
-Systems like [YunoHost](/faq/#yunohost) mutualise every part of the system for maximum resource efficiency - if there is a MySQL instance available on the system, then just make a new database there and share the MySQL instance instead of creating more.
+Systems like [YunoHost](#yunohost) mutualise every part of the system for maximum resource efficiency - if there is a MySQL instance available on the system, then just make a new database there and share the MySQL instance instead of creating more.
 
 However, as we see it, this creates a tight coupling between apps on the database level - running a migration on one app where you need to turn the database off takes down the other apps.
 
@@ -248,6 +248,18 @@ Yes! Horizontal scaling is one of the ways Co-op Cloud can really shine. `abra`
 
 ## Why only x86 support?
 
-We would love to do ARM support and hope to get there! We've been testing this and [ran into some issues](https://git.autonomic.zone/coop-cloud/organising/issues/25). The TLDR; is that a lot of upstream libre app developer communities are not publishing container builds that support ARM. If they are, there are typically subtle differences in the conventions used to build the image as they are mostly done by community members and not directly taken on by the upstream project themselves. Since one of the core goals is to coordinate and reuse upstream packaging work, we see that ARM support requires a lot of organising and community engagement. Perhaps projects themselves will not want to take on this burden? It is not the role of the Co-op Cloud to set up an entire ARM publishing work flow at this moment in time. We see the benefits of supporting ARM and if you've got ideas / thoughts / approaches for how to make progress here, [please get in touch](/intro/contact/).
+We would love to do ARM support and hope to get there! We've been testing this and [ran into some issues](https://git.coopcloud.tech/coop-cloud/organising/issues/25). The TLDR; is that a lot of upstream libre app developer communities are not publishing container builds that support ARM. If they are, there are typically subtle differences in the conventions used to build the image as they are mostly done by community members and not directly taken on by the upstream project themselves. Since one of the core goals is to coordinate and reuse upstream packaging work, we see that ARM support requires a lot of organising and community engagement. Perhaps projects themselves will not want to take on this burden? It is not the role of the Co-op Cloud to set up an entire ARM publishing work flow at this moment in time. We see the benefits of supporting ARM and if you've got ideas / thoughts / approaches for how to make progress here, [please get in touch](/intro/contact/).
 
 Update: [Can I run Co-op Cloud on ARM?](/operators/handbook/#can-i-run-co-op-cloud-on-arm)
+
+## Why would an activist group use Co-op Cloud infrastructure over private cloud infrastructure (e.g. AWS, Azure, GCP)?
+
+If your group is powerful enough to have generated opposition, it's not implausible that some law enforcement body may be trying to stymie your group's advances. To do this, law enforcement bodies may and probably will collaborate with big tech. Indeed, Big Tech has consistently shown a quick willingness to cooperate with Law Enforcement agencies (a la Snowden-revealed NSA subpoenas, [disallowing Signal to domain front](https://techcrunch.com/2018/05/02/signal-could-get-kicked-out-of-amazon-web-services/) and other such incidents where [Big Tech aided governments in hunting activists](http://discourse.leagueofconcernedusers.org/t/activist-infrastructures/69?u=themoonisblue)).
+
+If your group has ambitions that generate enough fury in your opposition, you should think twice about where you store your data and whose services you rely on to store your data.
+
+By using Co-op Cloud infrastructure over private cloud infrastructure, you create a few possibilities:
+
+- You may interact with a server provider that is more ethical than Big Tech. Although the server provider may still succumb to law enforcement, you might place more trust in some providers than in private cloud providers (e.g. AWS).
+
+- You may be able to situate your servers in locations that are relatively more impervious to law enforcement attempts to dismantle your infrastructure. Indeed, if you deployed your infrastructure in a relatively secure setting such as Switzerland, then you would weather a greater chance of keeping your infrastructure alive than if you deployed it in, say, the United States. Protonmail and [Extinction Rebellion (XR)](https://www.youtube.com/watch?v=I_O3zj3p52A) choose Switzerland for their servers, for reasons along these lines.

docs/maintainers/handbook.md

@@ -8,6 +8,15 @@ You can run `abra recipe new <recipe>` to generate a new `~/.abra/recipes/<recip
 
 ## Hacking on an existing recipe
 
+!!! warning
+
+    It is *very advisable* to disable any `healthcheck: ...` configuration
+    while hacking on new recipes. This is because it is very easy to mess up
+    and it will stop Traefik or other web proxies routing the app. You can
+    enable a specific healthcheck later when your recipe is stable. The default
+    "unconfigured" healthcheck behaviour is much less strict and it's faster to
+    get something up and running.
+
 If you want to make changes to an existing recipe then you can simply edit the files in `~/.abra/recipes/<recipe-name>` and run pass `--chaos` to the `deploy` command when deploying those changes. `abra` will not deploy unstaged changes to avoid instability but you can tell it to do so with `--chaos`. This means ou can simple hack away on the existing recipe files on your local file system and then when something is working, submit a change request to the recipe upstream.
 
 ## How is a recipe structured?
@@ -30,7 +39,7 @@ After docker creates the filesystem and copies files into a new container it run
 
 For a simple example check the [entrypoint.sh for `croc`](https://git.coopcloud.tech/coop-cloud/croc/src/commit/2f06e8aac52a3850d527434a26de0a242bea0c79/entrypoint.sh). In this case, `croc` needs the password to be exported as an environmental variable called `CROC_PASS`, and that is exactly what the entrypoint does before running vendor entrypoint.
 
-If you write your own entrypoint, it needs to be specified in the `config` section of compose.yml. See [this handbook entry](http://localhost:8000/maintainers/handbook/#entrypoints) for more.
+If you write your own entrypoint, it needs to be specified in the `config` section of compose.yml. See [this handbook entry](/maintainers/handbook/#how-do-i-set-a-custom-entrypoint) for more.
 
 ### `releases/` directory
 
@@ -102,6 +111,16 @@ You can also access it in your configs using the following syntax:
 {{ env "FOO" }}
 ```
 
+### Global environment variables
+
+- `TYPE`: specifies the recipe name
+- `DOMAIN`: specifies the app domain
+- `LETS_ENCRYPT_ENV`: TODO
+- `TIMEOUT`: specifies the time in seconds to wait until all services have started and passed the health checks
+- `ENABLE_AUTO_UPDATE`: if set to `true`, the auto-updater `kadabra` can update this app (see [this auto updater entry](/operators/tutorial/#automatic-upgrades) for more)
+- `POST_DEPLOY_CMDS="<container> <command> <arguments>|<container> <command> <arguments>|... "` specifies commands that should be executed after each `abra app deploy`
+- `POST_UPGRADE_CMDS="<container> <command> <arguments>|<container> <command> <arguments>|... "` specifies commands that should be executed after each `abra app upgrade`
+
 ## Manage secret data
 
 Adding a secret to your recipe is done:
@@ -224,11 +243,11 @@ file_env "DB_PASSWORD"
 Sometimes the containers don't even have Bash installed on them. You had better just use `/bin/sh` or, in your entrypoint script, install Bash :upside_down: The entrypoint secrets hack listed above doesn't work in this case (as it requires Bash), so instead you can just do `export FOO=$(cat /run/secrets/<secret-name>)`.
 
 
-## Reference services in configs?
+## How do I reference services in configs?
 
 When referencing an `app` service in a config file, you should prefix with the `STACK_NAME` to avoid namespace conflicts (because all these containers sit on the traefik overlay network). You might want to do something like this `{{ env "STACK_NAME" }}_app` (using  the often obscure dark magic of the Golang templating language). You can find examples of this approach used in the [Peertube recipe](https://git.coopcloud.tech/coop-cloud/peertube/src/commit/d1b297c5a6a23a06bf97bb954104ddfd7f736568/nginx.conf.tmpl#L9).
 
-## How are recipes are versioned?
+## How are recipes versioned?
 
 We'll use an example to work through this. Let's use [Gitea](https://hub.docker.com/r/gitea/gitea).
 
@@ -364,7 +383,7 @@ mkdir -p releases
 
 And then create a text file which corresponds to the version release, e.g. `1.1.0+5.9.0` and write some notes. `abra` will show these when another operator runs `abra app deploy` / `abra app upgrade`.
 
-## Generate the recipe catalogue
+## How do I generate the recipe catalogue
 
 To generate an entire new copy of the catalogue:
 
@@ -393,7 +412,7 @@ You can pass `--publish` to have `abra` automatically publish those changes.
 
     In order to have `abra` publish changes for you automatically, you'll have to have write permissons to the git.coopcloud.tech repository and your account must have a working SSH key configuration. `abra` will use the SSH based URL connection details for Git by automagically creating an `origin-ssh` remote in the repository and pushing to it.
 
-## Enable healthchecks
+## How do I enable healthchecks
 
 A healthcheck is an important and often overlooked part of the recipe configuration. It is part of the configuration that the runtime uses to figure out if a container is really up-and-running. You can tweak what command to run, how often and how many times to try until you assume the container is not up.
 
@@ -410,7 +429,7 @@ If you're just starting off with packaging a recipe, you can use `healthcheck: d
 
 `abra app errors -w <domain>` will show what errors are being reported from a failing healtcheck setup.
 
-## Tuning deploy configs
+## How do I tune deploy configs?
 
 A bit like healtchecks, there is no universal setup. A good default seems to be the following configuration:
 
@@ -431,13 +450,13 @@ Setting a restart policy is also good so that the runtime doesn't try to restart
 
 Best to [read](https://docs.docker.com/engine/reference/builder/#healthcheck) [the docs](https://docs.docker.com/compose/compose-file/compose-file-v3/#healthcheck) on this one.
 
-## Tuning resource limits
+## How do I tune resource limits?
 
-If you don't place resource limits on your app it will assume it can use the entire capacity of the server it is on. This can cause issues such as OOM eerors for your entire swarm.
+If you don't place resource limits on your app it will assume it can use the entire capacity of the server it is on. This can cause issues such as Out-Of Memory errors for your entire swarm.
 
 See the [Docker documentation](https://docs.docker.com/config/containers/resource_constraints/) to get into this topic and check the other recipes to see what other maintainers are doing.
 
-## Enable A+ SSL ratings
+## How do I enable A+ SSL ratings?
 
 If you want to get the highest rating on SSL certs, you can use the following traefik labels which use a tweaked Traefik configuration.
 
@@ -448,7 +467,7 @@ If you want to get the highest rating on SSL certs, you can use the following tr
 
 See [this PR](https://git.coopcloud.tech/coop-cloud/traefik/pulls/8/files) for the technical details
 
-## Tweaking secret generation length
+## How do I change secret generation length?
 
 It is possible to tell `abra` which length it should generate secrets with from your recipe config.
 
@@ -469,12 +488,13 @@ of passwords which admins have to type out in database shells.
 
 ## How are recipes added to the catalogue?
 
-> This is so far a manual process which requires a member of Autonomic. This is
-> a temporary situation, we want to open out this process & also introduce some
-> automation to support making thie process more convenient. Please nag us to
-> move things along.
+> This is so far a manual process which requires someone who's been added to the
+> `coop-cloud` "Organisation" on https://git.coopcloud.tech. This is a temporary
+> situation, we want to open out this process & also introduce some automation
+> to support making thie process more convenient. Please nag us to move things
+> along.
 
-- Publish your new recipe on the [git.coopcloud.tech](https://git.coopcloud.tech/coop-cloud) listing
+- Publish your new recipe on the [git.coopcloud.tech](https://git.coopcloud.tech/coop-cloud) "Organisation"
 - Run `abra catalogue generate <recipe> -p`
 - Run `cd ~/.abra/catalogue && make`
 
@@ -483,7 +503,7 @@ the [recipe release publishing dance](https://docs.coopcloud.tech/maintainers/ha
 which will then extend the `versions: [...]` section of the published JSON in the catalogue.
 
 Recipes that are not included in the catalogue can still be deployed. It is not
-required to add your recipes to the catalogue but this will improve the
+required to add your recipes to the catalogue, but this will improve the
 visibility for other co-op hosters & end-users.
 
 For now, it is best to [get in touch](https://docs.coopcloud.tech/intro/contact/) if you want to add your recipe to the catalogue.
@@ -502,14 +522,14 @@ Two of the current "blessed" options are
 [`backup-bot-two`](https://git.coopcloud.tech/coop-cloud/backup-bot-two) &
 [`abra`](https://git.coopcloud.tech/coop-cloud/abra).
 
-#### `abra`
-
-`abra` will read labels and store backups in `~/.abra/backups/...`.
-
 #### `backup-bot-two`
 
 Please see the [`README.md`](https://git.coopcloud.tech/coop-cloud/backup-bot-two#backupbot-ii) for the full docs.
 
+#### `abra`
+
+`abra` will read labels and store backups in `~/.abra/backups/...`.
+
 ### Backup
 
 For backup, here are the labels & some examples:
@@ -538,3 +558,162 @@ You can use [this `docker-compose` trick](https://docs.docker.com/compose/extend
 If you have a recipe that is using a `mysql` service and you'd like to use `postgresql` instead, you can create a `compose.psql.yml`!
 
 An example of this is the [`selfoss`](https://git.coopcloud.tech/coop-cloud/selfoss) recipe. The default is `sqlite` but there is a `postgresql` compose configuration there too.
+
+## How do I set a custom entrypoint?
+
+For more context, see the [`entrypoint.sh`](/maintainers/handbook/#entrypointsh) section. The following configuration example is ripped from the [`coop-cloud/peertube`](https://git.coopcloud.tech/coop-cloud/peertube) recipe but shortened down. Here are more or less the steps you need to take:
+
+Define a config:
+
+```yaml
+  app:
+    ...
+    configs:
+      - source: app_entrypoint
+        target: /docker-entrypoint.sh
+        mode: 0555
+    ...
+
+configs:
+  app_entrypoint:
+    name: ${STACK_NAME}_app_entrypoint_${APP_ENTRYPOINT_VERSION}
+    file: entrypoint.sh.tmpl
+    template_driver: golang
+```
+
+Define a `entrypoint.sh.tmpl`:
+
+```
+#!/bin/bash
+
+set -e
+
+file_env() {
+   local var="$1"
+   local fileVar="${var}_FILE"
+   local def="${2:-}"
+
+   if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+      echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+      exit 1
+   fi
+
+   local val="$def"
+
+   if [ "${!var:-}" ]; then
+      val="${!var}"
+   elif [ "${!fileVar:-}" ]; then
+      val="$(< "${!fileVar}")"
+   fi
+
+   export "$var"="$val"
+   unset "$fileVar"
+}
+
+file_env "PEERTUBE_DB_PASSWORD"
+
+{{ if eq (env "PEERTUBE_SMTP_ENABLED") "1" }}
+file_env "PEERTUBE_SMTP_PASSWORD"
+{{ end }}
+
+{{ if eq (env "PEERTUBE_LIVE_CHAT_ENABLED") "1" }}
+apt -y update && apt install -y prosody && apt -y clean
+mkdir -p /run/prosody && chown prosody:prosody /run/prosody
+{{ end }}
+
+# Copy the client files over to a named volume
+# so that they may be served by nginx directly
+cp -ar /app/client/dist /srv/client
+
+# upstream entrypoint
+# https://github.com/Chocobozzz/PeerTube/blob/66f77f63437c6774acbd72584a9839a7636ea167/support/docker/production/entrypoint.sh
+/usr/local/bin/entrypoint.sh "$@"
+```
+
+Please note:
+
+1. The `file_env` / `_FILE` hack is to pass secrets into the container runtime without exposing them in plaintext in the configuration. See [this entry](/maintainers/handbook/#exposing-secrets) for more.
+
+1. In order to pass execution back to the original entrypoint, it's a good idea to find the original entrypoint script and run it from your own entrypoint script. If there is none, you may want to reference the `CMD` definition or if that isn't working, try to actually specify `cmd: ...` in the `compose.yml` definition (there are other recipes which do this).
+
+1. If you're feeling reckless, you can also use the Golang templating engine to do things conditionally.
+
+Then, wire up the vendored config version:
+
+```
+# abra.sh
+export APP_ENTRYPOINT_VERSION=v5
+```
+
+You should be able to deploy this overriden configuration now.
+
+## Linting rules
+
+### R014: "invalid lightweight tag"
+
+This is an issue related to the way Git/`go-git` handle Git tags internally. We
+need to use "annotated tags" and not "lightweight tags" for our recipe versions
+tags. Otherwise, `abra` has a hard time parsing what is going on.
+
+The `R01O4` linting error happens because the recipe in question has a
+lightweight tag. This needs to be replaced. This is a manual process. Here's a
+practical example with the Gitea recipe when we had this issue.
+
+You can validate what kind of tag is which by running the following:
+
+```
+git for-each-ref refs/tags
+734045872a57d795cd54b1992a1753893a4934f1 tag    refs/tags/1.0.0+1.14.5-rootless
+b2cefa5ccf2f2f77dae54cf6c304cccecb3547ca tag    refs/tags/1.1.0+1.15.0-rootless
+6d669112d8caafcdcf4eb1485f2d6afdb54a8e30 tag    refs/tags/1.1.1+1.15.3-rootless
+64761ad187cc7a3984a37dd9abd4fa16979f97b9 tag    refs/tags/1.1.2+1.15.6-rootless
+1ccb1cb6a63a08eebf6ba5508b676eaaccba7ed8 tag    refs/tags/1.1.3+1.15.10-rootless
+b86e1f6dfef3c464b16736274b3cd95f8978f66b tag    refs/tags/1.2.0+1.16.3-rootless
+b1d22f3c39ca768a4efa1a0b9b9f780268c924b3 tag    refs/tags/1.2.1+1.16.8-rootless
+85a45aa749427822a73ef62b6362d57bae1a61af tag    refs/tags/1.3.0+1.17.2-rootless
+f35689989c0b57575b8362e1252476d8133dc961 commit refs/tags/1.3.1+1.17.3-rootless
+df015fae592fca7728a3f0835217e110da4dbafc tag    refs/tags/2.0.0+1.18.0-rootless
+71920adb0c25a59f7678894e39f1a705f0ad08dd tag    refs/tags/2.0.1+1.18.2-rootless
+1ab9a96922341c8e54bdb6d60850630cce4b9587 tag    refs/tags/2.1.0+1.18.5-rootless
+1e612d84a2ad7c9beb7aa064701a520c7e91eecc commit refs/tags/2.1.2+1.19.3-rootless
+0bee99615a8bbd534a66a315ee088af3124e054b tag    refs/tags/2.2.0+1.19.3-rootless
+699378f53501b2d5079fa62cc7f8e79930da7540 tag    refs/tags/2.3.0+1.20.1-rootless
+c0dc5f82930d875c0a6e29abc016b4f6a53b83dd tag    refs/tags/2.3.1+1.20.1-rootless
+```
+
+Where `f35689989c0b57575b8362e1252476d8133dc961` &
+`1e612d84a2ad7c9beb7aa064701a520c7e91eecc` need to be removed ("commit"). We
+will deal with `refs/tags/1.3.1+1.17.3-rootless` in this example.
+
+```
+# find the tag hash
+git show 1.3.1+1.17.3-rootless
+commit f35689989c0b57575b8362e1252476d8133dc961 (tag: 1.3.1+1.17.3-rootless)
+Merge: af97db8 1d4dc8e
+Author: decentral1se <decentral1se@noreply.git.coopcloud.tech>
+Date:   Sun Nov 13 21:54:01 2022 +0000
+
+    Merge pull request 'Adding Oauth2 options and up on versions' (#29) from javielico/gitea:master into master
+
+    Reviewed-on: https://git.coopcloud.tech/coop-cloud/gitea/pulls/29
+
+# delete the tag locally / remotely
+git tag -d 1.3.1+1.17.3-rootless
+git push origin 1.3.1+1.17.3-rootless --delete
+
+# re-tag, this time with `-a` (annotated)
+git checkout f35689989c0b57575b8362e1252476d8133dc961
+git tag -a 1.3.1+1.17.3-rootless
+
+# push new tag
+git checkout master # might be main on other recipes!
+git push origin master --tags
+
+# check everything works
+git for-each-ref refs/tags | grep 1.3.1+1.17.3-rootless
+964f1680000fbba6daa520aa8d533a53ad151ab8 tag refs/tags/1.3.1+1.17.3-rootless
+```
+
+That's it! Spread the word, use `-a` when tagging recipe versions manually! Or
+just use `abra` which should handle this issue automagically for you in all
+cases 🎉

docs/maintainers/tutorial.md

@@ -21,7 +21,7 @@ The idea scenario is when the upstream project provides both the packaged image
 - **Inspired**: Upstream image, someone else's compose file
 - **On fire**: Upstream image, upstream compose file
 
-### Writing the `compose.yml`
+### Writing / adapting the `compose.yml`
 
 Let's take a practical example, [Matomo web analytics](https://matomo.org/). We'll be making a Docker "swarm-mode" `compose.yml` file.
 
@@ -66,8 +66,8 @@ abra app new matomo --secrets \
  --server swarm.example.com
 ```
 
-Depending on whether you defined any extra environment variables, we didn't so
-far, in this example, you might want to run `abra app config swarm.example.com`
+Depending on whether you defined any extra environment variables -- we didn't so
+far, in this example -- you might want to run `abra app config swarm.example.com`
 to check the configuration.
 
 Otherwise, or once you've done that, go ahead and deploy the app:
@@ -80,4 +80,4 @@ Then, open the `DOMAIN` you configured (you might need to wait a while for Traef
 
 ### Finishing up
 
-You've probably got more questions, check out the [maintainers handbook](/maintainers/handbook)!
+You've probably got more questions, check out the [packaging handbook](/maintainers/handbook)!

docs/operators/handbook.md

@@ -82,6 +82,18 @@ Then, tell your collaborators (e.g. in the repository's `README.md`), to run `ma
 
     We don't currently recommend this, because it might set inaccurate expectations about the security model – remember that, by default, **any user who can deploy apps to a Docker Swarm can manage _any_ app in that swarm**.
 
+### Migrating a server into a repository
+
+Even if you've got your existing server configs in version control, by default, `abra server add` will define the server locally. To move it -- taking the example of `newserver.example.com`:
+
+```
+mv ~/.abra/servers/newserver.example.com ~/coop-cloud-apps/
+cd ~/coop-cloud-apps
+git add newserver.example.com
+git commit
+make link
+```
+
 ## Running abra server side
 
 If you're on an environment where it's hard to run Docker, or command-line programs in general, you might want to install `abra` on a server instead of your local work station.
@@ -236,9 +248,11 @@ usermod -aG docker $USER
 # setup swarm
 docker swarm init
 docker network create -d overlay proxy
-```
 
-`abra` will do this for you when you run `abra server add --provision`.
+# on debian machines as of 2023-02-17
+apt install apparmor
+systemctl restart docker containerd
+```
 
 ## Managing DNS entries
 
@@ -312,6 +326,10 @@ Running `server add` with `-d/--debug` should help you debug what is going on un
 
 If you need to run a command within a running container you can use `abra app run <domain> <service> <command>`. For example, you could run `abra app run cloud.lumbung.space app bash` to open a new bash terminal session inside your remote container.
 
+## How do I attach on a non-running container?
+
+If you need to run a command on a container that won't start (eg. the container is stuck in a restart loop) you can temporarily disable its default entrypoint by setting it in `compose.yml` to something like ['tail', '-f', '/dev/null'], then redeploy the stack (with `--force --chaos` so you don't need to commit), then [get into the now running container](#how-do-i-attach-to-a-running-container), do your business, and when done revert the compose.yml change and redeploy again.
+
 ## Can I run Co-op Cloud on ARM?
 
 `@Mayel`:
@@ -327,7 +345,132 @@ See [`#312`](https://git.coopcloud.tech/coop-cloud/organising/issues/312) for mo
 
 ## How do I backup/restore my app?
 
-If you're app [supports backup/restore](/handbook/#how-do-i-configure-backuprestore) then you have two options: [`backup-bot-two`](https://git.coopcloud.tech/coop-cloud/backup-bot-two) & [`abra`](https://git.coopcloud.tech/coop-cloud/abra).
+If you're app [supports backup/restore](/maintainers/handbook/#how-do-i-configure-backuprestore) then you have two options: [`backup-bot-two`](https://git.coopcloud.tech/coop-cloud/backup-bot-two) & [`abra`](https://git.coopcloud.tech/coop-cloud/abra).
 
 With `abra`, you can simply run `abra app backup ...` & `abra app restore ...`.
 Pass `-h` for more information on the specific flags & arguments.
+
+## How do I take a manual database backup?
+
+MySQL / MariaDB:
+
+```
+abra app run foo.bar.com db mysqldump -u root <database> | gzip > ~/.abra/backups/foo.bar.com_db_`date +%F`.sql.gz
+```
+
+Postgres:
+
+```
+abra app run foo.bar.com db pg_dump -u root <database> | gzip > ~/.abra/backups/foo.bar.com_db_`date +%F`.sql.gz
+```
+
+If you get errors about database access:
+- Make sure you've specified the right database user (`root` above) and db name
+- If you have a database password set, you might need to load it from a secret,
+    something like this:
+
+```
+abra app run foo.bar.com db bash -c 'mysqldump -u root -p"$(cat /run/secrets/db_oot_password)" <database>' | gzip > ~/.abra/backups/foo.bar.com_db_`date +%F`.sql.gz
+```
+
+## Can I deploy a recipe without `abra`?
+
+Yes! It's a design goal to keep the recipes not dependent on `abra` or any
+single tool that we develop. This means that the configuration commons can
+still be useful beyond this project. You can deploy a recipe with standard
+commands like so:
+
+```
+set -a
+source example.com.env
+cd ~/.abra/recipes/myrecipe
+docker stack deploy -c compose.yml example_com
+```
+
+`abra` makes all of this more convenient.
+
+## Proxying apps outside of Co-op Cloud with Traefik?
+
+It's possible! It's actually always been possible but we just didn't have
+spoons to investigate. Co-op Cloud can co-exist on the same server as bare
+metal apps, non-swarm containers (plain `docker-compose up` deployments!),
+Nginx installs etc. It's a bit gnarly with the networking but doable.
+
+Enable the following in your Traefik `$domain.env` configuration:
+
+```
+FILE_PROVIDER_DIRECTORY_ENABLED=1
+```
+
+You must also have host mode networking enabled for Traefik:
+
+```
+COMPOSE_FILE="$COMPOSE_FILE:compose.host.yml"
+```
+
+And re-deploy your `traefik` app. You now have full control over the [file
+provider](https://doc.traefik.io/traefik/providers/file/#directory)
+configuration of Traefik. This also means you lost the defaults of the
+`file-provider.yml.tmpl`, so this is a more involved approach.
+
+The main change is that there is now a `/etc/traefik/file-providers` volume
+being watched by Traefik for provider configurations. You can re-enable the
+recipe defaults by copying the original over to the volume (this assumes you've
+deployed `traefik` already without `FILE_PROVIDER_DIRECTORY_ENABLED`, which is
+required for the following command):
+
+```
+abra app run $your-traefik app \
+  cp /etc/traefik/file-provider.yml /etc/traefik/file-providers/
+```
+
+You don't need to re-deploy Traefik, it should automatically pick this up.
+
+You can route requests to a bare metal / non-docker service by making a
+`/etc/traefik/file-providers/$YOUR-SERVICE.yml` and putting something like this in
+it:
+
+```yaml
+http:
+  routers:
+    myservice:
+      rule: "Host(`my-service.example.com`)"
+      service: "myservice"
+      entryPoints:
+        - web-secure
+      tls:
+        certResolver: production
+
+  services:
+    myservice:
+      loadBalancer:
+        servers:
+          - url: "http://$YOUR-HOST-IP:8080/"
+```
+
+Where you should replace all instances of `myservice`.
+
+You must use your host level IP address (replace `$YOUR-HOST-IP` in the
+example). With host mode networking, your deployment can route out of the swarm
+to the host.
+
+If you're running a firewall (e.g. UFW) then it will likely block traffic from
+the swarm to the host. You can typically add a specific UFW to route from the
+swarm (typically, your `docker_gwbridge`) to the specific port of your bare
+metal / non-docker app:
+
+```
+docker network inspect docker_gwbridge --format='{{( index .IPAM.Config 0).Gateway}}'
+172.18.0.1
+ufw allow from 172.18.0.0/16 proto tcp to any port $YOUR-APP-PORT
+```
+
+Notice that we turn `172.18.0.1` into `172.18.0.0/16`. It's advised to open the
+firewall on a port by port case to avoid expanding your attack surface.
+
+Traefik should handle the usual automagic HTTPS certificate generation and
+route requests after. You're free to make as many `$whatever.yml` files in your
+`/etc/traefik/file-providers` directory. It should Just Work ™
+
+ Please note that we have to hardcode `production` and `web-secure` which are
+ typically configurable when not using `FILE_PROVIDER_DIRECTORY_ENABLED`.

docs/operators/tutorial.md

@@ -102,7 +102,7 @@ Most Co-op Cloud deployments have been run on Debian machines so far. Some exper
 
 You need to keep port `:80` and `:443` free on your server for web proxying to your apps. Typically, you don't need to keep any other ports free as the core web proxy ([Traefik](https://traefik.io)) keeps all app ports internal to its network. Sometimes however, you need to expose an app port when you need to use a transport which would perform better or more reliably without proxying.
 
-`abra` has support for both creating servers (`abra server new`) & provisioning them (passing `--provision` to `abra server add`) but those are more advanced automation options which are covered in the [handbook](/operators/handbook). For this tutorial, we'll focus on the basics. Assuming you've managed to create a testing VPS with some `$hosting_provider`, you'll need to install Docker, add your user to the Docker group & setup swarm mode:
+`abra` has support for creating servers (`abra server new`) but that is a more advanced automation feature which is covered in the [handbook](/operators/handbook). For this tutorial, we'll focus on the basics. Assuming you've managed to create a testing VPS with some `$hosting_provider`, you'll need to install Docker, add your user to the Docker group & setup swarm mode:
 
 ```
 # docker install convenience script
@@ -162,14 +162,13 @@ If you run into issues during installation, [please report a ticket](https://git
 
 #### Add your server
 
-Now you can connect `abra` with your server. You need to have a working SSH configuration before you can do this. That means you can run `ssh <server-domain>` on your command-line and everything Works :tm:.
+Now you can connect `abra` with your server. You should have a working SSH configuration before you can do this (e.g. a matching `Host <server-domain>` entry in `~/.ssh/config` with the correct SSH connection details). That means you can run `ssh <server-domain>` on your command-line and everything Works :tm:.
 
 ```bash
-abra server add <server-domain> -p
+ssh <server-domain> # make sure it works
+abra server add <server-domain>
 ```
 
-The `-p` or `--provision` flag means that `abra` will install Docker and initialise the [new single-host swarm](https://docs.docker.com/engine/swarm/key-concepts/) on your server. If you've already followed the steps in [the server setup](/operators/tutorial/#server-setup) step, then `abra` should not need to do any work.
-
 It is important to note that `<domain>` here is a publicy accessible domain name which points to your server IP address. `abra` does make sure this is the case and this is done to avoid issues with HTTPS certificate rate limiting.
 
 You will now have a new `~/.abra/` folder on your local file system which stores all the configuration of your Co-op Cloud instance.
@@ -184,10 +183,6 @@ abra server ls
 
     `abra` uses plain 'ol SSH under the hood and aims to make use of your existing SSH configurations in `~/.ssh/config` and interfaces with your running `ssh-agent` for password protected secret key files.
 
-    The `server add` command listed above assumes that that you make SSH connections on port 22 using your current username. If that is not he case, pass the new values as positional arguments. See `abra server add -h` for more on this.
-
-        abra server add <domain> <user> <port> -p
-
     Running `server add` with `-d/--debug` should help you debug what is going on under the hood. It's best to take a moment to read [this troubleshooting entry](/abra/trouble/#ssh-connection-issues) if you're running into SSH connection issues with `abra`.
 
 !!! question "How do I share my configs in `~/.abra`?"
@@ -211,7 +206,7 @@ Choose your newly registered server and specify a domain name.
 You will want to take a look at your generated configuration and tweak the `LETS_ENCRYPT_EMAIL` value. You can do that by running `abra app config`:
 
 ```bash
-abra app config traefik
+abra app config <traefik-domain>
 ```
 
 Every app you deploy will have one of these `.env` files, which contains variables which will be injected into app configurations when deployed. Variables starting with `#` are optional, others are required.
@@ -252,8 +247,45 @@ abra app errors -w <nextcloud-domain> # error catcher
 
 Your new `traefik` instance will detect that a new app is coming up and generate SSL certificates for it. You can see what `traefik` is up to using the same commands above but replacing `<netcloud-domain>` with the `<traefik-domain>` you chose earlier (`abra app ls` will remind you what domains you chose :grinning:).
 
+### Upgrade Nextcloud
+
+To upgrade an app manually to the newest available version run:
+
+```bash
+abra app upgrade <nextcloud-domain>
+```
+
+### Automatic Upgrades
+
+`kadabra` the auto-updater is still under development, use it with care and don't use it in production environments. To setup the auto-updater copy the `kadabra` binary to the server and configure a cronjob for regular app upgrades. The following script will configure ssmtp for email notifications and setup a cronjob. This cronjob checks daily for new app versions, notifies if any kind of update is available and upgrades all apps to the latest patch/minor version.
+
+
+```bash
+apt install ssmtp
+
+cat > /etc/ssmtp/ssmtp.conf << EOF
+mailhub=$MAIL_SERVER:587
+hostname=$MAIL_DOMAIN
+AuthUser=$USER
+AuthPass=$PASSWORD
+FromLineOverride=yes
+UseSTARTTLS=yes
+EOF
+
+cat > /etc/cron.d/abra_updater << EOF
+MAILTO=admin@example.com
+MAILFROM=noreply@example.com
+
+0  6 * * *       root    ~/kadabra notify --major
+30 4 * * *       root    ~/kadabra upgrade --all
+EOF
+
+```
+
+Add `ENABLE_AUTO_UPDATE=true` to the env config (`abra app config <app name>`) to enable the auto-updater for a specific app.
+
 ## Finishing up
 
 Hopefully you got something running! Well done! The [operators handbook](/operators/handbook) would probably be the next place to go check out if you're looking for more help. Especially on topics of ongoing maintenance.
 
-If not, please [get in touch](/intro/contact) or [raise a ticket](https://git.coopcloud.tech/coop-cloud/abra/issues/new) and we'll try to help out. We want our operator onboarding to be as smooth as possible, so we do appreciate any feedback we receive.
+If not, please [get in touch](/intro/contact) or [raise a ticket](https://git.coopcloud.tech/coop-cloud/organising/issues/new/choose) and we'll try to help out. We want our operator onboarding to be as smooth as possible, so we do appreciate any feedback we receive.

docs/recipes/index.md

@@ -15,6 +15,65 @@ It aims to be a helpful place to understand the status of apps, who is taking ca
 
 The recipe catalogue is available on [recipes.coopcloud.tech](https://recipes.coopcloud.tech/).
 
+## Status / features / scoring
+
+Each recipe README has a "metadata" section, to help communicate the overall status of the recipe, and which features are supported. Here's an example, from [the Wordpress recipe](https://git.coopcloud.tech/coop-cloud/wordpress/):
+
+```
+<!-- metadata -->
+
+* **Category**: Apps
+* **Status**: 3, stable
+* **Image**: [`wordpress`](https://hub.docker.com/_/wordpress), 4, upstream
+* **Healthcheck**: Yes
+* **Backups**: Yes
+* **Email**: 3
+* **Tests**: 2
+* **SSO**: No
+
+<!-- endmetadata -->
+```
+
+Currently, recipe maintainers need to update the scores in this section manually. The specific meanings of the scores are:
+
+### Status (overall score)
+
+- 5: everything in 4 + Single-Sign-On
+- 4: upstream image, backups, email, healthcheck, integration testing
+- 3: upstream image, missing 1-2 items from 4
+- 2: missing 3-4 items from 4 or no upstream image
+- 1: alpha
+
+### Image
+
+- 4: official upstream image
+- 3: semi-official / actively-maintained image
+- 2: 3rd-party image
+- 1: our own custom image
+
+### Email
+
+- 3: automatic (using environment variables)
+- 2: mostly automatic
+- 1: manual
+- 0: none
+- N/A: app doesn't send email
+
+### CI
+
+- 3: as 2, plus healthcheck
+- 2: auto secrets + networks
+- 1: basic deployment using `stack-ssh-deploy`, manual secrets + networks
+- 0: none
+
+### Single-Sign-On
+
+- 3: automatic (using environment variables)
+- 2: mostly automatic
+- 1: manual
+- 0: none
+- N/A: app doesn't support SSO
+
 ## Wishlist
 
 If you'd like to see a new recipe packaged, make a request on the [recipes-wishlist](https://git.coopcloud.tech/coop-cloud/recipes-wishlist) repository issue tracker.

mkdocs.yml

@@ -2,16 +2,20 @@
 site_author: Co-op Cloud
 site_name: "Co-op Cloud: Public Interest Infrastructure"
 site_url: https://docs.coopcloud.tech
+use_directory_urls: true
 
 theme:
   name: material
   features:
+    - content.action.edit
     - navigation.expand
+    - navigation.indexes
     - navigation.instant
+    - navigation.path
     - navigation.sections
     - navigation.tabs
     - navigation.tabs.sticky
-    - navigation.indexes
+    - navigation.tracking
   palette:
     primary: light pink
     accent: purple
@@ -19,7 +23,7 @@ theme:
   favicon: img/favicon.ico
   custom_dir: custom_theme/
 
-copyright: Copyleft 🄯 2022 Co-op Cloud
+copyright: Copyleft 2023 Co-op Cloud
 
 markdown_extensions:
   - meta
@@ -69,18 +73,43 @@ nav:
       - "Cheat Sheet": abra/cheat-sheet.md
   - "Get Involved":
       - get-involved/index.md
-  - "Democracy":
-      - democracy/index.md
-      - "Decisions": democracy/decisions.md
+  - "Federation":
+      - federation/index.md
+      - "FAQ": federation/faq.md
+      - "Resolutions":
+        - federation/resolutions/index.md
+        - "Passed":
+          - federation/resolutions/passed/001.md
+          - federation/resolutions/passed/002.md
+          - federation/resolutions/passed/003.md
+          - federation/resolutions/passed/004.md
+          - federation/resolutions/passed/005.md
+          - federation/resolutions/passed/006.md
+          - federation/resolutions/passed/007.md
+          - federation/resolutions/passed/008.md
+          - federation/resolutions/passed/009.md
+          - federation/resolutions/passed/010.md
+        - "In progress":
+          - federation/resolutions/in-progress/index.md
+          - federation/resolutions/in-progress/011.md
+        - "Draft":
+          - federation/resolutions/drafts/index.md
+      - "Finance": federation/finance.md
+      - "Membership": federation/membership.md
+      - "Minutes":
+        - federation/minutes/index.md
+        - "2022":
+          - federation/minutes/2022-03-03.md
+      - "Digital tools": federation/tools.md
   - "Glossary":
     - glossary/index.md
 
 plugins:
-  - search
   - awesome-pages
+  - search
 
 repo_name: coop-cloud/docs.coopcloud.tech
-repo_url: https://git.coopcloud.tech/coop-cloud/docs.coopcloud.tech
+repo_url: https://git.coopcloud.tech/coop-cloud/docs.coopcloud.tech/
 edit_uri: _edit/main/docs/
 
 extra_css:

renovate.json

@@ -1,3 +1,7 @@
 {
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json"
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "packageRules": [{
+    "matchUpdateTypes": ["minor", "patch"],
+    "automerge": true
+  }]
 }

requirements.txt

@@ -1,4 +1,4 @@
-mkdocs-awesome-pages-plugin==2.8.0
-mkdocs-material-extensions==1.0.3
-mkdocs-material==8.3.9
-mkdocs==1.3.1
+mkdocs-awesome-pages-plugin==2.9.1
+mkdocs-material-extensions==1.1.1
+mkdocs-material==9.1.21
+mkdocs==1.5.2