increase recipe fetch time

Reviewed-on: toolshed/alakazam#15

.dockerignore

@@ -0,0 +1,10 @@
+.git
+# the submodule's git dir is the only source for the abra version, the builder stage reads it
+!.git/modules/abra
+abra/.git
+.venv
+.pipeline
+__pycache__
+tests
+*.png
+*.drawio

.drone.yml

@@ -0,0 +1,45 @@
+---
+kind: pipeline
+type: docker
+name: tests
+
+steps:
+  - name: pytest
+    image: python:3.11
+    commands:
+      # not the slim image: MarkupSafe 1.1.1 has no wheel here and compiles a C extension
+      - pip install --quiet -r requirements-dev.txt
+      - pytest -q
+
+---
+kind: pipeline
+type: docker
+name: publish image
+
+depends_on:
+  - tests
+
+# the image builds abra from the submodule, and the default clone step leaves it empty
+clone:
+  disable: true
+
+steps:
+  - name: clone
+    image: plugins/git
+    settings:
+      recursive: true
+
+  - name: publish image
+    image: plugins/docker
+    settings:
+      registry: git.coopcloud.tech
+      repo: git.coopcloud.tech/toolshed/alakazam
+      username:
+        from_secret: git_coopcloud_tech_user
+      password:
+        from_secret: git_coopcloud_tech_token
+      auto_tag: true
+
+trigger:
+  event:
+    - tag

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+.venv/
+.pipeline/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "abra"]
+	path = abra
+	url = https://git.coopcloud.tech/toolshed/abra.git

Dockerfile

@@ -0,0 +1,51 @@
+# Builds an image that runs alakazam together with the abra version pinned by the submodule.
+
+# keep in step with GOVERSION in abra/Makefile, a FROM cannot read it from there
+ARG GO_VERSION=1.26
+
+FROM golang:${GO_VERSION} AS abra
+WORKDIR /src
+COPY abra/ ./
+COPY .git/modules/abra /gitdir
+# abra ships its dependencies in vendor/, so the build itself needs no network
+RUN set -eu; \
+  sed -i '/worktree =/d' /gitdir/config; \
+  version="$(git --git-dir=/gitdir describe --tags)"; \
+  commit="$(git --git-dir=/gitdir rev-parse HEAD)"; \
+  echo "building abra $version"; \
+  go build -mod=vendor -trimpath \
+  -ldflags "-s -w -X 'main.Version=$version' -X 'main.Commit=$commit'" \
+  -o /out/abra ./cmd/abra
+
+FROM python:3.11-slim AS deps
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends gcc libc6-dev \
+  && rm -rf /var/lib/apt/lists/*
+COPY requirements.txt /tmp/requirements.txt
+RUN python -m venv /opt/alakazam \
+  && /opt/alakazam/bin/pip install --no-cache-dir -r /tmp/requirements.txt
+
+FROM python:3.11-slim
+# git: alakazam syncs the recipe repos itself. openssh-client: abra reaches the servers over
+# ssh. argon2: recipes hash secrets locally in their abra.sh, vaultwarden's admin token does
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends git openssh-client make argon2 \
+  && rm -rf /var/lib/apt/lists/* \
+  && git config --global --add safe.directory '*'
+
+COPY --from=abra /out/abra /usr/local/bin/abra
+COPY --from=deps /opt/alakazam /opt/alakazam
+COPY alakazam.py /opt/alakazam/
+RUN printf '#!/bin/sh\nexec /opt/alakazam/bin/python /opt/alakazam/alakazam.py "$@"\n' \
+  > /usr/local/bin/alakazam \
+  && chmod +x /usr/local/bin/alakazam
+
+# PYTHONUNBUFFERED so the streamed hook output arrives while a command is still running.
+# LANG pins abra to its untranslated messages, which alakazam matches on to tell a missing
+# secret or an empty generate run apart from a real failure.
+ENV PYTHONUNBUFFERED=1 \
+  LANG=C.UTF-8 \
+  ABRA_DIR=/root/.abra
+
+WORKDIR /config
+CMD ["alakazam", "--help"]

LICENCE

@@ -0,0 +1,662 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+
+    Copyright (C) {{ year }}  {{ organization }}
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<http://www.gnu.org/licenses/>.

README.md

@@ -1,87 +1,389 @@
 # Alakazam
 
-Proof-of-concept meta-configuration app-connector abra wrapper.
+Alakazam is a meta-configuration app-connector and an abra wrapper, designed as a proof-of-concept to simplify the management of environment configuration files across multiple instances.
+
+> [!WARNING]
+> This software is in an experimental stage and has to be used with caution, expect things to break. 
+>
+> At the moment it is used in production for a [very tailored use case](https://wiki.local-it.cloud/s/kollicloud-wiki/doc/installation-wVp2LfBbg7).
+> We try to mainline features of this project to `abra` and make the tool more accessible by everyone.
+> Any feedback is highly appreciated.
 
 ## Problem Statement
 
-- managing a lot of env files can be error prone
-    - copy pasting env files
-- loosing the overview of configuration differences between instances
-- updating env files is a manual process of comparing the env files
-- no way to define default configurations that should always be applied
-- connecting two apps requires manual effort of specific env file changes and sharing secrets
 
-## Advantages
+Managing numerous environment (env) files can be prone to errors such as:
 
-- Have a global configuration that applies on every instance:
-    - avoid copy pasta errors
-    - avoid forgetting important configurations
-- Reduce manual configuration overhead for connecting multiple apps
-    - exchange domains
-    - share secrets
-    - set specific env vars
-- Have a minimal configuration file for each instance
-    - it should only contain parameters that differ from the defaults and the global configuration
-- Set a global subdomain convention
-    - each app will always have the same subdomain
-- Set up an instance with multiple connected apps in one run
-- Automatic updates of the env configurations 
-    - avoid manual env configuration at all
+- Frequent copy-pasting mistakes.
+- Difficulty maintaining an overview of configuration differences across instances.
+- Laborious manual processes required for updating env files, particularly when underlying recipe env configurations change.
+- Challenges in defining default configurations that should consistently apply across all apps and all instances.
+- Manual effort required for app integration (e.g., SSO), including configuring environment files and exchanging secrets.
+
+## Advantages of Alakazam
+
+- **Global Configuration**: Prevents errors related to manual copying and pasting and ensures no critical configurations are overlooked.
+- **Reduced Manual Configuration**: Facilitates the connection between multiple applications (i.E. SSO) through domain exchanges, secret sharing, and specific environmental variables.
+- **Minimal Configuration Files**: Each instance configuration only contains parameters that deviate from the defaults or global configurations, simplifying management.
+- **Standardized Subdomain Conventions**: Ensures consistent subdomain use across apps.
+- **Simplified Setup**: Allows for the setup of an instance with multiple connected apps in one run.
+- **Automatic Env Configuration Updates**: Eliminates manual env configuration.
+- **Automatic Batch Updates**: update all apps of a group of instances at once.
+- **Hierarchical Configuration Structure**: Supports configuration inheritance for ease of managing large-scale configurations.
+
+## How does it work
+
+`alakazam` serves as a wrapper for `abra`, automating the deployment process for multiple apps.
+It utilizes YAML configuration files to generate the necessary .env files and manage secret exchanges between applications.
+It supports the deployment or updating of multiple apps simultaneously and can execute post-deploy hooks.
+All configurations are hierarchically structured in YAML files, which can inherit properties from the layers above.
+
+## Tutorial
+
+### Install alakazam
+
+To install Alakazam, follow these steps to clone the repository and set up the executable.
+Replace `apt` with your package manager of choice.
+
+```bash
+sudo apt update
+sudo apt install curl git python3.11-venv
+# Install golang
+curl -sS https://webi.sh/golang | sh; \
+source ~/.config/envman/PATH.env
+# clone alakazam
+git clone --recursive https://git.coopcloud.tech/toolshed/alakazam.git
+# install alakazam
+cd alakazam
+ln -s $PWD/alakazam.sh ~/.local/bin/alakazam
+# Ensure `~/.local/bin` is in your `$PATH`
+alakazam install
+```
+
+Create a global `~/.config/alakazam.yml` that contains at least the `root` path, see [Global Settings](#global-settings)
+
+### Run in Docker
+
+The image contains alakazam together with the `abra` version the submodule points at, so nothing has to be installed on the host:
+
+```bash
+docker build -t alakazam .
+```
+
+```bash
+docker run --rm -it \
+    -v /path/to/instance/configs:/config \
+    -v ~/.config/alakazam.yml:/root/.config/alakazam.yml:ro \
+    -v ~/.abra:/root/.abra \
+    -v ~/.ssh:/root/.ssh \
+    alakazam alakazam /config/example.com.yml ls
+```
+
+```bash
+docker run --rm -it alakazam abra --version
+docker run --rm -it alakazam bash
+```
+
+- **`/config`** holds the instance configurations. Set `root: /config` in `alakazam.yml`, since the path is resolved inside the container.
+- **`/root/.abra`** keeps the recipes, the catalogue and the generated app `.env` files across runs. Without it every run starts by cloning every recipe again. The path comes from `ABRA_DIR`; in a CI where each step is a fresh container, point it at the shared workspace instead, so that a server added in one step still exists in the next.
+- **`/root/.ssh`** provides the keys `abra` connects with. Do not mount it read-only if you use `ControlMaster`, as ssh needs to create its socket there.
+
+The image sets `git config --global safe.directory '*'`, because a mounted `~/.abra` belongs to the host user and git would otherwise refuse to touch the recipe repositories. It also pins `LANG=C.UTF-8`: alakazam recognises some `abra` results by their message, and a translated `abra` would not match.
+
+### Create a new instance:
+
+To set up a new instance with Alakazam, begin by specifying the required applications in `example.com.yml`. The name of this file determines the domain for the instance.
+Here's an example configuration:
+
+```yaml
+traefik:
+backup-bot-two:
+authentik:
+nextcloud:
+onlyoffice:
+wordpress:
+vikunja:
+matrix-synapse:
+element-web:
+rallly:
+```
+
+Ensure that DNS records are set up for all your applications. The simplest approach is to create a `CNAME` record for the wildcard domain `*.example.com` to cover all subdomains used by the apps.
+
+For a more comprehensive configuration, refer to the example in [./examples](./examples), which can also serve as a template for single-instance configurations (`example.com.yml`).
+
+Steps to initialize the instance:
+
+1. **Generate .env Files**: `alakazam example.com.yml config`
+2. **Insert Secrets**: `alakazam example.com.yml secrets`
+3. **Deploy Applications**: `alakazam example.com.yml deploy -e` 
+    - the `-e` flag executes post-deployment hooks after each deployment
+
+Or let [`setup`](#setup-and-clean-deploy) do all of it in one command.
+
+### Setup and Clean Deploy
+
+`setup` builds a whole environment in one go:
+
+```
+alakazam example.com.yml setup -n
+```
+
+It writes the `.env` files, then walks the apps level by level in [dependency](#dependencies) order and runs secrets, deployment and [readiness hooks](#readiness-hooks) for each level, before waiting for every service to become healthy and running the `initial-hooks`. Apps of the same level do not wait for each other. Only a dependency makes an app wait, which is what removes the need to deploy an identity provider ahead of everything else by hand. Every failure along the way ends the run with a non-zero exit code, which is what makes it usable from a CI pipeline.
+
+`setup` runs on an **empty environment** only. 
+
+`clean-deploy` is the destructive variant — `undeploy`, `purge`, then `setup`:
+
+```
+alakazam example.com.yml clean-deploy -n
+```
+
+It deletes all data of the selected apps, so it has two safeguards: it insists on a single instance configuration file and refuses a group directory, and it asks for confirmation unless `-n` is given.
+
+### Updating All Instances
+
+To update all instances:
+
+1. **Update Environment Files**: `alakazam example.com.yml config` to refresh .env files. It's a good practice to keep these files under version control (e.g., in a Git repository at `~/.abra/example.com`) and review changes with `git diff` before proceeding.
+2. **Upgrade Applications**: `alakazam example.com.yml upgrade` to update all applications.
+
+### Rotate Secrets
+
+To replace secrets with freshly generated ones:
+
+```
+alakazam example.com.yml reinsert-secrets -s 'authentik:email_pass,nextcloud:smtp_password'
+```
+
+Secrets are addressed as `<RecipeName>:<SecretName>` pairs, comma separated, and `-s` can be specified multiple times.
+The command undeploys the affected apps, removes the listed secrets, creates them again the same way `secrets` does and redeploys the apps that were deployed before.
+
+- the `-e` flag executes post-deployment hooks after each deployment
+- a secret that is shared between apps has to be listed for every app holding a copy of it, otherwise the old value is copied back from the app that still has it
+
+Secrets are created in four variants: `conf` (values from the `secrets` configuration), `secret-hooks` (local `abra.sh` commands), `exchange` (`shared_secrets` between apps) and `generate` (everything left over, generated by abra).
+By default all four run; `-o`/`--only` limits them, both for `reinsert-secrets` and for `secrets`:
+
+```
+alakazam example.com.yml secrets --only conf,generate
+```
+
+To remove secrets without recreating them, `purge-secrets` uses the same notation:
+
+```
+alakazam example.com.yml purge-secrets -s 'authentik:email_pass'
+```
+
+Without `-s` all secrets of the recipes selected by `-r` are purged.
+
+### Waiting for Health
+
+`ps` shows the container status of the deployed apps. With `--wait` it blocks until every service has settled, which is what an automated deployment gates on:
+
+```
+alakazam example.com.yml ps --wait --timeout 300
+```
+
+### Excluding Recipes
+
+`-er`/`--exclude-recipe` leaves single recipes out of a run entirely:
+
+```
+alakazam -er traefik example.com.yml clean-deploy -n
+```
+
+### Hiding Secrets
+
+`secrets` and `setup` print the values they generate, which is right at a terminal and wrong in a pipeline whose logs are kept. `--hide-secrets` replaces them with `[hidden]`:
+
+```
+alakazam --hide-secrets example.com.yml clean-deploy -n
+```
+
+### Run CMDs
+
+Escaping can be akward:
+```
+alakazam example.com.yml cmd -c 'app run_occ ''"config:app:get sociallogin custom_providers"''' -r nextcloud
+```
+
+
+### Tests
+
+The test suite covers the pure logic and needs no `abra` installation:
+
+```
+pip install -r requirements-dev.txt
+pytest
+```
 
 ## Configuration
 
-### Concept
+### Instance Configuration
 
-This concept contains the following configuration files.
-Each configuration can be templated with jinja2 and global variables.
+Configuration files support templating with Jinja2 and global variables, facilitating dynamic adjustments. The primary types of configuration files include:
+
+1. **`alaka.yml`**/**`alaka-*.yml`**: Contains global configurations for a hierarchical layer, affecting all subordinate apps.
+2. **`example.com.yml`** (`<your-domain>.yml`): A minimalist configuration for each instance, specifying the apps to be installed, the instance domain and optionally instance specific configurations.
+    - an **instance** means a collection of apps that are integrated with each other (one server / vm can contain multiple instances)
+3. **`config-sets.yml`**/**`config-sets-*.yml`**: Optional files at the `root` path. They define named sets of per-app configurations, either switched on per instance or group, or applying by themselves as an integration between apps. See [Config-Sets](#config-sets).
+
+### Global Settings
+
+**`~/.config/alakazam.yml`**: The global settings for alakazam.
+It must contain the `root` path to all instance configurations. This path contains all the `alaka.yml`,`alaka-*.yml` and `example.com.yml` files.
+Further it can contain settings for an Uptime Kuma instance.
+
+```
+root: ~/root/path/to/my/instance/configurations
+uptime_kuma:
+    url: https://status.example.com
+    user: <username>
+    password: <password>
+    parameter:
+        interval: 300
+        retryInterval: 360
+        timeout: 48
+        maxretries: 5
+        ignoreTls: False
+        notificationIDList: [15, 16]
+        expiryNotification: True
+```
 
-1. `~/.abra/defaults.yml`
-    - It contains global configurations for each app that are applied on each instance
-        - i.e. smtp config, language...
-    - It contains a mapping for each app to a subdomain
-        - i.e. `cloud.example.com` for each nextcloud instance
-    - This configuration is for the operator to avoid copy pasta errors, reduce manual configurations and keep the same naming convention
-2. `./config.yml`
-    - a minimalist configuration per instance
-    - contains at least:
-        - the domain (and the server, if they differ)
-        - the apps to be installed
-    - can optionally contain instance specific configurations for each app, overwriting the `defaults.yml` configurations
-3. `combine.yml`
-    - contains the configuration steps that are required to combine apps with each other
-    - This configuration should not be touched by the operator
-        - It should be either split into the recipes repositories and maintained by the recipe maintainer or it should be maintained by the alakazam maintainer
 
 ### App Configuration
 
-`defaults.yml`, `config.yml` and `combine.yml` contain a similar structure to configure the individual apps.
-For each app the following `<app_configurations>` steps can be used:
+`alaka.yml`/`alaka-*.yml`, `example.com.yml` and the config-sets contain a similar configuration structure.
+For each app/recipe the following `<app_configurations>` can be used:
 
-- `uncomment:`
-    - uncomment each matching line
+- **`uncomment`/`comment`**: Manages the active status of lines in configuration files, such as uncommenting needed variables .
+    - it matches against parts of the line (i.E. `compose.smtp.yml`)
     - this is useful for env variables that are used multiple times like `COMPOSE_FILE`
-- `env:`
-    - set the value for env variables
-- `execute:`
-    - `abra.sh` commands that should be executed after deployment
-- `secrets:`
-    - insert a specific value into a secret
-    - for secrets that can not be generated, i.e. smtp passwords
-- `subdomain`:
-    - contains the subdomain that should be used for a specific app
+- **`env`**: Sets values for environment variables.
+- **`*-hooks`**: Specifies `abra.sh` commands or local scripts to run at specific stages.
+    - **`initial-hooks`**: commands for initialisation
+    - **`deploy-hooks`**: commands that should be run after each deployment
+    - **`upgrade-hooks`**: commands that should be run after each upgrade
+- **`readiness-hooks`**: Commands that decide whether an app is usable yet, repeated until one succeeds. See [Readiness Hooks](#readiness-hooks).
+- **`dependency`**: Names the apps that have to be set up before this one. See [Dependencies](#dependencies).
+- **`secrets`**: Inserts specific values (i.E. smtp passwords) into secrets; future updates will support encrypted file usage.
+- **`secret-hooks`**: Run `abra.sh` commands locally or local scripts for secrets that need to be generated.
+- **`shared_secrets`**: Shares a secret between two apps, as `<source_app>: {<target_secret>: <source_secret>}`.
+- **`subdomain`**: Specifies the subdomain scheme for individual recipes and apps. (not available in config-sets)
     - i.e. `cloud.example.com` for nextcloud
-    - (not available in `combine.yml`)
+- **`version`**: Controls the recipe version to deploy; if unspecified, the latest version is used. (not available in config-sets)
+    - a commit can be pinned with `<release>@<commit>`, for example `12.0.2+2026.5.2@be9ebb3`
+    - only the commit is deployed; the release in front of the `@` is there so that a dependency bot can follow the recipe's tags and offer patch updates
 
-The `combine.yml` configuration additionally contains
+### Config-Sets
 
-- `shared_secrets:`
-     - `<source_secret_name>:<target_secret_name>`
-     - a mapping to define which secrets should be shared between two apps
+A config-set is a named bundle of per-app configuration, defined in `config-sets.yml` or any `config-sets-*.yml` at the `root` path. The files are merged in the order `config-sets.yml`, then the `config-sets-*.yml` alphabetically, the same precedence the `alaka*.yml` of a directory have.
 
-Each configuration file can have a `GLOBALS` section, that cat be used to template the configuration with global variables.
+A set applies in one of two ways.
 
+**Switched on per instance or group**, via `CONFIG-SETS` in any `alaka.yml` or `example.com.yml`:
+
+```yaml
+CONFIG-SETS:
+    bbb: true
 ```
+
+This is for bundles of per-app configuration that an operator turns on deliberately, a feature for instance. See [examples/config-sets.yml](./examples/config-sets.yml).
+
+**By itself**, when it carries `apply-when` and every app named there is part of the instance:
+
+```yaml
+integrate-authentik-nextcloud:
+    apply-when: [authentik, nextcloud]
+    authentik:
+        uncomment: [compose.nextcloud.yml]
+    nextcloud:
+        uncomment: [compose.authentik.yml]
+        shared_secrets:
+            authentik:
+                nextcloud_secret: authentik_secret
+```
+
+That is what an integration between two apps is: configuration belonging to both of them that is pointless unless both are there. `apply-when` names the condition rather than deriving it from the configured apps, so an integration can configure only one side: `[matrix-synapse, traefik]` while configuring `traefik` alone. An instance can still switch such a set off with `CONFIG-SETS: {integrate-authentik-nextcloud: false}`.
+
+Merge priority: automatic sets < group < sets switched on explicitly < instance. An integration is the lowest layer of all, so a group configuration can adjust it — the precedence the `combine.yml` it replaces had. Its list entries land behind the ones an app configures for itself, which is what keeps an integration hook such as `app set_authentik` running after the `app core_install` that sets the app up in the first place.
+
+[examples/config-sets-combine.yml](./examples/config-sets-combine.yml) holds the integrations alakazam used to ship as `combine.yml`. Copy it into your `root` path to keep them.
+
+### Dependencies
+
+An app can name the apps it needs in place before itself:
+
+```yaml
+nextcloud:
+    dependency: [authentik]
+```
+
+Alakazam builds a directed graph from those entries and processes the apps in topological order. For the configuration, the secrets and the deployment alike. Apps unrelated by a dependency keep their configuration order. [`setup`](#setup-and-clean-deploy) additionally groups them into levels: everything without a dependency forms the first level, apps depending only on those the second, and so on.
+
+### Readiness Hooks
+
+An app can be deployed and healthy while still not being usable — authentik accepts connections long before it has applied its blueprints. A readiness hook is an `abra.sh` command of the recipe that answers that question and exits zero once the app is ready:
+
+```yaml
+authentik:
+    readiness-hooks:
+        - cmd: app check_blueprints
+          retries: 20
+          interval: 120
+          initial-delay: 60
+```
+
+`cmd` follows the same notation as the other hooks: the first token is the container to run in, or `local`. The first attempt starts after `initial-delay` seconds, further attempts follow every `interval` seconds, at most `retries` times. If none of them succeeds, the run aborts with a non-zero exit code instead of deploying on top of a half-ready app. The defaults are 20 retries, 60 seconds apart, with no initial delay.
+
+Readiness hooks run as part of [`setup`](#setup-and-clean-deploy) only, right after the app they belong to is deployed and before anything that depends on it. `deploy` does not run them, so it keeps returning as soon as the deployment is through.
+
+### \*-Hooks Command Formats
+
+**Abra command** — runs an abra.sh command inside a container:
+
+```yaml
+initial-hooks:
+  - app set_default_quota
+```
+
+**Local script** — runs a script on the local machine (for custom actions that aren't generic enough to be implemented as abra.sh commands):
+
+```yaml
+initial-hooks:
+  - script ./scripts/script.sh arg1
+```
+
+Relative paths resolve from the `root` path. The script receives `ALAKAZAM_APP_DOMAIN`, `ALAKAZAM_APP_SERVER`, and `ALAKAZAM_INSTANCE_DOMAIN` as environment variables.
+
+### Configuration Structure
+
+Configuration can be simplified into a single `example.com.yml` or expanded into multiple layered `alaka.yml`/`alaka-*.yml` files for complex deployments. This allows for easy maintenance of multiple instances or groups.
+These configurations are designed to modularize and simplify the management of app settings and integrations, making it easier to maintain and scale your deployments.
+
+1. **`alaka.yml`/`alaka-*.yml` and `example.com.yml`**
+    - **`alaka.yml`/`alaka-*.yml`**: Contain global app configurations (`<app_recipe>`) which are inherited by all configurations within its hierarchical layer. This ensures consistent settings across multiple applications. All `alaka-*.yml` files inside a directory are merged into one configuration. Avoid overrides between `alaka-*.yml` inside of one layer.
+    - **`example.com.yml`**: Specifies which apps should be deployed (`<app_recipe>` entries), focusing on the individual needs of each instance.
+    - Configurations within these files are structured as follows, with specific `<app_configuration>` settings detailed in the [App Configuration](#app-configuration) section:
+
+    ```
+    <app_recipe>:
+        <app_configurations>
+    ```
+
+2. **`config-sets.yml`**/**`config-sets-*.yml`**
+    - Optional files at the `root` path. Each top-level key is a config-set name; its value is a map of `<app_recipe>` to `<app_configurations>`.
+    - See [Config-Sets](#config-sets).
+
+
+### Templating Configurations
+
+Each configuration file can include a `GLOBALS` section that allows for templating with global variables, making it easier to manage repetitive values across different configurations. Here is how you can define and use these global variables:
+
+```yaml
 GLOBALS:
+    server: example-server.com
     smtp_user: noreply
     smtp_domain: example.org
 authentik:
@@ -92,78 +394,31 @@ nextcloud:
         MAIL_FROM_ADDRESS: "{{smtp_user}}"
 ```
 
-### Configuration Structure
+- **`server`**: Specifies the server where the instance will be deployed. If not provided, the instance domain is automatically taken from the `example.com.yml` file name.
 
-1. `defaults.yml`
-    - there is an entry for each `<app_recipe>` that should have a global configuration
-    - the `<app_configuration>` is configured according to [App Configuration](#app-configuration) above
+## Inheriting Configuration Structure
 
-    ```
-    <app_recipe>:
-        <app_configurations>
-    ```
+Alakazam supports a flexible inheritance model for configuration management, allowing for both simplicity and complexity depending on your deployment needs:
 
-2. `config.yml`
+- **Simple Configuration**: For straightforward setups, you can manage everything within a single `example.com.yml` file, placing all necessary configurations directly in this document.
+- **Multiple Similar Instances**: To manage multiple similar instances efficiently, common application configurations can be centralized in a global `alaka.yml`. This method reduces duplication by allowing shared settings across multiple instances. The global `alaka.yml` can also be split into multiple `alaka-*.yml` files which are merged together.
+- **Grouped Instances**: For more complex setups, instances can be organized into folders, each with its own `alaka.yml`/`alaka-*.yml`. This structure allows for group-specific configurations, making it easier to manage settings at different levels of your infrastructure hierarchy.
+- **Inherited Configurations**: In even more complex scenarios, each group can inherit configurations from the levels above it. This cascading setup ensures that changes at a higher level can be automatically applied to all subordinate groups and instances, maintaining consistency and ease of updates across your entire environment.
 
-    ```
-    domain: <instance_domain>
-    server: <instance_server>
-    apps:
-        <app_recipe>:
-            <app_configurations>
-    ```
+### Merge Order
 
-3. `combine.yml`
-    - To combine two deployed apps each `<target_app_configurations>` of a specific `<target_app_recipe>` is only applied if the `config.yml` also contains the belonging `<source_app_recipe>`
+Within one directory the configuration files are merged in a fixed order, from weakest to strongest:
 
-    ```
-    <target_app_recipe>:
-        <source_app_recipe>:
-            <target_app_configurations>
-    ```
+1. the merged configuration inherited from the parent directory
+2. `alaka.yml`
+3. the `alaka-*.yml` files in alphabetical order
+
+So a key set in `alaka-versions.yml` overrides the same key in `alaka.yml`, and both override the parent directory. Lists are the exception: they are not overridden but concatenated in merge order.
+
+This hierarchical approach to configuration is designed to scale with your system’s complexity and can be visualized through the following example diagram:
+
+![File Structure](./config_file_structure.png)
+
+![Configuration Structure](./config_structure.png)
 
 
-## Process
-
-Create a new instance:
-
-1. The `config.yml` is created by the operator to define the domain and required apps
-2. `./alakazam.py init-server`
-    - installs `traefik` and `backup-bot-two`
-3. `./alakazam.py setup-apps`
-    1. for each app the env files are created
-    2. for each app the `defaults.yml` configs are applied on the env files and secrets are inserted
-    3. for each app the `config.yml` configs are applied on the env files and secrets are inserted
-    4. for each app the `combine.yml` configs are applied on the env files and secrets are inserted
-        - the shared_secrets are generated
-        - `<source_app_recipe>.example.com` is replaced in the `<target_app_recipe>` env file according to `subdomain` configuration
-    5. all the remaining secrets are generated
-4. `./alakazam.py deploy-apps`
-    - each app is deployed
-    - the `execute` commands from `defaults.yml`, `configs.yml` and `combine.yml` are executed
-
-Step 3. (`./alakazam.py setup-apps`) should be executed every time the config is changed or the recipe env file is updated. It regenerates the env files and creates all the necessary secrets. 
-The env files itself shouldn't be edited manually anymore.
-
-## Conceptual Questions
-
-Please note any ideas and arguments you have: https://pad.local-it.org/dB0FsiomRCOCPs7rS40YSQ
-
-- Possible abra integration
-    - An independent wrapper for abra, that uses abras `--machine` output
-    - A go binary that is build on top of the abra library
-    - An abra command like `abra compose`
-
-- Put `config.yml` into `~/.abra/servers/<server>/` or let the user decide?
-
-- Should we move completely away from .env files?
-    - This would skip the translation step into env files
-    - It reduces the complexity!
-    - Recipes couldn't be run with `docker compose` only anymore
-        - Is this still possible?
-
-- Where should the `combine.yml` be maintained?
-    - Separated config for each recipe?
-        - better configuration versioning for different recipe versions
-        - harder to maintain the recipes: requires knowledge about the connectable apps
-    - Global config?

alakazam.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Get the directory of the script being run.
+SCRIPT_DIR="$( cd "$( dirname "$( readlink -f "${BASH_SOURCE[0]}" )" )" &> /dev/null && pwd )"
+VENV_PATH=$SCRIPT_DIR/.venv
+
+# check if venv exists
+if [ ! -d $VENV_PATH -o "$1" == "install" ]; then
+    cd $SCRIPT_DIR
+    git pull --recurse-submodules
+    echo install virtual environment: $VENV_PATH
+    rm -rf $VENV_PATH
+    python3 -m venv $VENV_PATH
+    source $VENV_PATH/bin/activate
+    pip3 install -r $SCRIPT_DIR/requirements.txt
+    deactivate
+    cd $SCRIPT_DIR/abra
+    make install
+    exit 0
+fi
+
+# Activate the virtual environment
+source $VENV_PATH/bin/activate
+python3 -u $SCRIPT_DIR/alakazam.py "$@"
+py_exit=$?
+deactivate
+exit "$py_exit"

combine.yml

@@ -1,193 +0,0 @@
-authentik:
-    nextcloud:
-        uncomment:
-            - compose.nextcloud.yml
-            - NEXTCLOUD_DOMAIN
-            - SECRET_NEXTCLOUD_ID_VERSION
-            - SECRET_NEXTCLOUD_SECRET_VERSION
-            - nextcloud.png
-    wordpress:
-        uncomment:
-            - compose.wordpress.yml
-            - WORDPRESS_DOMAIN
-            - WORDPRESS_GROUP
-            - SECRET_WORDPRESS_ID_VERSION
-            - SECRET_WORDPRESS_SECRET_VERSION
-            - wordpress.png
-    matrix-synapse:
-        uncomment:
-            - compose.matrix.yml
-            - ELEMENT_DOMAIN
-            - SECRET_MATRIX_ID_VERSION
-            - SECRET_MATRIX_SECRET_VERSION
-            - matrix.svg
-        secrets:
-            matrix_id: matrix
-    wekan:
-        uncomment:
-            - compose.wekan.yml
-            - WEKAN_DOMAIN
-            - SECRET_WEKAN_ID_VERSION
-            - SECRET_WEKAN_SECRET_VERSION
-            - wekan.png
-        secrets:
-            wekan_id: wekan
-    vikunja:
-        uncomment:
-            - compose.vikunja.yml
-            - VIKUNJA_DOMAIN
-            - SECRET_VIKUNJA_ID_VERSION
-            - SECRET_VIKUNJA_SECRET_VERSION
-            - vikunja.svg
-        secrets:
-            vikunja_id: vikunja
-    monitoring:
-        uncomment:
-            - compose.monitoring.yml
-            - MONITORING_DOMAIN
-            - SECRET_MONITORING_ID_VERSION
-            - SECRET_MONITORING_SECRET_VERSION
-            - monitoring.png
-    outline:
-        uncomment:
-            - compose.outline.yml
-            - OUTLINE_DOMAIN
-            - SECRET_OUTLINE_ID_VERSION
-            - SECRET_OUTLINE_SECRET_VERSION
-            - outline.png
-        secrets:
-            outline_id: outline
-    rallly:
-        uncomment:  
-            - compose.rallly.yml
-            - RALLLY_DOMAIN
-            - SECRET_RALLLY_ID_VERSION
-            - SECRET_RALLLY_SECRET_VERSION
-            - rallly.png
-        secrets:
-            rallly_id: rallly
-nextcloud:
-    authentik:
-        uncomment:
-            - compose.authentik.yml
-            - AUTHENTIK_USER_PREFIX
-            - AUTHENTIK_DOMAIN
-            - SECRET_AUTHENTIK_SECRET_VERSION
-            - SECRET_AUTHENTIK_ID_VERSION
-        execute:
-            - app set_authentik
-        shared_secrets:
-            nextcloud_secret: authentik_secret
-            nextcloud_id: authentik_id
-    onlyoffice:
-        uncomment:
-            - ONLYOFFICE_URL
-            - SECRET_ONLYOFFICE_JWT_VERSION
-        execute:
-            - app install_onlyoffice
-    collabora:
-        uncomment:
-            - COLLABORA_URL
-        execute:
-            - app install_collabora
-onlyoffice:
-    nextcloud:
-        uncomment:
-            - compose.jwt.yml
-            - SECRET_JWT_SECRET_VERSION
-        shared_secrets:
-            onlyoffice_jwt: jwt_secret
-outline:
-    authentik:
-        env: 
-            OIDC_CLIENT_ID: outline
-            OIDC_AUTH_URI: https://authentik.example.com/application/o/authorize/
-            OIDC_TOKEN_URI: https://authentik.example.com/application/o/token/
-            OIDC_USERINFO_URI: https://authentik.example.com/application/o/userinfo/
-            OIDC_DISPLAY_NAME: "Authentik"
-        uncomment:
-            - compose.oidc.yml
-            - OIDC_ENABLED
-            - OIDC_USERNAME_CLAIM
-            - OIDC_SCOPES
-            - SECRET_OIDC_CLIENT_SECRET_VERSION
-        shared_secrets:
-            outline_secret: oidc_client_secret
-wordpress:
-    authentik:
-        uncomment:
-            - compose.authentik.yml
-            - AUTHENTIK_DOMAIN
-            - SECRET_AUTHENTIK_SECRET_VERSION
-            - SECRET_AUTHENTIK_ID_VERSION
-            - LOGIN_TYPE
-        execute:
-            - app set_authentik
-        shared_secrets:
-            wordpress_secret: authentik_secret
-            wordpress_id: authentik_id
-vikunja:
-    authentik:
-        env:
-            OAUTH_NAME: authentik
-            OAUTH_URL: https://authentik.example.com/application/o/vikunja/
-            OAUTH_LOGOUT_URL: https://authentik.example.com/application/o/vikunja/end-session/
-            # TODO: set CLIENT_ID as secret
-            OAUTH_CLIENT_ID: vikunja
-        uncomment:
-            - compose.oauth.yml
-            - OAUTH_ENABLED
-            - SECRET_OAUTH_SECRET_VERSION
-        shared_secrets:
-            vikunja_secret: oauth_secret
-            #vikunja_id: oauth_id
-
-matrix-synapse:
-    authentik:
-        env:
-            KEYCLOAK_ID: authentik
-            KEYCLOAK_NAME: sso
-            KEYCLOAK_URL: https://authentik.example.com/application/o/matrix/
-            # TODO: correct client domain?
-            KEYCLOAK_CLIENT_DOMAIN: https://element-web.example.com
-            KEYCLOAK_ALLOW_EXISTING_USERS: "true"
-            # TODO: set CLIENT_ID as secret
-            KEYCLOAK_CLIENT_ID: matrix
-        uncomment:
-            - compose.keycloak.yml
-            - KEYCLOAK_ENABLED
-            - KEYCLOAK_CLIENT_ID
-            - SECRET_KEYCLOAK_CLIENT_SECRET_VERSION
-        shared_secrets:
-            matrix_secret: keycloak_client_secret
-rallly:
-    authentik:
-        env:
-            OIDC_NAME: "Authentik"
-            OIDC_DISCOVERY_URL: "https://authentik.example.com/application/o/Rallly/.well-known/openid-configuration"
-            OIDC_CLIENT_ID: rallly
-        uncomment: 
-            - compose.oidc.yml
-            - SECRET_OIDC_CLIENT_SECRET_VERSION
-            - OIDC_ENABLED
-wekan:
-    authentik:
-        env: 
-            OAUTH2_ENABLED: "true"
-            OAUTH2_SERVER_URL: https://authentik.example.com
-            # TODO: set CLIENT_ID as secret
-            OAUTH2_CLIENT_ID: wekan
-        uncomment:
-            - OAUTH2_LOGIN_STYLE
-            - OAUTH2_AUTH_ENDPOINT
-            - OAUTH2_USERINFO_ENDPOINT
-            - OAUTH2_TOKEN_ENDPOINT
-            - OAUTH2_REQUEST_PERMISSIONS
-            - OAUTH2_ID_MAP
-            - OAUTH2_USERNAME_MAP
-            - OAUTH2_FULLNAME_MAP
-            - OAUTH2_EMAIL_MAP
-            - PROPAGATE_OIDC_DATA
-            - OIDC_REDIRECTION_ENABLED
-        shared_secrets:
-            wekan_secret: oauth2_secret

examples/alaka-defaults.yml

@@ -0,0 +1,95 @@
+GLOBALS:
+    bbb_secret: your_super_secret_password
+    turn_secret: your_super_secret_password
+    basic_auth_password: your_super_secret_password
+    loki_url: https://loki.monitoring.example.com/loki/api/v1/push
+    basic_auth_hash: your_super_secret_password
+authentik:
+    env:
+        EXTRA_DOMAINS: ', `example.com`'
+        DEFAULT_LANGUAGE: de
+        APPLICATIONS: '{"Calendar": "https://nextcloud.example.com/apps/calendar/", "BBB": "https://nextcloud.example.com/apps/bbb"}'
+    uncomment:
+        - calendar.svg
+        - bbb.jpg
+    initial-hooks:
+       - worker set_admin_pass
+       - worker apply_blueprints
+       - worker add_applications
+       - local set_icons
+nextcloud:
+    env: 
+        DEFAULT_QUOTA: "100 MB"
+        APPS: "calendar"
+        BBB_URL: "https://talk.example.com/bigbluebutton/"
+    uncomment:
+        - compose.apps.yml
+        - SECRET_BBB_SECRET_VERSION
+    secrets:
+        bbb_secret: "{{bbb_secret}}"
+    initial-hooks:
+        - app install_bbb
+        - app set_default_quota
+        - app install_apps
+        - app run_occ 'app:disable dashboard'
+wordpress:
+    env:
+        LOCALE: 'de_DE'
+        DEFAULT_USER_ROLE: administrator
+    uncomment:
+        - EXTRA_DOMAINS
+    initial-hooks:
+        - app core_install
+matrix-synapse:
+    env: 
+        ENCRYPTED_BY_DEFAULT: off
+        TURN_URIS: '["turns:talk.example.com:5349?transport=tcp"]'
+        RETENTION_MAX_LIFETIME: 2y
+        ALLOWED_LIFETIME_MAX: 2y
+    uncomment:
+        - POST_DEPLOY_CMDS
+        - compose.turn.yml
+        - TURN_ENABLED
+        - TURN_ALLOW_GUESTS
+        - SECRET_TURN_SHARED_SECRET_VERSION
+    secrets:
+        turn_shared_secret: "{{turn_secret}}"
+wekan:
+    uncomment:
+        - PASSWORD_LOGIN_ENABLED
+backup-bot-two:
+    uncomment:
+        - SECRET_AWS_SECRET_ACCESS_KEY_VERSION
+        - compose.s3.yml
+onlyoffice:
+    env:
+        AUTO_ASSEMBLY_ENABLED: "true"     
+outline:
+    env:
+        DEFAULT_LANGUAGE: de_DE
+hedgedoc:
+    env:
+        CMD_ALLOW_ANONYMOUS: true
+        CMD_ALLOW_ANONYMOUS_EDITS: true
+        CMD_ALLOW_FREEURL: true
+        CMD_COOKIE_POLICY: none
+traefik:
+    env:
+        DASHBOARD_ENABLED: false
+        METRICS_ENABLED: 1
+        BASIC_AUTH: 1
+    secrets:
+        usersfile: "{{basic_auth_hash}}"
+    uncomment:
+        - compose.host.yml
+        - compose.metrics.yml
+        - compose.basicauth.yml
+        - SECRET_USERSFILE_VERSION
+monitoring-ng:
+    env:
+        LOKI_PUSH_URL: "{{loki_url}}"
+    uncomment:
+        - compose.promtail.yml
+        - SECRET_BASIC_AUTH_VERSION
+    secrets:
+        basic_auth: "{{basic_auth_password}}"

examples/alaka-smtp.yml

@@ -1,25 +1,17 @@
 GLOBALS:
-    smtp_password: top-secret
+    smtp_password: your_super_secret_password
+    smtp_password_urlencoded: your_super_secret_password
     smtp_user: noreply
     smtp_domain: example.com
     smtp_host: mail.example.com
 authentik:
-    subdomain: example.com
     env:
         AUTHENTIK_EMAIL__HOST: "{{smtp_host}}"
         AUTHENTIK_EMAIL__USERNAME: "{{smtp_user}}@{{smtp_domain}}"
         AUTHENTIK_EMAIL__FROM: "{{smtp_user}}@{{smtp_domain}}"
-        DEFAULT_LANGUAGE: de
-    execute:
-       - worker set_admin_pass
-       - worker apply_blueprints
-       - worker apply_blueprints
-       - worker add_applications # Todo: fix this command
-       - local set_icons
     secrets:
         email_pass: "{{smtp_password}}"
 nextcloud:
-    subdomain: cloud.example.com
     env: 
         SMTP_AUTHTYPE: LOGIN
         SMTP_HOST: "{{smtp_host}}"
@@ -28,35 +20,28 @@ nextcloud:
         SMTP_PORT: 587
         MAIL_FROM_ADDRESS: "{{smtp_user}}"
         MAIL_DOMAIN: "{{smtp_domain}}"
-        APPS: "calendar"
-        BBB_URL: "https://talk.example.com/bigbluebutton/"
     uncomment:
         - compose.smtp.yml
         - SECRET_SMTP_PASSWORD_VERSION
         - compose.apps.yml
-        - SECRET_BBB_SECRET_VERSION
     secrets:
         smtp_password: "{{smtp_password}}"
-    execute:
-        - app install_bbb
-        - app set_default_quota
-        - app install_apps
-        # - app run_occ 'app:disable dashboard' # Todo: fix this command
 wordpress:
     env:
-        LOCALE: 'de_DE'
-        DEFAULT_USER_ROLE: administrator
         SMTP_HOST: "{{smtp_host}}"
         MAIL_FROM: "{{smtp_user}}@{{smtp_domain}}"
+        SMTP_USER: "{{smtp_user}}@{{smtp_domain}}"
+        ADMIN_EMAIL: "{{smtp_user}}@{{smtp_domain}}"
     uncomment:
         - SMTP_PORT
         - SMTP_AUTH
+        - SMTP_OVERRIDE_FROM
         - SMTP_TLS
         - SECRET_SMTP_PASSWORD_VERSION
+        - compose.smtp.yml
+        - compose.mailrelay.yml
     secrets:
         smtp_password: "{{smtp_password}}"
-    execute:
-        - app core_install
 vikunja:
     env:
         SMTP_HOST: "{{smtp_host}}"
@@ -70,30 +55,53 @@ vikunja:
     secrets:
         smtp_password: "{{smtp_password}}"
 matrix-synapse:
-    subdomain: matrix.example.com
     env: 
+        ADMIN_EMAIL: "{{smtp_user}}@{{smtp_domain}}"
         SMTP_APP_NAME: mail
         SMTP_FROM: "{{smtp_user}}@{{smtp_domain}}"
         SMTP_HOST: "{{smtp_host}}"
         SMTP_PORT: 587
         SMTP_USER: "{{smtp_user}}@{{smtp_domain}}"
-        ENCRYPTED_BY_DEFAULT: "off"
     uncomment:
-        - POST_DEPLOY_CMDS
         - compose.smtp.yml
         - SMTP_ENABLED
         - SECRET_SMTP_PASSWORD_VERSION
     secrets:
         smtp_password: "{{smtp_password}}"
-element-web:
-    subdomain: chat.example.com
 wekan:
-    subdomain: board.example.com
     env:
-        # TODO fix format for smtp_password
-        MAIL_URL: "smtp://{{smtp_user}}%40{{smtp_domain}}%3A{{smtp_password}}@{{smtp_host}}:587/"
+        MAIL_URL: "smtp://{{smtp_user}}%40{{smtp_domain}}:{{smtp_password_urlencoded}}@{{smtp_host}}:587"
         MAIL_FROM: "Wekan Notifications <{{smtp_user}}@{{smtp_domain}}>"
+outline:
+    env:
+        SMTP_HOST: "{{smtp_host}}"
+        SMTP_PORT: 465
+        SMTP_USERNAME: "{{smtp_user}}@{{smtp_domain}}"
+        SMTP_FROM_EMAIL: "{{smtp_user}}@{{smtp_domain}}"
     uncomment:
-        - PASSWORD_LOGIN_ENABLED
-backup-bot-two:
-    subdomain: backup.example.com
+        - compose.smtp.yml
+        - SMTP_ENABLED
+        - SMTP_SECURE
+        - SECRET_SMTP_PASSWORD_VERSION
+    secrets:
+        smtp_password: "{{smtp_password}}"
+rallly:
+    env:
+        SUPPORT_EMAIL: "{{smtp_user}}@{{smtp_domain}}"
+        SMTP_HOST: "{{smtp_host}}"
+        SMTP_USER: "{{smtp_user}}@{{smtp_domain}}"
+    secrets:
+        smtp_pwd: "{{smtp_password}}"
+pretix:
+    env:
+        SMTP_FROM: "{{smtp_user}}@{{smtp_domain}}"
+        SMTP_HOST: "{{smtp_host}}"
+        SMTP_USER: "{{smtp_user}}@{{smtp_domain}}"
+        SMTP_PORT: 587
+        SMTP_TLS: on
+        SMTP_SSL: off
+    secrets: 
+        smtp_password: "{{smtp_password}}"
+traefik:
+    env:
+        LETS_ENCRYPT_EMAIL: "{{smtp_user}}@{{smtp_domain}}"

examples/alaka-subdomains.yml

@@ -0,0 +1,28 @@
+authentik:
+    subdomain: login.example.com
+nextcloud:
+    subdomain: files.example.com
+wordpress:
+    subdomain: blog.example.com
+vikunja:
+    subdomain: todo.example.com
+matrix-synapse:
+    subdomain: matrix.example.com
+element-web:
+    subdomain: chat.example.com
+wekan:
+    subdomain: board.example.com
+backup-bot-two:
+    subdomain: backup.example.com
+onlyoffice:
+    subdomain: office.example.com
+outline:
+    subdomain: outline.example.com
+rallly:
+    subdomain: poll.example.com
+hedgedoc:
+    subdomain: pad.example.com
+pretix:
+    subdomain: tickets.example.com
+monitoring-ng:
+    subdomain: monitoring.example.com

examples/alaka-versions.yml

@@ -0,0 +1,30 @@
+authentik:
+    version: 3.2.4+2023.6.1
+wekan:
+    version: 2.1.0+v7.30
+nextcloud:
+    version: 5.0.3+27.0.1-fpm
+onlyoffice:
+    version: 2.6.2+7.4.0
+collabora:
+    version: 2.7.1+23.05.10.1.1
+wordpress:
+    version: 2.5.2+6.3.0
+vikunja:
+    version: 0.4.0+0.22.1
+matrix-synapse:
+    version: 5.0.2+v1.93.0
+element-web:
+    version: 0.7.0+v1.11.63
+rallly:
+    version: 0.4.1+2.11.1
+hedgedoc:
+    version: 1.0.1+1.9.9
+pretix:
+    version: 1.0.0+2024.1.0
+traefik:
+    version: 2.4.3+v2.10.5
+backup-bot-two:
+    version: chaos
+monitoring-ng:
+    version: 1.1.0+v1.7.0

examples/config-sets-combine.yml

@@ -0,0 +1,395 @@
+# Integrations, converted from the combine.yml that alakazam used to ship.
+# A set applies as soon as every app of its 'apply-when' is part of an instance.
+# Copy this file into your alakazam root path, next to alaka-versions.yml.
+
+integrate-authentik-nextcloud:
+    apply-when:
+      - authentik
+      - nextcloud
+    authentik:
+        uncomment:
+          - compose.nextcloud.yml
+          - NEXTCLOUD_DOMAIN
+          - SECRET_NEXTCLOUD_ID_VERSION
+          - SECRET_NEXTCLOUD_SECRET_VERSION
+          - nextcloud.png
+    nextcloud:
+        uncomment:
+          - compose.authentik.yml
+          - AUTHENTIK_USER_PREFIX
+          - AUTHENTIK_DOMAIN
+          - SECRET_AUTHENTIK_SECRET_VERSION
+          - SECRET_AUTHENTIK_ID_VERSION
+        initial-hooks:
+          - app set_authentik
+        shared_secrets:
+            authentik:
+                nextcloud_secret: authentik_secret
+                nextcloud_id: authentik_id
+integrate-authentik-wordpress:
+    apply-when:
+      - authentik
+      - wordpress
+    authentik:
+        uncomment:
+          - compose.wordpress.yml
+          - WORDPRESS_DOMAIN
+          - WORDPRESS_GROUP
+          - SECRET_WORDPRESS_ID_VERSION
+          - SECRET_WORDPRESS_SECRET_VERSION
+          - wordpress.png
+    wordpress:
+        uncomment:
+          - compose.authentik.yml
+          - AUTHENTIK_DOMAIN
+          - SECRET_AUTHENTIK_SECRET_VERSION
+          - SECRET_AUTHENTIK_ID_VERSION
+          - LOGIN_TYPE
+        initial-hooks:
+          - app set_authentik
+        shared_secrets:
+            authentik:
+                wordpress_secret: authentik_secret
+                wordpress_id: authentik_id
+integrate-authentik-matrix-synapse:
+    apply-when:
+      - authentik
+      - matrix-synapse
+    authentik:
+        uncomment:
+          - compose.matrix.yml
+          - ELEMENT_DOMAIN
+          - MATRIX_DOMAIN
+          - SECRET_MATRIX_ID_VERSION
+          - SECRET_MATRIX_SECRET_VERSION
+          - matrix.svg
+        secrets:
+            matrix_id: matrix
+    matrix-synapse:
+        env:
+            KEYCLOAK_ID: authentik
+            KEYCLOAK_NAME: sso
+            KEYCLOAK_URL: https://authentik.example.com/application/o/matrix/
+            # TODO: correct client domain?
+            KEYCLOAK_CLIENT_DOMAIN: https://element-web.example.com
+            KEYCLOAK_ALLOW_EXISTING_USERS: "true"
+            # TODO: set CLIENT_ID as secret
+            KEYCLOAK_CLIENT_ID: matrix
+        uncomment:
+          - compose.keycloak.yml
+          - KEYCLOAK_ENABLED
+          - KEYCLOAK_CLIENT_ID
+          - SECRET_KEYCLOAK_CLIENT_SECRET_VERSION
+        shared_secrets:
+            authentik:
+                matrix_secret: keycloak_client_secret
+        dependency: [authentik]
+integrate-authentik-wekan:
+    apply-when:
+      - authentik
+      - wekan
+    authentik:
+        uncomment:
+          - compose.wekan.yml
+          - WEKAN_DOMAIN
+          - SECRET_WEKAN_ID_VERSION
+          - SECRET_WEKAN_SECRET_VERSION
+          - wekan.png
+        secrets:
+            wekan_id: wekan
+    wekan:
+        env:
+            OAUTH2_ENABLED: "true"
+            OAUTH2_SERVER_URL: https://authentik.example.com
+            # TODO: set CLIENT_ID as secret
+            OAUTH2_CLIENT_ID: wekan
+        uncomment:
+          - OAUTH2_LOGIN_STYLE
+          - OAUTH2_AUTH_ENDPOINT
+          - OAUTH2_USERINFO_ENDPOINT
+          - OAUTH2_TOKEN_ENDPOINT
+          - OAUTH2_REQUEST_PERMISSIONS
+          - OAUTH2_ID_MAP
+          - OAUTH2_USERNAME_MAP
+          - OAUTH2_FULLNAME_MAP
+          - OAUTH2_EMAIL_MAP
+          - PROPAGATE_OIDC_DATA
+          - OIDC_REDIRECTION_ENABLED
+        shared_secrets:
+            authentik:
+                wekan_secret: oauth2_secret
+integrate-authentik-vikunja:
+    apply-when:
+      - authentik
+      - vikunja
+    authentik:
+        uncomment:
+          - compose.vikunja.yml
+          - VIKUNJA_DOMAIN
+          - SECRET_VIKUNJA_ID_VERSION
+          - SECRET_VIKUNJA_SECRET_VERSION
+          - vikunja.svg
+        secrets:
+            vikunja_id: vikunja
+    vikunja:
+        env:
+            OAUTH_NAME: authentik
+            OAUTH_URL: https://authentik.example.com/application/o/vikunja/
+            OAUTH_LOGOUT_URL: https://authentik.example.com/application/o/vikunja/end-session/
+            # TODO: set CLIENT_ID as secret
+            OAUTH_CLIENT_ID: vikunja
+        uncomment:
+          - compose.oauth.yml
+          - OAUTH_ENABLED
+          - SECRET_OAUTH_SECRET_VERSION
+        shared_secrets:
+            #vikunja_id: oauth_id
+            authentik:
+                vikunja_secret: oauth_secret
+integrate-authentik-kimai:
+    apply-when:
+      - authentik
+      - kimai
+    authentik:
+        uncomment:
+          - compose.kimai.yml
+          - KIMAI_DOMAIN
+          - SECRET_KIMAI_ID_VERSION
+          - SECRET_KIMAI_SECRET_VERSION
+          - kimai_logo.png
+          - KIMAI_GROUP
+    kimai:
+        uncomment:
+          - SSO_ENABLED
+          - SSO_PROVIDER_URL
+          - SSO_SAML_URL
+          - SSO_LOGOUT_URL
+          - SSO_ADMIN_GROUP_NAME
+        secret_hooks:
+          - insert_authentik_certificate
+        dependency: [authentik]
+integrate-authentik-zammad:
+    apply-when:
+      - authentik
+      - zammad
+    authentik:
+        uncomment:
+          - compose.zammad.yml
+          - ZAMMAD_DOMAIN
+          - zammad.svg
+    zammad:
+        uncomment:
+          - SSO_PROVIDER_DOMAIN
+          - IDP_SSO_TARGET_URL
+          - IDP_SLO_SERVICE_URL
+        initial-hooks:
+          - local enable_authentik_sso
+        dependency: [authentik]
+integrate-authentik-monitoring-ng:
+    apply-when:
+      - authentik
+      - monitoring-ng
+    authentik:
+        uncomment:
+          - compose.monitoring.yml
+          - MONITORING_DOMAIN
+          - SECRET_MONITORING_ID_VERSION
+          - SECRET_MONITORING_SECRET_VERSION
+          - monitoring.svg
+        secrets:
+            monitoring_id: monitoring
+    monitoring-ng:
+        env:
+            GF_SERVER_ROOT_URL: https://monitoring-ng.example.com
+            OIDC_CLIENT_ID: monitoring
+            OIDC_AUTH_URL: https://authentik.example.com/application/o/authorize/
+            OIDC_API_URL: https://authentik.example.com/application/o/userinfo/
+            OIDC_TOKEN_URL: https://authentik.example.com/application/o/token/
+        uncomment:
+          - compose.prometheus.yml
+          - PROMETHEUS_RETENTION_TIME
+          - compose.loki.yml
+          - LOKI_RETENTION_PERIOD
+          - LOKI_STORAGE_FILESYSTEM
+          - compose.grafana.yml
+          - OIDC_ENABLED
+          - SECRET_GRAFANA_ADMIN_PASSWORD_VERSION
+          - SECRET_GRAFANA_OIDC_CLIENT_SECRET_VERSION
+          - SECRET_GRAFANA_SMTP_PASSWORD_VERSION
+        shared_secrets:
+            authentik:
+                monitoring_secret: grafana_oidc_client_secret
+        dependency: [authentik]
+integrate-authentik-outline:
+    apply-when:
+      - authentik
+      - outline
+    authentik:
+        uncomment:
+          - compose.outline.yml
+          - OUTLINE_DOMAIN
+          - SECRET_OUTLINE_ID_VERSION
+          - SECRET_OUTLINE_SECRET_VERSION
+          - outline.png
+        secrets:
+            outline_id: outline
+    outline:
+        env:
+            OIDC_CLIENT_ID: outline
+            OIDC_AUTH_URI: https://authentik.example.com/application/o/authorize/
+            OIDC_TOKEN_URI: https://authentik.example.com/application/o/token/
+            OIDC_USERINFO_URI: https://authentik.example.com/application/o/userinfo/
+            OIDC_DISPLAY_NAME: "Authentik"
+        uncomment:
+          - compose.oidc.yml
+          - OIDC_ENABLED
+          - OIDC_USERNAME_CLAIM
+          - OIDC_SCOPES
+          - SECRET_OIDC_CLIENT_SECRET_VERSION
+        shared_secrets:
+            authentik:
+                outline_secret: oidc_client_secret
+integrate-authentik-rallly:
+    apply-when:
+      - authentik
+      - rallly
+    authentik:
+        uncomment:
+          - compose.rallly.yml
+          - RALLLY_DOMAIN
+          - SECRET_RALLLY_ID_VERSION
+          - SECRET_RALLLY_SECRET_VERSION
+          - rallly.png
+        secrets:
+            rallly_id: rallly
+    rallly:
+        env:
+            OIDC_NAME: "Authentik"
+            OIDC_DISCOVERY_URL: "https://authentik.example.com/application/o/rallly/.well-known/openid-configuration"
+            OIDC_ISSUER_URL: "https://authentik.example.com/application/o/rallly/"
+            OIDC_CLIENT_ID: rallly
+        uncomment:
+          - compose.oidc.yml
+          - SECRET_OIDC_CLIENT_SECRET_VERSION
+          - OIDC_ENABLED
+        shared_secrets:
+            authentik:
+                rallly_secret: oidc_client_secret
+integrate-authentik-hedgedoc:
+    apply-when:
+      - authentik
+      - hedgedoc
+    authentik:
+        uncomment:
+          - compose.hedgedoc.yml
+          - HEDGEDOC_DOMAIN
+          - SECRET_HEDGEDOC_ID_VERSION
+          - SECRET_HEDGEDOC_SECRET_VERSION
+          - hedgedoc.png
+        secrets:
+            hedgedoc_id: hedgedoc
+    hedgedoc:
+        env:
+            CMD_OAUTH2_USER_PROFILE_URL: https://authentik.example.com/application/o/userinfo/
+            CMD_OAUTH2_USER_PROFILE_USERNAME_ATTR: preferred_username
+            CMD_OAUTH2_USER_PROFILE_DISPLAY_NAME_ATTR: name
+            CMD_OAUTH2_USER_PROFILE_EMAIL_ATTR: email
+            CMD_OAUTH2_TOKEN_URL: https://authentik.example.com/application/o/token/
+            CMD_OAUTH2_AUTHORIZATION_URL: https://authentik.example.com/application/o/authorize/
+            # TODO: set CLIENT_ID as secret
+            CMD_OAUTH2_CLIENT_ID: hedgedoc
+            CMD_OAUTH2_PROVIDERNAME: Authentik
+        uncomment:
+          - compose.oauth.yml
+          - SECRET_OAUTH_KEY_VERSION
+        shared_secrets:
+            authentik:
+                hedgedoc_secret: oauth_key
+        dependency: [authentik]
+integrate-authentik-pretix:
+    apply-when:
+      - authentik
+      - pretix
+    authentik:
+        env:
+            APPLICATIONS:
+                Pretix:
+                    url: https://pretix.example.com/control/
+                    group:
+            EXTRA_ICONS:
+                Pretix: ~/.abra/recipes/authentik/icons/pretix.svg
+integrate-authentik-vaultwarden:
+    apply-when:
+      - authentik
+      - vaultwarden
+    authentik:
+        env:
+            APPLICATIONS:
+                Vaultwarden:
+                    url: https://vaultwarden.example.com/
+                    group:
+            EXTRA_ICONS:
+                Vaultwarden: ~/.abra/recipes/authentik/icons/vaultwarden.svg
+integrate-authentik-mila:
+    apply-when:
+      - authentik
+      - mila
+    authentik:
+        uncomment:
+          - compose.mila.yml
+          - MILA_DOMAIN
+          - MILA_GROUP
+          - SECRET_MILA_ID_VERSION
+          - SECRET_MILA_SECRET_VERSION
+          - mila.svg
+        secrets:
+            mila_id: mila
+    mila:
+        env:
+            OIDC_CLIENT_ID: mila
+            OIDC_BASE_URL: https://authentik.example.com/application/o/mila
+            OIDC_REDIRECT_URI: https://mila.example.com/auth/user/oidc/callback
+            OIDC_ADMIN_GROUP_NAME: mv_admin
+            OIDC_GROUPS_CLAIM: groups
+            OIDC_ONLY: "true"
+        uncomment:
+          - compose.oidc.yml
+          - SECRET_OIDC_CLIENT_SECRET_VERSION
+        shared_secrets:
+            authentik:
+                mila_secret: oidc_client_secret
+integrate-nextcloud-onlyoffice:
+    apply-when:
+      - nextcloud
+      - onlyoffice
+    nextcloud:
+        uncomment:
+          - compose.onlyoffice.yml
+          - ONLYOFFICE_URL
+          - SECRET_ONLYOFFICE_JWT_VERSION
+        initial-hooks:
+          - app install_onlyoffice
+    onlyoffice:
+        uncomment:
+          - compose.jwt.yml
+          - SECRET_JWT_SECRET_VERSION
+        shared_secrets:
+            nextcloud:
+                onlyoffice_jwt: jwt_secret
+integrate-collabora-nextcloud:
+    apply-when:
+      - collabora
+      - nextcloud
+    nextcloud:
+        uncomment:
+          - COLLABORA_URL
+        initial-hooks:
+          - app install_collabora
+integrate-matrix-synapse-traefik:
+    apply-when:
+      - matrix-synapse
+      - traefik
+    traefik:
+        uncomment:
+          - compose.matrix.yml
+          - MATRIX_FEDERATION_ENABLED

examples/config-sets.yml

@@ -0,0 +1,29 @@
+# config-sets.yml
+# Place this file at your alakazam root path.
+# Each top-level key is a named config-set containing per-app configurations.
+# Activate a set in any instance .yml or alaka.yml:
+#
+#   CONFIG-SETS:
+#     bbb: true
+#     calendar: true
+#
+# Disable a set inherited from a group alaka.yml by setting it to false:
+#
+#   CONFIG-SETS:
+#     bbb: false
+#
+
+bbb:
+  authentik:
+    env:
+      APPLICATIONS:
+        BBB: https://nextcloud.example.com/apps/bbb
+  nextcloud:
+    initial-hooks:
+      - app install_bbb
+
+calendar:
+  authentik:
+    env:
+      APPLICATIONS:
+        Calendar: https://nextcloud.example.com/apps/calendar/

examples/config.yml

@@ -1,18 +0,0 @@
-domain: example-domain.com
-server: example-server.com
-apps:
-    authentik:
-        env:
-            WELCOME_MESSAGE: "Welcome to CoopCloud"
-    wekan:
-    nextcloud:
-    onlyoffice:
-    wordpress:
-        subdomain: "blog.special-domain.com"
-    vikunja:
-        subdomain: "todo.example.com"
-    matrix-synapse:
-    element-web:
-        env:
-            SERVER_NAME: testmatrix
-            BRAND: testmatrix

examples/example.com.yml

@@ -0,0 +1,25 @@
+GLOBALS: 
+    smtp_password: my_secret_password
+    smtp_password_urlencoded: my_secret_password
+    smtp_user: support
+    smtp_domain: maildomain.com
+    smtp_host: maymailhost.com
+    bbb_secret: my_secret_password
+    turn_secret: my_secret_password
+    basic_auth_password: my_secret_password
+    basic_auth_hash: my_secret_password
+traefik:
+backup-bot-two:
+monitoring-ng:
+authentik:
+nextcloud:
+onlyoffice:
+wordpress:
+vikunja:
+wekan:
+matrix-synapse:
+element-web:
+rallly:
+outline:
+hedgedoc:
+pretix:

requirements-dev.txt

@@ -0,0 +1,2 @@
+-r requirements.txt
+pytest==9.1.1

requirements.txt

@@ -4,3 +4,7 @@ ruamel.yaml==0.18.3
 Jinja2==2.11.3
 MarkupSafe==1.1.1
 python-dotenv==1.0.0
+icecream==2.1.3
+packaging==24.0 
+GitPython==3.1.43
+uptime-kuma-api2==2.7.0

tests/test_backup.py

@@ -0,0 +1,22 @@
+"""Tests for the backupbot lookup of the backup command."""
+
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from click.testing import CliRunner
+
+
+class TestBackupBotLookup:
+    def test_a_server_without_a_backupbot_is_reported_not_crashed(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "get_server_apps",
+                            lambda apps=None, all=False: {} if all else {"a.org": ["login.a.org"]})
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        from click.testing import CliRunner
+        result = CliRunner().invoke(alakazam.backup, [], standalone_mode=False)
+        assert result.exit_code == 0, result.exception

tests/test_config_sets.py

@@ -0,0 +1,191 @@
+"""Tests for config-sets, in particular the ones that apply by themselves."""
+
+import logging
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import get_config_set_app_configs, merge_instance_configs, read_config_sets
+
+INTEGRATION = {
+    "integrate-authentik-nextcloud": {
+        "apply-when": ["authentik", "nextcloud"],
+        "authentik": {"uncomment": ["compose.nextcloud.yml"]},
+        "nextcloud": {"uncomment": ["compose.authentik.yml"],
+                      "shared_secrets": {"authentik": {"nextcloud_secret": "authentik_secret"}}},
+    }
+}
+
+
+def apply(config_sets, instance_apps, active=None):
+    """The effective configuration, with the automatic sets below the explicit ones."""
+    automatic, explicit = get_config_set_app_configs(active or {}, config_sets, instance_apps)
+    return alakazam.merge_dict(automatic, explicit)
+
+
+class TestApplyWhen:
+    def test_it_applies_when_every_app_is_there(self):
+        result = apply(INTEGRATION, ["authentik", "nextcloud", "traefik"])
+        assert result["authentik"]["uncomment"] == ["compose.nextcloud.yml"]
+        assert result["nextcloud"]["shared_secrets"] == {"authentik": {"nextcloud_secret": "authentik_secret"}}
+
+    def test_it_stays_out_when_one_app_is_missing(self):
+        assert apply(INTEGRATION, ["authentik", "traefik"]) == {}
+
+    def test_apply_when_is_not_passed_on_as_an_app(self):
+        """It is a condition, not configuration, and would end up in an .env otherwise."""
+        assert "apply-when" not in apply(INTEGRATION, ["authentik", "nextcloud"])
+
+    def test_an_instance_can_switch_it_off(self):
+        assert apply(INTEGRATION, ["authentik", "nextcloud"],
+                     active={"integrate-authentik-nextcloud": False}) == {}
+
+    def test_a_one_sided_integration_configures_only_one_app(self):
+        """4 of the 17 pairs in the shipped combine.yml were one-sided."""
+        sets = {"integrate-matrix-synapse-traefik": {
+            "apply-when": ["matrix-synapse", "traefik"],
+            "traefik": {"uncomment": ["compose.matrix.yml"]}}}
+        assert list(apply(sets, ["matrix-synapse", "traefik"])) == ["traefik"]
+        assert apply(sets, ["traefik"]) == {}
+
+
+class TestExplicitSets:
+    SET = {"bbb": {"authentik": {"env": {"A": "b"}}}}
+
+    def test_an_enabled_set_applies(self):
+        assert apply(self.SET, ["authentik"], active={"bbb": True})["authentik"]["env"] == {"A": "b"}
+
+    def test_a_set_that_is_not_enabled_does_not_apply(self):
+        assert apply(self.SET, ["authentik"]) == {}
+
+    def test_an_unknown_name_is_reported(self, caplog):
+        with caplog.at_level(logging.WARNING):
+            apply(self.SET, ["authentik"], active={"typo": True})
+        assert "'typo' is enabled but not defined" in caplog.text
+
+    def test_an_explicit_set_refines_an_automatic_one(self):
+        """Automatic integrations are the base, an explicit set may still override them."""
+        sets = dict(INTEGRATION)
+        sets["override"] = {"authentik": {"env": {"SOURCE": "explicit"}}}
+        result = apply(sets, ["authentik", "nextcloud"], active={"override": True})
+        assert result["authentik"]["env"] == {"SOURCE": "explicit"}
+        assert result["authentik"]["uncomment"] == ["compose.nextcloud.yml"]
+
+
+class TestReadConfigSets:
+    def write(self, root, name, text):
+        (root / name).write_text(text)
+
+    def test_the_base_file_is_read(self, tmp_path):
+        self.write(tmp_path, "config-sets.yml", "bbb:\n    authentik:\n        env:\n            A: b\n")
+        assert read_config_sets(tmp_path)["bbb"]["authentik"]["env"] == {"A": "b"}
+
+    def test_further_files_are_merged(self, tmp_path):
+        self.write(tmp_path, "config-sets.yml", "bbb:\n    authentik: {}\n")
+        self.write(tmp_path, "config-sets-authentik.yml", "integrate:\n    apply-when: [a, b]\n")
+        assert sorted(read_config_sets(tmp_path)) == ["bbb", "integrate"]
+
+    def test_the_base_file_is_merged_first(self, tmp_path):
+        self.write(tmp_path, "config-sets.yml", "bbb:\n    authentik:\n        env:\n            A: base\n")
+        self.write(tmp_path, "config-sets-later.yml", "bbb:\n    authentik:\n        env:\n            A: later\n")
+        assert read_config_sets(tmp_path)["bbb"]["authentik"]["env"]["A"] == "later"
+
+    def test_a_root_without_config_sets(self, tmp_path):
+        assert read_config_sets(tmp_path) == {}
+
+    def test_a_key_defined_twice_is_reported(self, tmp_path, caplog):
+        self.write(tmp_path, "config-sets.yml", "bbb:\n    authentik:\n        env:\n            A: base\n")
+        self.write(tmp_path, "config-sets-later.yml", "bbb:\n    authentik:\n        env:\n            A: later\n")
+        with caplog.at_level(logging.WARNING):
+            read_config_sets(tmp_path)
+        assert "bbb.authentik.env.A" in caplog.text
+
+
+class TestIntegrationReachesTheMergedConfig:
+    """The instance config is what every command reads, the integration has to arrive there."""
+
+    def merge(self, apps):
+        return merge_instance_configs({}, "example.com", {app: None for app in apps}, INTEGRATION)
+
+    def test_both_apps_get_their_side(self):
+        merged = self.merge(["authentik", "nextcloud"])
+        assert merged["authentik"]["uncomment"] == ["compose.nextcloud.yml"]
+        assert merged["nextcloud"]["uncomment"] == ["compose.authentik.yml"]
+
+    def test_a_lone_app_gets_nothing(self):
+        assert "uncomment" not in self.merge(["authentik"])["authentik"]
+
+
+class TestUnknownAppKeys:
+    """A misspelt app key drops its configuration, which is only visible if it is reported."""
+
+    def test_a_typo_in_an_automatic_set_is_reported(self, caplog):
+        sets = {"integrate": {"apply-when": ["authentik", "nextcloud"],
+                              "nextcoud": {"env": {"A": "b"}}}}
+        with caplog.at_level(logging.WARNING):
+            merge_instance_configs({}, "example.com", {"authentik": None, "nextcloud": None}, sets)
+        assert "'nextcoud' is not listed" in caplog.text
+
+    def test_a_typo_in_an_explicit_set_is_reported(self, caplog):
+        sets = {"bbb": {"nextcoud": {"env": {"A": "b"}}}}
+        with caplog.at_level(logging.WARNING):
+            merge_instance_configs({}, "example.com",
+                                   {"nextcloud": None, "CONFIG-SETS": {"bbb": True}}, sets)
+        assert "'nextcoud' is not listed" in caplog.text
+
+    def test_a_correct_app_key_is_not_reported(self, caplog):
+        sets = {"integrate": {"apply-when": ["authentik", "nextcloud"],
+                              "nextcloud": {"env": {"A": "b"}}}}
+        with caplog.at_level(logging.WARNING):
+            merge_instance_configs({}, "example.com", {"authentik": None, "nextcloud": None}, sets)
+        assert caplog.text == ""
+
+
+class TestPrecedence:
+    """An integration is the lowest layer, as combine.yml was before it."""
+
+    def test_the_group_configuration_beats_an_integration(self):
+        group = {"nextcloud": {"env": {"SOURCE": "group"}}}
+        sets = {"integrate": {"apply-when": ["authentik", "nextcloud"],
+                              "nextcloud": {"env": {"SOURCE": "integration"}}}}
+        merged = merge_instance_configs(group, "example.com", {"authentik": None, "nextcloud": None}, sets)
+        assert merged["nextcloud"]["env"]["SOURCE"] == "group"
+
+    def test_the_instance_beats_everything(self):
+        group = {"nextcloud": {"env": {"SOURCE": "group"}}}
+        sets = {"integrate": {"apply-when": ["authentik", "nextcloud"],
+                              "nextcloud": {"env": {"SOURCE": "integration"}}}}
+        merged = merge_instance_configs(
+            group, "example.com", {"authentik": None, "nextcloud": {"env": {"SOURCE": "instance"}}}, sets)
+        assert merged["nextcloud"]["env"]["SOURCE"] == "instance"
+
+    def test_an_integration_still_adds_what_nobody_else_sets(self):
+        group = {"nextcloud": {"env": {"OTHER": "group"}}}
+        sets = {"integrate": {"apply-when": ["authentik", "nextcloud"],
+                              "nextcloud": {"env": {"SOURCE": "integration"}}}}
+        merged = merge_instance_configs(group, "example.com", {"authentik": None, "nextcloud": None}, sets)
+        assert merged["nextcloud"]["env"] == {"SOURCE": "integration", "OTHER": "group"}
+
+
+class TestHookOrder:
+    """An integration adds to an app that already configures itself, and must come after it."""
+
+    SETS = {"integrate-authentik-wordpress": {
+        "apply-when": ["authentik", "wordpress"],
+        "wordpress": {"initial-hooks": ["app set_authentik"]}}}
+
+    def test_the_integration_hook_runs_last(self):
+        """set_authentik configures SSO in a WordPress that core_install has to create first."""
+        group = {"wordpress": {"initial-hooks": ["app core_install", "app enable_auto_updates"]}}
+        merged = merge_instance_configs(group, "example.com", {"authentik": None, "wordpress": None}, self.SETS)
+        assert merged["wordpress"]["initial-hooks"] == [
+            "app core_install", "app enable_auto_updates", "app set_authentik"]
+
+    def test_an_instance_hook_also_comes_first(self):
+        instance = {"authentik": None, "wordpress": {"initial-hooks": ["app from_instance"]}}
+        merged = merge_instance_configs({}, "example.com", instance, self.SETS)
+        assert merged["wordpress"]["initial-hooks"] == ["app from_instance", "app set_authentik"]

tests/test_dependencies.py

@@ -0,0 +1,150 @@
+"""Tests for the dependency ordering that replaces the special casing of authentik."""
+
+import logging
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import get_apps, group_apps_by_dependency, sort_apps_by_dependency
+
+
+def instance(**apps):
+    """Builds an instance configuration, values are the 'dependency' lists."""
+    return {name: {"app_domain": f"{name}.a.org", "dependency": list(dependencies)} for name, dependencies in apps.items()}
+
+
+class TestSortAppsByDependency:
+    def test_a_dependency_comes_first(self):
+        config = instance(nextcloud=["authentik"], authentik=[])
+        assert sort_apps_by_dependency(["nextcloud", "authentik"], config) == ["authentik", "nextcloud"]
+
+    def test_order_is_independent_of_the_input_order(self):
+        config = instance(nextcloud=["authentik"], authentik=[])
+        assert sort_apps_by_dependency(["authentik", "nextcloud"], config) == ["authentik", "nextcloud"]
+
+    def test_several_apps_share_one_dependency(self):
+        config = instance(nextcloud=["authentik"], wekan=["authentik"], authentik=[])
+        ordered = sort_apps_by_dependency(["nextcloud", "wekan", "authentik"], config)
+        assert ordered.index("authentik") == 0
+
+    def test_transitive_dependencies(self):
+        config = instance(c=["b"], b=["a"], a=[])
+        assert sort_apps_by_dependency(["c", "b", "a"], config) == ["a", "b", "c"]
+
+    def test_unrelated_apps_keep_their_configuration_order(self):
+        config = instance(traefik=[], authentik=[], nextcloud=[])
+        apps = ["traefik", "authentik", "nextcloud"]
+        assert sort_apps_by_dependency(apps, config) == apps
+
+    def test_apps_without_a_dependency_key(self):
+        config = {"authentik": {"app_domain": "login.a.org"}, "nextcloud": {"app_domain": "cloud.a.org"}}
+        assert sort_apps_by_dependency(["nextcloud", "authentik"], config) == ["nextcloud", "authentik"]
+
+    def test_every_app_is_returned_exactly_once(self):
+        config = instance(d=["a", "b"], c=["a"], b=["a"], a=[])
+        ordered = sort_apps_by_dependency(["d", "c", "b", "a"], config)
+        assert sorted(ordered) == ["a", "b", "c", "d"]
+        assert len(ordered) == 4
+
+    def test_empty_selection(self):
+        assert sort_apps_by_dependency([], instance(a=[])) == []
+
+
+class TestGroupAppsByDependency:
+    """Levels express what a flat order cannot: which apps may proceed without waiting."""
+
+    def test_apps_sharing_one_dependency_end_up_in_one_level(self):
+        config = instance(zammad=["authentik"], kimai=["authentik"], hedgedoc=["authentik"],
+                          authentik=[], traefik=[])
+        levels = group_apps_by_dependency(["zammad", "kimai", "hedgedoc", "authentik", "traefik"], config)
+        assert levels == [["authentik", "traefik"], ["zammad", "kimai", "hedgedoc"]]
+
+    def test_unrelated_apps_are_all_in_the_first_level(self):
+        config = instance(traefik=[], authentik=[], nextcloud=[])
+        assert group_apps_by_dependency(["traefik", "authentik", "nextcloud"], config) == [
+            ["traefik", "authentik", "nextcloud"]
+        ]
+
+    def test_a_transitive_chain_yields_one_level_each(self):
+        config = instance(c=["b"], b=["a"], a=[])
+        assert group_apps_by_dependency(["c", "b", "a"], config) == [["a"], ["b"], ["c"]]
+
+    def test_the_deepest_dependency_decides_the_level(self):
+        """An app waits for all of its dependencies, so it lands behind the latest of them."""
+        config = instance(d=["a", "c"], c=["b"], b=["a"], a=[])
+        assert group_apps_by_dependency(["d", "c", "b", "a"], config) == [["a"], ["b"], ["c"], ["d"]]
+
+    def test_configuration_order_is_kept_within_a_level(self):
+        config = instance(wekan=["authentik"], nextcloud=["authentik"], authentik=[])
+        assert group_apps_by_dependency(["wekan", "nextcloud", "authentik"], config)[1] == ["wekan", "nextcloud"]
+
+    def test_a_dependency_outside_the_selection_does_not_add_a_level(self):
+        config = instance(nextcloud=["authentik"], authentik=[])
+        assert group_apps_by_dependency(["nextcloud"], config) == [["nextcloud"]]
+
+    def test_empty_selection(self):
+        assert group_apps_by_dependency([], instance(a=[])) == []
+
+    def test_every_app_appears_exactly_once(self):
+        config = instance(d=["a", "b"], c=["a"], b=["a"], a=[])
+        levels = group_apps_by_dependency(["d", "c", "b", "a"], config)
+        assert sorted(app for level in levels for app in level) == ["a", "b", "c", "d"]
+
+    def test_a_cycle_is_rejected(self):
+        config = instance(nextcloud=["authentik"], authentik=["nextcloud"])
+        with pytest.raises(click.ClickException):
+            group_apps_by_dependency(["nextcloud", "authentik"], config)
+
+
+class TestCycles:
+    def test_a_cycle_is_rejected(self):
+        config = instance(nextcloud=["authentik"], authentik=["nextcloud"])
+        with pytest.raises(click.ClickException) as excinfo:
+            sort_apps_by_dependency(["nextcloud", "authentik"], config)
+        assert excinfo.value.exit_code != 0
+
+    def test_the_cycle_is_named(self):
+        config = instance(nextcloud=["authentik"], authentik=["nextcloud"])
+        with pytest.raises(click.ClickException) as excinfo:
+            sort_apps_by_dependency(["nextcloud", "authentik"], config)
+        assert "nextcloud -> authentik -> nextcloud" in excinfo.value.message
+
+    def test_a_longer_cycle_is_named(self):
+        config = instance(a=["c"], b=["a"], c=["b"])
+        with pytest.raises(click.ClickException) as excinfo:
+            sort_apps_by_dependency(["a", "b", "c"], config)
+        assert "a -> c -> b -> a" in excinfo.value.message
+
+    def test_an_app_depending_on_itself(self):
+        config = instance(a=["a"])
+        with pytest.raises(click.ClickException) as excinfo:
+            sort_apps_by_dependency(["a"], config)
+        assert "a -> a" in excinfo.value.message
+
+
+class TestDependenciesOutsideTheSelection:
+    def test_a_dependency_not_selected_is_skipped(self, caplog):
+        """Filtering a run down to a few recipes must not pull in anything else."""
+        config = instance(nextcloud=["authentik"], authentik=[])
+        with caplog.at_level(logging.WARNING):
+            assert sort_apps_by_dependency(["nextcloud"], config) == ["nextcloud"]
+        assert caplog.text == ""
+
+class TestGetAppsOrdering:
+    """The order has to reach the commands, not just the helper."""
+
+    def test_get_apps_returns_dependencies_first(self, monkeypatch):
+        config = instance(nextcloud=["authentik"], authentik=[])
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", {"a.org": config})
+        assert [app for app, _ in get_apps()["a.org"]] == ["authentik", "nextcloud"]
+
+    def test_an_explicit_recipe_filter_is_ordered_too(self, monkeypatch):
+        config = instance(nextcloud=["authentik"], authentik=[])
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", {"a.org": config})
+        apps = get_apps(("nextcloud", "authentik"))["a.org"]
+        assert [app for app, _ in apps] == ["authentik", "nextcloud"]

tests/test_env_updates.py

@@ -0,0 +1,76 @@
+"""Tests for how update_configs() applies a config's env/comment/uncomment keys to a .env file."""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from alakazam import update_configs
+
+
+class TestUnknownConfigKeys:
+    def test_an_unknown_key_never_reaches_the_env_file(self, tmp_path):
+        """A promotion mechanism sets 'hold: <reason>' on a recipe, which must pass through untouched."""
+        env = tmp_path / "app.env"
+        env.write_text("#SMTP_HOST=mail.example.com\n")
+        update_configs(env, {"hold": "waiting for the upstream fix", "env": {"SMTP_HOST": "mail.a.org"}})
+        content = env.read_text()
+        assert "SMTP_HOST=mail.a.org" in content
+        assert "hold" not in content
+        assert "waiting for the upstream fix" not in content
+
+    def test_an_unknown_key_alone_changes_nothing(self, tmp_path):
+        env = tmp_path / "app.env"
+        env.write_text("#SMTP_HOST=mail.example.com\n")
+        update_configs(env, {"hold": "waiting for the upstream fix"})
+        assert env.read_text() == "#SMTP_HOST=mail.example.com\n"
+
+
+class TestExtendingEnvKeys:
+    """
+    Some env keys are repeated on multiple lines, relying on bash accumulation
+    ("COMPOSE_FILE=\"$COMPOSE_FILE:compose.mariadb.yml\"") to build up a value.
+    A config that sets such a key should extend that accumulation, not replace
+    or uncomment every line sharing the key.
+    """
+
+    ACCUMULATING_ENV = (
+        'COMPOSE_FILE="compose.yml"\n'
+        'COMPOSE_FILE="$COMPOSE_FILE:compose.mariadb.yml"\n'
+        '# COMPOSE_FILE="$COMPOSE_FILE:compose.onlyoffice.yml"\n'
+    )
+
+    def test_a_key_sharing_a_prefix_is_not_matched_by_substring(self, tmp_path):
+        """COMPOSE must not be treated as matching lines whose key is COMPOSE_FILE."""
+        env = tmp_path / "app.env"
+        env.write_text(self.ACCUMULATING_ENV)
+        update_configs(env, {"env": {"COMPOSE": "compose.custom.yml"}})  # not self-referencing
+        lines = env.read_text().splitlines()
+        assert lines[:-1] == self.ACCUMULATING_ENV.splitlines()  # every original line untouched
+        assert lines[-1] == 'COMPOSE=compose.custom.yml'  # set as its own, unrelated key
+
+    def test_a_self_referencing_value_is_appended_without_touching_other_lines(self, tmp_path):
+        env = tmp_path / "app.env"
+        env.write_text(self.ACCUMULATING_ENV)
+        update_configs(env, {"env": {"COMPOSE_FILE": "$COMPOSE_FILE:../../customoverride.yml"}})
+        lines = env.read_text().splitlines()
+        assert lines[:-1] == self.ACCUMULATING_ENV.splitlines()  # every original line untouched
+        assert lines[-1] == 'COMPOSE_FILE="$COMPOSE_FILE:../../customoverride.yml"'  # appended
+
+    def test_setting_the_same_self_referencing_value_twice_is_idempotent(self, tmp_path):
+        env = tmp_path / "app.env"
+        env.write_text(self.ACCUMULATING_ENV)
+        config = {"env": {"COMPOSE_FILE": "$COMPOSE_FILE:../../customoverride.yml"}}
+        update_configs(env, config)
+        update_configs(env, config)
+        lines = env.read_text().splitlines()
+        assert lines.count('COMPOSE_FILE="$COMPOSE_FILE:../../customoverride.yml"') == 1
+
+    def test_a_self_referencing_value_matching_an_existing_commented_line_uncomments_it_in_place(self, tmp_path):
+        env = tmp_path / "app.env"
+        env.write_text(self.ACCUMULATING_ENV)
+        update_configs(env, {"env": {"COMPOSE_FILE": "$COMPOSE_FILE:compose.onlyoffice.yml"}})
+        lines = env.read_text().splitlines()
+        assert lines[:-1] == self.ACCUMULATING_ENV.splitlines()[:-1]  # untouched, incl. the already-active line
+        assert lines[-1] == 'COMPOSE_FILE="$COMPOSE_FILE:compose.onlyoffice.yml"'  # uncommented in place
+        assert len(lines) == len(self.ACCUMULATING_ENV.splitlines())  # no new line appended

tests/test_exclude_recipes.py

@@ -0,0 +1,89 @@
+"""Tests for leaving single recipes out of a run."""
+
+import logging
+import os
+import sys
+
+import pytest
+from click.testing import CliRunner
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import exclude_from_configs
+
+CONFIGS = {
+    "a.org": {"authentik": {"app_domain": "login.a.org"}, "traefik": {"app_domain": "a.org"}},
+    "b.org": {"traefik": {"app_domain": "b.org"}, "nextcloud": {"app_domain": "cloud.b.org"}},
+}
+
+
+class TestExcludeFromConfigs:
+    def test_nothing_excluded_returns_the_configs_unchanged(self):
+        assert exclude_from_configs(CONFIGS, ()) is CONFIGS
+
+    def test_the_recipe_is_gone_from_every_instance(self):
+        result = exclude_from_configs(CONFIGS, ("traefik",))
+        assert sorted(result["a.org"]) == ["authentik"]
+        assert sorted(result["b.org"]) == ["nextcloud"]
+
+    def test_several_recipes(self):
+        result = exclude_from_configs(CONFIGS, ("traefik", "nextcloud"))
+        assert result["b.org"] == {}
+
+    def test_the_original_is_not_modified(self):
+        exclude_from_configs(CONFIGS, ("traefik",))
+        assert "traefik" in CONFIGS["a.org"]
+
+    def test_an_unknown_recipe_is_reported(self, caplog):
+        """A typo would otherwise exclude nothing and look like it worked."""
+        with caplog.at_level(logging.WARNING):
+            exclude_from_configs(CONFIGS, ("treafik",))
+        assert "'treafik' is excluded but not configured" in caplog.text
+
+    def test_a_known_recipe_is_not_reported(self, caplog):
+        with caplog.at_level(logging.WARNING):
+            exclude_from_configs(CONFIGS, ("traefik",))
+        assert caplog.text == ""
+
+
+class TestExcludeReachesTheCommands:
+    """One filter in cli() has to cover every consumer of INSTANCE_CONFIGS."""
+
+    @pytest.fixture
+    def env(self, tmp_path, monkeypatch):
+        root = tmp_path / "root"
+        (root / "group").mkdir(parents=True)
+        (root / "alaka.yml").write_text(
+            "authentik:\n    version: 1.0.0\ntraefik:\n    version: 2.0.0\n")
+        (root / "group" / "example.com.yml").write_text("authentik:\ntraefik:\n")
+        (root / "alakazam.yml").write_text(f"root: {root}\n")
+        monkeypatch.setattr(alakazam, "get_settings_path", lambda: str(root / "alakazam.yml"))
+        monkeypatch.setattr(alakazam, "get_abra_dir", lambda: root / ".abra")
+        monkeypatch.setattr(alakazam, "fetch_recipes", lambda *a, **k: None)
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        return root
+
+    def invoke(self, root, args):
+        return CliRunner().invoke(alakazam.cli, args + [str(root / "group"), "ls"])
+
+    def test_without_the_flag_both_recipes_are_listed(self, env):
+        result = self.invoke(env, [])
+        assert result.exit_code == 0, result.output
+        assert "authentik" in result.output
+        assert "traefik" in result.output
+
+    def test_the_excluded_recipe_disappears(self, env):
+        result = self.invoke(env, ["-er", "traefik"])
+        assert result.exit_code == 0, result.output
+        assert "authentik" in result.output
+        assert "traefik" not in result.output
+
+    def test_the_long_option_works_too(self, env):
+        result = self.invoke(env, ["--exclude-recipe", "traefik"])
+        assert "traefik" not in result.output
+
+    def test_it_can_be_given_more_than_once(self, env):
+        result = self.invoke(env, ["-er", "traefik", "-er", "authentik"])
+        assert "traefik" not in result.output
+        assert "authentik" not in result.output

tests/test_fetch.py

@@ -0,0 +1,148 @@
+"""Tests for the recipe refresh that keeps the offline abra calls working on current recipes."""
+
+import os
+import sys
+from pathlib import Path
+
+import pytest
+from git import Repo
+from git.exc import GitCommandError
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import fetch_recipes, sync_repo
+
+INSTANCE_CONFIGS = {"a.org": {"authentik": {}, "nextcloud": {}}}
+
+
+def commit(repo, name, content="x"):
+    """Adds a file and commits it, returning the new commit."""
+    path = Path(repo.working_dir) / name
+    path.write_text(content)
+    repo.index.add([name])
+    return repo.index.commit(f"add {name}")
+
+
+@pytest.fixture
+def remote_and_clone(tmp_path):
+    """A bare-ish upstream repository plus a clone of it, as abra would leave one behind."""
+    upstream = Repo.init(tmp_path / "upstream", initial_branch="main")
+    upstream.config_writer().set_value("user", "name", "test").release()
+    upstream.config_writer().set_value("user", "email", "test@example.org").release()
+    commit(upstream, "compose.yml")
+    clone = Repo.clone_from(str(tmp_path / "upstream"), str(tmp_path / "clone"))
+    clone.config_writer().set_value("user", "name", "test").release()
+    clone.config_writer().set_value("user", "email", "test@example.org").release()
+    return upstream, clone
+
+
+class TestSyncRepo:
+    def test_fast_forwards_to_the_remote(self, remote_and_clone):
+        upstream, clone = remote_and_clone
+        commit(upstream, "abra.sh")
+        sync_repo(Path(clone.working_dir))
+        assert (Path(clone.working_dir) / "abra.sh").exists()
+
+    def test_fetches_tags(self, remote_and_clone):
+        """A pinned recipe version is a tag, so abra needs them even without a fast-forward."""
+        upstream, clone = remote_and_clone
+        upstream.create_tag("1.2.3")
+        sync_repo(Path(clone.working_dir))
+        assert "1.2.3" in [tag.name for tag in clone.tags]
+
+    def test_writes_the_fetch_head_marker(self, remote_and_clone):
+        _, clone = remote_and_clone
+        marker = Path(clone.working_dir) / ".git" / "FETCH_HEAD"
+        marker.unlink(missing_ok=True)
+        sync_repo(Path(clone.working_dir))
+        assert marker.exists()
+
+    def test_a_checked_out_version_is_left_alone(self, remote_and_clone):
+        """abra checks a pinned version out itself, moving it here would fight with that."""
+        upstream, clone = remote_and_clone
+        upstream.create_tag("1.2.3")
+        sync_repo(Path(clone.working_dir))
+        clone.git.checkout("1.2.3")
+        commit(upstream, "abra.sh")
+        sync_repo(Path(clone.working_dir))
+        assert clone.head.is_detached
+        assert not (Path(clone.working_dir) / "abra.sh").exists()
+
+    def test_a_dirty_repository_is_left_alone(self, remote_and_clone, caplog):
+        upstream, clone = remote_and_clone
+        (Path(clone.working_dir) / "compose.yml").write_text("local change")
+        commit(upstream, "abra.sh")
+        sync_repo(Path(clone.working_dir))
+        assert (Path(clone.working_dir) / "compose.yml").read_text() == "local change"
+        assert "uncommitted changes" in caplog.text
+
+    def test_a_repository_without_a_remote_is_skipped(self, tmp_path):
+        repo = Repo.init(tmp_path / "solo", initial_branch="main")
+        repo.config_writer().set_value("user", "name", "test").release()
+        repo.config_writer().set_value("user", "email", "test@example.org").release()
+        commit(repo, "compose.yml")
+        sync_repo(tmp_path / "solo")
+
+    def test_an_unreachable_remote_raises(self, tmp_path, remote_and_clone):
+        """fetch_recipes has to catch this, a broken forge must not end the run."""
+        _, clone = remote_and_clone
+        clone.remotes[0].set_url(str(tmp_path / "gone"))
+        with pytest.raises(GitCommandError):
+            sync_repo(Path(clone.working_dir))
+
+
+class TestFetchRecipes:
+    @pytest.fixture
+    def abra_dir(self, tmp_path, monkeypatch):
+        monkeypatch.setattr(alakazam, "ABRA_DIR", tmp_path)
+        synced = []
+        monkeypatch.setattr(alakazam, "sync_repo", synced.append)
+        for name in ("catalogue", "recipes/authentik", "recipes/nextcloud"):
+            (tmp_path / name / ".git").mkdir(parents=True)
+        return synced
+
+    def test_syncs_the_catalogue_and_every_used_recipe(self, tmp_path, abra_dir):
+        """The catalogue feeds 'abra recipe versions', leaving it out freezes version lookups."""
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert [p.name for p in abra_dir] == ["catalogue", "authentik", "nextcloud"]
+
+    def test_a_recently_fetched_repository_is_skipped(self, tmp_path, abra_dir):
+        (tmp_path / "recipes/authentik/.git/FETCH_HEAD").touch()
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert "authentik" not in [p.name for p in abra_dir]
+
+    def test_a_stale_marker_does_not_skip(self, tmp_path, abra_dir):
+        marker = tmp_path / "recipes/authentik/.git/FETCH_HEAD"
+        marker.touch()
+        os.utime(marker, (0, 0))
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert "authentik" in [p.name for p in abra_dir]
+
+    def test_the_marker_is_written_even_without_a_remote(self, tmp_path, abra_dir):
+        """A fetch writes FETCH_HEAD itself, this is the fallback for the paths that do not fetch."""
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert (tmp_path / "recipes/authentik/.git/FETCH_HEAD").exists()
+
+    def test_a_failing_repository_does_not_stop_the_others(self, tmp_path, monkeypatch):
+        monkeypatch.setattr(alakazam, "ABRA_DIR", tmp_path)
+        synced = []
+
+        def failing(path):
+            synced.append(path)
+            if path.name == "authentik":
+                raise GitCommandError("git fetch", 128, b"could not resolve host")
+
+        monkeypatch.setattr(alakazam, "sync_repo", failing)
+        for name in ("catalogue", "recipes/authentik", "recipes/nextcloud"):
+            (tmp_path / name / ".git").mkdir(parents=True)
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert [p.name for p in synced] == ["catalogue", "authentik", "nextcloud"]
+
+    def test_a_recipe_that_was_never_cloned_is_tolerated(self, tmp_path, monkeypatch):
+        """abra clones a missing recipe itself, EnsureExists runs before its offline check."""
+        monkeypatch.setattr(alakazam, "ABRA_DIR", tmp_path)
+        monkeypatch.setattr(alakazam, "sync_repo", alakazam.sync_repo)
+        (tmp_path / "catalogue" / ".git").mkdir(parents=True)
+        fetch_recipes(INSTANCE_CONFIGS)
+        assert not (tmp_path / "recipes/authentik").exists()

tests/test_group_configs.py

@@ -0,0 +1,194 @@
+"""Tests for the group configuration merge order and its collision reporting."""
+
+import logging
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from alakazam import (
+    deduplicate,
+    iter_leaf_paths,
+    merge_all_group_configs,
+    sort_group_config_files,
+    warn_on_key_collisions,
+)
+
+
+def write_config(directory, name, content):
+    """Writes a group configuration file and returns its path."""
+    path = directory / name
+    path.write_text(content)
+    return path
+
+
+class TestSortGroupConfigFiles:
+    def test_selects_only_group_configs(self):
+        files = ["alaka.yml", "example.com.yml", "config-sets.yml", "alaka-smtp.yaml", "notes.md"]
+        assert sort_group_config_files(files) == ["alaka.yml", "alaka-smtp.yaml"]
+
+    def test_base_config_comes_first(self):
+        files = ["alaka-versions.yml", "alaka.yml", "alaka-defaults.yml"]
+        assert sort_group_config_files(files) == ["alaka.yml", "alaka-defaults.yml", "alaka-versions.yml"]
+
+    def test_order_is_independent_of_the_input_order(self):
+        files = ["alaka.yml", "alaka-a.yml", "alaka-b.yml"]
+        assert sort_group_config_files(files) == sort_group_config_files(list(reversed(files)))
+
+    def test_empty_listing(self):
+        assert sort_group_config_files([]) == []
+
+
+class TestIterLeafPaths:
+    def test_nested_dicts_become_key_paths(self):
+        leaves = dict(iter_leaf_paths({"authentik": {"env": {"SMTP_HOST": "mail.example.com"}}}))
+        assert leaves == {("authentik", "env", "SMTP_HOST"): "mail.example.com"}
+
+    def test_lists_are_leaves(self):
+        leaves = dict(iter_leaf_paths({"authentik": {"initial-hooks": ["app one", "app two"]}}))
+        assert leaves == {("authentik", "initial-hooks"): ["app one", "app two"]}
+
+
+class TestDeduplicate:
+    def test_keeps_the_first_occurrence(self):
+        assert deduplicate(["a", "b", "a", "c", "b"]) == ["a", "b", "c"]
+
+    def test_lists_of_mappings(self):
+        """A readiness-hooks entry is a mapping, which is unhashable and needs equality."""
+        hooks = [{"cmd": "app one"}, {"cmd": "app two"}, {"cmd": "app one"}]
+        assert deduplicate(hooks) == [{"cmd": "app one"}, {"cmd": "app two"}]
+
+    def test_mappings_differing_in_one_field_are_kept(self):
+        hooks = [{"cmd": "app one", "retries": 1}, {"cmd": "app one", "retries": 2}]
+        assert deduplicate(hooks) == hooks
+
+    def test_empty_list(self):
+        assert deduplicate([]) == []
+
+
+class TestMergeOrder:
+    def test_specialisations_override_the_base_config(self, tmp_path):
+        write_config(tmp_path, "alaka.yml", "authentik:\n    version: 1.0.0\n")
+        write_config(tmp_path, "alaka-versions.yml", "authentik:\n    version: 2.0.0\n")
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(tmp_path)]["authentik"]["version"] == "2.0.0"
+
+    def test_later_specialisation_wins_alphabetically(self, tmp_path):
+        write_config(tmp_path, "alaka-a.yml", "authentik:\n    version: 1.0.0\n")
+        write_config(tmp_path, "alaka-b.yml", "authentik:\n    version: 2.0.0\n")
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(tmp_path)]["authentik"]["version"] == "2.0.0"
+
+    def test_result_is_stable_across_directory_orderings(self, tmp_path):
+        """The same content laid down in a different creation order must merge identically."""
+        results = []
+        for order in (("alaka.yml", "alaka-smtp.yml"), ("alaka-smtp.yml", "alaka.yml")):
+            directory = tmp_path / "_".join(order)
+            directory.mkdir()
+            contents = {
+                "alaka.yml": "authentik:\n    version: 1.0.0\n    env:\n        SMTP_HOST: base\n",
+                "alaka-smtp.yml": "authentik:\n    env:\n        SMTP_HOST: special\n",
+            }
+            for name in order:
+                write_config(directory, name, contents[name])
+            results.append(merge_all_group_configs(directory)[str(directory)])
+        assert results[0] == results[1]
+        assert results[0]["authentik"]["env"]["SMTP_HOST"] == "special"
+
+    def test_subdirectory_inherits_and_overrides_the_parent(self, tmp_path):
+        write_config(tmp_path, "alaka.yml", "authentik:\n    version: 1.0.0\n    server: parent\n")
+        child = tmp_path / "group"
+        child.mkdir()
+        write_config(child, "alaka.yml", "authentik:\n    version: 2.0.0\n")
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(child)]["authentik"] == {"version": "2.0.0", "server": "parent"}
+
+    def test_directory_without_group_config_inherits_the_parent(self, tmp_path):
+        write_config(tmp_path, "alaka.yml", "authentik:\n    version: 1.0.0\n")
+        child = tmp_path / "group"
+        child.mkdir()
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(child)] == merged[str(tmp_path)]
+
+    def test_lists_of_sibling_files_are_concatenated_in_merge_order(self, tmp_path):
+        write_config(tmp_path, "alaka.yml", "authentik:\n    initial-hooks:\n        - app first\n")
+        write_config(tmp_path, "alaka-extra.yml", "authentik:\n    initial-hooks:\n        - app second\n")
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(tmp_path)]["authentik"]["initial-hooks"] == ["app first", "app second"]
+
+
+class TestMergingMappingLists:
+    """'readiness-hooks' is the first configuration key whose list holds mappings."""
+
+    def test_a_child_directory_adds_to_the_parents_hooks(self, tmp_path):
+        write_config(tmp_path, "alaka.yml",
+                     "authentik:\n    readiness-hooks:\n        - cmd: app check_blueprints\n          retries: 20\n")
+        child = tmp_path / "group"
+        child.mkdir()
+        write_config(child, "alaka.yml",
+                     "authentik:\n    readiness-hooks:\n        - cmd: app check_users\n")
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(child)]["authentik"]["readiness-hooks"] == [
+            {"cmd": "app check_blueprints", "retries": 20},
+            {"cmd": "app check_users"},
+        ]
+
+    def test_an_identical_hook_is_not_repeated(self, tmp_path):
+        hook = "authentik:\n    readiness-hooks:\n        - cmd: app check_blueprints\n"
+        write_config(tmp_path, "alaka.yml", hook)
+        child = tmp_path / "group"
+        child.mkdir()
+        write_config(child, "alaka.yml", hook)
+        merged = merge_all_group_configs(tmp_path)
+        assert merged[str(child)]["authentik"]["readiness-hooks"] == [{"cmd": "app check_blueprints"}]
+
+
+class TestKeyCollisions:
+    def test_conflicting_scalar_is_reported(self, caplog):
+        defined_by = {}
+        warn_on_key_collisions({"authentik": {"version": "1.0.0"}}, defined_by, "alaka.yml", "/root")
+        with caplog.at_level(logging.WARNING):
+            warn_on_key_collisions({"authentik": {"version": "2.0.0"}}, defined_by, "alaka-versions.yml", "/root")
+        assert "authentik.version" in caplog.text
+        assert "alaka-versions.yml takes precedence" in caplog.text
+
+    def test_concatenated_lists_are_not_reported(self, caplog):
+        """Composing one list from several files is intentional, not an override."""
+        defined_by = {}
+        warn_on_key_collisions({"authentik": {"initial-hooks": ["app a"]}}, defined_by, "alaka.yml", "/root")
+        with caplog.at_level(logging.WARNING):
+            warn_on_key_collisions({"authentik": {"initial-hooks": ["app b"]}}, defined_by, "alaka-extra.yml", "/root")
+        assert caplog.text == ""
+
+    def test_a_list_replacing_a_scalar_is_reported(self, caplog):
+        """merge_dict only concatenates when both sides are lists, otherwise the value is replaced."""
+        defined_by = {}
+        warn_on_key_collisions({"authentik": {"version": "1.0.0"}}, defined_by, "alaka.yml", "/root")
+        with caplog.at_level(logging.WARNING):
+            warn_on_key_collisions({"authentik": {"version": ["2.0.0"]}}, defined_by, "alaka-versions.yml", "/root")
+        assert "authentik.version" in caplog.text
+
+    def test_identical_values_are_not_reported(self, caplog):
+        defined_by = {}
+        warn_on_key_collisions({"authentik": {"version": "1.0.0"}}, defined_by, "alaka.yml", "/root")
+        with caplog.at_level(logging.WARNING):
+            warn_on_key_collisions({"authentik": {"version": "1.0.0"}}, defined_by, "alaka-versions.yml", "/root")
+        assert caplog.text == ""
+
+    def test_distinct_keys_are_not_reported(self, caplog):
+        defined_by = {}
+        warn_on_key_collisions({"authentik": {"version": "1.0.0"}}, defined_by, "alaka.yml", "/root")
+        with caplog.at_level(logging.WARNING):
+            warn_on_key_collisions({"nextcloud": {"version": "2.0.0"}}, defined_by, "alaka-nextcloud.yml", "/root")
+        assert caplog.text == ""
+
+    def test_overriding_a_parent_directory_is_not_reported(self, tmp_path, caplog):
+        write_config(tmp_path, "alaka.yml", "authentik:\n    version: 1.0.0\n")
+        child = tmp_path / "group"
+        child.mkdir()
+        write_config(child, "alaka.yml", "authentik:\n    version: 2.0.0\n")
+        with caplog.at_level(logging.WARNING):
+            merge_all_group_configs(tmp_path)
+        assert caplog.text == ""

tests/test_hide_secrets.py

@@ -0,0 +1,186 @@
+"""Tests for keeping secret values out of the output of a run whose logs are kept."""
+
+import logging
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import generate_all_secrets, insert_secret, is_secret_command
+
+VALUE = "hunter2-do-not-log-me"
+
+
+@pytest.fixture(params=[False, True], ids=["visible", "hidden"])
+def hide(request, monkeypatch):
+    monkeypatch.setattr(alakazam, "HIDE_SECRETS", request.param)
+    return request.param
+
+
+class TestIsSecretCommand:
+    @pytest.mark.parametrize("args", [
+        ("app", "secret", "insert", "login.a.org", "db_password", "v1", VALUE),
+        ("app", "secret", "generate", "-a", "login.a.org"),
+    ])
+    def test_commands_carrying_a_value(self, args):
+        assert is_secret_command(args)
+
+    @pytest.mark.parametrize("args", [
+        ("app", "secret", "ls", "login.a.org"),
+        ("app", "secret", "rm", "login.a.org", "db_password"),
+        ("app", "ls"),
+        ("app", "secret"),
+    ])
+    def test_commands_without_a_value(self, args):
+        assert not is_secret_command(args)
+
+
+class TestGeneratedValues:
+    def install(self, monkeypatch, created=False):
+        def abra(*args, **kwargs):
+            if args[:3] == ("app", "secret", "ls"):
+                return [{"name": "db_password", "created on server": str(created).lower()}]
+            if args[:3] == ("app", "secret", "generate"):
+                return [{"name": "db_password", "value": VALUE}]
+            raise AssertionError(f"unexpected: {args}")
+        monkeypatch.setattr(alakazam, "abra", abra)
+
+    def test_the_name_is_always_reported(self, hide, monkeypatch, capsys):
+        self.install(monkeypatch)
+        generate_all_secrets("login.a.org")
+        assert "db_password" in capsys.readouterr().out
+
+    def test_the_value_follows_the_switch(self, hide, monkeypatch, capsys):
+        self.install(monkeypatch)
+        generate_all_secrets("login.a.org")
+        out = capsys.readouterr().out
+        assert (VALUE in out) is not hide
+        assert ("[hidden]" in out) is hide
+
+
+class TestDebugLog:
+    """-l debug is the level a pipeline reaches for when something breaks."""
+
+    def run_abra(self, monkeypatch, args):
+        class Process:
+            returncode = 0
+            stdout = f'[{{"name":"db_password","value":"{VALUE}"}}]'.encode()
+            stderr = b""
+        monkeypatch.setattr(alakazam.subprocess, "run", lambda cmd, capture_output: Process())
+        return alakazam.abra(*args)
+
+    def test_the_command_line_is_not_logged_when_hidden(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        with caplog.at_level(logging.DEBUG):
+            self.run_abra(monkeypatch, ("app", "secret", "insert", "login.a.org", "db_password", "v1", VALUE))
+        assert VALUE not in caplog.text
+
+    def test_the_generated_output_is_not_logged_when_hidden(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        with caplog.at_level(logging.DEBUG):
+            self.run_abra(monkeypatch, ("app", "secret", "generate", "-a", "login.a.org"))
+        assert VALUE not in caplog.text
+
+    def test_other_commands_are_still_logged(self, monkeypatch, caplog):
+        """Hiding secrets must not turn the debug log off altogether."""
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        with caplog.at_level(logging.DEBUG):
+            self.run_abra(monkeypatch, ("app", "ls"))
+        assert "run command" in caplog.text
+
+    def test_insert_logs_the_name_but_not_the_value(self, monkeypatch, caplog):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        with caplog.at_level(logging.DEBUG):
+            insert_secret("login.a.org", "db_password", VALUE)
+        assert "db_password" in caplog.text
+        assert VALUE not in caplog.text
+
+
+class TestLocalHookOutput:
+    """A recipe may print the secret it created, vaultwarden's admin token does exactly that."""
+
+    TOKEN = "vaultwarden-admin-token-in-plain"
+
+    def install(self, monkeypatch, fail=False):
+        seen = {}
+
+        class Process:
+            returncode = 1 if fail else 0
+            stdout = TestLocalHookOutput.TOKEN.encode()
+            stderr = b""
+
+        def run(cmd, capture_output=False, **kwargs):
+            seen["capture_output"] = capture_output
+            return Process()
+
+        monkeypatch.setattr(alakazam.subprocess, "run", run)
+        def streamed(cmd):
+            seen["streamed"] = True
+            return Process()
+
+        monkeypatch.setattr(alakazam, "run_streamed", streamed)
+        return seen
+
+    def hook(self, strict=False):
+        alakazam.run_secret_hooks("login.a.org", {"secret_hooks": ["insert_admin_token"], "server": "a.org"}, strict=strict)
+
+    def test_the_hook_is_streamed_when_not_hiding(self, monkeypatch):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", False)
+        seen = self.install(monkeypatch)
+        self.hook()
+        assert seen.get("streamed")
+
+    def test_the_hook_is_not_streamed_when_hiding(self, monkeypatch, capsys):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        seen = self.install(monkeypatch)
+        self.hook()
+        assert not seen.get("streamed")
+        assert self.TOKEN not in capsys.readouterr().out
+
+    def test_a_failing_hook_does_not_leak_through_the_error(self, monkeypatch):
+        """The captured output lands in the exception, which is where it would escape."""
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        self.install(monkeypatch, fail=True)
+        with pytest.raises(alakazam.click.ClickException) as excinfo:
+            self.hook(strict=True)
+        assert self.TOKEN not in excinfo.value.message
+        assert "withheld by --hide-secrets" in excinfo.value.message
+
+    def test_a_failing_hook_still_reports_its_output_when_not_hiding(self, monkeypatch):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", False)
+        self.install(monkeypatch, fail=True)
+        with pytest.raises(alakazam.click.ClickException) as excinfo:
+            self.hook(strict=True)
+        assert self.TOKEN in excinfo.value.message
+
+
+class TestLocalScriptOutput:
+    def install(self, monkeypatch, tmp_path):
+        script = tmp_path / "hook.sh"
+        script.write_text("#!/bin/sh\necho secret\n")
+        script.chmod(0o755)
+        seen = {}
+
+        class Result:
+            returncode = 0
+
+        monkeypatch.setattr(alakazam, "resolve_path", lambda p, base=None: script)
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda cmd, **kw: seen.update(kw) or Result())
+        return seen
+
+    def test_the_script_output_is_captured_when_hiding(self, monkeypatch, tmp_path):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", True)
+        seen = self.install(monkeypatch, tmp_path)
+        alakazam.run_local_script(["script", "hook.sh"], "login.a.org", "a.org", "a.org")
+        assert seen["capture_output"] is True
+
+    def test_the_script_output_is_passed_through_when_not_hiding(self, monkeypatch, tmp_path):
+        monkeypatch.setattr(alakazam, "HIDE_SECRETS", False)
+        seen = self.install(monkeypatch, tmp_path)
+        alakazam.run_local_script(["script", "hook.sh"], "login.a.org", "a.org", "a.org")
+        assert seen["capture_output"] is False

tests/test_jinja_substitution.py

@@ -0,0 +1,33 @@
+"""Tests for substitute_jinja_variable()'s recursion into dicts, lists, and scalars."""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from alakazam import substitute_jinja_variable
+
+
+class TestSubstituteJinjaVariable:
+    def test_a_scalar_in_a_dict_substitutes(self):
+        config = {"nextcloud": {"env": {"DOMAIN": "{{ domain }}"}}}
+        substitute_jinja_variable(config, {"domain": "nc.example.com"})
+        assert config["nextcloud"]["env"]["DOMAIN"] == "nc.example.com"
+
+    def test_a_scalar_in_a_list_substitutes(self):
+        config = {"nextcloud": {"initial-hooks": [
+            "app install_apps",
+            "app set_app_config files default_quota {{ quota }}",
+            "app run_occ",
+        ]}}
+        substitute_jinja_variable(config, {"quota": "0"})
+        hooks = config["nextcloud"]["initial-hooks"]
+        assert isinstance(hooks, list) and len(hooks) == 3
+        assert hooks[0] == "app install_apps"
+        assert "0" in hooks[1]
+        assert hooks[2] == "app run_occ"
+
+    def test_a_scalar_in_a_list_without_a_template_is_left_untouched(self):
+        config = {"nextcloud": {"initial-hooks": ["app run_occ 'app:install groupfolders'"]}}
+        substitute_jinja_variable(config, {"quota": "0"})
+        assert config["nextcloud"]["initial-hooks"] == ["app run_occ 'app:install groupfolders'"]

tests/test_local_scripts.py

@@ -0,0 +1,184 @@
+"""Tests for local script functionality in hooks."""
+
+import logging
+import os
+import stat
+import sys
+
+import click
+import pytest
+from click.testing import CliRunner
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import (create_secrets, deploy_apps, execute_cmds, get_abra_dir,
+                       resolve_path, run_local_script, run_secret_hooks)
+
+
+class FakeProcess:
+    def __init__(self, returncode):
+        self.returncode = returncode
+
+
+def make_script(tmp_path, name="script.sh", executable=True):
+    """Write a script file under tmp_path, executable by default."""
+    path = tmp_path / name
+    path.write_text("#!/bin/sh\nexit 0\n")
+    if executable:
+        path.chmod(path.stat().st_mode | stat.S_IXUSR)
+    return path
+
+
+class TestResolvePath:
+    def test_an_absolute_path_passes_through_unchanged(self, tmp_path):
+        absolute = tmp_path / "script.sh"
+        assert resolve_path(str(absolute)) == absolute
+
+    def test_a_tilde_path_expands_against_home(self, monkeypatch, tmp_path):
+        monkeypatch.setenv("HOME", str(tmp_path))
+        assert resolve_path("~/script.sh") == tmp_path / "script.sh"
+
+    def test_a_relative_path_resolves_against_the_given_base(self, tmp_path):
+        base = tmp_path / "instance"
+        assert resolve_path("scripts/script.sh", base) == base / "scripts/script.sh"
+
+    def test_a_relative_path_resolves_against_root_path_by_default(self, monkeypatch, tmp_path):
+        monkeypatch.setattr(alakazam, "ROOT_PATH", tmp_path)
+        assert resolve_path("scripts/script.sh") == tmp_path / "scripts/script.sh"
+
+
+class TestRunLocalScript:
+    def test_the_script_receives_its_env_vars_and_arguments(self, monkeypatch, tmp_path):
+        script = make_script(tmp_path)
+        calls = []
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda cmd, env, **kwargs: calls.append((cmd, env)) or FakeProcess(0))
+        run_local_script(["script", str(script), "arg1"], "login.a.org", "a.org", "a.org")
+        [(cmd, env)] = calls
+        assert cmd == [str(script), "arg1"]
+        assert env["ALAKAZAM_APP_DOMAIN"] == "login.a.org"
+        assert env["ALAKAZAM_APP_SERVER"] == "a.org"
+        assert env["ALAKAZAM_INSTANCE_DOMAIN"] == "a.org"
+
+    def test_a_non_executable_script_is_not_run(self, monkeypatch, tmp_path, caplog):
+        script = make_script(tmp_path, executable=False)
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda *a, **k: pytest.fail("a non-executable script must not run"))
+        with caplog.at_level(logging.ERROR):
+            run_local_script(["script", str(script)], "login.a.org", "a.org", "a.org")
+        assert "not executable" in caplog.text
+
+    def test_dry_run_does_not_execute_the_script(self, monkeypatch, tmp_path, capsys):
+        script = make_script(tmp_path)
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda *a, **k: pytest.fail("dry_run must not execute anything"))
+        run_local_script(["script", str(script)], "login.a.org", "a.org", "a.org", dry_run=True)
+        assert "Run local script" in capsys.readouterr().out
+
+    def test_a_non_zero_exit_only_warns_by_default(self, monkeypatch, tmp_path, caplog):
+        script = make_script(tmp_path)
+        monkeypatch.setattr(alakazam.subprocess, "run", lambda *a, **k: FakeProcess(1))
+        with caplog.at_level(logging.WARNING):
+            run_local_script(["script", str(script)], "login.a.org", "a.org", "a.org")
+        assert "exited with code 1" in caplog.text
+
+    def test_a_non_zero_exit_aborts_in_strict_mode(self, monkeypatch, tmp_path):
+        script = make_script(tmp_path)
+        monkeypatch.setattr(alakazam.subprocess, "run", lambda *a, **k: FakeProcess(1))
+        with pytest.raises(click.ClickException) as excinfo:
+            run_local_script(["script", str(script)], "login.a.org", "a.org", "a.org", strict=True)
+        assert "exited with code 1" in excinfo.value.message
+
+    def test_a_non_executable_script_aborts_in_strict_mode(self, monkeypatch, tmp_path):
+        script = make_script(tmp_path, executable=False)
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda *a, **k: pytest.fail("a non-executable script must not run"))
+        with pytest.raises(click.ClickException) as excinfo:
+            run_local_script(["script", str(script)], "login.a.org", "a.org", "a.org", strict=True)
+        assert "not executable" in excinfo.value.message
+
+
+class TestScriptHookDispatch:
+    def test_a_script_secret_hook_dispatches_to_run_local_script(self, monkeypatch, tmp_path):
+        script = make_script(tmp_path)
+        calls = []
+        monkeypatch.setattr(alakazam, "run_local_script", lambda *a, **k: calls.append((a, k)))
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: pytest.fail("should not call abra"))
+        run_secret_hooks("login.a.org", {"secret_hooks": [f"script {script}"], "server": "a.org"}, "a.org")
+        [(args, kwargs)] = calls
+        assert args == (["script", str(script)], "login.a.org", "a.org", "a.org")
+
+    def test_a_normal_secret_hook_goes_through_abra(self, monkeypatch):
+        calls = []
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: calls.append((a, k)) or "")
+        run_secret_hooks("login.a.org", {"secret_hooks": ["insert_cert"], "server": "a.org"})
+        assert calls, "a non-script hook must still run through abra"
+
+    def test_a_script_command_dispatches_to_run_local_script(self, monkeypatch, tmp_path):
+        script = make_script(tmp_path)
+        calls = []
+        monkeypatch.setattr(alakazam, "run_local_script", lambda *a, **k: calls.append((a, k)))
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: pytest.fail("should not call abra"))
+        execute_cmds({"app_domain": "login.a.org", "server": "a.org",
+                       "initial-hooks": [f"script {script} arg1"]}, initial=True)
+        [(args, kwargs)] = calls
+        assert args == (["script", str(script), "arg1"], "login.a.org", "a.org", "", False, False)
+
+    def test_a_normal_command_goes_through_abra(self, monkeypatch):
+        calls = []
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: calls.append((a, k)) or "")
+        execute_cmds({"app_domain": "login.a.org", "server": "a.org",
+                       "initial-hooks": ["app set_default_quota"]}, initial=True)
+        assert calls, "a non-script command must still run through abra"
+
+    def test_a_script_token_with_a_missing_file_falls_back_to_abra(self, monkeypatch, tmp_path):
+        """A 'script' hook whose path does not exist is not a script hook at all, just an abra
+        command whose container happens to be named 'script'."""
+        missing = tmp_path / "does-not-exist.sh"
+        calls = []
+        monkeypatch.setattr(alakazam, "run_local_script",
+                            lambda *a, **k: pytest.fail("should not run a missing script"))
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: calls.append((a, k)) or "")
+        execute_cmds({"app_domain": "login.a.org", "server": "a.org",
+                       "initial-hooks": [f"script {missing}"]}, initial=True)
+        assert calls, "a missing script path must fall back to a normal abra command"
+
+
+class TestInstanceDomainThreading:
+    """create_secrets(), deploy_apps() and the 'cmd' CLI command each thread instance_domain
+    into run_secret_hooks()/execute_cmds() for script hooks."""
+
+    @pytest.fixture
+    def app_config(self, monkeypatch):
+        """A single 'authentik' app on instance 'a.org', shared by tests below."""
+        config = {"app_domain": "login.a.org", "server": "a.org"}
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", {"a.org": {"authentik": config}})
+        return config
+
+    def test_create_secrets_threads_the_instance_domain(self, monkeypatch, app_config):
+        calls = []
+        monkeypatch.setattr(alakazam, "run_secret_hooks", lambda *a, **k: calls.append((a, k)))
+        create_secrets(recipes=(), variants={"secret-hooks"})
+        [(args, kwargs)] = calls
+        assert args == ("login.a.org", app_config, "a.org")
+
+    def test_deploy_apps_threads_the_instance_domain(self, monkeypatch, app_config):
+        app_config["version"] = "1.0.0"
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        calls = []
+        monkeypatch.setattr(alakazam, "execute_cmds", lambda app_config, **k: calls.append(k))
+        deploy_apps({"a.org": [["authentik", "login.a.org"]]}, execute_hooks=True)
+        [kwargs] = calls
+        assert kwargs["deploy"] is True
+        assert kwargs["instance_domain"] == "a.org"
+
+    def test_cmd_threads_the_instance_domain(self, monkeypatch, app_config):
+        monkeypatch.setattr(alakazam, "get_deployed_apps", lambda apps: {"login.a.org": "1.0.0"})
+        calls = []
+        monkeypatch.setattr(alakazam, "execute_cmds", lambda app_config, **k: calls.append(k))
+        result = CliRunner().invoke(alakazam.cmd, ["-i"], standalone_mode=False)
+        assert result.exception is None
+        [kwargs] = calls
+        assert kwargs["initial"] is True
+        assert kwargs["instance_domain"] == "a.org"

tests/test_preflight.py

@@ -0,0 +1,228 @@
+"""Tests for the configuration preflight that runs before any abra call."""
+
+import os
+import sys
+
+import click
+import pytest
+from click.testing import CliRunner
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import check_config_readable, get_relevant_config_paths, preflight_configs
+
+# a git-crypt encrypted file starts with this marker and is not valid UTF-8
+GIT_CRYPT_HEADER = b"\x00GITCRYPT\x00\xeb\x1c\x9a\x7f\x3d\x00\xff\xfe"
+
+
+@pytest.fixture
+def config_root(tmp_path):
+    """A root path holding one group with one instance, mirroring the documented layout."""
+    root = tmp_path / "root"
+    group = root / "group"
+    group.mkdir(parents=True)
+    (root / "alaka.yml").write_text("authentik:\n    version: 1.0.0\n")
+    (root / "config-sets.yml").write_text("bbb:\n    authentik:\n        env:\n            A: b\n")
+    (group / "alaka-versions.yml").write_text("authentik:\n    version: 2.0.0\n")
+    (group / "example.com.yml").write_text("authentik:\n    server: example.com\n")
+    return root
+
+
+class TestCheckConfigReadable:
+    def test_valid_config(self, tmp_path):
+        path = tmp_path / "alaka.yml"
+        path.write_text("authentik:\n    version: 1.0.0\n")
+        assert check_config_readable(path) is None
+
+    def test_missing_file_is_not_a_failure(self, tmp_path):
+        """A configuration that simply is not there is a legitimate state."""
+        assert check_config_readable(tmp_path / "absent.yml") is None
+
+    def test_empty_file_is_not_a_failure(self, tmp_path):
+        path = tmp_path / "alaka.yml"
+        path.write_text("")
+        assert check_config_readable(path) is None
+
+    def test_encrypted_file(self, tmp_path):
+        path = tmp_path / "alaka-secrets.yml"
+        path.write_bytes(GIT_CRYPT_HEADER)
+        assert "codec can't decode" in check_config_readable(path)
+
+    def test_malformed_yaml(self, tmp_path):
+        path = tmp_path / "alaka.yml"
+        path.write_text("authentik:\n  - version: 1.0.0\n    broken: [\n")
+        assert check_config_readable(path)
+
+    def test_reason_is_a_single_line(self, tmp_path):
+        """Multi-line parser output would break up the failure report."""
+        path = tmp_path / "alaka.yml"
+        path.write_text("authentik:\n  - version: 1.0.0\n    broken: [\n")
+        assert "\n" not in check_config_readable(path)
+
+
+class TestReadConfigSeverity:
+    """An unreadable file that reaches read_config is out of scope by construction."""
+
+    def test_an_unreadable_config_warns_rather_than_errors(self, config_root, caplog):
+        import logging
+        broken = config_root / "foreign" / "alaka-secrets.yml"
+        broken.parent.mkdir()
+        broken.write_bytes(GIT_CRYPT_HEADER)
+        with caplog.at_level(logging.DEBUG):
+            assert alakazam.read_config(str(broken)) == {}
+        assert [r.levelname for r in caplog.records] == ["WARNING"]
+        assert "is skipped" in caplog.text
+
+
+class TestPreflightConfigs:
+    def test_passes_for_readable_configs(self, config_root):
+        preflight_configs([config_root / "alaka.yml", config_root / "config-sets.yml"])
+
+    def test_aborts_on_an_unreadable_config(self, config_root):
+        broken = config_root / "alaka-secrets.yml"
+        broken.write_bytes(GIT_CRYPT_HEADER)
+        with pytest.raises(click.ClickException) as excinfo:
+            preflight_configs([config_root / "alaka.yml", broken])
+        assert str(broken) in excinfo.value.message
+        assert "git crypt unlock" in excinfo.value.message
+
+    def test_reports_every_failure_at_once(self, config_root):
+        """A CI run should see all broken files, not just the first one."""
+        for name in ("alaka-a.yml", "alaka-b.yml"):
+            (config_root / name).write_bytes(GIT_CRYPT_HEADER)
+        with pytest.raises(click.ClickException) as excinfo:
+            preflight_configs([config_root / "alaka-a.yml", config_root / "alaka-b.yml"])
+        assert "alaka-a.yml" in excinfo.value.message
+        assert "alaka-b.yml" in excinfo.value.message
+        assert excinfo.value.message.startswith("2 ")
+
+    def test_exit_code_is_non_zero(self):
+        assert click.ClickException("").exit_code != 0
+
+
+class TestRelevantConfigPaths:
+    def test_covers_the_inheritance_chain_and_the_instances(self, config_root):
+        paths = get_relevant_config_paths(config_root, config_root / "group", [])
+        assert config_root / "alaka.yml" in paths
+        assert config_root / "config-sets.yml" in paths
+        assert config_root / "group" / "alaka-versions.yml" in paths
+        assert config_root / "group" / "example.com.yml" in paths
+
+    def test_covers_every_config_set_file(self, config_root):
+        (config_root / "config-sets-authentik.yml").write_text("integrate:\n    apply-when: [a]\n")
+        paths = get_relevant_config_paths(config_root, config_root / "group", [])
+        assert config_root / "config-sets.yml" in paths
+        assert config_root / "config-sets-authentik.yml" in paths
+
+    def test_a_single_instance_file_pulls_in_its_ancestors(self, config_root):
+        paths = get_relevant_config_paths(config_root, config_root / "group" / "example.com.yml", [])
+        assert config_root / "alaka.yml" in paths
+        assert config_root / "group" / "alaka-versions.yml" in paths
+        assert config_root / "group" / "example.com.yml" in paths
+
+    def test_leaves_out_foreign_groups(self, config_root):
+        """A group that cannot change this run's result stays out of the strict check."""
+        foreign = config_root / "foreign"
+        foreign.mkdir()
+        (foreign / "alaka.yml").write_text("nextcloud:\n    version: 1.0.0\n")
+        (foreign / "other.com.yml").write_text("nextcloud:\n    server: other.com\n")
+        paths = get_relevant_config_paths(config_root, config_root / "group", [])
+        assert foreign / "alaka.yml" not in paths
+        assert foreign / "other.com.yml" not in paths
+
+    def test_leaves_out_paths_above_the_root(self, config_root):
+        (config_root.parent / "alaka.yml").write_text("authentik:\n    version: 0.0.1\n")
+        paths = get_relevant_config_paths(config_root, config_root / "group", [])
+        assert config_root.parent / "alaka.yml" not in paths
+
+    def test_honours_exclude_paths(self, config_root):
+        excluded = config_root / "group" / "excluded"
+        excluded.mkdir()
+        (excluded / "skip.com.yml").write_text("authentik:\n    server: skip.com\n")
+        paths = get_relevant_config_paths(config_root, config_root / "group", [str(excluded)])
+        assert excluded / "skip.com.yml" not in paths
+
+    def test_has_no_duplicates(self, config_root):
+        paths = get_relevant_config_paths(config_root, config_root / "group", [])
+        assert len(paths) == len(set(paths))
+
+
+class TestPreflightInCli:
+    """The preflight has to abort before the first abra call, not halfway through a run."""
+
+    @pytest.fixture
+    def cli_env(self, config_root, monkeypatch):
+        calls = []
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: calls.append(a) or "")
+        monkeypatch.setattr(alakazam, "fetch_recipes", lambda *a, **k: None)
+        monkeypatch.setattr(alakazam, "get_settings_path", lambda: str(config_root / "alakazam.yml"))
+        monkeypatch.setattr(alakazam, "get_abra_dir", lambda: config_root / ".abra")
+        (config_root / "alakazam.yml").write_text(f"root: {config_root}\n")
+        return calls
+
+    def test_clean_configs_run_through(self, config_root, cli_env):
+        result = CliRunner().invoke(alakazam.cli, [str(config_root / "group"), "ls"])
+        assert result.exit_code == 0, result.output
+
+    def test_encrypted_config_aborts_without_calling_abra(self, config_root, cli_env):
+        (config_root / "group" / "alaka-secrets.yml").write_bytes(GIT_CRYPT_HEADER)
+        result = CliRunner().invoke(alakazam.cli, [str(config_root / "group"), "ls"])
+        assert result.exit_code != 0
+        assert "alaka-secrets.yml" in result.output
+        assert "git crypt unlock" in result.output
+        assert cli_env == []
+
+    def test_encrypted_config_of_a_foreign_group_does_not_abort(self, config_root, cli_env):
+        foreign = config_root / "foreign"
+        foreign.mkdir()
+        (foreign / "alaka-secrets.yml").write_bytes(GIT_CRYPT_HEADER)
+        result = CliRunner().invoke(alakazam.cli, [str(config_root / "group"), "ls"])
+        assert result.exit_code == 0, result.output
+
+    def test_broken_settings_file_aborts(self, config_root, cli_env):
+        (config_root / "alakazam.yml").write_bytes(GIT_CRYPT_HEADER)
+        result = CliRunner().invoke(alakazam.cli, [str(config_root / "group"), "ls"])
+        assert result.exit_code != 0
+        assert "alakazam.yml" in result.output
+
+
+class TestLogLevel:
+    """-l has to win over the handler that an early logging.warning() installs by itself."""
+
+    @pytest.fixture
+    def probe(self, config_root, monkeypatch):
+        import logging
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        monkeypatch.setattr(alakazam, "fetch_recipes", lambda *a, **k: None)
+        monkeypatch.setattr(alakazam, "get_settings_path", lambda: str(config_root / "alakazam.yml"))
+        monkeypatch.setattr(alakazam, "get_abra_dir", lambda: config_root / ".abra")
+        (config_root / "alakazam.yml").write_text(f"root: {config_root}\n")
+        # a missing config-sets.yml makes read_config warn before cli() is through
+        (config_root / "config-sets.yml").unlink()
+        levels = []
+
+        @alakazam.cli.command()
+        def probe():
+            levels.append(logging.getLogger().getEffectiveLevel())
+
+        monkeypatch.setattr(alakazam, "GROUP_PATH", None)
+        return levels
+
+    def run(self, config_root, level):
+        return CliRunner().invoke(alakazam.cli, ["-l", level, str(config_root / "group"), "probe"])
+
+    def test_debug_reaches_the_root_logger(self, config_root, probe):
+        import logging
+        result = self.run(config_root, "DEBUG")
+        assert result.exit_code == 0, result.output
+        assert probe == [logging.DEBUG]
+
+    def test_info_reaches_the_root_logger(self, config_root, probe):
+        import logging
+        self.run(config_root, "INFO")
+        assert probe == [logging.INFO]
+
+    def test_an_invalid_level_is_rejected(self, config_root, probe):
+        result = self.run(config_root, "LOUD")
+        assert result.exit_code != 0

tests/test_ps_wait.py

@@ -0,0 +1,154 @@
+"""Tests for the health gate of 'ps --wait'."""
+
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import get_pending_containers, wait_for_healthy_apps
+
+
+def container(service, status, version="1.0.0"):
+    return {"service": service, "status": status, "version": version, "chaos": "false", "state": "running", "image": "img"}
+
+
+class TestPendingContainers:
+    def test_healthy_containers_are_not_pending(self):
+        containers = [("a.org", container("app", "healthy")), ("a.org", container("db", "running"))]
+        assert get_pending_containers(containers) == []
+
+    @pytest.mark.parametrize("status", ["starting", "unknown", "unhealthy", "starting (health: starting)"])
+    def test_unsettled_statuses_are_pending(self, status):
+        containers = [("a.org", container("app", status))]
+        assert len(get_pending_containers(containers)) == 1
+
+    def test_init_containers_are_exempt(self):
+        """Init containers run once and exit, they never report a healthy status."""
+        containers = [("a.org", container("authentik-init", "unknown")), ("a.org", container("app", "healthy"))]
+        assert get_pending_containers(containers) == []
+
+    def test_an_init_container_does_not_hide_a_real_one(self):
+        containers = [("a.org", container("authentik-init", "unknown")), ("a.org", container("app", "starting"))]
+        pending = get_pending_containers(containers)
+        assert [c["service"] for _, c in pending] == ["app"]
+
+
+class FakeCluster:
+    """Serves a scripted sequence of container states, one per health check."""
+
+    def __init__(self, rounds, deployed=True):
+        self.rounds = list(rounds)
+        self.deployed = deployed
+        self.checks = 0
+        self.slept = 0
+
+    def install(self, monkeypatch):
+        monkeypatch.setattr(alakazam, "get_apps_by_deployment", self.get_apps_by_deployment)
+        monkeypatch.setattr(alakazam, "get_app_containers", self.get_app_containers)
+        monkeypatch.setattr(alakazam, "sleep", self.sleep)
+        return self
+
+    def get_apps_by_deployment(self, recipes, deployed=True):
+        return {"a.org": [["authentik", "login.a.org", "1.0.0"]]} if self.deployed else {}
+
+    def get_app_containers(self, instance_apps):
+        self.checks += 1
+        return self.rounds[min(self.checks - 1, len(self.rounds) - 1)]
+
+    def sleep(self, seconds):
+        self.slept += seconds
+
+
+HEALTHY = [("login.a.org", container("app", "healthy"))]
+STARTING = [("login.a.org", container("app", "starting"))]
+
+
+class TestWaitForHealthyApps:
+    def test_returns_immediately_when_everything_is_healthy(self, monkeypatch, capsys):
+        cluster = FakeCluster([HEALTHY]).install(monkeypatch)
+        wait_for_healthy_apps((), timeout=300)
+        assert cluster.checks == 1
+        assert cluster.slept == 0
+        assert "All services are healthy" in capsys.readouterr().out
+
+    def test_waits_until_the_services_settle(self, monkeypatch):
+        cluster = FakeCluster([STARTING, STARTING, HEALTHY]).install(monkeypatch)
+        wait_for_healthy_apps((), timeout=300)
+        assert cluster.checks == 3
+        assert cluster.slept == 2 * alakazam.PS_WAIT_INTERVAL
+
+    def test_timeout_aborts_and_names_the_services(self, monkeypatch):
+        cluster = FakeCluster([STARTING]).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            wait_for_healthy_apps((), timeout=0)
+        assert "login.a.org app: starting" in excinfo.value.message
+        assert excinfo.value.exit_code != 0
+
+    def test_timeout_checks_at_least_once(self, monkeypatch):
+        """A zero timeout still has to look, otherwise it could not report anything."""
+        cluster = FakeCluster([STARTING]).install(monkeypatch)
+        with pytest.raises(click.ClickException):
+            wait_for_healthy_apps((), timeout=0)
+        assert cluster.checks == 1
+
+    def test_nothing_deployed_is_a_failure(self, monkeypatch):
+        """Reporting the health of an environment that does not exist would turn a broken deploy green."""
+        FakeCluster([HEALTHY], deployed=False).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            wait_for_healthy_apps((), timeout=300)
+        assert "nothing to wait for" in excinfo.value.message
+
+    def test_an_init_container_alone_does_not_block(self, monkeypatch):
+        rounds = [[("login.a.org", container("authentik-init", "unknown")), ("login.a.org", container("app", "healthy"))]]
+        cluster = FakeCluster(rounds).install(monkeypatch)
+        wait_for_healthy_apps((), timeout=300)
+        assert cluster.checks == 1
+
+
+class TestPsCommand:
+    """Without --wait the command must behave exactly as it did before the flag existed."""
+
+    @pytest.fixture
+    def cluster(self, monkeypatch):
+        cluster = FakeCluster([STARTING]).install(monkeypatch)
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", {"a.org": {"authentik": {"app_domain": "login.a.org"}}})
+        return cluster
+
+    def invoke(self, args):
+        from click.testing import CliRunner
+        return CliRunner().invoke(alakazam.ps, args, standalone_mode=False)
+
+    def test_plain_ps_checks_once_and_does_not_wait(self, cluster):
+        result = self.invoke([])
+        assert result.exit_code == 0, result.output
+        assert cluster.checks == 1
+        assert cluster.slept == 0
+
+    def test_plain_ps_prints_the_table(self, cluster):
+        result = self.invoke([])
+        assert "Service" in result.output
+        assert "login.a.org" in result.output
+        assert "starting" in result.output
+
+    def test_plain_ps_succeeds_on_unhealthy_services(self, cluster):
+        """Reporting a status is not the same as gating on it."""
+        assert self.invoke([]).exit_code == 0
+
+    def test_plain_ps_without_deployed_apps(self, cluster, monkeypatch):
+        cluster.deployed = False
+        result = self.invoke([])
+        assert result.exit_code == 0
+        assert "No apps deployed" in result.output
+
+    def test_wait_and_watch_are_mutually_exclusive(self, cluster):
+        result = self.invoke(["--wait", "--watch"])
+        assert isinstance(result.exception, click.UsageError)
+
+    def test_wait_gates_on_the_status(self, cluster):
+        result = self.invoke(["--wait", "--timeout", "0"])
+        assert isinstance(result.exception, click.ClickException)
+        assert "still not healthy after 0s" in result.exception.message

tests/test_purge.py

@@ -0,0 +1,92 @@
+"""Tests for the removal of apps and everything they left on their servers."""
+
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import purge_app_secrets, purge_apps
+
+INSTANCE_APPS = {"a.org": [["authentik", "login.a.org"]]}
+CONFIG = {"a.org": {"authentik": {"app_domain": "login.a.org", "server": "a.org"}}}
+
+
+@pytest.fixture
+def env_file(tmp_path, monkeypatch):
+    """Places a local app configuration where purge_apps looks for it."""
+    path = tmp_path / "login.a.org.env"
+    path.write_text("")
+    monkeypatch.setattr(alakazam, "get_env_path", lambda server, domain: path)
+    monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", CONFIG)
+    return path
+
+
+
+class TestPurgeApps:
+    def test_a_failing_removal_aborts(self, env_file, monkeypatch):
+        """Reporting a purge that did not happen would make a later setup build on leftovers."""
+        def failing(*args, **kwargs):
+            raise RuntimeError("FATA login.a.org is still deployed")
+        monkeypatch.setattr(alakazam, "abra", failing)
+        with pytest.raises(click.ClickException) as excinfo:
+            purge_apps(INSTANCE_APPS)
+        assert "could not purge login.a.org" in excinfo.value.message
+        assert excinfo.value.exit_code != 0
+
+    def test_a_successful_removal_reports_it(self, env_file, monkeypatch, capsys):
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "removed")
+        purge_apps(INSTANCE_APPS)
+        assert "login.a.org purged" in capsys.readouterr().out
+
+    def test_an_app_without_a_local_config_is_skipped(self, tmp_path, monkeypatch):
+        """abra addresses secrets and volumes through the .env, without it there is nothing to remove."""
+        calls = []
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", CONFIG)
+        monkeypatch.setattr(alakazam, "get_env_path", lambda server, domain: tmp_path / "absent.env")
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: calls.append(a) or "")
+        purge_apps(INSTANCE_APPS)
+        assert calls == []
+
+
+class TestPurgeAppSecrets:
+    @pytest.fixture
+    def config(self, monkeypatch):
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", CONFIG)
+
+    def install(self, monkeypatch, error=None):
+        calls = []
+
+        def abra(*args, **kwargs):
+            calls.append(args)
+            if error:
+                raise RuntimeError(error)
+            return ""
+
+        monkeypatch.setattr(alakazam, "abra", abra)
+        return calls
+
+    def test_all_secrets_of_a_recipe_are_removed(self, config, monkeypatch, capsys):
+        calls = self.install(monkeypatch)
+        purge_app_secrets({"authentik": []})
+        assert calls == [("app", "secret", "rm", "-a", "login.a.org")]
+        assert "purged" in capsys.readouterr().out
+
+    def test_an_app_without_secrets_is_not_a_failure(self, config, monkeypatch, capsys):
+        """abra exits non-zero when it found nothing to remove, which says the job is already done."""
+        self.install(monkeypatch, error="FATA no secrets to remove?")
+        purge_app_secrets({"authentik": []})
+        assert "has no secrets on its server" in capsys.readouterr().out
+
+    def test_a_real_failure_still_propagates(self, config, monkeypatch):
+        self.install(monkeypatch, error="FATA error during connect: no route to host")
+        with pytest.raises(RuntimeError):
+            purge_app_secrets({"authentik": []})
+
+    def test_a_named_secret_that_is_absent_is_skipped(self, config, monkeypatch, capsys):
+        self.install(monkeypatch, error="FATA email_pass doesn't exist on server?")
+        purge_app_secrets({"authentik": ["email_pass"]})
+        assert "is not stored on the server" in capsys.readouterr().out

tests/test_readiness_hooks.py

@@ -0,0 +1,160 @@
+"""Tests for the readiness hooks that gate dependent apps on an app becoming usable."""
+
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+import logging
+
+from alakazam import abra, build_app_cmd, run_readiness_hook, run_readiness_hooks, run_streamed
+
+
+class FakeAbra:
+    """Fails a scripted number of times before succeeding, recording every call."""
+
+    def __init__(self, failures=0, always_fail=False):
+        self.failures = failures
+        self.always_fail = always_fail
+        self.calls = []
+        self.slept = []
+
+    def install(self, monkeypatch):
+        monkeypatch.setattr(alakazam, "abra", self)
+        monkeypatch.setattr(alakazam, "sleep", self.slept.append)
+        return self
+
+    def __call__(self, *args, **kwargs):
+        self.calls.append(args)
+        if self.always_fail or len(self.calls) <= self.failures:
+            raise RuntimeError("FATA blueprint not applied")
+        return "ok"
+
+
+class TestBuildAppCmd:
+    def test_container_command(self):
+        assert build_app_cmd("login.a.org", "app check_blueprints") == [
+            "app", "cmd", "-T", "", "login.a.org", "app", "--", "check_blueprints",
+        ]
+
+    def test_local_command(self):
+        assert build_app_cmd("login.a.org", "local enable_sso") == [
+            "app", "cmd", "--local", "", "login.a.org", "--", "enable_sso",
+        ]
+
+    def test_arguments_are_passed_through(self):
+        assert build_app_cmd("login.a.org", "app set_flag on")[-2:] == ["set_flag", "on"]
+
+    def test_chaos_flag(self):
+        assert "-C" in build_app_cmd("login.a.org", "app check_blueprints", "-C")
+
+
+class TestRunReadinessHook:
+    def test_succeeds_on_the_first_attempt(self, monkeypatch, capsys):
+        abra = FakeAbra().install(monkeypatch)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints"})
+        assert len(abra.calls) == 1
+        assert abra.slept == []
+        assert "is ready" in capsys.readouterr().out
+
+    def test_retries_until_it_succeeds(self, monkeypatch):
+        abra = FakeAbra(failures=2).install(monkeypatch)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "interval": 5})
+        assert len(abra.calls) == 3
+        assert abra.slept == [5, 5]
+
+    def test_gives_up_after_the_configured_retries(self, monkeypatch):
+        abra = FakeAbra(always_fail=True).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "retries": 3, "interval": 5})
+        assert len(abra.calls) == 3
+        assert "failed 3 times" in excinfo.value.message
+        assert excinfo.value.exit_code != 0
+
+    def test_does_not_sleep_after_the_last_attempt(self, monkeypatch):
+        abra = FakeAbra(always_fail=True).install(monkeypatch)
+        with pytest.raises(click.ClickException):
+            run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "retries": 3, "interval": 5})
+        assert abra.slept == [5, 5]
+
+    def test_initial_delay_precedes_the_first_attempt(self, monkeypatch):
+        abra = FakeAbra().install(monkeypatch)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "initial-delay": 60})
+        assert abra.slept == [60]
+
+    def test_defaults_apply_when_only_a_command_is_given(self, monkeypatch):
+        abra = FakeAbra(always_fail=True).install(monkeypatch)
+        with pytest.raises(click.ClickException):
+            run_readiness_hook("login.a.org", {"cmd": "app check_blueprints"})
+        assert len(abra.calls) == alakazam.READINESS_RETRIES
+        assert abra.slept == [alakazam.READINESS_INTERVAL] * (alakazam.READINESS_RETRIES - 1)
+
+    def test_a_hook_without_a_command_is_rejected(self, monkeypatch):
+        FakeAbra().install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            run_readiness_hook("login.a.org", {"retries": 3})
+        assert "no 'cmd'" in excinfo.value.message
+
+    def test_the_last_failure_is_reported(self, monkeypatch):
+        FakeAbra(always_fail=True).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "retries": 1})
+        assert "blueprint not applied" in excinfo.value.message
+
+
+class TestVisibility:
+    """A hook can block for half an hour, so silence is indistinguishable from a hang."""
+
+    def test_the_command_output_is_streamed(self):
+        result = run_streamed(["bash", "-c", "echo first; echo second >&2; exit 3"])
+        assert result.returncode == 3
+        assert result.stdout.decode().splitlines() == ["first", "second"]
+
+    def test_the_hook_output_is_not_buffered(self, monkeypatch):
+        """abra is asked to stream, otherwise nothing appears until the command has finished."""
+        streamed = []
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **kw: streamed.append(kw.get("stream")) or "ok")
+        monkeypatch.setattr(alakazam, "sleep", lambda seconds: None)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints"})
+        assert streamed == [True]
+
+    def test_progress_is_printed(self, monkeypatch, capsys):
+        """Progress is what the run is doing, not a diagnostic, and logging.info needs -l to show."""
+        FakeAbra(failures=1).install(monkeypatch)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "interval": 5})
+        out = capsys.readouterr().out
+        assert "attempt 1/20" in out
+        assert "retry in 5s" in out
+
+    def test_the_initial_delay_is_announced(self, monkeypatch, capsys):
+        FakeAbra().install(monkeypatch)
+        run_readiness_hook("login.a.org", {"cmd": "app check_blueprints", "initial-delay": 60})
+        assert "Wait 60s" in capsys.readouterr().out
+
+    def test_streaming_and_machine_output_are_incompatible(self):
+        """Streaming merges abra's log lines into stdout, which would break the JSON parsing."""
+        with pytest.raises(ValueError):
+            abra("app", "ls", machine_output=True, stream=True)
+
+
+class TestRunReadinessHooks:
+    def test_no_hooks_configured_is_a_no_op(self, monkeypatch):
+        abra = FakeAbra().install(monkeypatch)
+        run_readiness_hooks("login.a.org", {"version": "1.0.0"})
+        assert abra.calls == []
+
+    def test_hooks_run_in_configuration_order(self, monkeypatch):
+        abra = FakeAbra().install(monkeypatch)
+        run_readiness_hooks("login.a.org", {"readiness-hooks": [{"cmd": "app first"}, {"cmd": "app second"}]})
+        assert [call[-1] for call in abra.calls] == ["first", "second"]
+
+    def test_a_failing_hook_stops_the_following_ones(self, monkeypatch):
+        """Dependent apps must not proceed on a half ready app."""
+        abra = FakeAbra(always_fail=True).install(monkeypatch)
+        with pytest.raises(click.ClickException):
+            run_readiness_hooks("login.a.org", {"readiness-hooks": [{"cmd": "app first", "retries": 1}, {"cmd": "app second"}]})
+        assert all(call[-1] == "first" for call in abra.calls)

tests/test_retry.py

@@ -0,0 +1,85 @@
+"""Tests for which abra failures may be retried."""
+
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import abra, is_connection_error
+
+
+class FakeProcess:
+    def __init__(self, returncode, stderr=b""):
+        self.returncode = returncode
+        self.stdout = b""
+        self.stderr = stderr
+
+
+UNREACHABLE = b"FATA error during connect: kex_exchange_identification: Connection closed by remote host"
+REJECTED = b"FATA login.a.org is still deployed"
+
+
+class TestRetryLoop:
+    """A server that drops the ssh handshake is usually throttling, so the retries have to spread out."""
+
+    @pytest.fixture
+    def runs(self, monkeypatch):
+        calls = {"results": [], "slept": []}
+        monkeypatch.setattr(alakazam, "sleep", calls["slept"].append)
+        monkeypatch.setattr(alakazam.subprocess, "run",
+                            lambda cmd, capture_output: calls["results"].pop(0))
+        return calls
+
+    def test_a_successful_command_does_not_retry(self, runs):
+        runs["results"] = [FakeProcess(0)]
+        abra("app", "ls")
+        assert runs["slept"] == []
+
+    def test_the_delay_doubles_between_attempts(self, runs):
+        runs["results"] = [FakeProcess(1, UNREACHABLE)] * alakazam.ABRA_RETRIES
+        with pytest.raises(RuntimeError):
+            abra("app", "ls")
+        expected = [alakazam.ABRA_RETRY_DELAY * 2 ** n for n in range(alakazam.ABRA_RETRIES - 1)]
+        assert runs["slept"] == expected
+
+    def test_it_stops_retrying_once_the_command_succeeds(self, runs):
+        runs["results"] = [FakeProcess(1, UNREACHABLE), FakeProcess(1, UNREACHABLE), FakeProcess(0)]
+        abra("app", "ls")
+        assert runs["slept"] == [alakazam.ABRA_RETRY_DELAY, alakazam.ABRA_RETRY_DELAY * 2]
+
+    def test_a_rejected_operation_is_not_repeated(self, runs):
+        """Repeating a command the server answered would only waste the whole backoff."""
+        runs["results"] = [FakeProcess(1, REJECTED)]
+        with pytest.raises(RuntimeError):
+            abra("app", "ls")
+        assert runs["slept"] == []
+
+
+class TestConnectionErrors:
+    """Only unreachable servers may be retried, a rejected operation must not be repeated."""
+
+    @pytest.mark.parametrize("output", [
+        'error during connect: Get "http://docker.example.com/v1.51/info"',
+        "kex_exchange_identification: Connection closed by remote host",
+        "Connection closed by 152.53.133.16 port 22",
+        "connection reset by peer",
+        "ssh: connect to host example.com port 22: Connection refused",
+        "Connection timed out",
+        "No route to host",
+        "write: broken pipe",
+    ])
+    def test_unreachable_server(self, output):
+        assert is_connection_error(output)
+
+    @pytest.mark.parametrize("output", [
+        "FATA email_pass doesn't exist on server?",
+        "FATA login.a.org is still deployed",
+        "FATA secret is in use",
+        "",
+    ])
+    def test_operational_failures_are_not_retried(self, output):
+        assert not is_connection_error(output)

tests/test_secrets.py

@@ -0,0 +1,67 @@
+"""Tests for the secret generation, in particular its behaviour on a repeated run."""
+
+import logging
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import generate_all_secrets
+
+# what 'abra app secret generate -a' reports when every secret is already on the server, exiting 1
+NOTHING_LEFT = RuntimeError(
+    "abra -o app secret generate -a login.a.org -m \n STDOUT: \n \n STDERR: "
+    "WARN login_a_org_db_password_v1 already exists\nWARN no secrets generated"
+)
+
+
+class FakeAbra:
+    def __init__(self, stored, generate_error=None):
+        self.stored = stored
+        self.generate_error = generate_error
+        self.generated = False
+
+    def __call__(self, *args, **kwargs):
+        if args[:3] == ("app", "secret", "ls"):
+            return [{"name": name, "created on server": str(created).lower()}
+                    for name, created in self.stored.items()]
+        if args[:3] == ("app", "secret", "generate"):
+            self.generated = True
+            if self.generate_error:
+                raise self.generate_error
+            return [{"name": "db_password", "value": "s3cret"}]
+        raise AssertionError(f"unexpected abra call: {args}")
+
+
+class TestGenerateAllSecrets:
+    def test_nothing_runs_when_every_secret_exists(self, monkeypatch):
+        abra = FakeAbra({"db_password": True, "email_pass": True})
+        monkeypatch.setattr(alakazam, "abra", abra)
+        generate_all_secrets("login.a.org")
+        assert not abra.generated
+
+    def test_missing_secrets_are_generated(self, monkeypatch, capsys):
+        abra = FakeAbra({"db_password": False})
+        monkeypatch.setattr(alakazam, "abra", abra)
+        generate_all_secrets("login.a.org")
+        assert abra.generated
+        assert "db_password: s3cret" in capsys.readouterr().out
+
+    def test_an_empty_generate_run_is_not_a_failure(self, monkeypatch, caplog):
+        """A retry after a half finished run finds every secret in place, abra still exits 1."""
+        abra = FakeAbra({"db_password": False}, generate_error=NOTHING_LEFT)
+        monkeypatch.setattr(alakazam, "abra", abra)
+        with caplog.at_level(logging.WARNING):
+            generate_all_secrets("login.a.org")
+        assert "generated no secrets" in caplog.text
+
+    def test_a_real_failure_still_propagates(self, monkeypatch):
+        """Only the 'nothing to do' case is tolerated, an unreachable server is not."""
+        abra = FakeAbra({"db_password": False},
+                        generate_error=RuntimeError("STDERR: error during connect: no route to host"))
+        monkeypatch.setattr(alakazam, "abra", abra)
+        with pytest.raises(RuntimeError):
+            generate_all_secrets("login.a.org")

tests/test_setup.py

@@ -0,0 +1,256 @@
+"""Tests for 'setup' and 'clean-deploy', the commands that replace the external deploy script."""
+
+import os
+import sys
+from pathlib import Path
+
+import click
+import pytest
+from click.testing import CliRunner
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import get_storage_traces, require_empty_environment, setup_environment
+
+
+class FakeServer:
+    """Answers the abra calls of a setup run and records the order of the steps."""
+
+    def __init__(self, deployed=(), secrets=(), volumes=()):
+        self.deployed = set(deployed)
+        self.secrets = dict(secrets)
+        self.volumes = set(volumes)
+        self.steps = []
+
+    def install(self, monkeypatch, config=None):
+        if config is None:
+            config = {
+                    "authentik": {"app_domain": "login.a.org", "server": "a.org", "dependency": []},
+                "nextcloud": {"app_domain": "cloud.a.org", "server": "a.org", "dependency": ["authentik"],
+                              "readiness-hooks": [{"cmd": "app ready", "retries": 1}]},
+            }
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", {"a.org": config})
+        monkeypatch.setattr(alakazam, "abra", self.abra)
+        monkeypatch.setattr(alakazam, "sleep", lambda seconds: None)
+        monkeypatch.setattr(alakazam, "configure_apps", lambda recipes: self.steps.append("config"))
+        monkeypatch.setattr(alakazam, "create_secrets", self.create_secrets)
+        monkeypatch.setattr(alakazam, "deploy_apps", self.deploy_apps)
+        monkeypatch.setattr(alakazam, "undeploy_apps", self.undeploy_apps)
+        monkeypatch.setattr(alakazam, "purge_apps", self.purge_apps)
+        monkeypatch.setattr(alakazam, "execute_cmds", lambda app_config, **kw: self.steps.append(f"init:{app_config['app_domain']}"))
+        return self
+
+    def create_secrets(self, recipes, variants=None, **kwargs):
+        label = "conf" if variants == {"conf"} else "secrets"
+        self.steps.append(f"{label}:{','.join(recipes) if recipes else 'all'}")
+
+    def deploy_apps(self, instance_apps, **kwargs):
+        for _, apps in instance_apps.items():
+            for app, domain, *_ in apps:
+                self.steps.append(f"deploy:{app}")
+                self.deployed.add(domain)
+
+    def undeploy_apps(self, instance_apps):
+        self.steps.append("undeploy")
+        self.deployed.clear()
+
+    def purge_apps(self, instance_apps):
+        self.steps.append("purge")
+        self.secrets.clear()
+        self.volumes.clear()
+
+    def abra(self, *args, machine_output=False, **kwargs):
+        if args[:3] == ("app", "secret", "ls"):
+            return [{"name": name, "created on server": "true"} for name in self.secrets.get(args[3], [])]
+        if args[:3] == ("app", "volume", "ls"):
+            return "NAME  ON SERVER\ndata  /var/lib\n" if args[4] in self.volumes else ""
+        if args[:2] == ("app", "ls"):
+            apps = [{"appName": d, "status": "deployed", "chaos": "false", "version": "1.0.0"} for d in sorted(self.deployed)]
+            return {"a.org": {"apps": apps}}
+        if args[:2] == ("app", "ps"):
+            self.steps.append(f"ps:{args[-1]}")
+            return {"app": {"service": "app", "status": "healthy", "version": "1.0.0",
+                            "chaos": "false", "state": "running", "image": "img"}}
+        if args[:3] == ("app", "cmd", "-T"):
+            self.steps.append(f"ready:{args[4]}")
+            return "ok"
+        raise AssertionError(f"unexpected abra call: {args}")
+
+
+class TestStorageTraces:
+    def test_an_empty_environment_leaves_no_traces(self, monkeypatch):
+        FakeServer().install(monkeypatch)
+        assert get_storage_traces({"a.org": [["authentik", "login.a.org"]]}) == []
+
+    def test_leftover_secrets_are_found(self, monkeypatch):
+        FakeServer(secrets={"login.a.org": ["email_pass", "admin_pass"]}).install(monkeypatch)
+        traces = get_storage_traces({"a.org": [["authentik", "login.a.org"]]})
+        assert "2 secret(s)" in traces[0]
+        assert "admin_pass, email_pass" in traces[0]
+
+    def test_leftover_volumes_are_found(self, monkeypatch):
+        FakeServer(volumes=["login.a.org"]).install(monkeypatch)
+        traces = get_storage_traces({"a.org": [["authentik", "login.a.org"]]})
+        assert traces == ["login.a.org still has volumes on its server"]
+
+    def test_an_unreadable_volume_list_aborts(self, monkeypatch):
+        """Guessing that a failing read means "no volumes" would deploy on top of existing data."""
+        server = FakeServer().install(monkeypatch)
+        original = server.abra
+
+        def failing(*args, **kwargs):
+            if args[:3] == ("app", "volume", "ls"):
+                raise RuntimeError("abra -o app volume ls -d login.a.org\n STDERR: Error: no such server")
+            return original(*args, **kwargs)
+
+        monkeypatch.setattr(alakazam, "abra", failing)
+        with pytest.raises(click.ClickException) as excinfo:
+            get_storage_traces({"a.org": [["authentik", "login.a.org"]]})
+        assert "could not read the volumes of login.a.org" in excinfo.value.message
+
+    def test_uncreated_secrets_do_not_count(self, monkeypatch):
+        """A secret the recipe declares but the server does not hold is not a leftover."""
+        server = FakeServer().install(monkeypatch)
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: [{"name": "x", "created on server": "false"}]
+                            if a[:3] == ("app", "secret", "ls") else "")
+        assert get_storage_traces({"a.org": [["authentik", "login.a.org"]]}) == []
+
+
+class TestRequireEmptyEnvironment:
+    def test_no_traces_passes(self):
+        require_empty_environment([])
+
+    def test_traces_abort_with_a_hint(self):
+        with pytest.raises(click.ClickException) as excinfo:
+            require_empty_environment(["login.a.org is deployed"])
+        assert "login.a.org is deployed" in excinfo.value.message
+        assert "clean-deploy" in excinfo.value.message
+        assert excinfo.value.exit_code != 0
+
+
+class TestSetupEnvironment:
+    def test_runs_every_phase_per_dependency_level(self, monkeypatch):
+        config = {
+            "authentik": {"app_domain": "login.a.org", "server": "a.org", "dependency": []},
+            "traefik": {"app_domain": "a.org", "server": "a.org", "dependency": []},
+            "nextcloud": {"app_domain": "cloud.a.org", "server": "a.org", "dependency": ["authentik"],
+                          "readiness-hooks": [{"cmd": "app ready", "retries": 1}]},
+            "wekan": {"app_domain": "boards.a.org", "server": "a.org", "dependency": ["authentik"]},
+        }
+        server = FakeServer().install(monkeypatch, config=config)
+        setup_environment(())
+        assert server.steps[:8] == [
+            "config",
+            "conf:all",
+            "secrets:authentik,traefik", "deploy:authentik", "deploy:traefik",
+            "secrets:nextcloud,wekan", "deploy:nextcloud", "deploy:wekan",
+        ]
+
+    def test_apps_of_one_level_are_deployed_before_that_level_is_awaited(self, monkeypatch):
+        """Nothing in a level waits for a sibling, the readiness hooks close the level."""
+        server = FakeServer().install(monkeypatch)
+        setup_environment(())
+        assert server.steps.index("deploy:nextcloud") < server.steps.index("ready:cloud.a.org")
+
+    def test_configured_secrets_are_inserted_before_any_exchange(self, monkeypatch):
+        """Otherwise an exchange generates a random value that shadows the configured one."""
+        server = FakeServer().install(monkeypatch)
+        setup_environment(())
+        assert server.steps[1] == "conf:all"
+        assert all(step != "conf:all" for step in server.steps[2:])
+
+    def test_initial_hooks_run_after_the_health_check(self, monkeypatch):
+        server = FakeServer().install(monkeypatch)
+        setup_environment(())
+        assert server.steps[-2:] == ["init:login.a.org", "init:cloud.a.org"]
+        assert server.steps.index("ps:login.a.org") < server.steps.index("init:login.a.org")
+
+    def test_a_deployed_app_aborts_before_deploying(self, monkeypatch):
+        server = FakeServer(deployed=["login.a.org"]).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            setup_environment(())
+        assert "login.a.org is deployed" in excinfo.value.message
+        assert server.steps == ["config"]
+
+    def test_leftover_secrets_abort_the_run(self, monkeypatch):
+        """An undeploy without a purge must not look like an empty environment."""
+        server = FakeServer(secrets={"login.a.org": ["email_pass"]}).install(monkeypatch)
+        with pytest.raises(click.ClickException) as excinfo:
+            setup_environment(())
+        assert "email_pass" in excinfo.value.message
+        assert server.steps == ["config"]
+
+    def test_leftover_volumes_abort_the_run(self, monkeypatch):
+        server = FakeServer(volumes=["cloud.a.org"]).install(monkeypatch)
+        with pytest.raises(click.ClickException):
+            setup_environment(())
+        assert "deploy:authentik" not in server.steps
+
+    def test_nothing_configured_is_a_failure(self, monkeypatch):
+        FakeServer().install(monkeypatch, config={})
+        with pytest.raises(click.ClickException) as excinfo:
+            setup_environment(())
+        assert "nothing to set up" in excinfo.value.message
+
+    def test_a_failing_readiness_hook_stops_the_run(self, monkeypatch):
+        server = FakeServer().install(monkeypatch)
+        original = server.abra
+
+        def failing(*args, **kwargs):
+            if args[:3] == ("app", "cmd", "-T"):
+                raise RuntimeError("FATA blueprints missing")
+            return original(*args, **kwargs)
+
+        monkeypatch.setattr(alakazam, "abra", failing)
+        with pytest.raises(click.ClickException) as excinfo:
+            setup_environment(())
+        assert "is not ready" in excinfo.value.message
+        assert "ps:login.a.org" not in server.steps
+
+
+class TestCleanDeployCommand:
+    def invoke(self, args, group_path, monkeypatch):
+        monkeypatch.setattr(alakazam, "GROUP_PATH", Path(group_path))
+        return CliRunner().invoke(alakazam.clean_deploy, args, standalone_mode=False)
+
+    def test_rebuilds_an_instance(self, monkeypatch, tmp_path):
+        server = FakeServer(deployed=["login.a.org"], secrets={"login.a.org": ["email_pass"]}).install(monkeypatch)
+        instance = tmp_path / "example.com.yml"
+        instance.write_text("")
+        result = self.invoke(["-n"], instance, monkeypatch)
+        assert result.exit_code == 0, result.exception
+        assert server.steps[:4] == ["config", "undeploy", "purge", "config"]
+
+    def test_the_apps_are_configured_before_they_are_purged(self, monkeypatch, tmp_path):
+        """purge_apps() skips an app without a local .env, so a fresh checkout would purge nothing."""
+        server = FakeServer(deployed=["login.a.org"]).install(monkeypatch)
+        instance = tmp_path / "example.com.yml"
+        instance.write_text("")
+        self.invoke(["-n"], instance, monkeypatch)
+        assert server.steps.index("config") < server.steps.index("purge")
+
+    def test_the_apps_are_configured_again_after_the_purge(self, monkeypatch, tmp_path):
+        """'abra app rm' deletes the local .env as well, so setup has to write it a second time."""
+        server = FakeServer(deployed=["login.a.org"]).install(monkeypatch)
+        instance = tmp_path / "example.com.yml"
+        instance.write_text("")
+        self.invoke(["-n"], instance, monkeypatch)
+        assert server.steps.count("config") == 2
+        assert server.steps.index("purge") < len(server.steps) - 1 - server.steps[::-1].index("config")
+
+    def test_a_group_directory_is_rejected(self, monkeypatch, tmp_path):
+        server = FakeServer().install(monkeypatch)
+        result = self.invoke(["-n"], tmp_path, monkeypatch)
+        assert isinstance(result.exception, click.UsageError)
+        assert "single instance configuration file" in str(result.exception)
+        assert server.steps == []
+
+    def test_the_confirmation_can_be_declined(self, monkeypatch, tmp_path):
+        server = FakeServer().install(monkeypatch)
+        instance = tmp_path / "example.com.yml"
+        instance.write_text("")
+        monkeypatch.setattr(alakazam, "GROUP_PATH", instance)
+        result = CliRunner().invoke(alakazam.clean_deploy, [], input="no\n", standalone_mode=False)
+        assert result.exit_code == 0
+        assert server.steps == []

tests/test_strict_hooks.py

@@ -0,0 +1,74 @@
+"""Tests for the strict hook mode that lets a scratch build fail on a broken hook."""
+
+import os
+import stat
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import execute_cmds, run_secret_hooks
+
+
+def failing_script(tmp_path):
+    """A tiny local script that exits non-zero, for testing 'script' hook failure handling."""
+    script = tmp_path / "fail.sh"
+    script.write_text("#!/bin/sh\nexit 1\n")
+    script.chmod(script.stat().st_mode | stat.S_IXUSR)
+    return script
+
+
+class TestStrictHooks:
+    def test_a_secret_hook_failure_is_tolerated_by_default(self, monkeypatch):
+        """'secrets' has always continued past a failing hook, that must not change."""
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        run_secret_hooks("login.a.org", {"secret_hooks": ["insert_cert"]})
+
+    def test_a_secret_hook_failure_aborts_in_strict_mode(self, monkeypatch):
+        def failing(*args, ignore_error=False, **kwargs):
+            assert not ignore_error
+            assert kwargs.get("stream"), "a hook can run for minutes, its output must not be buffered"
+            raise RuntimeError("FATA no such command")
+        monkeypatch.setattr(alakazam, "abra", failing)
+        with pytest.raises(click.ClickException) as excinfo:
+            run_secret_hooks("login.a.org", {"secret_hooks": ["insert_cert"]}, strict=True)
+        assert "secret hook 'insert_cert' failed" in excinfo.value.message
+
+    def test_a_command_failure_is_tolerated_by_default(self, monkeypatch):
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: "")
+        execute_cmds({"app_domain": "login.a.org", "server": "a.org", "initial-hooks": ["app init"]}, initial=True)
+
+    def test_a_command_failure_aborts_in_strict_mode(self, monkeypatch):
+        def failing(*args, ignore_error=False, **kwargs):
+            raise RuntimeError("FATA container not found")
+        monkeypatch.setattr(alakazam, "abra", failing)
+        with pytest.raises(click.ClickException) as excinfo:
+            execute_cmds({"app_domain": "login.a.org", "server": "a.org", "initial-hooks": ["app init"]}, initial=True, strict=True)
+        assert "command 'app init' failed" in excinfo.value.message
+
+    def test_a_script_secret_hook_failure_is_tolerated_by_default(self, tmp_path):
+        """A failing local script must be tolerated by default too, same as a failing abra.sh hook."""
+        script = failing_script(tmp_path)
+        run_secret_hooks("login.a.org", {"secret_hooks": [f"script {script}"], "server": "a.org"})
+
+    def test_a_script_secret_hook_failure_aborts_in_strict_mode(self, tmp_path):
+        script = failing_script(tmp_path)
+        with pytest.raises(click.ClickException) as excinfo:
+            run_secret_hooks("login.a.org", {"secret_hooks": [f"script {script}"], "server": "a.org"}, strict=True)
+        assert str(script) in excinfo.value.message
+        assert "failed" in excinfo.value.message
+
+    def test_a_script_command_failure_is_tolerated_by_default(self, tmp_path):
+        script = failing_script(tmp_path)
+        execute_cmds({"app_domain": "login.a.org", "server": "a.org", "initial-hooks": [f"script {script}"]}, initial=True)
+
+    def test_a_script_command_failure_aborts_in_strict_mode(self, tmp_path):
+        script = failing_script(tmp_path)
+        with pytest.raises(click.ClickException) as excinfo:
+            execute_cmds({"app_domain": "login.a.org", "server": "a.org", "initial-hooks": [f"script {script}"]},
+                         initial=True, strict=True)
+        assert str(script) in excinfo.value.message
+        assert "failed" in excinfo.value.message

tests/test_upgrade_secrets.py

@@ -0,0 +1,85 @@
+"""Tests for filling in secrets a recipe added, before the upgrade that would fail on them."""
+
+import os
+import sys
+
+import pytest
+from click.testing import CliRunner
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import get_missing_secrets
+
+CONFIG = {"a.org": {"nextcloud": {"app_domain": "cloud.a.org", "server": "a.org", "version": "2.0.0"}}}
+
+
+class TestGetMissingSecrets:
+    def install(self, monkeypatch, stored):
+        monkeypatch.setattr(alakazam, "abra", lambda *a, **k: stored)
+
+    def test_a_complete_app_reports_nothing(self, monkeypatch):
+        self.install(monkeypatch, [{"name": "db_password", "created on server": "true"}])
+        assert get_missing_secrets("cloud.a.org") == []
+
+    def test_the_missing_names_are_returned(self, monkeypatch):
+        self.install(monkeypatch, [
+            {"name": "db_password", "created on server": "true"},
+            {"name": "oidc_secret", "created on server": "false"},
+        ])
+        assert get_missing_secrets("cloud.a.org") == ["oidc_secret"]
+
+    def test_an_app_without_any_secrets(self, monkeypatch):
+        """abra answers with an empty document rather than a list when nothing is stored."""
+        self.install(monkeypatch, {})
+        assert get_missing_secrets("cloud.a.org") == []
+
+
+class TestUpgradeFillsThemIn:
+    @pytest.fixture
+    def steps(self, monkeypatch):
+        steps = []
+        monkeypatch.setattr(alakazam, "INSTANCE_CONFIGS", CONFIG)
+        monkeypatch.setattr(alakazam, "sleep", lambda s: None)
+        monkeypatch.setattr(alakazam, "get_apps_by_deployment",
+                            lambda recipes, deployed=True: {"a.org": [["nextcloud", "cloud.a.org", "1.0.0"]]})
+        monkeypatch.setattr(alakazam, "create_secrets",
+                            lambda recipes, **kw: steps.append(f"secrets:{recipes[0]}"))
+        return steps
+
+    def install_abra(self, monkeypatch, steps, missing):
+        def abra(*args, **kwargs):
+            if args[:3] == ("app", "secret", "ls"):
+                return [{"name": n, "created on server": str(n not in missing).lower()}
+                        for n in ("db_password", "oidc_secret")]
+            steps.append(" ".join(a for a in args if a))
+            return ""
+        monkeypatch.setattr(alakazam, "abra", abra)
+
+    def invoke(self):
+        return CliRunner().invoke(alakazam.upgrade, ["-n"], standalone_mode=False)
+
+    def test_a_missing_secret_is_created_before_the_upgrade(self, monkeypatch, steps):
+        self.install_abra(monkeypatch, steps, missing={"oidc_secret"})
+        result = self.invoke()
+        assert result.exit_code == 0, result.exception
+        # the planning phase fetches release notes with the same command plus -r, skip that one
+        upgrade = next(i for i, s in enumerate(steps)
+                       if s.startswith("app upgrade") and " -r " not in s)
+        assert steps.index("secrets:nextcloud") < upgrade
+
+    def test_the_missing_name_is_reported(self, monkeypatch, steps):
+        """Silently repairing secrets would hide which one the recipe added."""
+        self.install_abra(monkeypatch, steps, missing={"oidc_secret"})
+        assert "missing 1 secret(s): oidc_secret" in self.invoke().output
+
+    def test_a_complete_app_does_not_run_the_secrets(self, monkeypatch, steps):
+        self.install_abra(monkeypatch, steps, missing=set())
+        self.invoke()
+        assert not any(s.startswith("secrets:") for s in steps)
+
+    def test_a_dry_run_creates_nothing(self, monkeypatch, steps):
+        """The check sits behind the confirmation, so --dry-run must not reach it."""
+        self.install_abra(monkeypatch, steps, missing={"oidc_secret"})
+        CliRunner().invoke(alakazam.upgrade, ["-n", "--dry-run"], standalone_mode=False)
+        assert not any(s.startswith("secrets:") for s in steps)

tests/test_versions.py

@@ -0,0 +1,61 @@
+"""Tests for the classification of recipe version strings."""
+
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from alakazam import classify_version, is_hash, is_version
+
+# a coop cloud version is '<recipe-version>+<upstream-tag>'
+COOP_CLOUD_VERSION = "1.2.3+4.5.6-fpm"
+
+
+class TestIsHash:
+    @pytest.mark.parametrize("value", ["deadbeef", "DEADBEEF", "1234", "a" * 64])
+    def test_hashes(self, value):
+        assert is_hash(value)
+
+    @pytest.mark.parametrize("value", ["1.2.3", COOP_CLOUD_VERSION, "chaos", "latest", None, "", "abc", "g" * 8, "a" * 65])
+    def test_non_hashes(self, value):
+        assert not is_hash(value)
+
+
+class TestIsVersion:
+    @pytest.mark.parametrize("value", ["1.2.3", COOP_CLOUD_VERSION, "1234"])
+    def test_versions(self, value):
+        assert is_version(value)
+
+    @pytest.mark.parametrize("value", ["chaos", "latest", None, "", "deadbeef"])
+    def test_non_versions(self, value):
+        """None must not raise, packaging answers it with a TypeError rather than InvalidVersion."""
+        assert not is_version(value)
+
+
+class TestClassifyVersion:
+    @pytest.mark.parametrize(
+        "value,expected",
+        [
+            ("1.2.3", "version"),
+            (COOP_CLOUD_VERSION, "version"),
+            ("chaos", "chaos"),
+            ("latest", "chaos"),
+            (None, "chaos"),
+            ("deadbeef", "hash"),
+            ("", "unknown"),
+            ("no version at all", "unknown"),
+        ],
+    )
+    def test_classification(self, value, expected):
+        assert classify_version(value) == expected
+
+    def test_hash_takes_precedence_over_version(self):
+        """A purely numeric commit hash must not be mistaken for a release."""
+        assert is_version("1234") and is_hash("1234")
+        assert classify_version("1234") == "hash"
+
+    def test_upstream_tag_does_not_change_the_class(self):
+        """packaging treats everything after '+' as build metadata, the full string still classifies."""
+        assert classify_version("1.2.3") == classify_version(COOP_CLOUD_VERSION)

tests/test_versions_pinned.py

@@ -0,0 +1,79 @@
+"""Tests for a version that pins a commit while naming the release it is based on."""
+
+import logging
+import os
+import sys
+
+import click
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import alakazam
+from alakazam import classify_version, merge_instance_configs, resolve_version
+
+
+class TestResolveVersion:
+    def test_a_pinned_commit_wins_over_the_release(self):
+        assert resolve_version("12.0.2+2026.5.2@be9ebb3", "nextcloud") == "be9ebb3"
+
+    @pytest.mark.parametrize("version", ["1.2.3", "12.0.2+2026.5.2", "chaos", "latest", "be9ebb3"])
+    def test_a_version_without_a_commit_is_untouched(self, version):
+        assert resolve_version(version, "nextcloud") == version
+
+    @pytest.mark.parametrize("version", [None, 1.2, True])
+    def test_non_strings_are_untouched(self, version):
+        """The configuration is user supplied and does not have to hold a string."""
+        assert resolve_version(version, "nextcloud") is version
+
+    def test_a_trailing_at_is_rejected(self):
+        """Silently deploying the release instead of the intended commit would be worse."""
+        with pytest.raises(click.ClickException) as excinfo:
+            resolve_version("12.0.2@", "nextcloud")
+        assert "without a commit" in excinfo.value.message
+        assert "nextcloud" in excinfo.value.message
+
+    def test_the_release_is_kept_in_the_debug_log(self, caplog):
+        """It is the only place the base release survives, and CI failures are read there."""
+        with caplog.at_level(logging.DEBUG):
+            resolve_version("12.0.2+2026.5.2@be9ebb3", "nextcloud")
+        assert "based on 12.0.2+2026.5.2" in caplog.text
+
+    def test_the_last_at_separates(self):
+        assert resolve_version("a@b@be9ebb3", "nextcloud") == "be9ebb3"
+
+
+class TestClassification:
+    def test_the_resolved_commit_classifies_as_a_hash(self):
+        """upgrade branches on this, a release would take the wrong abra command."""
+        assert classify_version(resolve_version("12.0.2+2026.5.2@be9ebb3", "nextcloud")) == "hash"
+
+    def test_an_unpinned_release_still_classifies_as_a_version(self):
+        assert classify_version(resolve_version("12.0.2+2026.5.2", "nextcloud")) == "version"
+
+
+class TestMergedConfig:
+    """The resolution has to happen once, where the configuration is built."""
+
+    def merge(self, version):
+        return merge_instance_configs(
+            {}, "example.com", {"nextcloud": {"version": version}}, {})
+
+    def test_the_merged_config_carries_the_commit(self):
+        assert self.merge("12.0.2+2026.5.2@be9ebb3")["nextcloud"]["version"] == "be9ebb3"
+
+    def test_a_plain_version_survives_the_merge(self):
+        assert self.merge("12.0.2")["nextcloud"]["version"] == "12.0.2"
+
+    def test_an_app_without_a_version(self):
+        assert "version" not in self.merge(None)["nextcloud"] or \
+            self.merge(None)["nextcloud"]["version"] is None
+
+    def test_a_templated_version_is_resolved_after_substitution(self):
+        """Jinja runs first, so a version coming from GLOBALS is pinned as well."""
+        merged = merge_instance_configs(
+            {"GLOBALS": {"pin": "12.0.2+2026.5.2@be9ebb3"}},
+            "example.com",
+            {"nextcloud": {"version": "{{pin}}"}},
+            {})
+        assert merged["nextcloud"]["version"] == "be9ebb3"