test: improve semi-manual testing

tests/integration/README.md

@@ -2,3 +2,10 @@
 
 - `cp .envrc.sample .envrc` (fill out values && `direnv allow`)
 - `TARGET=install.sh make` (ensure `docker context use default`)
+
+`testfunctions.sh` contains the functions necessary to save and manipulate logs
+run `test_all.sh logdir` to run tests specified in that file and save the logs to `logdir`
+when creating new tests, make sure the test command is a one-liner (you can use `;` to separate commands). include `testfunctions.sh` and then write your tests like this:
+`run_test '$ABRA other stuff here'`
+
+by default, the testing script will ask after every command if the execution succeeded. If you reply `n`, it will log the test in the `logdir`. If you want all tests to run without questions, run `export logall=yes` before executing the test script

tests/integration/app.sh

@@ -1,15 +1,16 @@
 #!/bin/bash
 
+source ./testfunctions.sh
 source ./common.sh
 
 echo "all apps, all servers"
-$ABRA app ls
+run_test '$ABRA app ls'
 printf "\\n\\n\\n"
 
 echo "all wordpress apps, all servers"
-$ABRA app ls --type wordpress
+run_test '$ABRA app ls --type wordpress'
 printf "\\n\\n\\n"
 
 echo "all wordpress apps, only server2"
-$ABRA app ls --type wordpress --server server2
+run_test '$ABRA app ls --type wordpress --server server2'
 printf "\\n\\n\\n"

tests/integration/autocomplete.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
 
+source ./testfunctions.sh
 source ./common.sh
 
-$ABRA autocomplete bash
+run_test '$ABRA autocomplete bash'
 
-$ABRA autocomplete fizsh
+run_test '$ABRA autocomplete fizsh'
 
-$ABRA autocomplete zsh
+run_test '$ABRA autocomplete zsh'

tests/integration/catalogue.sh

@@ -1,7 +1,8 @@
 #!/bin/bash
 
+source ./testfunctions.sh
 source ./common.sh
 
-$ABRA catalogue generate --debug
+run_test '$ABRA --debug catalogue generate'
 
-$ABRA catalogue generate gitea --debug
+run_test '$ABRA --debug catalogue generate gitea'

tests/integration/install.sh

@@ -1,15 +1,12 @@
 #!/bin/bash
 
+source ./testfunctions.sh
 source ./common.sh
 
-wget -O- https://install.abra.autonomic.zone | bash
-~/.local/bin/abra -v
+run_test 'wget -O- https://install.abra.autonomic.zone | bash; ~/.local/bin/abra -v'
 
-wget -O- https://install.abra.autonomic.zone | bash -s -- --rc
-~/.local/bin/abra -v
+run_test 'wget -O- https://install.abra.autonomic.zone | bash -s -- --rc; ~/.local/bin/abra -v'
 
-$ABRA upgrade
-~/.local/bin/abra -v
+run_test '$ABRA upgrade; ~/.local/bin/abra -v'
 
-$ABRA upgrade --rc
-~/.local/bin/abra -v
+run_test '$ABRA upgrade --rc; ~/.local/bin/abra -v'

tests/integration/recipe.sh

@@ -1,12 +1,13 @@
 #!/bin/bash
 
+source ./testfunctions.sh
 source ./common.sh
 
-$ABRA recipe new testrecipe
+run_test '$ABRA recipe new testrecipe'
 
-$ABRA recipe list
-$ABRA recipe list -p cloud
+run_test '$ABRA recipe list'
+run_test '$ABRA recipe list -p cloud'
 
-$ABRA recipe versions peertube
+run_test '$ABRA recipe versions peertube'
 
-$ABRA recipe lint gitea
+run_test '$ABRA recipe lint gitea'

tests/integration/test_all.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+if [ -z $1 ]; then
+    echo "usage: ./test_all.sh logdir"
+    exit
+fi
+
+res_dir=$1/
+if [[ ! -d "$res_dir" ]]; then
+    mkdir "$res_dir"
+fi
+
+# Usage: run_test [number] [name] [command]
+run_test () {
+    logfile="$res_dir/$1-$2.log"
+    echo $logfile
+}
+
+testScripts=("app.sh" "recipe.sh" "install.sh")
+
+for i in "${testScripts[@]}"; do
+    cmd="./$i $res_dir${i/sh/log}"
+    eval $cmd
+done

tests/integration/testfunctions.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+if [ -z $1 ]; then
+    logfile=/dev/null
+else
+    logfile=$1
+fi
+
+if [ -z $logall ]; then
+    logall=no
+fi
+
+run_test () {
+    if [ -z "$@" ]; then
+        echo "run_test needs a command to run"
+    else
+        tempLogfile=$(mktemp)
+        cmd=$(eval echo "$@")
+        echo "------------ INPUT -------------------" | tee -a $tempLogfile
+        echo "$" "$cmd"  | tee -a $tempLogfile
+        echo "------------ OUTPUT ------------------" | tee -a $tempLogfile
+        eval $cmd 2>&1 | tee -a $tempLogfile
+        if [ $logall = "yes" ]; then
+            cat $tempLogfile >> $logfile
+            echo -e "\\n\\n" >> $logfile
+        else
+            read -N 1 -p "Did the test pass? [y/n]: " pass
+            if [ $pass = 'n' ]; then
+                cat $tempLogfile >> $logfile
+                echo -e "\\n\\n" >> $logfile
+            fi
+        fi
+        rm $tempLogfile
+    fi
+}