fix: reproducable ssh-add

.gitignore

@@ -7,3 +7,4 @@
 /bin
 dist/
 tests/integration/.bats
+tests/resources/path_to_repo

Makefile

@@ -84,3 +84,12 @@ build-mo:
 
 release:
 	@goreleaser release --clean
+
+# TEST_INTEGRATION_FILES overrides the default of running all test files
+ifdef TEST_INTEGRATION_FILES
+test_files := $(TEST_INTEGRATION_FILES);
+else
+test_files := *;
+endif
+test-integration: build
+	@bats -Tp --verbose-run tests/integration/$(test_files)

flake.lock

@@ -18,6 +18,27 @@
         "type": "github"
       }
     },
+    "microvm": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "spectrum": "spectrum"
+      },
+      "locked": {
+        "lastModified": 1784666190,
+        "narHash": "sha256-xgfS6slV7J3baMooNN1UuBi51RIgg9y0DbCxfSA0668=",
+        "owner": "astro",
+        "repo": "microvm.nix",
+        "rev": "fa5340ac684cdce8a22b6d4a0bcebb0cc999275e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "astro",
+        "repo": "microvm.nix",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
       "locked": {
         "lastModified": 1778443072,
@@ -37,9 +58,26 @@
     "root": {
       "inputs": {
         "flake-utils": "flake-utils",
+        "microvm": "microvm",
         "nixpkgs": "nixpkgs"
       }
     },
+    "spectrum": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1783694892,
+        "narHash": "sha256-xO8f7Qng+18FK2UlB9vcrkxCaQMCt5WjCH24aW/11eg=",
+        "ref": "refs/heads/main",
+        "rev": "24c4346e30fdea8d8e80f34aec3554a15a667d24",
+        "revCount": 1410,
+        "type": "git",
+        "url": "https://spectrum-os.org/git/spectrum"
+      },
+      "original": {
+        "type": "git",
+        "url": "https://spectrum-os.org/git/spectrum"
+      }
+    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,

flake.nix

@@ -4,34 +4,103 @@
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable";
     flake-utils.url = "github:numtide/flake-utils";
+    microvm = {
+      url = "github:astro/microvm.nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
   outputs =
     {
       self,
       nixpkgs,
+      microvm,
       flake-utils,
     }:
-    flake-utils.lib.eachDefaultSystem (
-      system:
-      let
-        pkgs = nixpkgs.legacyPackages.${system};
-      in
-      {
-        packages = rec {
-          abra = pkgs.callPackage ./package.nix { };
-          default = abra;
+    let
+      system = "x86_64-linux";
+      pkgs = nixpkgs.legacyPackages.${system};
+      publicKey = builtins.readFile ./tests/resources/local_integration_ssh.pub;
+      privateKey = builtins.readFile ./tests/resources/local_integration_ssh;
+      pathToRepo = builtins.readFile ./tests/resources/path_to_repo;
+      # local DNS aliases for the server host
+      extraHosts = builtins.readFile ./tests/resources/extra_hosts;
+      username = "abra";
+      password = "abra";
+      defaultDNS = [
+        # Quad9.net
+        "9.9.9.9"
+        "149.112.112.112"
+        "2620:fe::fe"
+        "2620:fe::9"
+      ];
+    in
+    {
+      nixosConfigurations = {
+        abra-client = nixpkgs.lib.nixosSystem {
+          system = "x86_64-linux";
+          modules = [
+            microvm.nixosModules.microvm
+            ./nix/hosts/client/configuration.nix
+          ];
+          specialArgs = {
+            inherit
+              publicKey
+              privateKey
+              username
+              password
+              defaultDNS
+              extraHosts
+              pathToRepo
+              ;
+          };
         };
-        apps = rec {
-          abra = flake-utils.lib.mkApp { drv = self.packages.${system}.abra; };
-          default = abra;
+        abra-server = nixpkgs.lib.nixosSystem {
+          system = "x86_64-linux";
+          modules = [
+            microvm.nixosModules.microvm
+            ./nix/hosts/server/configuration.nix
+          ];
+          specialArgs = {
+            inherit
+              publicKey
+              privateKey
+              username
+              password
+              defaultDNS
+              ;
+          };
         };
-        devShells.default = pkgs.mkShell {
-          packages = with pkgs; [
-            go_1_26
-            gnumake
+      };
+      nixosModules = rec {
+        host = {
+          imports = [
+            microvm.nixosModules.host
+            ./nix/modules/host.nix
           ];
         };
-      }
-    );
+        default = host;
+      };
+      packages = rec {
+        abra = pkgs.callPackage ./nix/package.nix { };
+        default = abra;
+      };
+      apps = rec {
+        abra = flake-utils.lib.mkApp { drv = self.packages.${system}.abra; };
+        default = abra;
+      };
+      devShells.${system}.default = pkgs.mkShell {
+        # testing env variables
+        BATS_LIB_PATH = "~/.local/share/bats/";
+        TEST_SERVER = "abra.local";
+        ABRA_DIR = "$HOME/.abra_test";
+
+        packages = with pkgs; [
+          go_1_26
+          gnumake
+          gopls
+        ];
+      };
+
+    };
 }

nix/Makefile

@@ -0,0 +1,43 @@
+GIT_ROOT := $$(git rev-parse --show-toplevel)
+
+test-integration:
+	@$(GIT_ROOT)/nix/test_integration.sh
+
+test-integration-hosts:
+	$(GIT_ROOT)/scripts/tests/extra_hosts
+
+CLIENT_VM := abra-client
+vm-client-status:
+	sudo systemctl status microvm@$(CLIENT_VM).service
+vm-client-create:test-integration-hosts
+	cd $(GIT_ROOT) && sudo microvm -f git+file://$(GIT_ROOT) -c $(CLIENT_VM)
+vm-client-start:
+	sudo systemctl start microvm@$(CLIENT_VM).service
+vm-client-update:test-integration-hosts
+	cd $(GIT_ROOT) && sudo microvm -R -f git+file://$(GIT_ROOT) -u $(CLIENT_VM) 
+vm-client-run:
+	sudo microvm -f git+file://$(GIT_ROOT) -r $(CLIENT_VM)
+vm-client-stop:
+	sudo systemctl stop microvm@$(CLIENT_VM)
+vm-client-delete: vm-client-stop
+	sudo rm -rf /var/lib/microvms/$(CLIENT_VM)
+vm-client-connect:
+	ssh abra@10.0.0.2 -i $(GIT_ROOT)/tests/resources/local_integration_ssh
+
+SERVER_VM := abra-server
+vm-server-status:
+	sudo systemctl status microvm@$(SERVER_VM).service
+vm-server-create:
+	cd $(GIT_ROOT) && sudo microvm -f git+file://$(GIT_ROOT) -c $(SERVER_VM)
+vm-server-start:
+	sudo systemctl start microvm@$(SERVER_VM).service
+vm-server-update:
+	cd $(GIT_ROOT) && sudo microvm -R -f git+file://$(GIT_ROOT) -u $(SERVER_VM) 
+vm-server-run:
+	cd $(GIT_ROOT) && sudo microvm -f git+file://$(GIT_ROOT) -r $(SERVER_VM)
+vm-server-stop:
+	sudo systemctl stop microvm@$(SERVER_VM)
+vm-server-delete: vm-server-stop
+	sudo rm -rf /var/lib/microvms/$(SERVER_VM)
+vm-server-connect:
+	ssh abra@10.0.0.3 -i $(GIT_ROOT)/tests/resources/local_integration_ssh

nix/hosts/README.md

@@ -0,0 +1,48 @@
+# Hosts
+These hosts are VMs used for running the integration tests on your local machine.
+The hosts contain a client where abra with the tests will be run and a server
+to simulate a remote machine.
+
+## Prepare the host
+For this setup to work you need a machine with NixOS and flakes enabled.
+
+1. Import the abra flake
+
+The install example is based on the [using nix flakes wiki page](https://nixos.wiki/wiki/flakes#Using_nix_flakes_with_NixOS).
+```nix
+inputs = {
+  abra = {
+    url = "git+https://git.coopcloud.tech/toolshed/abra.git";
+  };
+};
+```
+2. Add the host module to your configuration
+
+Now add the host module to your configuration. At the toplevel of a flake it could look like this:
+```nix
+desktop = inputs.nixpkgs.lib.nixosSystem {
+  system = "x86_64-linux";
+  modules = [
+    abra.nixosModules.host
+    ./configuration.nix
+  ];
+  specialArgs = inputs;
+};
+```
+3. Adjust the required config
+```nix
+abra.testing = {
+  enable = true; # loads the required config, when set true
+  externalInterface = "eth1"; # adjust this to your network interface that has access to the internet
+};
+```
+You can look up the interface with `ip link`. After that rebuild your system and you are ready to go.
+
+## Get started
+After preparing the host run:
+```
+make test-integration
+```
+This runs the automatic setup of the VMs and network. You will be prompted if you want to run all
+or specific files.
+

nix/hosts/client/configuration.nix

@@ -0,0 +1,145 @@
+{
+  lib,
+  pkgs,
+  publicKey,
+  username,
+  pathToRepo,
+  defaultDNS,
+  extraHosts,
+  ...
+}:
+let
+  index = 2;
+  mac = "00:00:00:00:00:01";
+  serverIp = "10.0.0.3";
+in
+{
+  imports = [
+    ../../modules/ssh.nix
+    ../../modules/user.nix
+    ../../modules/docker.nix
+  ];
+
+  microvm = {
+    vcpu = 4;
+    mem = 2049; # see issue related in microvm repo with 2048, so add 1MB
+    interfaces = [
+      {
+        id = "vm${toString index}";
+        type = "tap";
+        inherit mac;
+      }
+    ];
+    shares = [
+      {
+        proto = "virtiofs";
+        tag = "repo";
+        # Source path can be absolute or relative
+        # to /var/lib/microvms/$hostName
+        source = "${lib.trim pathToRepo}";
+        mountPoint = "/home/${username}/abra";
+      }
+    ];
+  };
+
+  boot.tmp = {
+    useTmpfs = true;
+    tmpfsSize = "2G";
+  };
+
+  networking = {
+    hostName = "abra-client";
+    useNetworkd = true;
+  };
+  systemd.network.networks."10-eth" = {
+    matchConfig.MACAddress = mac;
+    # Static IP configuration
+    address = [
+      "10.0.0.${toString index}/32"
+      "fec0::${lib.toHexString index}/128"
+    ];
+    routes = [
+      {
+        # A route to the host
+        Destination = "10.0.0.0/32";
+        GatewayOnLink = true;
+      }
+      {
+        # Route to server
+        Destination = "${serverIp}/32";
+        Gateway = "10.0.0.0";
+        GatewayOnLink = true;
+      }
+      {
+        # Default route
+        Destination = "0.0.0.0/0";
+        Gateway = "10.0.0.0";
+        GatewayOnLink = true;
+      }
+      {
+        # Default route
+        Destination = "::/0";
+        Gateway = "fec0::";
+        GatewayOnLink = true;
+      }
+    ];
+    networkConfig = {
+      # DNS servers no longer come from DHCP nor Router
+      # Advertisements. Perhaps you want to change the defaults:
+      DNS = defaultDNS;
+    };
+  };
+
+  # open the ports to allow ssh access from the host
+  networking.firewall.allowedTCPPorts = [ 22 ];
+  networking.firewall.allowedUDPPorts = [ 22 ];
+
+  networking.extraHosts = extraHosts;
+
+  environment.variables = {
+    CGO_ENABLED = 0;
+    TEST_SERVER = "abra.local";
+    ABRA_DIR = "$HOME/.abra_test";
+  };
+
+  programs.ssh = {
+    startAgent = true;
+    knownHostsFiles = [
+      (pkgs.writeText "local.keys" ''
+        abra.local ${publicKey} 
+      '')
+    ];
+    extraConfig = ''
+      Host abra.local
+        HostName abra.local
+        Port 22
+        User ${username}
+    '';
+  };
+  programs.git = {
+    enable = true;
+    # many recipe commands need a preset git user
+    config = [
+      {
+        user = {
+          name = "abra dev";
+          email = "helo@coopcloud.tech";
+        };
+      }
+    ];
+  };
+
+  environment.systemPackages = with pkgs; [
+    # build dependencies, copied from flake.nix
+    go_1_26
+    gnumake
+    # testing dependencies
+    (bats.withLibraries (p: [
+      p.bats-assert
+      p.bats-file
+      p.bats-support
+    ]))
+    jq
+    wget
+  ];
+}

nix/hosts/client/home.nix

@@ -0,0 +1,42 @@
+{
+  serverIp,
+  username,
+  ...
+}:
+{
+  # Home Manager needs a bit of information about you and the
+  # paths it should manage.
+  home.username = username;
+  home.homeDirectory = "/home/${username}";
+
+  # This value determines the Home Manager release that your
+  # configuration is compatible with. This helps avoid breakage
+  # when a new Home Manager release introduces backwards
+  # incompatible changes.
+  #
+  # You can update Home Manager without changing this value. See
+  # the Home Manager release notes for a list of state version
+  # changes in each release.
+  home.stateVersion = "26.05";
+
+  # Let Home Manager install and manage itself.
+  programs.home-manager.enable = true;
+  programs.ssh = {
+    enable = true;
+    enableDefaultConfig = false;
+    settings = {
+      "abra.local" = {
+        HostName = serverIp;
+        User = username;
+        Port = 22;
+        IdentityFile = "/home/${username}/abra/tests/resources/local_integration_ssh";
+      };
+      "*.abra.local" = {
+        HostName = serverIp;
+        User = username;
+        Port = 22;
+        IdentityFile = "/home/${username}/abra/tests/resources/local_integration_sshn";
+      };
+    };
+  };
+}

nix/hosts/server/configuration.nix

@@ -0,0 +1,91 @@
+{
+  lib,
+  defaultDNS,
+  ...
+}:
+let
+  index = 3;
+  mac = "00:00:00:00:00:02";
+  clientIp = "10.0.0.2";
+in
+{
+  imports = [
+    ../../modules/ssh.nix
+    ../../modules/user.nix
+    ../../modules/docker.nix
+  ];
+
+  microvm = {
+    vcpu = 4;
+    mem = 8097; # see issue related in microvm repo with 2048, so add 1MB
+    interfaces = [
+      {
+        id = "vm${toString index}";
+        type = "tap";
+        inherit mac;
+      }
+    ];
+  };
+
+  boot.tmp = {
+    useTmpfs = true;
+    tmpfsSize = "8G";
+  };
+
+  networking = {
+    hostName = "abra-server";
+    useNetworkd = true;
+  };
+  systemd.network.networks."11-eth" = {
+    matchConfig.MACAddress = mac;
+    # Static IP configuration
+    address = [
+      "10.0.0.${toString index}/32"
+      "fec0::${lib.toHexString index}/128"
+    ];
+    routes = [
+      {
+        # A route to the host
+        Destination = "10.0.0.0/32";
+        GatewayOnLink = true;
+      }
+      {
+        # Route to server
+        Destination = "${clientIp}/32";
+        Gateway = "10.0.0.0";
+        GatewayOnLink = true;
+      }
+      {
+        # Default route
+        Destination = "0.0.0.0/0";
+        Gateway = "10.0.0.0";
+        GatewayOnLink = true;
+      }
+      {
+        # Default route
+        Destination = "::/0";
+        Gateway = "fec0::";
+        GatewayOnLink = true;
+      }
+    ];
+    networkConfig = {
+      # DNS servers no longer come from DHCP nor Router
+      # Advertisements. Perhaps you want to change the defaults:
+      DNS = defaultDNS;
+    };
+  };
+
+  # open the ports to allow ssh access from the host
+  networking.firewall.allowedTCPPorts = [
+    22
+    80
+    443
+    1312 # deploy with udp and tcp on same port test
+  ];
+  networking.firewall.allowedUDPPorts = [
+    22
+    80
+    443
+    1312 # deploy with udp and tcp on same port test
+  ];
+}

nix/modules/docker.nix

@@ -0,0 +1,7 @@
+{ ... }:
+{
+  virtualisation.docker = {
+    enable = true;
+    liveRestore = false;
+  };
+}

nix/modules/host.nix

@@ -0,0 +1,59 @@
+{
+  lib,
+  config,
+  ...
+}:
+let
+  maxVMs = 3;
+in
+{
+  options = {
+    abra.testing.enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Enable the abra integration testing framework";
+    };
+    abra.testing.externalInterface = lib.mkOption {
+      type = lib.types.str;
+      default = null;
+      description = "Change this to the interface with upstream Internet access";
+    };
+  };
+
+  config = lib.mkIf config.abra.testing.enable {
+    networking.useNetworkd = true;
+    systemd.network.wait-online.enable = false;
+    systemd.network.networks = builtins.listToAttrs (
+      map (index: {
+        name = "30-vm${toString index}";
+        value = {
+          matchConfig.Name = "vm${toString index}";
+          # Host's addresses
+          address = [
+            "10.0.0.0/32"
+            "fec0::/128"
+          ];
+          # Setup routes to the VM
+          routes = [
+            {
+              Destination = "10.0.0.${toString index}/32";
+            }
+            {
+              Destination = "fec0::${lib.toHexString index}/128";
+            }
+          ];
+          # Enable routing
+          networkConfig = {
+            IPv4Forwarding = true;
+            IPv6Forwarding = true;
+          };
+        };
+      }) (lib.genList (i: i + 1) maxVMs)
+    );
+    networking.nat = {
+      enable = true;
+      internalIPs = [ "10.0.0.0/24" ];
+      externalInterface = config.abra.testing.externalInterface;
+    };
+  };
+}

nix/modules/ssh.nix

@@ -0,0 +1,33 @@
+{
+  privateKey,
+  publicKey,
+  username,
+  ...
+}:
+{
+  # default host key location
+  environment.etc."ssh/ssh_host_ed25519_key" = {
+    text = privateKey;
+    mode = "0600";
+  };
+  environment.etc."ssh/ssh_host_ed25519_key.pub" = {
+    text = publicKey;
+    mode = "0644";
+  };
+
+  services.openssh = {
+    enable = true;
+    settings = {
+      PasswordAuthentication = false;
+      PermitRootLogin = "no";
+      AllowUsers = [ "${username}" ];
+    };
+    generateHostKeys = false;
+    hostKeys = [
+      {
+        path = "/etc/ssh/ssh_host_ed25519_key";
+        type = "ed25519";
+      }
+    ];
+  };
+}

nix/modules/user.nix

@@ -0,0 +1,22 @@
+{
+  username,
+  password,
+  publicKey,
+  ...
+}:
+{
+  users.users."${username}" = {
+    isNormalUser = true;
+    password = password;
+    home = "/home/${username}";
+    description = "Abra user";
+    extraGroups = [
+      "wheel"
+      "networkmanager"
+      "docker"
+    ];
+    openssh.authorizedKeys.keys = [
+      publicKey
+    ];
+  };
+}

nix/test_integration.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+GIT_ROOT=$(git rev-parse --show-toplevel)
+# write path to repo
+echo "$GIT_ROOT" > "$GIT_ROOT/tests/resources/path_to_repo" 
+
+# create or update a VM, start it and wait for successful ping
+start_vm () {
+  if [ -d "/var/lib/microvms/abra-$1" ]; then
+    make "vm-$1-update"
+  else
+    make "vm-$1-create"
+  fi
+  make "vm-$1-start"
+
+  host_available=0
+  while [ $host_available == 0 ]; do
+    if ping -c 1 "$2" &> /dev/null
+    then
+      host_available=1
+    else
+	    echo "could not ping $1, waiting for 5 seconds..."
+      sleep 5
+    fi
+  done
+}
+
+start_vm "client" "10.0.0.2"
+start_vm "server" "10.0.0.3"
+
+# revert dynamic content of file to prevent commiting changes
+git restore -- "$GIT_ROOT/tests/resources/path_to_repo"
+
+read -rp "Press enter to run all tests or specify test files: " TEST_INTEGRATION_FILES
+# -t option is needed for stdin output that is checked in integration tests
+ssh -t abra@10.0.0.2 -i "$GIT_ROOT/tests/resources/local_integration_ssh" "cd abra; chmod 600 ./tests/resources/local_integration_ssh; ssh-add ./tests/resources/local_integration_ssh; make test-integration TEST_INTEGRATION_FILES=$TEST_INTEGRATION_FILES" 
+
+
+

scripts/tests/extra_hosts

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# This file generates local DNS aliases for the client VM for integration testing
+
+GIT_ROOT=$(git rev-parse --show-toplevel)
+echo "generating hosts file..."
+hosts_file_path="$GIT_ROOT/tests/resources/extra_hosts"
+echo "removing old file..."
+rm -f "$hosts_file_path"
+
+# the server domain
+server_ip="10.0.0.3"
+domain="abra.local"
+echo "$server_ip $domain" >> "$hosts_file_path"
+# static subdomains used in integration tests
+subdomains=("gitea" "zammad" "custom-html" "foo" "foobar")
+
+for subdomain in "${subdomains[@]}"
+do
+  echo "$server_ip $subdomain.$domain" >> "$hosts_file_path"
+done
+
+search_dir="$GIT_ROOT/tests/integration"
+for entry in "$search_dir"/*.bats
+do
+	file_name=$(basename "${entry}")
+	# converting file names to testing subdomains
+	subdomain=$(echo "$file_name" | tr . _)
+	echo "$server_ip $subdomain.$domain" >> "$hosts_file_path"
+done
+
+echo "writing extra hosts success"

tests/README.md

@@ -7,4 +7,5 @@
 
 * Integration tests are in `./tests/integration`. Please see [these
   docs](https://docs.coopcloud.tech/abra/hack/#integration-tests) for
-  instructions and tips on how to run them.
+  instructions and tips on how to run them. If you want to run it locally via nix,
+  read the [nix testing docs](../nix/hosts/README.md). 

tests/integration/app_deploy.bats

@@ -598,8 +598,7 @@ teardown(){
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
   assert_success
 
-  run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
-    $(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
+ 	_inspect_env_test_app 
   assert_success
   assert_output --partial "WITH_COMMENT=foo"
 
@@ -610,8 +609,7 @@ teardown(){
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input --force
   assert_success
 
-  run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
-    $(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
+ 	_inspect_env_test_app 
   assert_success
   refute_output --partial "WITH_COMMENT=foo"
   assert_output --partial "WITH_COMMENT=bar"
@@ -629,8 +627,14 @@ teardown(){
   run $ABRA app deploy "$TEST_APP_DOMAIN" --no-input
   assert_success
 
-  run docker service inspect  --format '{{ range .Endpoint.Ports }}{{ .Protocol }}={{ .PublishedPort }}{{ end }}' \
+	if _is_local; 
+	then 
+		run docker service inspect --format "{{ range .Endpoint.Ports }}{{ .Protocol }}={{ .PublishedPort }}{{ end }}" \
     "${TEST_APP_DOMAIN//./_}_app"
+	else 
+		run ssh "$TEST_SERVER" "docker service inspect --format '{{ range .Endpoint.Ports }}{{ .Protocol }}={{ .PublishedPort }}{{ end }}' ${TEST_APP_DOMAIN//./_}_app"
+	fi
+  
   assert_success
   assert_output --partial "tcp=1312"
   assert_output --partial "udp=1312"

tests/integration/app_deploy_env_version.bats

@@ -133,8 +133,7 @@ teardown(){
   run $ABRA app deploy "$TEST_APP_DOMAIN" "0.1.0+1.20.0" --no-input
   assert_success
 
-  run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
-    $(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
+ 	_inspect_env_test_app 
   assert_success
   assert_output --partial "$TEST_RECIIPE:0.1.0+1.20.0"
 }

tests/integration/app_remove.bats

@@ -140,15 +140,22 @@ teardown(){
   _undeploy_app
 
   sanitisedDomainName="${TEST_APP_DOMAIN//./_}"
-  run docker config create "${sanitisedDomainName}_test_conf_v99" "$ABRA_DIR/recipes/abra-test-recipe/abra.sh"
-  assert_success
+	remote_ssh_command=""
+	if _is_local; 
+	then 
+  	run docker config create "${sanitisedDomainName}_test_conf_v99" "$ABRA_DIR/recipes/abra-test-recipe/abra.sh"
+	else
+		remote_ssh_command="ssh '$TEST_SERVER'"
+		run cat "$ABRA_DIR/recipes/abra-test-recipe/abra.sh" | ssh "$TEST_SERVER" docker config create "${sanitisedDomainName}_test_conf_v99 -"
+		assert_success
+	fi
 
-  assert bash -c "docker config ls | grep -q test_conf_v99"
+  assert bash -c "$remote_ssh_command docker config ls | grep -q test_conf_v99"
 
   run $ABRA app rm "$TEST_APP_DOMAIN" --no-input
   assert_success
 
-  refute bash -c "docker config ls | grep -q test_conf_v99"
+  refute bash -c "$remote_ssh_command docker config ls | grep -q test_conf_v99"
 }
 
 @test "remove .env file" {

tests/integration/app_rollback_env_version.bats

@@ -54,8 +54,7 @@ teardown(){
     --no-input
   assert_success
 
-  run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
-    $(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
+	_inspect_env_test_app
   assert_success
   assert_output --partial "$TEST_RECIIPE:0.1.0+1.20.0"
 }

tests/integration/app_upgrade_env_version.bats

@@ -53,8 +53,7 @@ teardown(){
   run $ABRA app upgrade "$TEST_APP_DOMAIN" "0.2.0+1.21.0" --no-input
   assert_success
 
-  run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' \
-    $(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)
+ 	_inspect_env_test_app 
   assert_success
-  assert_output --partial "$TEST_RECIIPE:0.2.0+1.21.0"
+  assert_output --partial "$TEST_RECIPE:0.2.0+1.21.0"
 }

tests/integration/helpers/common.bash

@@ -31,9 +31,18 @@ _ensure_ssh_agent() {
     exit 1
   fi
 
-  export SSH_AUTH_SOCK="$HOME/.ssh/ssh_auth_sock"
-  if [ ! -S ~/.ssh/ssh_auth_sock ]; then
+  if [[ ! -v SSH_AUTH_SOCK ]]; then
+		export SSH_AUTH_SOCK="$HOME/.ssh/ssh_auth_sock"
     eval `ssh-agent`
     ln -sf "$SSH_AUTH_SOCK" ~/.ssh/ssh_auth_sock
   fi
 }
+
+_is_local() {
+	if [[ "$TEST_SERVER" == "default" ]];
+  then
+    return 0 
+  else
+    return 1
+  fi
+}

tests/integration/helpers/docker.bash

@@ -1,13 +1,37 @@
 #!/usr/bin/env bash
 
 _ensure_swarm() {
-  if [ "$(docker info | grep Swarm | sed 's/Swarm: //g' | tr -d ' ')" == "inactive" ]; then
-    run docker swarm init --advertise-addr 127.0.0.1:2377
-    assert_success
+	if [ "$(docker info | grep Swarm | sed 's/Swarm: //g' | tr -d ' ')" == "inactive" ]; then
+   	run docker swarm init --advertise-addr 127.0.0.1:2377
+   	assert_success
   fi
 
-  if ! $(docker network ls | grep -q 'proxy'); then
-    run docker network create -d overlay proxy
-    assert_success
+  if ! docker network ls | grep -q 'proxy'; then
+   	run docker network create -d overlay proxy
+   	assert_success
   fi
+
+	if ! _is_local; 
+	then 
+		if [ "$(ssh "$TEST_SERVER" docker info | grep Swarm | sed 's/Swarm: //g' | tr -d ' ')" == "inactive" ]; then
+    	run ssh "$TEST_SERVER" docker swarm init --advertise-addr 127.0.0.1:2377
+    	assert_success
+  	fi
+
+  	if ! ssh "$TEST_SERVER" docker network ls | grep -q 'proxy'; then
+    	run ssh "$TEST_SERVER" docker network create -d overlay proxy
+    	assert_success
+  	fi
+	fi
+}
+
+_inspect_env_test_app() {
+	if _is_local; 
+	then 
+		containerId="$(docker ps -f name="$TEST_APP_DOMAIN_$TEST_SERVER" -q)" 
+		run docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' "$containerId"
+	else 
+		containerId="$(ssh "$TEST_SERVER" docker ps -f name="$TEST_APP_DOMAIN" -q)"
+		run ssh "$TEST_SERVER" "docker inspect --format='{{range .Config.Env}}{{println .}}{{end}}' '$containerId'"
+	fi
 }

tests/integration/recipe_upgrade.bats

@@ -38,8 +38,6 @@ teardown(){
 
   run $ABRA recipe upgrade "custom-html" --no-input
   assert_success
-  assert_output --partial 'can upgrade service: app'
-
   assert_exists "$ABRA_DIR/recipes/custom-html"
 }
 

tests/integration/server_add.bats

@@ -27,7 +27,7 @@ teardown(){
   assert bash -c "docker context ls | grep -q $TEST_SERVER"
 
   server_dir_perms=$(stat -c "%a" "$ABRA_DIR/servers/$TEST_SERVER")
-  assert_equal $server_dir_perms "700"
+  assert_equal "$server_dir_perms" "700"
 }
 
 @test "error if using name and --local together" {
@@ -43,8 +43,8 @@ teardown(){
   assert bash -c "docker context ls | grep -q default"
   assert_output --partial 'local server successfully added'
 
-  server_dir_perms=$(stat -c "%a" "$ABRA_DIR/servers/$TEST_SERVER")
-  assert_equal $server_dir_perms "700"
+  server_dir_perms=$(stat -c "%a" "$ABRA_DIR/servers/default")
+  assert_equal "$server_dir_perms" "700"
 }
 
 @test "create local server fails when no docker swarm" {

tests/resources/extra_hosts

@@ -0,0 +1,57 @@
+10.0.0.3 abra.local
+10.0.0.3 gitea.abra.local
+10.0.0.3 zammad.abra.local
+10.0.0.3 custom-html.abra.local
+10.0.0.3 foo.abra.local
+10.0.0.3 foobar.abra.local
+10.0.0.3 app_check_bats.abra.local
+10.0.0.3 app_cmd_bats.abra.local
+10.0.0.3 app_config_bats.abra.local
+10.0.0.3 app_cp_bats.abra.local
+10.0.0.3 app_deploy_bats.abra.local
+10.0.0.3 app_deploy_env_version_bats.abra.local
+10.0.0.3 app_deploy_overview_bats.abra.local
+10.0.0.3 app_deploy_remote_recipes_bats.abra.local
+10.0.0.3 app_env_bats.abra.local
+10.0.0.3 app_env_version_bats.abra.local
+10.0.0.3 app_labels_bats.abra.local
+10.0.0.3 app_list_bats.abra.local
+10.0.0.3 app_logs_bats.abra.local
+10.0.0.3 app_move_bats.abra.local
+10.0.0.3 app_new_bats.abra.local
+10.0.0.3 app_ps_bats.abra.local
+10.0.0.3 app_remove_bats.abra.local
+10.0.0.3 app_restart_bats.abra.local
+10.0.0.3 app_rollback_bats.abra.local
+10.0.0.3 app_rollback_env_version_bats.abra.local
+10.0.0.3 app_rollback_overview_bats.abra.local
+10.0.0.3 app_run_bats.abra.local
+10.0.0.3 app_secret_bats.abra.local
+10.0.0.3 app_secret_env_version_bats.abra.local
+10.0.0.3 app_services_bats.abra.local
+10.0.0.3 app_undeploy_bats.abra.local
+10.0.0.3 app_undeploy_env_version_bats.abra.local
+10.0.0.3 app_undeploy_overview_bats.abra.local
+10.0.0.3 app_upgrade_bats.abra.local
+10.0.0.3 app_upgrade_env_version_bats.abra.local
+10.0.0.3 app_upgrade_overview_bats.abra.local
+10.0.0.3 app_volume_bats.abra.local
+10.0.0.3 autocomplete_bats.abra.local
+10.0.0.3 catalogue_bats.abra.local
+10.0.0.3 dirs_bats.abra.local
+10.0.0.3 install_bats.abra.local
+10.0.0.3 recipe_diff_bats.abra.local
+10.0.0.3 recipe_fetch_bats.abra.local
+10.0.0.3 recipe_lint_bats.abra.local
+10.0.0.3 recipe_list_bats.abra.local
+10.0.0.3 recipe_new_bats.abra.local
+10.0.0.3 recipe_release_bats.abra.local
+10.0.0.3 recipe_reset_bats.abra.local
+10.0.0.3 recipe_upgrade_bats.abra.local
+10.0.0.3 recipe_version_bats.abra.local
+10.0.0.3 server_add_bats.abra.local
+10.0.0.3 server_list_bats.abra.local
+10.0.0.3 server_prune_bats.abra.local
+10.0.0.3 server_remove_bats.abra.local
+10.0.0.3 upgrade_bats.abra.local
+10.0.0.3 version_bats.abra.local

tests/resources/local_integration_ssh

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACBQ967UhxmFkq71WXvVRkEmtehCGsEnBVppAXxgtHHm7wAAAJhFGwTnRRsE
+5wAAAAtzc2gtZWQyNTUxOQAAACBQ967UhxmFkq71WXvVRkEmtehCGsEnBVppAXxgtHHm7w
+AAAEAnHEZcf2NeRQEcJC/aVgUWsdOz+vQgEG9ZY+3ErCeaKFD3rtSHGYWSrvVZe9VGQSa1
+6EIawScFWmkBfGC0cebvAAAAFWFicmEgaW50ZWdyYXRpb24gdGVzdA==
+-----END OPENSSH PRIVATE KEY-----

tests/resources/local_integration_ssh.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFD3rtSHGYWSrvVZe9VGQSa16EIawScFWmkBfGC0cebv abra integration test

tests/resources/path_to_repo

@@ -0,0 +1 @@
+/path/to/this/repo