rix/abra
0
0
Fork 0
forked from toolshed/abra
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor!: consolidate SSH handling

Browse Source 
Closes coop-cloud/organising#389.
Closes coop-cloud/organising#341.
Closes coop-cloud/organising#326.
Closes coop-cloud/organising#380.
Closes coop-cloud/organising#360.
This commit is contained in:
decentral1se
2023-01-31 16:09:09 +01:00
committed by Gitea
parent f20fbbc913
commit 7c1a97be72
23 changed files with 253 additions and 1249 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
cli/app/backup.go
View File

@ -20,6 +20,7 @@ import (
"github.com/docker/cli/cli/command"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
dockerClient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/system"
"github.com/klauspost/pgzip"
@ -72,6 +73,11 @@ This single file can be used to restore your app. See "abra app restore" for mor
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
recipe, err := recipe.Get(app.Recipe)
if err != nil {
logrus.Fatal(err)
@ -115,7 +121,7 @@ This single file can be used to restore your app. See "abra app restore" for mor
logrus.Infof("running backup for the %s service", serviceName)
if err := runBackup(app, serviceName, backupConfig); err != nil {
if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
logrus.Fatal(err)
}
} else {
@ -126,7 +132,7 @@ This single file can be used to restore your app. See "abra app restore" for mor
for serviceName, backupConfig := range backupConfigs {
logrus.Infof("running backup for the %s service", serviceName)
if err := runBackup(app, serviceName, backupConfig); err != nil {
if err := runBackup(cl, app, serviceName, backupConfig); err != nil {
logrus.Fatal(err)
}
}
@ -143,16 +149,11 @@ func TimeStamp() string {
}
// runBackup does the actual backup logic.
func runBackup(app config.App, serviceName string, bkConfig backupConfig) error {
func runBackup(cl *dockerClient.Client, app config.App, serviceName string, bkConfig backupConfig) error {
if len(bkConfig.backupPaths) == 0 {
return fmt.Errorf("backup paths are empty for %s?", serviceName)
}
cl, err := client.New(app.Server)
if err != nil {
return err
}
// FIXME: avoid instantiating a new CLI
dcli, err := command.NewDockerCli()
if err != nil {

15
cli/app/cmd.go
View File

@ -20,6 +20,7 @@ import (
"github.com/docker/cli/cli/command"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
dockerClient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/archive"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
@ -52,6 +53,11 @@ Example:
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
if internal.LocalCmd && internal.RemoteUser != "" {
internal.ShowSubcommandHelpAndError(c, errors.New("cannot use --local & --user together"))
}
@ -129,7 +135,7 @@ Example:
logrus.Debug("did not detect any command arguments")
}
if err := runCmdRemote(app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
if err := runCmdRemote(cl, app, abraSh, targetServiceName, cmdName, parsedCmdArgs); err != nil {
logrus.Fatal(err)
}
}
@ -170,12 +176,7 @@ func ensureCommand(abraSh, recipeName, execCmd string) error {
return nil
}
func runCmdRemote(app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
cl, err := client.New(app.Server)
if err != nil {
return err
}
func runCmdRemote(cl *dockerClient.Client, app config.App, abraSh, serviceName, cmdName, cmdArgs string) error {
filters := filters.NewArgs()
filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), serviceName))

18
cli/app/cp.go
View File

@ -14,6 +14,7 @@ import (
"coopcloud.tech/abra/pkg/formatter"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
dockerClient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/archive"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
@ -43,6 +44,11 @@ And if you want to copy that file back to your current working directory locally
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
src := c.Args().Get(1)
dst := c.Args().Get(2)
if src == "" {
@ -88,28 +94,24 @@ And if you want to copy that file back to your current working directory locally
logrus.Fatalf("%s does not exist locally?", dstPath)
}
}
err := configureAndCp(c, app, srcPath, dstPath, service, isToContainer)
if err != nil {
if err := configureAndCp(c, cl, app, srcPath, dstPath, service, isToContainer); err != nil {
logrus.Fatal(err)
}
return nil
return nil
},
BashComplete: autocomplete.AppNameComplete,
}
func configureAndCp(
c *cli.Context,
cl *dockerClient.Client,
app config.App,
srcPath string,
dstPath string,
service string,
isToContainer bool) error {
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
filters := filters.NewArgs()
filters.Add("name", fmt.Sprintf("^%s_%s", app.StackName(), service))

12
cli/app/list.go
View File

@ -8,10 +8,8 @@ import (
"coopcloud.tech/abra/cli/internal"
"coopcloud.tech/abra/pkg/config"
"coopcloud.tech/abra/pkg/context"
"coopcloud.tech/abra/pkg/formatter"
"coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/ssh"
"coopcloud.tech/tagcmp"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
@ -98,13 +96,6 @@ can take some time.
alreadySeen := make(map[string]bool)
for _, app := range apps {
if _, ok := alreadySeen[app.Server]; !ok {
if err := context.HasDockerContext(app.Name, app.Server); err != nil {
logrus.Fatal(err)
}
if err := ssh.EnsureHostKey(app.Server); err != nil {
logrus.Fatal(fmt.Sprintf(internal.SSHFailMsg, app.Server))
}
alreadySeen[app.Server] = true
}
}
@ -114,7 +105,6 @@ can take some time.
logrus.Fatal(err)
}
var err error
catl, err = recipe.ReadRecipeCatalogue()
if err != nil {
logrus.Fatal(err)
@ -212,6 +202,7 @@ can take some time.
}
allStats[app.Server] = stats
}
if internal.MachineReadable {
jsonstring, err := json.Marshal(allStats)
if err != nil {
@ -221,6 +212,7 @@ can take some time.
}
return nil
}
alreadySeen := make(map[string]bool)
for _, app := range apps {
if _, ok := alreadySeen[app.Server]; ok {

16
cli/app/restore.go
View File

@ -16,6 +16,7 @@ import (
"github.com/docker/cli/cli/command"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
dockerClient "github.com/docker/docker/client"
"github.com/docker/docker/pkg/archive"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
@ -55,6 +56,11 @@ Example:
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
serviceName := c.Args().Get(1)
if serviceName == "" {
internal.ShowSubcommandHelpAndError(c, errors.New("missing <service>?"))
@ -104,7 +110,8 @@ Example:
if !ok {
rsConfig = restoreConfig{}
}
if err := runRestore(app, backupPath, serviceName, rsConfig); err != nil {
if err := runRestore(cl, app, backupPath, serviceName, rsConfig); err != nil {
logrus.Fatal(err)
}
@ -113,12 +120,7 @@ Example:
}
// runRestore does the actual restore logic.
func runRestore(app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
cl, err := client.New(app.Server)
if err != nil {
return err
}
func runRestore(cl *dockerClient.Client, app config.App, backupPath, serviceName string, rsConfig restoreConfig) error {
// FIXME: avoid instantiating a new CLI
dcli, err := command.NewDockerCli()
if err != nil {

14
cli/app/secret.go
View File

@ -48,6 +48,11 @@ var appSecretGenerateCommand = cli.Command{
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
if len(c.Args()) == 1 && !allSecrets {
err := errors.New("missing arguments <secret>/<version> or '--all'")
internal.ShowSubcommandHelpAndError(c, err)
@ -79,7 +84,7 @@ var appSecretGenerateCommand = cli.Command{
}
}
secretVals, err := secret.GenerateSecrets(secretsToCreate, app.StackName(), app.Server)
secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
if err != nil {
logrus.Fatal(err)
}
@ -135,6 +140,11 @@ Example:
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
if len(c.Args()) != 4 {
internal.ShowSubcommandHelpAndError(c, errors.New("missing arguments?"))
}
@ -144,7 +154,7 @@ Example:
data := c.Args().Get(3)
secretName := fmt.Sprintf("%s_%s_%s", app.StackName(), name, version)
if err := client.StoreSecret(secretName, data, app.Server); err != nil {
if err := client.StoreSecret(cl, secretName, data, app.Server); err != nil {
logrus.Fatal(err)
}

10
cli/app/upgrade.go
View File

@ -53,6 +53,11 @@ recipes.
app := internal.ValidateApp(c)
stackName := app.StackName()
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
if !internal.Chaos {
if err := recipe.EnsureUpToDate(app.Recipe); err != nil {
logrus.Fatal(err)
@ -68,11 +73,6 @@ recipes.
logrus.Fatal(err)
}
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("checking whether %s is already deployed", stackName)
isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, stackName)

16
cli/app/volume.go
View File

@ -26,12 +26,17 @@ var appVolumeListCommand = cli.Command{
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
filters, err := app.Filters(false, true)
if err != nil {
logrus.Fatal(err)
}
volumeList, err := client.GetVolumes(context.Background(), app.Server, filters)
volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
if err != nil {
logrus.Fatal(err)
}
@ -80,12 +85,17 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
Action: func(c *cli.Context) error {
app := internal.ValidateApp(c)
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
filters, err := app.Filters(false, true)
if err != nil {
logrus.Fatal(err)
}
volumeList, err := client.GetVolumes(context.Background(), app.Server, filters)
volumeList, err := client.GetVolumes(cl, context.Background(), app.Server, filters)
if err != nil {
logrus.Fatal(err)
}
@ -109,7 +119,7 @@ Passing "--force/-f" will select all volumes for removal. Be careful.
volumesToRemove = volumeNames
}
err = client.RemoveVolumes(context.Background(), app.Server, volumesToRemove, internal.Force)
err = client.RemoveVolumes(cl, context.Background(), app.Server, volumesToRemove, internal.Force)
if err != nil {
logrus.Fatal(err)
}

67
cli/internal/cli.go
View File

@ -378,73 +378,6 @@ var RemoteUserFlag = &cli.StringFlag{
Destination: &RemoteUser,
}
// SSHFailMsg is a hopefully helpful SSH failure message
var SSHFailMsg = `
Woops, Abra is unable to connect to connect to %s.
Here are a few tips for debugging your local SSH config. Abra uses plain 'ol
SSH to make connections to servers, so if your SSH config is working, Abra is
working.
In the first place, Abra will always try to read your Docker context connection
string for SSH connection details. You can view your server context configs
with the following command. Are they correct?
abra server ls
Is your ssh-agent running? You can start it by running the following command:
eval "$(ssh-agent)"
If your SSH private key loaded? You can check by running the following command:
ssh-add -L
If, you can add it with:
ssh-add ~/.ssh/<private-key-part>
If you are using a non-default public/private key, you can configure this in
your ~/.ssh/config file which Abra will read in order to figure out connection
details:
Host foo.coopcloud.tech
Hostname foo.coopcloud.tech
User bar
Port 12345
IdentityFile ~/.ssh/bar@foo.coopcloud.tech
If you're only using password authentication, you can use the following config:
Host foo.coopcloud.tech
Hostname foo.coopcloud.tech
User bar
Port 12345
PreferredAuthentications=password
PubkeyAuthentication=no
Good luck!
`
var ServerAddFailMsg = `
Failed to add server %s.
This could be caused by two things.
Abra isn't picking up your SSH configuration or you need to specify it on the
command-line (e.g you use a non-standard port or username to connect). Run
"server add" with "-d/--debug" to learn more about what Abra is doing under the
hood.
Docker is not installed on your server. You can pass "-p/--provision" to
install Docker and initialise Docker Swarm mode. See help output for "server
add"
See "abra server add -h" for more.
`
// SubCommandBefore wires up pre-action machinery (e.g. --debug handling).
func SubCommandBefore(c *cli.Context) error {
if Debug {

9
cli/internal/deploy.go
View File

@ -8,7 +8,6 @@ import (
"path"
"strings"
"coopcloud.tech/abra/pkg/client"
"coopcloud.tech/abra/pkg/config"
"coopcloud.tech/abra/pkg/dns"
"coopcloud.tech/abra/pkg/formatter"
@ -17,12 +16,13 @@ import (
"coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/upstream/stack"
"github.com/AlecAivazis/survey/v2"
dockerClient "github.com/docker/docker/client"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
// DeployAction is the main command-line action for this package
func DeployAction(c *cli.Context) error {
func DeployAction(c *cli.Context, cl *dockerClient.Client) error {
app := ValidateApp(c)
if !Chaos {
@ -40,11 +40,6 @@ func DeployAction(c *cli.Context) error {
logrus.Fatal(err)
}
cl, err := client.New(app.Server)
if err != nil {
logrus.Fatal(err)
}
logrus.Debugf("checking whether %s is already deployed", app.StackName())
isDeployed, deployedVersion, err := stack.IsDeployed(context.Background(), cl, app.StackName())

24
cli/internal/new.go
View File

@ -5,15 +5,15 @@ import (
"path"
"coopcloud.tech/abra/pkg/app"
"coopcloud.tech/abra/pkg/client"
"coopcloud.tech/abra/pkg/config"
"coopcloud.tech/abra/pkg/context"
"coopcloud.tech/abra/pkg/formatter"
"coopcloud.tech/abra/pkg/jsontable"
"coopcloud.tech/abra/pkg/recipe"
recipePkg "coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/secret"
"coopcloud.tech/abra/pkg/ssh"
"github.com/AlecAivazis/survey/v2"
dockerClient "github.com/docker/docker/client"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
@ -25,7 +25,7 @@ type AppSecrets map[string]string
var RecipeName string
// createSecrets creates all secrets for a new app.
func createSecrets(sanitisedAppName string) (AppSecrets, error) {
func createSecrets(cl *dockerClient.Client, sanitisedAppName string) (AppSecrets, error) {
appEnvPath := path.Join(config.ABRA_DIR, "servers", NewAppServer, fmt.Sprintf("%s.env", Domain))
appEnv, err := config.ReadEnv(appEnvPath)
if err != nil {
@ -33,7 +33,7 @@ func createSecrets(sanitisedAppName string) (AppSecrets, error) {
}
secretEnvVars := secret.ReadSecretEnvVars(appEnv)
secrets, err := secret.GenerateSecrets(secretEnvVars, sanitisedAppName, NewAppServer)
secrets, err := secret.GenerateSecrets(cl, secretEnvVars, sanitisedAppName, NewAppServer)
if err != nil {
return nil, err
}
@ -144,19 +144,15 @@ func NewAction(c *cli.Context) error {
logrus.Fatal(err)
}
cl, err := client.New(NewAppServer)
if err != nil {
logrus.Fatal(err)
}
var secrets AppSecrets
var secretTable *jsontable.JSONTable
if Secrets {
if err := context.HasDockerContext(sanitisedAppName, NewAppServer); err != nil {
logrus.Fatal(err)
}
if err := ssh.EnsureHostKey(NewAppServer); err != nil {
logrus.Fatal(err)
}
var err error
secrets, err = createSecrets(sanitisedAppName)
secrets, err := createSecrets(cl, sanitisedAppName)
if err != nil {
logrus.Fatal(err)
}

10
cli/internal/validate.go
View File

@ -8,9 +8,7 @@ import (
"coopcloud.tech/abra/pkg/app"
"coopcloud.tech/abra/pkg/config"
"coopcloud.tech/abra/pkg/context"
"coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/ssh"
"github.com/AlecAivazis/survey/v2"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
@ -142,14 +140,6 @@ func ValidateApp(c *cli.Context) config.App {
logrus.Fatal(err)
}
if err := context.HasDockerContext(app.Name, app.Server); err != nil {
logrus.Fatal(err)
}
if err := ssh.EnsureHostKey(app.Server); err != nil {
logrus.Fatal(err)
}
logrus.Debugf("validated %s as app argument", appName)
return app

446
cli/server/add.go
View File

@ -1,14 +1,9 @@
package server
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
"os/user"
"path/filepath"
"strings"
"coopcloud.tech/abra/cli/internal"
"coopcloud.tech/abra/pkg/client"
@ -16,34 +11,11 @@ import (
contextPkg "coopcloud.tech/abra/pkg/context"
"coopcloud.tech/abra/pkg/dns"
"coopcloud.tech/abra/pkg/server"
"coopcloud.tech/abra/pkg/ssh"
"github.com/AlecAivazis/survey/v2"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/swarm"
dockerClient "github.com/docker/docker/client"
sshPkg "coopcloud.tech/abra/pkg/ssh"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)
var (
dockerInstallMsg = `
A docker installation cannot be found on %s. This is a required system
dependency for running Co-op Cloud apps on your server. If you would like, Abra
can attempt to install Docker for you using the upstream non-interactive
installation script.
See the following documentation for more:
https://docs.docker.com/engine/install/debian/#install-using-the-convenience-script
N.B Docker doesn't recommend it for production environments but many use it for
such purposes. Docker stable is now installed by default by this script. The
source for this script can be seen here:
https://github.com/docker/docker-install
`
)
var local bool
var localFlag = &cli.BoolFlag{
Name: "local, l",
@ -51,35 +23,15 @@ var localFlag = &cli.BoolFlag{
Destination: &local,
}
var provision bool
var provisionFlag = &cli.BoolFlag{
Name: "provision, p",
Usage: "Provision server so it can deploy apps",
Destination: &provision,
}
var sshAuth string
var sshAuthFlag = &cli.StringFlag{
Name: "ssh-auth, s",
Value: "identity-file",
Usage: "Select SSH authentication method (identity-file, password)",
Destination: &sshAuth,
}
var askSudoPass bool
var askSudoPassFlag = &cli.BoolFlag{
Name: "ask-sudo-pass, a",
Usage: "Ask for sudo password",
Destination: &askSudoPass,
}
func cleanUp(domainName string) {
logrus.Warnf("cleaning up context for %s", domainName)
if err := client.DeleteContext(domainName); err != nil {
logrus.Fatal(err)
if domainName != "default" {
logrus.Infof("cleaning up context for %s", domainName)
if err := client.DeleteContext(domainName); err != nil {
logrus.Fatal(err)
}
}
logrus.Warnf("cleaning up server directory for %s", domainName)
logrus.Infof("attempting to clean up server directory for %s", domainName)
serverDir := filepath.Join(config.SERVERS_DIR, domainName)
files, err := config.GetAllFilesInDirectory(serverDir)
@ -97,72 +49,10 @@ func cleanUp(domainName string) {
}
}
func installDockerLocal(c *cli.Context) error {
fmt.Println(fmt.Sprintf(dockerInstallMsg, "this local server"))
response := false
prompt := &survey.Confirm{
Message: fmt.Sprintf("attempt install docker on local server?"),
}
if err := survey.AskOne(prompt, &response); err != nil {
return err
}
if !response {
logrus.Fatal("exiting as requested")
}
for _, exe := range []string{"wget", "bash"} {
exists, err := ensureLocalExecutable(exe)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("%s missing, please install it", exe)
}
}
cmd := exec.Command("bash", "-c", "wget -O- https://get.docker.com | bash")
if err := internal.RunCmd(cmd); err != nil {
return err
}
return nil
}
func newLocalServer(c *cli.Context, domainName string) error {
if err := createServerDir(domainName); err != nil {
return err
}
cl, err := newClient(c, domainName)
if err != nil {
return err
}
if provision {
exists, err := ensureLocalExecutable("docker")
if err != nil {
return err
}
if !exists {
if err := installDockerLocal(c); err != nil {
return err
}
}
if err := initSwarmLocal(c, cl, domainName); err != nil {
if !strings.Contains(err.Error(), "proxy already exists") {
logrus.Fatal(err)
}
}
}
logrus.Info("local server has been added")
return nil
}
// newContext creates a new internal Docker context for a server. This is how
// Docker manages SSH connection details. These are stored to disk in
// ~/.docker. Abra can manage this completely for the user, so it's an
// implementation detail.
func newContext(c *cli.Context, domainName, username, port string) error {
store := contextPkg.NewDefaultDockerContextStore()
contexts, err := store.Store.List()
@ -186,187 +76,7 @@ func newContext(c *cli.Context, domainName, username, port string) error {
return nil
}
func newClient(c *cli.Context, domainName string) (*dockerClient.Client, error) {
cl, err := client.New(domainName)
if err != nil {
return &dockerClient.Client{}, err
}
return cl, nil
}
func installDocker(c *cli.Context, cl *dockerClient.Client, sshCl *ssh.Client, domainName string) error {
exists, err := ensureRemoteExecutable("docker", sshCl)
if err != nil {
return err
}
if !exists {
fmt.Println(fmt.Sprintf(dockerInstallMsg, domainName))
response := false
prompt := &survey.Confirm{
Message: fmt.Sprintf("attempt install docker on %s?", domainName),
}
if err := survey.AskOne(prompt, &response); err != nil {
return err
}
if !response {
logrus.Fatal("exiting as requested")
}
exes := []string{"wget", "bash"}
if askSudoPass {
exes = append(exes, "ssh-askpass")
}
for _, exe := range exes {
exists, err := ensureRemoteExecutable(exe, sshCl)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("%s missing on remote, please install it", exe)
}
}
var sudoPass string
if askSudoPass {
cmd := "wget -O- https://get.docker.com | bash"
prompt := &survey.Password{
Message: "sudo password?",
}
if err := survey.AskOne(prompt, &sudoPass); err != nil {
return err
}
logrus.Debugf("running %s on %s now with sudo password", cmd, domainName)
if sudoPass == "" {
return fmt.Errorf("missing sudo password but requested --ask-sudo-pass?")
}
logrus.Warn("installing docker, this could take some time...")
if err := ssh.RunSudoCmd(cmd, sudoPass, sshCl); err != nil {
fmt.Print(fmt.Sprintf(`
Abra was unable to bootstrap Docker, see below for logs:
%s
If nothing works, you can try running the Docker install script manually on your server:
wget -O- https://get.docker.com | bash
`, string(err.Error())))
logrus.Fatal("Process exited with status 1")
}
logrus.Infof("docker is installed on %s", domainName)
remoteUser := sshCl.SSHClient.Conn.User()
logrus.Infof("adding %s to docker group", remoteUser)
permsCmd := fmt.Sprintf("sudo usermod -aG docker %s", remoteUser)
if err := ssh.RunSudoCmd(permsCmd, sudoPass, sshCl); err != nil {
return err
}
} else {
cmd := "wget -O- https://get.docker.com | bash"
logrus.Debugf("running %s on %s now without sudo password", cmd, domainName)
logrus.Warn("installing docker, this could take some time...")
if out, err := sshCl.Exec(cmd); err != nil {
fmt.Print(fmt.Sprintf(`
Abra was unable to bootstrap Docker, see below for logs:
%s
This could be due to several reasons. One of the most common is that your
server user account does not have sudo access, and if it does, you need to pass
"--ask-sudo-pass" in order to supply Abra with your password.
If nothing works, you try running the Docker install script manually on your server:
wget -O- https://get.docker.com | bash
`, string(out)))
logrus.Fatal(err)
}
logrus.Infof("docker is installed on %s", domainName)
}
}
return nil
}
func initSwarmLocal(c *cli.Context, cl *dockerClient.Client, domainName string) error {
initReq := swarm.InitRequest{ListenAddr: "0.0.0.0:2377"}
if _, err := cl.SwarmInit(context.Background(), initReq); err != nil {
if strings.Contains(err.Error(), "is already part of a swarm") ||
strings.Contains(err.Error(), "must specify a listening address") {
logrus.Infof("swarm mode already initialised on %s", domainName)
} else {
return err
}
} else {
logrus.Infof("initialised swarm mode on local server")
}
netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
if _, err := cl.NetworkCreate(context.Background(), "proxy", netOpts); err != nil {
if !strings.Contains(err.Error(), "proxy already exists") {
return err
}
logrus.Info("swarm overlay network already created on local server")
} else {
logrus.Infof("swarm overlay network created on local server")
}
return nil
}
func initSwarm(c *cli.Context, cl *dockerClient.Client, domainName string) error {
ipv4, err := dns.EnsureIPv4(domainName)
if err != nil {
return err
}
initReq := swarm.InitRequest{
ListenAddr: "0.0.0.0:2377",
AdvertiseAddr: ipv4,
}
if _, err := cl.SwarmInit(context.Background(), initReq); err != nil {
if strings.Contains(err.Error(), "is already part of a swarm") ||
strings.Contains(err.Error(), "must specify a listening address") {
logrus.Infof("swarm mode already initialised on %s", domainName)
} else {
return err
}
} else {
logrus.Infof("initialised swarm mode on %s", domainName)
}
netOpts := types.NetworkCreate{Driver: "overlay", Scope: "swarm"}
if _, err := cl.NetworkCreate(context.Background(), "proxy", netOpts); err != nil {
if !strings.Contains(err.Error(), "proxy already exists") {
return err
}
logrus.Infof("swarm overlay network already created on %s", domainName)
} else {
logrus.Infof("swarm overlay network created on %s", domainName)
}
return nil
}
// createServerDir creates the ~/.abra/servers/... directory for a new server.
func createServerDir(domainName string) error {
if err := server.CreateServerDir(domainName); err != nil {
if !os.IsExist(err) {
@ -374,6 +84,7 @@ func createServerDir(domainName string) error {
}
logrus.Debugf("server dir for %s already created", domainName)
}
return nil
}
@ -382,34 +93,21 @@ var serverAddCommand = cli.Command{
Aliases: []string{"a"},
Usage: "Add a server to your configuration",
Description: `
Add a new server to your configuration so that it can be managed by Abra. This
command can also provision your server ("--provision/-p") with a Docker
installation so that it is capable of hosting Co-op Cloud apps.
Add a new server to your configuration so that it can be managed by Abra.
Abra will default to expecting that you have a running ssh-agent and are using
SSH keys to connect to your new server. Abra will also read your SSH config
(matching "Host" as <domain>). SSH connection details precedence follows as
such: command-line > SSH config > guessed defaults.
Abra uses the SSH command-line to discover connection details for your server.
It is advised to configure an entry per-host in your ~/.ssh/config for each
server. For example:
If you have no SSH key configured for this host and are instead using password
authentication, you may pass "--ssh-auth password" to have Abra ask you for the
password. "--ask-sudo-pass" may be passed if you run your provisioning commands
via sudo privilege escalation.
Host example.com
Hostname example.com
User exampleUser
Port 12345
IdentityFile ~/.ssh/example@somewhere
The <domain> argument must be a publicy accessible domain name which points to
your server. You should have working SSH access to this server already, Abra
will assume port 22 and will use your current system username to make an
initial connection. You can use the <user> and <port> arguments to adjust this.
Abra can then load SSH connection details from this configuratiion with:
Example:
abra server add varia.zone glodemodem 12345 -p
Abra will construct the following SSH connection and Docker context:
ssh://globemodem@varia.zone:12345
All communication between Abra and the server will use this SSH connection.
abra server add example.com
If "--local" is passed, then Abra assumes that the current local server is
intended as the target server. This is useful when you want to have your entire
@ -420,104 +118,64 @@ developer machine.
internal.DebugFlag,
internal.NoInputFlag,
localFlag,
provisionFlag,
sshAuthFlag,
askSudoPassFlag,
},
Before: internal.SubCommandBefore,
ArgsUsage: "<domain> [<user>] [<port>]",
ArgsUsage: "<domain>",
Action: func(c *cli.Context) error {
if len(c.Args()) > 0 && local || !internal.ValidateSubCmdFlags(c) {
err := errors.New("cannot use <domain> and --local together")
internal.ShowSubcommandHelpAndError(c, err)
}
if sshAuth != "password" && sshAuth != "identity-file" {
err := errors.New("--ssh-auth only accepts identity-file or password")
internal.ShowSubcommandHelpAndError(c, err)
var domainName string
if local {
domainName = "default"
} else {
domainName = internal.ValidateDomain(c)
}
domainName := internal.ValidateDomain(c)
if local {
if err := newLocalServer(c, "default"); err != nil {
if err := createServerDir(domainName); err != nil {
logrus.Fatal(err)
}
logrus.Infof("attempting to create client for %s", domainName)
if _, err := client.New(domainName); err != nil {
cleanUp(domainName)
logrus.Fatal(err)
}
logrus.Info("local server added")
return nil
}
username := c.Args().Get(1)
if username == "" {
systemUser, err := user.Current()
if err != nil {
return err
}
username = systemUser.Username
}
port := c.Args().Get(2)
if port == "" {
port = "22"
if _, err := dns.EnsureIPv4(domainName); err != nil {
logrus.Fatal(err)
}
if err := createServerDir(domainName); err != nil {
logrus.Fatal(err)
}
if err := newContext(c, domainName, username, port); err != nil {
hostConfig, err := sshPkg.GetHostConfig(domainName)
if err != nil {
logrus.Fatal(err)
}
cl, err := newClient(c, domainName)
if err != nil {
if err := newContext(c, domainName, hostConfig.User, hostConfig.Port); err != nil {
logrus.Fatal(err)
}
logrus.Infof("attempting to create client for %s", domainName)
if _, err := client.New(domainName); err != nil {
cleanUp(domainName)
logrus.Debugf("failed to construct client for %s, saw %s", domainName, err.Error())
logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
logrus.Fatal(sshPkg.Fatal(domainName, err))
}
if provision {
logrus.Debugf("attempting to construct SSH client for %s", domainName)
sshCl, err := ssh.New(domainName, sshAuth, username, port)
if err != nil {
cleanUp(domainName)
logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
}
defer sshCl.Close()
logrus.Debugf("successfully created SSH client for %s", domainName)
if err := installDocker(c, cl, sshCl, domainName); err != nil {
logrus.Fatal(err)
}
if err := initSwarm(c, cl, domainName); err != nil {
logrus.Fatal(err)
}
}
if _, err := cl.Info(context.Background()); err != nil {
cleanUp(domainName)
logrus.Fatalf(fmt.Sprintf(internal.ServerAddFailMsg, domainName))
}
logrus.Infof("%s added", domainName)
return nil
},
}
// ensureLocalExecutable ensures that an executable is present on the local machine
func ensureLocalExecutable(exe string) (bool, error) {
out, err := exec.Command("which", exe).Output()
if err != nil {
return false, err
}
return string(out) != "", nil
}
// ensureRemoteExecutable ensures that an executable is present on a remote machine
func ensureRemoteExecutable(exe string, sshCl *ssh.Client) (bool, error) {
out, err := sshCl.Exec(fmt.Sprintf("which %s", exe))
if err != nil && string(out) != "" {
return false, err
}
return string(out) != "", nil
}

31
pkg/client/client.go
View File

@ -2,22 +2,29 @@
package client
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"time"
contextPkg "coopcloud.tech/abra/pkg/context"
sshPkg "coopcloud.tech/abra/pkg/ssh"
commandconnPkg "coopcloud.tech/abra/pkg/upstream/commandconn"
"github.com/docker/docker/client"
"github.com/sirupsen/logrus"
)
// New initiates a new Docker client.
func New(contextName string) (*client.Client, error) {
// New initiates a new Docker client. New client connections are validated so
// that we ensure connections via SSH to the daemon can succeed. It takes into
// account that you may only want the local client and not communicate via SSH.
// For this use-case, please pass "default" as the contextName.
func New(serverName string) (*client.Client, error) {
var clientOpts []client.Opt
if contextName != "default" {
context, err := GetContext(contextName)
if serverName != "default" {
context, err := GetContext(serverName)
if err != nil {
return nil, err
}
@ -33,7 +40,6 @@ func New(contextName string) (*client.Client, error) {
}
httpClient := &http.Client{
// No tls, no proxy
Transport: &http.Transport{
DialContext: helper.Dialer,
IdleConnTimeout: 30 * time.Second,
@ -59,7 +65,20 @@ func New(contextName string) (*client.Client, error) {
return nil, err
}
logrus.Debugf("created client for %s", contextName)
logrus.Debugf("created client for %s", serverName)
info, err := cl.Info(context.Background())
if err != nil {
return cl, sshPkg.Fatal(serverName, err)
}
if info.Swarm.LocalNodeState == "inactive" {
if serverName != "default" {
return cl, fmt.Errorf("swarm mode not enabled on %s?", serverName)
} else {
return cl, errors.New("swarm mode not enabled on local server?")
}
}
return cl, nil
}

12
pkg/client/secret.go
View File

@ -4,20 +4,14 @@ import (
"context"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client"
)
func StoreSecret(secretName, secretValue, server string) error {
cl, err := New(server)
if err != nil {
return err
}
ctx := context.Background()
func StoreSecret(cl *client.Client, secretName, secretValue, server string) error {
ann := swarm.Annotations{Name: secretName}
spec := swarm.SecretSpec{Annotations: ann, Data: []byte(secretValue)}
// We don't bother with the secret IDs for now
if _, err := cl.SecretCreate(ctx, spec); err != nil {
if _, err := cl.SecretCreate(context.Background(), spec); err != nil {
return err
}

15
pkg/client/volumes.go
View File

@ -5,14 +5,10 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
)
func GetVolumes(ctx context.Context, server string, fs filters.Args) ([]*types.Volume, error) {
cl, err := New(server)
if err != nil {
return nil, err
}
func GetVolumes(cl *client.Client, ctx context.Context, server string, fs filters.Args) ([]*types.Volume, error) {
volumeListOKBody, err := cl.VolumeList(ctx, fs)
volumeList := volumeListOKBody.Volumes
if err != nil {
@ -32,12 +28,7 @@ func GetVolumeNames(volumes []*types.Volume) []string {
return volumeNames
}
func RemoveVolumes(ctx context.Context, server string, volumeNames []string, force bool) error {
cl, err := New(server)
if err != nil {
return err
}
func RemoveVolumes(cl *client.Client, ctx context.Context, server string, volumeNames []string, force bool) error {
for _, volName := range volumeNames {
err := cl.VolumeRemove(ctx, volName, force)
if err != nil {

20
pkg/config/app.go
View File

@ -10,6 +10,7 @@ import (
"github.com/schollz/progressbar/v3"
"coopcloud.tech/abra/pkg/client"
"coopcloud.tech/abra/pkg/formatter"
"coopcloud.tech/abra/pkg/upstream/convert"
loader "coopcloud.tech/abra/pkg/upstream/stack"
@ -368,16 +369,27 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str
}
}
var bar *progressbar.ProgressBar
for server := range servers {
// validate that all server connections work
if _, err := client.New(server); err != nil {
return statuses, err
}
}
var bar *progressbar.ProgressBar
if !MachineReadable {
bar = formatter.CreateProgressbar(len(servers), "querying remote servers...")
}
ch := make(chan stack.StackStatus, len(servers))
for server := range servers {
cl, err := client.New(server)
if err != nil {
return statuses, err
}
go func(s string) {
ch <- stack.GetAllDeployedServices(s)
ch <- stack.GetAllDeployedServices(cl, s)
if !MachineReadable {
bar.Add(1)
}
@ -386,6 +398,10 @@ func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]str
for range servers {
status := <-ch
if status.Err != nil {
return statuses, status.Err
}
for _, service := range status.Services {
result := make(map[string]string)
name := service.Spec.Labels[convert.LabelNamespace]

66
pkg/context/context.go
View File

@ -2,7 +2,6 @@ package context
import (
"errors"
"fmt"
"github.com/docker/cli/cli/command"
dConfig "github.com/docker/cli/cli/config"
@ -43,68 +42,3 @@ func GetContextEndpoint(ctx contextStore.Metadata) (string, error) {
func newContextStore(dir string, config contextStore.Config) contextStore.Store {
return contextStore.New(dir, config)
}
// missingContextMsg helps end-user debug missing docker context issues. This
// version of the message has no app domain name included. This is due to the
// code paths being unable to determine which app is requesting a server
// connection at certain points. It is preferred to use
// missingContextWithAppMsg where possible and only use missingContextMsg when
// the call path is located too deep in the SSH stack.
var missingContextMsg = `unable to find Docker context for %s?
Please run "abra server ls -p" to confirm. If you see "unknown" in the table
output then you need to run the following command:
abra server add %s <args>
See "abra server add --help" for more.
`
// missingContextWithAppMsg helps end-users debug missing docker context
// issues. The app name is included in this message for extra clarity. See
// missingContextMsg docs for alternative usage.
var missingContextWithAppMsg = `unable to find Docker context for %s?
%s (app) is deployed on %s (server).
Please run "abra server ls -p" to confirm. If you see "unknown" in the table
output then you need to run the following command:
abra server add %s <args>
See "abra server add --help" for more.
`
// HasDockerContext figures out if a local setup has a working docker context
// configuration or not. This usually tells us if they'll be able to make a SSH
// connection to a server or not and can be a useful way to signal to end-users
// that they need to fix something up if missing.
func HasDockerContext(appName, serverName string) error {
dockerContextStore := NewDefaultDockerContextStore()
contexts, err := dockerContextStore.Store.List()
if err != nil {
return err
}
for _, ctx := range contexts {
if ctx.Name == serverName {
return nil
}
}
if appName != "" {
return fmt.Errorf(
missingContextWithAppMsg,
serverName,
appName,
serverName,
serverName,
)
}
return fmt.Errorf(
missingContextMsg,
serverName,
serverName,
)
}

30
pkg/dns/common.go
View File

@ -1,7 +1,6 @@
package dns
import (
"context"
"fmt"
"net"
"os"
@ -32,35 +31,12 @@ func NewToken(provider, providerTokenEnvVar string) (string, error) {
// EnsureIPv4 ensures that an ipv4 address is set for a domain name
func EnsureIPv4(domainName string) (string, error) {
var ipv4 string
// comrade librehosters DNS resolver -> https://www.privacy-handbuch.de/handbuch_93d.htm
freifunkDNS := "5.1.66.255:53"
resolver := &net.Resolver{
PreferGo: false,
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
d := net.Dialer{
Timeout: time.Millisecond * time.Duration(10000),
}
return d.DialContext(ctx, "udp", freifunkDNS)
},
}
ctx := context.Background()
ips, err := resolver.LookupIPAddr(ctx, domainName)
ipv4, err := net.ResolveIPAddr("ip", domainName)
if err != nil {
return ipv4, err
return "", err
}
if len(ips) == 0 {
return ipv4, fmt.Errorf("unable to retrieve ipv4 address for %s", domainName)
}
ipv4 = ips[0].IP.To4().String()
logrus.Debugf("%s points to %s (resolver: %s)", domainName, ipv4, freifunkDNS)
return ipv4, nil
return ipv4.String(), nil
}
// EnsureDomainsResolveSameIPv4 ensures that domains resolve to the same ipv4 address

7
pkg/secret/secret.go
View File

@ -13,6 +13,7 @@ import (
"coopcloud.tech/abra/pkg/client"
"coopcloud.tech/abra/pkg/config"
"github.com/decentral1se/passgen"
dockerClient "github.com/docker/docker/client"
"github.com/sirupsen/logrus"
)
@ -117,7 +118,7 @@ func ParseSecretEnvVarValue(secret string) (secretValue, error) {
}
// GenerateSecrets generates secrets locally and sends them to a remote server for storage.
func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (map[string]string, error) {
func GenerateSecrets(cl *dockerClient.Client, secretEnvVars map[string]string, appName, server string) (map[string]string, error) {
secrets := make(map[string]string)
var mutex sync.Mutex
@ -146,7 +147,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
return
}
if err := client.StoreSecret(secretRemoteName, passwords[0], server); err != nil {
if err := client.StoreSecret(cl, secretRemoteName, passwords[0], server); err != nil {
if strings.Contains(err.Error(), "AlreadyExists") {
logrus.Warnf("%s already exists, moving on...", secretRemoteName)
ch <- nil
@ -166,7 +167,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
return
}
if err := client.StoreSecret(secretRemoteName, passphrases[0], server); err != nil {
if err := client.StoreSecret(cl, secretRemoteName, passphrases[0], server); err != nil {
if strings.Contains(err.Error(), "AlreadyExists") {
logrus.Warnf("%s already exists, moving on...", secretRemoteName)
ch <- nil

557
pkg/ssh/ssh.go
View File

@ -1,37 +1,13 @@
package ssh
import (
"bufio"
"bytes"
"crypto/sha256"
"encoding/base64"
"fmt"
"io"
"net"
"os"
"os/user"
"path/filepath"
"os/exec"
"strings"
"sync"
"time"
"coopcloud.tech/abra/pkg/context"
"github.com/AlecAivazis/survey/v2"
dockerSSHPkg "github.com/docker/cli/cli/connhelper/ssh"
sshPkg "github.com/gliderlabs/ssh"
"github.com/kevinburke/ssh_config"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
"golang.org/x/crypto/ssh/knownhosts"
)
var KnownHostsPath = filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts")
type Client struct {
SSHClient *ssh.Client
}
// HostConfig is a SSH host config.
type HostConfig struct {
Host string
@ -40,509 +16,66 @@ type HostConfig struct {
User string
}
// Exec cmd on the remote host and return stderr and stdout
func (c *Client) Exec(cmd string) ([]byte, error) {
session, err := c.SSHClient.NewSession()
if err != nil {
return nil, err
}
defer session.Close()
return session.CombinedOutput(cmd)
}
// Close the underlying SSH connection
func (c *Client) Close() error {
return c.SSHClient.Close()
}
// New creates a new SSH client connection.
func New(domainName, sshAuth, username, port string) (*Client, error) {
var client *Client
ctxConnDetails, err := GetContextConnDetails(domainName)
if err != nil {
return client, nil
}
if sshAuth == "identity-file" {
var err error
client, err = connectWithAgentTimeout(
ctxConnDetails.Host,
ctxConnDetails.User,
ctxConnDetails.Port,
5*time.Second,
)
if err != nil {
return client, err
}
} else {
password := ""
prompt := &survey.Password{
Message: "SSH password?",
}
if err := survey.AskOne(prompt, &password); err != nil {
return client, err
}
var err error
client, err = connectWithPasswordTimeout(
ctxConnDetails.Host,
ctxConnDetails.User,
ctxConnDetails.Port,
password,
5*time.Second,
)
if err != nil {
return client, err
}
}
return client, nil
}
// sudoWriter supports sudo command handling
type sudoWriter struct {
b bytes.Buffer
pw string
stdin io.Writer
m sync.Mutex
}
// Write satisfies the write interface for sudoWriter
func (w *sudoWriter) Write(p []byte) (int, error) {
if strings.Contains(string(p), "sudo_password") {
w.stdin.Write([]byte(w.pw + "\n"))
w.pw = ""
return len(p), nil
}
w.m.Lock()
defer w.m.Unlock()
return w.b.Write(p)
}
// RunSudoCmd runs SSH commands and streams output
func RunSudoCmd(cmd, passwd string, cl *Client) error {
session, err := cl.SSHClient.NewSession()
if err != nil {
return err
}
defer session.Close()
sudoCmd := fmt.Sprintf("SSH_ASKPASS=/usr/bin/ssh-askpass; sudo -p sudo_password -S %s", cmd)
w := &sudoWriter{pw: passwd}
w.stdin, err = session.StdinPipe()
if err != nil {
return err
}
session.Stdout = w
session.Stderr = w
modes := ssh.TerminalModes{
ssh.ECHO: 0,
ssh.TTY_OP_ISPEED: 14400,
ssh.TTY_OP_OSPEED: 14400,
}
err = session.RequestPty("xterm", 80, 40, modes)
if err != nil {
return err
}
if err := session.Run(sudoCmd); err != nil {
return fmt.Errorf("%s", string(w.b.Bytes()))
}
return nil
}
// EnsureKnowHostsFiles ensures that ~/.ssh/known_hosts is created
func EnsureKnowHostsFiles() error {
if _, err := os.Stat(KnownHostsPath); os.IsNotExist(err) {
logrus.Debugf("missing %s, creating now", KnownHostsPath)
file, err := os.OpenFile(KnownHostsPath, os.O_CREATE, 0600)
if err != nil {
return err
}
file.Close()
}
return nil
}
// GetHostKey checks if a host key is registered in the ~/.ssh/known_hosts file
func GetHostKey(hostname string) (bool, sshPkg.PublicKey, error) {
var hostKey sshPkg.PublicKey
ctxConnDetails, err := GetContextConnDetails(hostname)
if err != nil {
return false, hostKey, err
}
if err := EnsureKnowHostsFiles(); err != nil {
return false, hostKey, err
}
file, err := os.Open(KnownHostsPath)
if err != nil {
return false, hostKey, err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
fields := strings.Split(scanner.Text(), " ")
if len(fields) != 3 {
continue
}
hostnameAndPort := fmt.Sprintf("%s:%s", ctxConnDetails.Host, ctxConnDetails.Port)
hashed := knownhosts.Normalize(hostnameAndPort)
if strings.Contains(fields[0], hashed) {
var err error
hostKey, _, _, _, err = ssh.ParseAuthorizedKey(scanner.Bytes())
if err != nil {
return false, hostKey, fmt.Errorf("error parsing server SSH host key %q: %v", fields[2], err)
}
break
}
}
if hostKey != nil {
logrus.Debugf("server SSH host key present in ~/.ssh/known_hosts for %s", hostname)
return true, hostKey, nil
}
return false, hostKey, nil
}
// InsertHostKey adds a new host key to the ~/.ssh/known_hosts file
func InsertHostKey(hostname string, remote net.Addr, pubKey ssh.PublicKey) error {
file, err := os.OpenFile(KnownHostsPath, os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer file.Close()
hashedHostname := knownhosts.Normalize(hostname)
lineHostname := knownhosts.Line([]string{hashedHostname}, pubKey)
_, err = file.WriteString(fmt.Sprintf("%s\n", lineHostname))
if err != nil {
return err
}
hashedRemote := knownhosts.Normalize(remote.String())
lineRemote := knownhosts.Line([]string{hashedRemote}, pubKey)
_, err = file.WriteString(fmt.Sprintf("%s\n", lineRemote))
if err != nil {
return err
}
logrus.Debugf("SSH host key generated: %s", lineHostname)
logrus.Debugf("SSH host key generated: %s", lineRemote)
return nil
}
// HostKeyAddCallback ensures server ssh host keys are handled
func HostKeyAddCallback(hostnameAndPort string, remote net.Addr, pubKey ssh.PublicKey) error {
exists, _, err := GetHostKey(hostnameAndPort)
if err != nil {
return err
}
if exists {
hostname := strings.Split(hostnameAndPort, ":")[0]
logrus.Debugf("server SSH host key found for %s", hostname)
return nil
}
if !exists {
hostname := strings.Split(hostnameAndPort, ":")[0]
parsedPubKey := FingerprintSHA256(pubKey)
fmt.Printf(fmt.Sprintf(`
You are attempting to make an SSH connection to a server but there is no entry
in your ~/.ssh/known_hosts file which confirms that you have already validated
that this is indeed the server you want to connect to. Please take a moment to
validate the following SSH host key, it is important.
Host: %s
Fingerprint: %s
If this is confusing to you, you can read the article below and learn how to
validate this fingerprint safely. Thanks to the comrades at cyberia.club for
writing this extensive guide <3
https://sequentialread.com/understanding-the-secure-shell-protocol-ssh/
`, hostname, parsedPubKey))
response := false
prompt := &survey.Confirm{
Message: "are you sure you trust this host key?",
}
if err := survey.AskOne(prompt, &response); err != nil {
return err
}
if !response {
logrus.Fatal("exiting as requested")
}
logrus.Debugf("attempting to insert server SSH host key for %s, %s", hostnameAndPort, remote)
if err := InsertHostKey(hostnameAndPort, remote, pubKey); err != nil {
return err
}
logrus.Infof("successfully added server SSH host key for %s", hostname)
}
return nil
}
// connect makes the SSH connection
func connect(username, host, port string, authMethod ssh.AuthMethod, timeout time.Duration) (*Client, error) {
config := &ssh.ClientConfig{
User: username,
Auth: []ssh.AuthMethod{authMethod},
HostKeyCallback: HostKeyAddCallback, // the main reason why we fork
}
hostnameAndPort := fmt.Sprintf("%s:%s", host, port)
logrus.Debugf("tcp dialing %s", hostnameAndPort)
var conn net.Conn
var err error
conn, err = net.DialTimeout("tcp", hostnameAndPort, timeout)
if err != nil {
logrus.Debugf("tcp dialing %s failed, trying via ~/.ssh/config", hostnameAndPort)
hostConfig, err := GetHostConfig(host, username, port, true)
if err != nil {
return nil, err
}
conn, err = net.DialTimeout("tcp", fmt.Sprintf("%s:%s", hostConfig.Host, hostConfig.Port), timeout)
if err != nil {
return nil, err
}
}
sshConn, chans, reqs, err := ssh.NewClientConn(conn, hostnameAndPort, config)
if err != nil {
return nil, err
}
client := ssh.NewClient(sshConn, chans, reqs)
c := &Client{SSHClient: client}
return c, nil
}
func connectWithAgentTimeout(host, username, port string, timeout time.Duration) (*Client, error) {
logrus.Debugf("using ssh-agent to make an SSH connection for %s", host)
sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
if err != nil {
return nil, err
}
agentCl := agent.NewClient(sshAgent)
authMethod := ssh.PublicKeysCallback(agentCl.Signers)
loadedKeys, err := agentCl.List()
if err != nil {
return nil, err
}
var convertedKeys []string
for _, key := range loadedKeys {
convertedKeys = append(convertedKeys, key.String())
}
if len(convertedKeys) > 0 {
logrus.Debugf("ssh-agent has these keys loaded: %s", strings.Join(convertedKeys, ","))
} else {
logrus.Debug("ssh-agent has no keys loaded")
}
return connect(username, host, port, authMethod, timeout)
}
func connectWithPasswordTimeout(host, username, port, pass string, timeout time.Duration) (*Client, error) {
authMethod := ssh.Password(pass)
return connect(username, host, port, authMethod, timeout)
}
// EnsureHostKey ensures that a host key trusted and added to the ~/.ssh/known_hosts file
func EnsureHostKey(hostname string) error {
if hostname == "default" || hostname == "local" {
logrus.Debugf("not checking server SSH host key against local/default target")
return nil
}
exists, _, err := GetHostKey(hostname)
if err != nil {
return err
}
if exists {
return nil
}
ctxConnDetails, err := GetContextConnDetails(hostname)
if err != nil {
return err
}
_, err = connectWithAgentTimeout(
ctxConnDetails.Host,
ctxConnDetails.User,
ctxConnDetails.Port,
5*time.Second,
// String presents a human friendly output for the HostConfig.
func (h HostConfig) String() string {
return fmt.Sprintf(
"{host: %s, username: %s, port: %s, identityfile: %s}",
h.Host,
h.User,
h.Port,
h.IdentityFile,
)
if err != nil {
return err
}
return nil
}
// FingerprintSHA256 generates the SHA256 fingerprint for a server SSH host key
func FingerprintSHA256(key ssh.PublicKey) string {
hash := sha256.Sum256(key.Marshal())
b64hash := base64.StdEncoding.EncodeToString(hash[:])
trimmed := strings.TrimRight(b64hash, "=")
return fmt.Sprintf("SHA256:%s", trimmed)
}
// GetContextConnDetails retrieves SSH connection details from a docker context endpoint
func GetContextConnDetails(serverName string) (*dockerSSHPkg.Spec, error) {
dockerContextStore := context.NewDefaultDockerContextStore()
contexts, err := dockerContextStore.Store.List()
if err != nil {
return &dockerSSHPkg.Spec{}, err
}
if strings.Contains(serverName, ":") {
serverName = strings.Split(serverName, ":")[0]
}
for _, ctx := range contexts {
endpoint, err := context.GetContextEndpoint(ctx)
if err != nil && strings.Contains(err.Error(), "does not exist") {
// No local context found, we can continue safely
continue
}
if ctx.Name == serverName {
ctxConnDetails, err := dockerSSHPkg.ParseURL(endpoint)
if err != nil {
return &dockerSSHPkg.Spec{}, err
}
logrus.Debugf("found context connection details %v for %s", ctxConnDetails, serverName)
return ctxConnDetails, nil
}
}
hostConfig, err := GetHostConfig(serverName, "", "", false)
if err != nil {
return &dockerSSHPkg.Spec{}, err
}
logrus.Debugf("couldn't find a docker context matching %s", serverName)
logrus.Debugf("searching ~/.ssh/config for a Host entry for %s", serverName)
connDetails := &dockerSSHPkg.Spec{
Host: hostConfig.Host,
User: hostConfig.User,
Port: hostConfig.Port,
}
logrus.Debugf("using %v from ~/.ssh/config for connection details", connDetails)
return connDetails, nil
}
// GetHostConfig retrieves a ~/.ssh/config config for a host.
func GetHostConfig(hostname, username, port string, override bool) (HostConfig, error) {
// GetHostConfig retrieves a ~/.ssh/config config for a host using /usr/bin/ssh
// directly. We therefore maintain consistent interop with this standard
// tooling. This is useful because SSH confuses a lot of people and having to
// learn how two tools (`ssh` and `abra`) handle SSH connection details instead
// of one (just `ssh`) is Not Cool. Here's to less bug reports on this topic!
func GetHostConfig(hostname string) (HostConfig, error) {
var hostConfig HostConfig
if hostname == "" || override {
if sshHost := ssh_config.Get(hostname, "Hostname"); sshHost != "" {
hostname = sshHost
}
out, err := exec.Command("ssh", "-G", hostname).Output()
if err != nil {
return hostConfig, err
}
if username == "" || override {
if sshUser := ssh_config.Get(hostname, "User"); sshUser != "" {
username = sshUser
} else {
systemUser, err := user.Current()
if err != nil {
return hostConfig, err
for _, line := range strings.Split(string(out), "\n") {
entries := strings.Split(line, " ")
for idx, entry := range entries {
if entry == "hostname" {
hostConfig.Host = entries[idx+1]
}
username = systemUser.Username
}
}
if port == "" || override {
if sshPort := ssh_config.Get(hostname, "Port"); sshPort != "" {
// skip override probably correct port with dummy default value from
// ssh_config which is 22. only when the original port number is empty
// should we try this default. this might not cover all cases
// unfortunately.
if port != "" && sshPort != "22" {
port = sshPort
if entry == "user" {
hostConfig.User = entries[idx+1]
}
if entry == "port" {
hostConfig.Port = entries[idx+1]
}
if entry == "identityfile" {
if hostConfig.IdentityFile == "" {
hostConfig.IdentityFile = entries[idx+1]
}
}
}
}
if idf := ssh_config.Get(hostname, "IdentityFile"); idf != "" && idf != "~/.ssh/identity" {
var err error
idf, err = identityFileAbsPath(idf)
if err != nil {
return hostConfig, err
}
hostConfig.IdentityFile = idf
} else {
hostConfig.IdentityFile = ""
}
hostConfig.Host = hostname
hostConfig.Port = port
hostConfig.User = username
logrus.Debugf("constructed SSH config %s for %s", hostConfig, hostname)
logrus.Debugf("retrieved ssh config for %s: %s", hostname, hostConfig.String())
return hostConfig, nil
}
func identityFileAbsPath(relPath string) (string, error) {
var err error
var absPath string
if strings.HasPrefix(relPath, "~/") {
systemUser, err := user.Current()
if err != nil {
return absPath, err
}
absPath = filepath.Join(systemUser.HomeDir, relPath[2:])
// Fatal is a error output wrapper which aims to make SSH failures easier to
// parse through re-wording.
func Fatal(hostname string, err error) error {
out := err.Error()
if strings.Contains(out, "Host key verification failed.") {
return fmt.Errorf("SSH host key verification failed for %s", hostname)
} else if strings.Contains(out, "Could not resolve hostname") {
return fmt.Errorf("could not resolve hostname for %s", hostname)
} else if strings.Contains(out, "Connection timed out") {
return fmt.Errorf("connection timed out for %s", hostname)
} else {
absPath, err = filepath.Abs(relPath)
if err != nil {
return absPath, err
}
return err
}
logrus.Debugf("resolved %s to %s to read the ssh identity file", relPath, absPath)
return absPath, nil
}

27
pkg/upstream/commandconn/connection.go
View File

@ -2,19 +2,15 @@ package commandconn
import (
"context"
"fmt"
"net"
"net/url"
ctxPkg "coopcloud.tech/abra/pkg/context"
sshPkg "coopcloud.tech/abra/pkg/ssh"
"github.com/docker/cli/cli/connhelper"
"github.com/docker/cli/cli/connhelper/ssh"
"github.com/docker/cli/cli/context/docker"
dCliContextStore "github.com/docker/cli/cli/context/store"
dClient "github.com/docker/docker/client"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
// GetConnectionHelper returns Docker-specific connection helper for the given URL.
@ -37,29 +33,6 @@ func getConnectionHelper(daemonURL string, sshFlags []string) (*connhelper.Conne
return nil, errors.Wrap(err, "ssh host connection is not valid")
}
if err := ctxPkg.HasDockerContext("", ctxConnDetails.Host); err != nil {
return nil, err
}
if err := sshPkg.EnsureHostKey(ctxConnDetails.Host); err != nil {
return nil, err
}
hostConfig, err := sshPkg.GetHostConfig(
ctxConnDetails.Host,
ctxConnDetails.User,
ctxConnDetails.Port,
false,
)
if err != nil {
return nil, err
}
if hostConfig.IdentityFile != "" {
msg := "discovered %s as identity file for %s, using for ssh connection"
logrus.Debugf(msg, hostConfig.IdentityFile, ctxConnDetails.Host)
sshFlags = append(sshFlags, fmt.Sprintf("-o IdentityFile=%s", hostConfig.IdentityFile))
}
return &connhelper.ConnectionHelper{
Dialer: func(ctx context.Context, network, addr string) (net.Conn, error) {
return New(ctx, "ssh", append(sshFlags, ctxConnDetails.Args("docker", "system", "dial-stdio")...)...)

63
pkg/upstream/stack/stack.go
View File

@ -8,7 +8,6 @@ import (
"strings"
"time"
abraClient "coopcloud.tech/abra/pkg/client"
"coopcloud.tech/abra/pkg/upstream/convert"
"github.com/docker/cli/cli/command/service/progress"
composetypes "github.com/docker/cli/cli/compose/types"
@ -18,7 +17,7 @@ import (
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/api/types/versions"
"github.com/docker/docker/client"
dockerclient "github.com/docker/docker/client"
dockerClient "github.com/docker/docker/client"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@ -57,20 +56,10 @@ func GetStackServices(ctx context.Context, dockerclient client.APIClient, namesp
}
// GetDeployedServicesByLabel filters services by label
func GetDeployedServicesByLabel(contextName string, label string) StackStatus {
cl, err := abraClient.New(contextName)
if err != nil {
if strings.Contains(err.Error(), "does not exist") {
// No local context found, bail out gracefully
return StackStatus{[]swarm.Service{}, nil}
}
return StackStatus{[]swarm.Service{}, err}
}
ctx := context.Background()
func GetDeployedServicesByLabel(cl *dockerClient.Client, contextName string, label string) StackStatus {
filters := filters.NewArgs()
filters.Add("label", label)
services, err := cl.ServiceList(ctx, types.ServiceListOptions{Filters: filters})
services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: filters})
if err != nil {
return StackStatus{[]swarm.Service{}, err}
}
@ -78,18 +67,8 @@ func GetDeployedServicesByLabel(contextName string, label string) StackStatus {
return StackStatus{services, nil}
}
func GetAllDeployedServices(contextName string) StackStatus {
cl, err := abraClient.New(contextName)
if err != nil {
if strings.Contains(err.Error(), "does not exist") {
// No local context found, bail out gracefully
return StackStatus{[]swarm.Service{}, nil}
}
return StackStatus{[]swarm.Service{}, err}
}
ctx := context.Background()
services, err := cl.ServiceList(ctx, types.ServiceListOptions{Filters: getAllStacksFilter()})
func GetAllDeployedServices(cl *dockerClient.Client, contextName string) StackStatus {
services, err := cl.ServiceList(context.Background(), types.ServiceListOptions{Filters: getAllStacksFilter()})
if err != nil {
return StackStatus{[]swarm.Service{}, err}
}
@ -98,7 +77,7 @@ func GetAllDeployedServices(contextName string) StackStatus {
}
// GetDeployedServicesByName filters services by name
func GetDeployedServicesByName(ctx context.Context, cl *dockerclient.Client, stackName, serviceName string) StackStatus {
func GetDeployedServicesByName(ctx context.Context, cl *dockerClient.Client, stackName, serviceName string) StackStatus {
filters := filters.NewArgs()
filters.Add("name", fmt.Sprintf("%s_%s", stackName, serviceName))
@ -111,7 +90,7 @@ func GetDeployedServicesByName(ctx context.Context, cl *dockerclient.Client, sta
}
// IsDeployed chekcks whether an appp is deployed or not.
func IsDeployed(ctx context.Context, cl *dockerclient.Client, stackName string) (bool, string, error) {
func IsDeployed(ctx context.Context, cl *dockerClient.Client, stackName string) (bool, string, error) {
version := "unknown"
isDeployed := false
@ -142,7 +121,7 @@ func IsDeployed(ctx context.Context, cl *dockerclient.Client, stackName string)
}
// pruneServices removes services that are no longer referenced in the source
func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace convert.Namespace, services map[string]struct{}) {
func pruneServices(ctx context.Context, cl *dockerClient.Client, namespace convert.Namespace, services map[string]struct{}) {
oldServices, err := GetStackServices(ctx, cl, namespace.Name())
if err != nil {
logrus.Infof("Failed to list services: %s\n", err)
@ -158,9 +137,7 @@ func pruneServices(ctx context.Context, cl *dockerclient.Client, namespace conve
}
// RunDeploy is the swarm implementation of docker stack deploy
func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, appName string, dontWait bool) error {
ctx := context.Background()
func RunDeploy(cl *dockerClient.Client, opts Deploy, cfg *composetypes.Config, appName string, dontWait bool) error {
if err := validateResolveImageFlag(&opts); err != nil {
return err
}
@ -170,7 +147,7 @@ func RunDeploy(cl *dockerclient.Client, opts Deploy, cfg *composetypes.Config, a
opts.ResolveImage = ResolveImageNever
}
return deployCompose(ctx, cl, opts, cfg, appName, dontWait)
return deployCompose(context.Background(), cl, opts, cfg, appName, dontWait)
}
// validateResolveImageFlag validates the opts.resolveImage command line option
@ -183,7 +160,7 @@ func validateResolveImageFlag(opts *Deploy) error {
}
}
func deployCompose(ctx context.Context, cl *dockerclient.Client, opts Deploy, config *composetypes.Config, appName string, dontWait bool) error {
func deployCompose(ctx context.Context, cl *dockerClient.Client, opts Deploy, config *composetypes.Config, appName string, dontWait bool) error {
namespace := convert.NewNamespace(opts.Namespace)
if opts.Prune {
@ -241,7 +218,7 @@ func getServicesDeclaredNetworks(serviceConfigs []composetypes.ServiceConfig) ma
return serviceNetworks
}
func validateExternalNetworks(ctx context.Context, client dockerclient.NetworkAPIClient, externalNetworks []string) error {
func validateExternalNetworks(ctx context.Context, client dockerClient.NetworkAPIClient, externalNetworks []string) error {
for _, networkName := range externalNetworks {
if !container.NetworkMode(networkName).IsUserDefined() {
// Networks that are not user defined always exist on all nodes as
@ -250,7 +227,7 @@ func validateExternalNetworks(ctx context.Context, client dockerclient.NetworkAP
}
network, err := client.NetworkInspect(ctx, networkName, types.NetworkInspectOptions{})
switch {
case dockerclient.IsErrNotFound(err):
case dockerClient.IsErrNotFound(err):
return errors.Errorf("network %q is declared as external, but could not be found. You need to create a swarm-scoped network before the stack is deployed", networkName)
case err != nil:
return err
@ -261,7 +238,7 @@ func validateExternalNetworks(ctx context.Context, client dockerclient.NetworkAP
return nil
}
func createSecrets(ctx context.Context, cl *dockerclient.Client, secrets []swarm.SecretSpec) error {
func createSecrets(ctx context.Context, cl *dockerClient.Client, secrets []swarm.SecretSpec) error {
for _, secretSpec := range secrets {
secret, _, err := cl.SecretInspectWithRaw(ctx, secretSpec.Name)
switch {
@ -270,7 +247,7 @@ func createSecrets(ctx context.Context, cl *dockerclient.Client, secrets []swarm
if err := cl.SecretUpdate(ctx, secret.ID, secret.Meta.Version, secretSpec); err != nil {
return errors.Wrapf(err, "failed to update secret %s", secretSpec.Name)
}
case dockerclient.IsErrNotFound(err):
case dockerClient.IsErrNotFound(err):
// secret does not exist, then we create a new one.
logrus.Infof("Creating secret %s\n", secretSpec.Name)
if _, err := cl.SecretCreate(ctx, secretSpec); err != nil {
@ -283,7 +260,7 @@ func createSecrets(ctx context.Context, cl *dockerclient.Client, secrets []swarm
return nil
}
func createConfigs(ctx context.Context, cl *dockerclient.Client, configs []swarm.ConfigSpec) error {
func createConfigs(ctx context.Context, cl *dockerClient.Client, configs []swarm.ConfigSpec) error {
for _, configSpec := range configs {
config, _, err := cl.ConfigInspectWithRaw(ctx, configSpec.Name)
switch {
@ -292,7 +269,7 @@ func createConfigs(ctx context.Context, cl *dockerclient.Client, configs []swarm
if err := cl.ConfigUpdate(ctx, config.ID, config.Meta.Version, configSpec); err != nil {
return errors.Wrapf(err, "failed to update config %s", configSpec.Name)
}
case dockerclient.IsErrNotFound(err):
case dockerClient.IsErrNotFound(err):
// config does not exist, then we create a new one.
logrus.Infof("Creating config %s\n", configSpec.Name)
if _, err := cl.ConfigCreate(ctx, configSpec); err != nil {
@ -305,7 +282,7 @@ func createConfigs(ctx context.Context, cl *dockerclient.Client, configs []swarm
return nil
}
func createNetworks(ctx context.Context, cl *dockerclient.Client, namespace convert.Namespace, networks map[string]types.NetworkCreate) error {
func createNetworks(ctx context.Context, cl *dockerClient.Client, namespace convert.Namespace, networks map[string]types.NetworkCreate) error {
existingNetworks, err := getStackNetworks(ctx, cl, namespace.Name())
if err != nil {
return err
@ -335,7 +312,7 @@ func createNetworks(ctx context.Context, cl *dockerclient.Client, namespace conv
func deployServices(
ctx context.Context,
cl *dockerclient.Client,
cl *dockerClient.Client,
services map[string]swarm.ServiceSpec,
namespace convert.Namespace,
sendAuth bool,
@ -469,7 +446,7 @@ func getStackConfigs(ctx context.Context, dockerclient client.APIClient, namespa
// https://github.com/docker/cli/blob/master/cli/command/service/helpers.go
// https://github.com/docker/cli/blob/master/cli/command/service/progress/progress.go
func WaitOnService(ctx context.Context, cl *dockerclient.Client, serviceID, appName string) error {
func WaitOnService(ctx context.Context, cl *dockerClient.Client, serviceID, appName string) error {
errChan := make(chan error, 1)
pipeReader, pipeWriter := io.Pipe()