package config import ( "fmt" "io/ioutil" "os" "path" "strconv" "strings" "github.com/schollz/progressbar/v3" "coopcloud.tech/abra/pkg/client" "coopcloud.tech/abra/pkg/formatter" "coopcloud.tech/abra/pkg/upstream/convert" loader "coopcloud.tech/abra/pkg/upstream/stack" stack "coopcloud.tech/abra/pkg/upstream/stack" composetypes "github.com/docker/cli/cli/compose/types" "github.com/docker/docker/api/types/filters" "github.com/sirupsen/logrus" ) // Type aliases to make code hints easier to understand // AppEnv is a map of the values in an apps env config type AppEnv = map[string]string // AppModifiers is a map of modifiers in an apps env config type AppModifiers = map[string]map[string]string // AppName is AppName type AppName = string // AppFile represents app env files on disk without reading the contents type AppFile struct { Path string Server string } // AppFiles is a slice of appfiles type AppFiles map[AppName]AppFile // App reprents an app with its env file read into memory type App struct { Name AppName Recipe string Domain string Env AppEnv Server string Path string } // See documentation of config.StackName func (a App) StackName() string { if _, exists := a.Env["STACK_NAME"]; exists { return a.Env["STACK_NAME"] } stackName := StackName(a.Name) a.Env["STACK_NAME"] = stackName return stackName } // StackName gets whatever the docker safe (uses the right delimiting // character, e.g. "_") stack name is for the app. In general, you don't want // to use this to show anything to end-users, you want use a.Name instead. func StackName(appName string) string { stackName := SanitiseAppName(appName) if len(stackName) > MAX_SANITISED_APP_NAME_LENGTH { logrus.Debugf("trimming %s to %s to avoid runtime limits", stackName, stackName[:MAX_SANITISED_APP_NAME_LENGTH]) stackName = stackName[:MAX_SANITISED_APP_NAME_LENGTH] } return stackName } // Filters retrieves app filters for querying the container runtime. By default // it filters on all services in the app. It is also possible to pass an // otional list of service names, which get filtered instead. // // Due to upstream issues, filtering works different depending on what you're // querying. So, for example, secrets don't work with regex! The caller needs // to implement their own validation that the right secrets are matched. In // order to handle these cases, we provide the `appendServiceNames` / // `exactMatch` modifiers. func (a App) Filters(appendServiceNames, exactMatch bool, services ...string) (filters.Args, error) { filters := filters.NewArgs() if len(services) > 0 { for _, serviceName := range services { filters.Add("name", ServiceFilter(a.StackName(), serviceName, exactMatch)) } return filters, nil } // When not appending the service name, just add one filter for the whole // stack. if !appendServiceNames { f := fmt.Sprintf("%s", a.StackName()) if exactMatch { f = fmt.Sprintf("^%s", f) } filters.Add("name", f) return filters, nil } composeFiles, err := GetComposeFiles(a.Recipe, a.Env) if err != nil { return filters, err } opts := stack.Deploy{Composefiles: composeFiles} compose, err := GetAppComposeConfig(a.Recipe, opts, a.Env) if err != nil { return filters, err } for _, service := range compose.Services { f := ServiceFilter(a.StackName(), service.Name, exactMatch) filters.Add("name", f) } return filters, nil } // ServiceFilter creates a filter string for filtering a service in the docker // container runtime. When exact match is true, it uses regex to match the // string exactly. func ServiceFilter(stack, service string, exact bool) string { if exact { return fmt.Sprintf("^%s_%s", stack, service) } return fmt.Sprintf("%s_%s", stack, service) } // ByServer sort a slice of Apps type ByServer []App func (a ByServer) Len() int { return len(a) } func (a ByServer) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByServer) Less(i, j int) bool { return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server) } // ByServerAndRecipe sort a slice of Apps type ByServerAndRecipe []App func (a ByServerAndRecipe) Len() int { return len(a) } func (a ByServerAndRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByServerAndRecipe) Less(i, j int) bool { if a[i].Server == a[j].Server { return strings.ToLower(a[i].Recipe) < strings.ToLower(a[j].Recipe) } return strings.ToLower(a[i].Server) < strings.ToLower(a[j].Server) } // ByRecipe sort a slice of Apps type ByRecipe []App func (a ByRecipe) Len() int { return len(a) } func (a ByRecipe) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByRecipe) Less(i, j int) bool { return strings.ToLower(a[i].Recipe) < strings.ToLower(a[j].Recipe) } // ByName sort a slice of Apps type ByName []App func (a ByName) Len() int { return len(a) } func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByName) Less(i, j int) bool { return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name) } func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) { env, err := ReadEnv(appFile.Path) if err != nil { return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error()) } logrus.Debugf("read env %s from %s", env, appFile.Path) app, err := NewApp(env, name, appFile) if err != nil { return App{}, fmt.Errorf("env file for %s has issues: %s", name, err.Error()) } return app, nil } // NewApp creates new App object func NewApp(env AppEnv, name string, appFile AppFile) (App, error) { domain := env["DOMAIN"] recipe, exists := env["RECIPE"] if !exists { recipe, exists = env["TYPE"] if !exists { return App{}, fmt.Errorf("%s is missing the TYPE env var?", name) } } return App{ Name: name, Domain: domain, Recipe: recipe, Env: env, Server: appFile.Server, Path: appFile.Path, }, nil } // LoadAppFiles gets all app files for a given set of servers or all servers. func LoadAppFiles(servers ...string) (AppFiles, error) { appFiles := make(AppFiles) if len(servers) == 1 { if servers[0] == "" { // Empty servers flag, one string will always be passed var err error servers, err = GetAllFoldersInDirectory(SERVERS_DIR) if err != nil { return appFiles, err } } } logrus.Debugf("collecting metadata from %v servers: %s", len(servers), strings.Join(servers, ", ")) for _, server := range servers { serverDir := path.Join(SERVERS_DIR, server) files, err := GetAllFilesInDirectory(serverDir) if err != nil { return appFiles, fmt.Errorf("server %s doesn't exist? Run \"abra server ls\" to check", server) } for _, file := range files { appName := strings.TrimSuffix(file.Name(), ".env") appFilePath := path.Join(SERVERS_DIR, server, file.Name()) appFiles[appName] = AppFile{ Path: appFilePath, Server: server, } } } return appFiles, nil } // GetApp loads an apps settings, reading it from file, in preparation to use // it. It should only be used when ready to use the env file to keep IO // operations down. func GetApp(apps AppFiles, name AppName) (App, error) { appFile, exists := apps[name] if !exists { return App{}, fmt.Errorf("cannot find app with name %s", name) } app, err := ReadAppEnvFile(appFile, name) if err != nil { return App{}, err } return app, nil } // GetApps returns a slice of Apps with their env files read from a given // slice of AppFiles. func GetApps(appFiles AppFiles, recipeFilter string) ([]App, error) { var apps []App for name := range appFiles { app, err := GetApp(appFiles, name) if err != nil { return nil, err } if recipeFilter != "" { if app.Recipe == recipeFilter { apps = append(apps, app) } } else { apps = append(apps, app) } } return apps, nil } // GetAppServiceNames retrieves a list of app service names. func GetAppServiceNames(appName string) ([]string, error) { var serviceNames []string appFiles, err := LoadAppFiles("") if err != nil { return serviceNames, err } app, err := GetApp(appFiles, appName) if err != nil { return serviceNames, err } composeFiles, err := GetComposeFiles(app.Recipe, app.Env) if err != nil { return serviceNames, err } opts := stack.Deploy{Composefiles: composeFiles} compose, err := GetAppComposeConfig(app.Recipe, opts, app.Env) if err != nil { return serviceNames, err } for _, service := range compose.Services { serviceNames = append(serviceNames, service.Name) } return serviceNames, nil } // GetAppNames retrieves a list of app names. func GetAppNames() ([]string, error) { var appNames []string appFiles, err := LoadAppFiles("") if err != nil { return appNames, err } apps, err := GetApps(appFiles, "") if err != nil { return appNames, err } for _, app := range apps { appNames = append(appNames, app.Name) } return appNames, nil } // TemplateAppEnvSample copies the example env file for the app into the users // env files. func TemplateAppEnvSample(recipeName, appName, server, domain string) error { envSamplePath := path.Join(RECIPES_DIR, recipeName, ".env.sample") envSample, err := ioutil.ReadFile(envSamplePath) if err != nil { return err } appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName)) if _, err := os.Stat(appEnvPath); !os.IsNotExist(err) { return fmt.Errorf("%s already exists?", appEnvPath) } err = ioutil.WriteFile(appEnvPath, envSample, 0o664) if err != nil { return err } read, err := ioutil.ReadFile(appEnvPath) if err != nil { return err } newContents := strings.Replace(string(read), recipeName+".example.com", domain, -1) err = ioutil.WriteFile(appEnvPath, []byte(newContents), 0) if err != nil { return err } logrus.Debugf("copied & templated %s to %s", envSamplePath, appEnvPath) return nil } // SanitiseAppName makes a app name usable with Docker by replacing illegal // characters. func SanitiseAppName(name string) string { return strings.ReplaceAll(name, ".", "_") } // GetAppStatuses queries servers to check the deployment status of given apps. func GetAppStatuses(apps []App, MachineReadable bool) (map[string]map[string]string, error) { statuses := make(map[string]map[string]string) servers := make(map[string]struct{}) for _, app := range apps { if _, ok := servers[app.Server]; !ok { servers[app.Server] = struct{}{} } } var bar *progressbar.ProgressBar if !MachineReadable { bar = formatter.CreateProgressbar(len(servers), "querying remote servers...") } ch := make(chan stack.StackStatus, len(servers)) for server := range servers { cl, err := client.New(server) if err != nil { return statuses, err } go func(s string) { ch <- stack.GetAllDeployedServices(cl, s) if !MachineReadable { bar.Add(1) } }(server) } for range servers { status := <-ch if status.Err != nil { return statuses, status.Err } for _, service := range status.Services { result := make(map[string]string) name := service.Spec.Labels[convert.LabelNamespace] if _, ok := statuses[name]; !ok { result["status"] = "deployed" } labelKey := fmt.Sprintf("coop-cloud.%s.chaos", name) chaos, ok := service.Spec.Labels[labelKey] if ok { result["chaos"] = chaos } labelKey = fmt.Sprintf("coop-cloud.%s.chaos-version", name) if chaosVersion, ok := service.Spec.Labels[labelKey]; ok { result["chaosVersion"] = chaosVersion } labelKey = fmt.Sprintf("coop-cloud.%s.autoupdate", name) if autoUpdate, ok := service.Spec.Labels[labelKey]; ok { result["autoUpdate"] = autoUpdate } else { result["autoUpdate"] = "false" } labelKey = fmt.Sprintf("coop-cloud.%s.version", name) if version, ok := service.Spec.Labels[labelKey]; ok { result["version"] = version } else { continue } statuses[name] = result } } logrus.Debugf("retrieved app statuses: %s", statuses) return statuses, nil } // ensurePathExists ensures that a path exists. func ensurePathExists(path string) error { if _, err := os.Stat(path); err != nil && os.IsNotExist(err) { return err } return nil } // GetComposeFiles gets the list of compose files for an app (or recipe if you // don't already have an app) which should be merged into a composetypes.Config // while respecting the COMPOSE_FILE env var. func GetComposeFiles(recipe string, appEnv AppEnv) ([]string, error) { var composeFiles []string composeFileEnvVar, ok := appEnv["COMPOSE_FILE"] if !ok { path := fmt.Sprintf("%s/%s/compose.yml", RECIPES_DIR, recipe) if err := ensurePathExists(path); err != nil { return composeFiles, err } logrus.Debugf("no COMPOSE_FILE detected, loading default: %s", path) composeFiles = append(composeFiles, path) return composeFiles, nil } if !strings.Contains(composeFileEnvVar, ":") { path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, composeFileEnvVar) if err := ensurePathExists(path); err != nil { return composeFiles, err } logrus.Debugf("COMPOSE_FILE detected, loading %s", path) composeFiles = append(composeFiles, path) return composeFiles, nil } numComposeFiles := strings.Count(composeFileEnvVar, ":") + 1 envVars := strings.SplitN(composeFileEnvVar, ":", numComposeFiles) if len(envVars) != numComposeFiles { return composeFiles, fmt.Errorf("COMPOSE_FILE (=\"%s\") parsing failed?", composeFileEnvVar) } for _, file := range envVars { path := fmt.Sprintf("%s/%s/%s", RECIPES_DIR, recipe, file) if err := ensurePathExists(path); err != nil { return composeFiles, err } composeFiles = append(composeFiles, path) } logrus.Debugf("COMPOSE_FILE detected (%s), loading %s", composeFileEnvVar, strings.Join(envVars, ", ")) logrus.Debugf("retrieved %s configs for %s", strings.Join(composeFiles, ", "), recipe) return composeFiles, nil } // GetAppComposeConfig retrieves a compose specification for a recipe. This // specification is the result of a merge of all the compose.**.yml files in // the recipe repository. func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*composetypes.Config, error) { compose, err := loader.LoadComposefile(opts, appEnv) if err != nil { return &composetypes.Config{}, err } logrus.Debugf("retrieved %s for %s", compose.Filename, recipe) return compose, nil } // ExposeAllEnv exposes all env variables to the app container func ExposeAllEnv(stackName string, compose *composetypes.Config, appEnv AppEnv) { for _, service := range compose.Services { if service.Name == "app" { logrus.Debugf("Add the following environment to the app service config of %s:", stackName) for k, v := range appEnv { _, exists := service.Environment[k] if !exists { value := v service.Environment[k] = &value logrus.Debugf("Add Key: %s Value: %s to %s", k, value, stackName) } } } } } // SetRecipeLabel adds the label 'coop-cloud.${STACK_NAME}.recipe=${RECIPE}' to the app container // to signal which recipe is connected to the deployed app func SetRecipeLabel(compose *composetypes.Config, stackName string, recipe string) { for _, service := range compose.Services { if service.Name == "app" { logrus.Debugf("set recipe label 'coop-cloud.%s.recipe' to %s for %s", stackName, recipe, stackName) labelKey := fmt.Sprintf("coop-cloud.%s.recipe", stackName) service.Deploy.Labels[labelKey] = recipe } } } // SetChaosLabel adds the label 'coop-cloud.${STACK_NAME}.chaos=true/false' to the app container // to signal if the app is deployed in chaos mode func SetChaosLabel(compose *composetypes.Config, stackName string, chaos bool) { for _, service := range compose.Services { if service.Name == "app" { logrus.Debugf("set label 'coop-cloud.%s.chaos' to %v for %s", stackName, chaos, stackName) labelKey := fmt.Sprintf("coop-cloud.%s.chaos", stackName) service.Deploy.Labels[labelKey] = strconv.FormatBool(chaos) } } } // SetChaosVersionLabel adds the label 'coop-cloud.${STACK_NAME}.chaos-version=$(GIT_COMMIT)' to the app container func SetChaosVersionLabel(compose *composetypes.Config, stackName string, chaosVersion string) { for _, service := range compose.Services { if service.Name == "app" { logrus.Debugf("set label 'coop-cloud.%s.chaos-version' to %v for %s", stackName, chaosVersion, stackName) labelKey := fmt.Sprintf("coop-cloud.%s.chaos-version", stackName) service.Deploy.Labels[labelKey] = chaosVersion } } } // SetUpdateLabel adds env ENABLE_AUTO_UPDATE as label to enable/disable the // auto update process for this app. The default if this variable is not set is to disable // the auto update process. func SetUpdateLabel(compose *composetypes.Config, stackName string, appEnv AppEnv) { for _, service := range compose.Services { if service.Name == "app" { enable_auto_update, exists := appEnv["ENABLE_AUTO_UPDATE"] if !exists { enable_auto_update = "false" } logrus.Debugf("set label 'coop-cloud.%s.autoupdate' to %s for %s", stackName, enable_auto_update, stackName) labelKey := fmt.Sprintf("coop-cloud.%s.autoupdate", stackName) service.Deploy.Labels[labelKey] = enable_auto_update } } } // GetLabel reads docker labels in the format of "coop-cloud.${STACK_NAME}.${LABEL}" from the local compose files func GetLabel(compose *composetypes.Config, stackName string, label string) string { for _, service := range compose.Services { if service.Name == "app" { labelKey := fmt.Sprintf("coop-cloud.%s.%s", stackName, label) logrus.Debugf("get label '%s'", labelKey) if labelValue, ok := service.Deploy.Labels[labelKey]; ok { return labelValue } } } logrus.Debugf("no %s label found for %s", label, stackName) return "" } // GetTimeoutFromLabel reads the timeout value from docker label "coop-cloud.${STACK_NAME}.TIMEOUT" and returns 50 as default value func GetTimeoutFromLabel(compose *composetypes.Config, stackName string) (int, error) { timeout := 50 // Default Timeout var err error = nil if timeoutLabel := GetLabel(compose, stackName, "timeout"); timeoutLabel != "" { logrus.Debugf("timeout label: %s", timeoutLabel) timeout, err = strconv.Atoi(timeoutLabel) } return timeout, err }