refactor: use pkg directory structure

This commit is contained in:
2021-09-05 21:37:03 +02:00
parent f59380a35e
commit b7742d5e18
55 changed files with 55 additions and 55 deletions

357
pkg/config/app.go Normal file
View File

@ -0,0 +1,357 @@
package config
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"coopcloud.tech/abra/pkg/client/convert"
loader "coopcloud.tech/abra/pkg/client/stack"
stack "coopcloud.tech/abra/pkg/client/stack"
composetypes "github.com/docker/cli/cli/compose/types"
"github.com/docker/distribution/reference"
"github.com/sirupsen/logrus"
)
// Type aliases to make code hints easier to understand
// AppEnv is a map of the values in an apps env config
type AppEnv = map[string]string
// AppName is AppName
type AppName = string
// AppFile represents app env files on disk without reading the contents
type AppFile struct {
Path string
Server string
}
// AppFiles is a slice of appfiles
type AppFiles map[AppName]AppFile
// App reprents an app with its env file read into memory
type App struct {
Name AppName
Type string
Domain string
Env AppEnv
File AppFile
}
// StackName gets what the docker safe stack name is for the app
func (a App) StackName() string {
return SanitiseAppName(a.Name)
}
// SORTING TYPES
// ByServer sort a slice of Apps
type ByServer []App
func (a ByServer) Len() int { return len(a) }
func (a ByServer) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByServer) Less(i, j int) bool {
return strings.ToLower(a[i].File.Server) < strings.ToLower(a[j].File.Server)
}
// ByServerAndType sort a slice of Apps
type ByServerAndType []App
func (a ByServerAndType) Len() int { return len(a) }
func (a ByServerAndType) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByServerAndType) Less(i, j int) bool {
if a[i].File.Server == a[j].File.Server {
return strings.ToLower(a[i].Type) < strings.ToLower(a[j].Type)
}
return strings.ToLower(a[i].File.Server) < strings.ToLower(a[j].File.Server)
}
// ByType sort a slice of Apps
type ByType []App
func (a ByType) Len() int { return len(a) }
func (a ByType) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByType) Less(i, j int) bool {
return strings.ToLower(a[i].Type) < strings.ToLower(a[j].Type)
}
// ByName sort a slice of Apps
type ByName []App
func (a ByName) Len() int { return len(a) }
func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByName) Less(i, j int) bool {
return strings.ToLower(a[i].Name) < strings.ToLower(a[j].Name)
}
func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
env, err := ReadEnv(appFile.Path)
if err != nil {
return App{}, fmt.Errorf("env file for '%s' couldn't be read: %s", name, err.Error())
}
app, err := newApp(env, name, appFile)
if err != nil {
return App{}, fmt.Errorf("env file for '%s' has issues: %s", name, err.Error())
}
return app, nil
}
// newApp creates new App object
func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
// Checking for type as it is required - apps wont work without it
domain := env["DOMAIN"]
apptype, ok := env["TYPE"]
if !ok {
return App{}, errors.New("missing TYPE variable")
}
return App{
Name: name,
Domain: domain,
Type: apptype,
Env: env,
File: appFile,
}, nil
}
// LoadAppFiles gets all app files for a given set of servers or all servers
func LoadAppFiles(servers ...string) (AppFiles, error) {
appFiles := make(AppFiles)
if len(servers) == 1 {
if servers[0] == "" {
// Empty servers flag, one string will always be passed
var err error
servers, err = getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
if err != nil {
return nil, err
}
}
}
for _, server := range servers {
serverDir := path.Join(ABRA_SERVER_FOLDER, server)
files, err := getAllFilesInDirectory(serverDir)
if err != nil {
return nil, err
}
for _, file := range files {
appName := strings.TrimSuffix(file.Name(), ".env")
appFilePath := path.Join(ABRA_SERVER_FOLDER, server, file.Name())
appFiles[appName] = AppFile{
Path: appFilePath,
Server: server,
}
}
}
return appFiles, nil
}
// GetApp loads an apps settings, reading it from file, in preparation to use it
//
// ONLY use when ready to use the env file to keep IO down
func GetApp(apps AppFiles, name AppName) (App, error) {
appFile, exists := apps[name]
if !exists {
return App{}, fmt.Errorf("cannot find app file with name '%s'", name)
}
app, err := readAppEnvFile(appFile, name)
if err != nil {
return App{}, err
}
return app, nil
}
// GetApps returns a slice of Apps with their env files read from a given slice of AppFiles
func GetApps(appFiles AppFiles) ([]App, error) {
var apps []App
for name := range appFiles {
app, err := GetApp(appFiles, name)
if err != nil {
return nil, err
}
apps = append(apps, app)
}
return apps, nil
}
// CopyAppEnvSample copies the example env file for the app into the users env files
func CopyAppEnvSample(appType, appName, server string) error {
envSamplePath := path.Join(ABRA_DIR, "apps", appType, ".env.sample")
envSample, err := ioutil.ReadFile(envSamplePath)
if err != nil {
return err
}
appEnvPath := path.Join(ABRA_DIR, "servers", server, fmt.Sprintf("%s.env", appName))
if _, err := os.Stat(appEnvPath); err == nil {
return fmt.Errorf("%s already exists?", appEnvPath)
}
err = ioutil.WriteFile(appEnvPath, envSample, 0755)
if err != nil {
return err
}
return nil
}
// SanitiseAppName makes a app name usable with Docker by replacing illegal characters
func SanitiseAppName(name string) string {
return strings.ReplaceAll(name, ".", "_")
}
// GetAppStatuses queries servers to check the deployment status of given apps
func GetAppStatuses(appFiles AppFiles) (map[string]string, error) {
servers := appFiles.GetServers()
ch := make(chan stack.StackStatus, len(servers))
for _, server := range servers {
go func(s string) { ch <- stack.GetAllDeployedServices(s) }(server)
}
statuses := map[string]string{}
for range servers {
status := <-ch
for _, service := range status.Services {
name := service.Spec.Labels[convert.LabelNamespace]
if _, ok := statuses[name]; !ok {
statuses[name] = "deployed"
}
}
}
return statuses, nil
}
// GetAppComposeFiles gets the list of compose files for an app which should be
// merged into a composetypes.Config while respecting the COMPOSE_FILE env var.
func GetAppComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
if _, ok := appEnv["COMPOSE_FILE"]; !ok {
pattern := fmt.Sprintf("%s/%s/compose**yml", APPS_DIR, recipe)
composeFiles, err := filepath.Glob(pattern)
if err != nil {
return composeFiles, err
}
return composeFiles, nil
}
var composeFiles []string
composeFileEnvVar := appEnv["COMPOSE_FILE"]
for _, file := range strings.Split(composeFileEnvVar, ":") {
path := fmt.Sprintf("%s/%s/%s", APPS_DIR, recipe, file)
composeFiles = append(composeFiles, path)
}
return composeFiles, nil
}
// GetAppComposeConfig retrieves a compose specification for a recipe. This
// specification is the result of a merge of all the compose.**.yml files in
// the recipe repository.
func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*composetypes.Config, error) {
compose, err := loader.LoadComposefile(opts, appEnv)
if err != nil {
return &composetypes.Config{}, err
}
return compose, nil
}
func UpdateAppComposeTag(recipe, image, tag string, appEnv AppEnv) error {
pattern := fmt.Sprintf("%s/%s/compose**yml", APPS_DIR, recipe)
composeFiles, err := filepath.Glob(pattern)
if err != nil {
return err
}
for _, composeFile := range composeFiles {
opts := stack.Deploy{Composefiles: []string{composeFile}}
compose, err := loader.LoadComposefile(opts, appEnv)
if err != nil {
return err
}
for _, service := range compose.Services {
if service.Image == "" {
continue // may be a compose.$optional.yml file
}
img, _ := reference.ParseNormalizedNamed(service.Image)
if err != nil {
logrus.Fatal(err)
}
composeImage := reference.Path(img)
if strings.Contains(composeImage, "library") {
// ParseNormalizedNamed prepends 'library' to images like nginx:<tag>,
// postgres:<tag>, i.e. images which do not have a username in the
// first position of the string
composeImage = strings.Split(composeImage, "/")[1]
}
composeTag := img.(reference.NamedTagged).Tag()
if image == composeImage {
bytes, err := ioutil.ReadFile(composeFile)
if err != nil {
logrus.Fatal(err)
}
old := fmt.Sprintf("%s:%s", composeImage, composeTag)
new := fmt.Sprintf("%s:%s", composeImage, tag)
replacedBytes := strings.Replace(string(bytes), old, new, -1)
if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
return err
}
}
}
}
return nil
}
func UpdateAppComposeLabel(recipe, serviceName, newLabel string, appEnv AppEnv) error {
pattern := fmt.Sprintf("%s/%s/compose**yml", APPS_DIR, recipe)
composeFiles, err := filepath.Glob(pattern)
if err != nil {
return err
}
for _, composeFile := range composeFiles {
opts := stack.Deploy{Composefiles: []string{composeFile}}
compose, err := loader.LoadComposefile(opts, appEnv)
if err != nil {
return err
}
serviceExists := false
var service composetypes.ServiceConfig
for _, s := range compose.Services {
if s.Name == serviceName {
service = s
serviceExists = true
}
}
if !serviceExists {
continue
}
for oldLabel, value := range service.Deploy.Labels {
if strings.HasPrefix(oldLabel, "coop-cloud") {
bytes, err := ioutil.ReadFile(composeFile)
if err != nil {
return err
}
old := fmt.Sprintf("coop-cloud.${STACK_NAME}.%s.version=%s", service.Name, value)
replacedBytes := strings.Replace(string(bytes), old, newLabel, -1)
if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
return err
}
}
}
}
return nil
}

37
pkg/config/app_test.go Normal file
View File

@ -0,0 +1,37 @@
package config
import (
"reflect"
"testing"
)
func TestNewApp(t *testing.T) {
app, err := newApp(expectedAppEnv, appName, expectedAppFile)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(app, expectedApp) {
t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
}
}
func TestReadAppEnvFile(t *testing.T) {
app, err := readAppEnvFile(expectedAppFile, appName)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(app, expectedApp) {
t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
}
}
func TestGetApp(t *testing.T) {
// TODO: Test failures as well as successes
app, err := GetApp(expectedAppFiles, appName)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(app, expectedApp) {
t.Fatalf("did not get expected app type. Expected: %s; Got: %s", app, expectedApp)
}
}

141
pkg/config/env.go Normal file
View File

@ -0,0 +1,141 @@
package config
import (
"bufio"
"fmt"
"io/fs"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"github.com/Autonomic-Cooperative/godotenv"
"github.com/sirupsen/logrus"
)
var ABRA_DIR = os.ExpandEnv("$HOME/.abra")
var ABRA_SERVER_FOLDER = path.Join(ABRA_DIR, "servers")
var APPS_JSON = path.Join(ABRA_DIR, "apps.json")
var APPS_DIR = path.Join(ABRA_DIR, "apps")
var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
func (a AppFiles) GetServers() []string {
var unique []string
servers := make(map[string]struct{})
for _, appFile := range a {
if _, ok := servers[appFile.Server]; !ok {
servers[appFile.Server] = struct{}{}
unique = append(unique, appFile.Server)
}
}
return unique
}
func ReadEnv(filePath string) (AppEnv, error) {
var envFile AppEnv
envFile, err := godotenv.Read(filePath)
if err != nil {
return nil, err
}
return envFile, nil
}
func ReadServerNames() ([]string, error) {
serverNames, err := getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
if err != nil {
return nil, err
}
return serverNames, nil
}
// getAllFilesInDirectory returns filenames of all files in directory
func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
var realFiles []fs.FileInfo
files, err := ioutil.ReadDir(directory)
if err != nil {
return nil, err
}
for _, file := range files {
// Follow any symlinks
filePath := path.Join(directory, file.Name())
realPath, err := filepath.EvalSymlinks(filePath)
if err != nil {
logrus.Warningf("broken symlink in your abra config folders: '%s'", filePath)
} else {
realFile, err := os.Stat(realPath)
if err != nil {
return nil, err
}
if !realFile.IsDir() {
realFiles = append(realFiles, file)
}
}
}
return realFiles, nil
}
// getAllFoldersInDirectory returns both folder and symlink paths
func getAllFoldersInDirectory(directory string) ([]string, error) {
var folders []string
files, err := ioutil.ReadDir(directory)
if err != nil {
return nil, err
}
if len(files) == 0 {
return nil, fmt.Errorf("directory is empty: '%s'", directory)
}
for _, file := range files {
// Check if file is directory or symlink
if file.IsDir() || file.Mode()&fs.ModeSymlink != 0 {
filePath := path.Join(directory, file.Name())
realDir, err := filepath.EvalSymlinks(filePath)
if err != nil {
logrus.Warningf("broken symlink in your abra config folders: '%s'", filePath)
} else if stat, err := os.Stat(realDir); err == nil && stat.IsDir() {
// path is a directory
folders = append(folders, file.Name())
}
}
}
return folders, nil
}
// EnsureAbraDirExists checks for the abra config folder and throws error if not
func EnsureAbraDirExists() error {
if _, err := os.Stat(ABRA_DIR); os.IsNotExist(err) {
if err := os.Mkdir(ABRA_DIR, 0777); err != nil {
return err
}
}
return nil
}
func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
envVars := make(map[string]string)
file, err := os.Open(abraSh)
if err != nil {
if os.IsNotExist(err) {
return envVars, fmt.Errorf("'%s' does not exist?", abraSh)
}
return envVars, err
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "export") {
splitVals := strings.Split(line, "export ")
envVarDef := splitVals[len(splitVals)-1]
keyVal := strings.Split(envVarDef, "=")
if len(keyVal) != 2 {
return envVars, fmt.Errorf("couldn't parse %s", line)
}
envVars[keyVal[0]] = keyVal[1]
}
}
return envVars, nil
}

83
pkg/config/env_test.go Normal file
View File

@ -0,0 +1,83 @@
package config
import (
"os"
"path"
"reflect"
"strings"
"testing"
)
var testFolder = os.ExpandEnv("$PWD/../tests/resources/test_folder")
var validAbraConf = os.ExpandEnv("$PWD/../tests/resources/valid_abra_config")
// make sure these are in alphabetical order
var tFolders = []string{"folder1", "folder2"}
var tFiles = []string{"bar", "foo"}
var appName = "ecloud"
var serverName = "evil.corp"
var expectedAppEnv = AppEnv{
"DOMAIN": "ecloud.evil.corp",
"TYPE": "ecloud",
}
var expectedApp = App{
Name: appName,
Type: expectedAppEnv["TYPE"],
Domain: expectedAppEnv["DOMAIN"],
Env: expectedAppEnv,
File: expectedAppFile,
}
var expectedAppFile = AppFile{
Path: path.Join(validAbraConf, "servers", serverName, appName+".env"),
Server: serverName,
}
var expectedAppFiles = map[string]AppFile{
appName: expectedAppFile,
}
// var expectedServerNames = []string{"evil.corp"}
func TestGetAllFoldersInDirectory(t *testing.T) {
folders, err := getAllFoldersInDirectory(testFolder)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(folders, tFolders) {
t.Fatalf("did not get expected folders. Expected: (%s), Got: (%s)", strings.Join(tFolders, ","), strings.Join(folders, ","))
}
}
func TestGetAllFilesInDirectory(t *testing.T) {
files, err := getAllFilesInDirectory(testFolder)
if err != nil {
t.Fatal(err)
}
var fileNames []string
for _, file := range files {
fileNames = append(fileNames, file.Name())
}
if !reflect.DeepEqual(fileNames, tFiles) {
t.Fatalf("did not get expected files. Expected: (%s), Got: (%s)", strings.Join(tFiles, ","), strings.Join(fileNames, ","))
}
}
func TestReadEnv(t *testing.T) {
env, err := ReadEnv(expectedAppFile.Path)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(env, expectedAppEnv) {
t.Fatalf(
"did not get expected application settings. Expected: DOMAIN=%s TYPE=%s; Got: DOMAIN=%s TYPE=%s",
expectedAppEnv["DOMAIN"],
expectedAppEnv["TYPE"],
env["DOMAIN"],
env["TYPE"],
)
}
}