codegod100/abra
0
0
Fork 0
forked from toolshed/abra
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: debug logging

Browse Source 
Closes coop-cloud/organising#164.
This commit is contained in:
decentral1se
2021-09-11 00:54:02 +02:00
parent 27d665c3be
commit 9fcdc45851
38 changed files with 305 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
pkg/app/app.go
View File

@ -8,6 +8,7 @@ import (
"coopcloud.tech/abra/pkg/client/stack"
"coopcloud.tech/abra/pkg/config"
apiclient "github.com/docker/docker/client"
"github.com/sirupsen/logrus"
)
// Get retrieves an app
@ -22,6 +23,8 @@ func Get(appName string) (config.App, error) {
return config.App{}, err
}
logrus.Debugf("retrieved '%s' for '%s'", app, appName)
return app, nil
}
@ -51,6 +54,14 @@ func DeployedVersions(ctx context.Context, cl *apiclient.Client, app config.App)
}
}
deployed := len(services) > 0
if deployed {
logrus.Debugf("detected '%s' as deployed versions of '%s'", appSpec, app.Name)
} else {
logrus.Debugf("detected '%s' as not deployed", app.Name)
}
return appSpec, len(services) > 0, nil
}
@ -58,11 +69,17 @@ func DeployedVersions(ctx context.Context, cl *apiclient.Client, app config.App)
func ParseVersionLabel(label string) (string, string) {
// versions may look like v4.2-abcd or v4.2-alpine-abcd
idx := strings.LastIndex(label, "-")
return label[:idx], label[idx+1:]
version := label[:idx]
digest := label[idx+1:]
logrus.Debugf("parsed '%s' as version from '%s'", version, label)
logrus.Debugf("parsed '%s' as digest from '%s'", digest, label)
return version, digest
}
// ParseVersionName parses a $STACK_NAME_$SERVICE_NAME service label.
func ParseServiceName(label string) string {
idx := strings.LastIndex(label, "_")
return label[idx+1:]
serviceName := label[idx+1:]
logrus.Debugf("parsed '%s' as service name from '%s'", serviceName, label)
return serviceName
}

23
pkg/catalogue/catalogue.go
View File

@ -15,6 +15,7 @@ import (
"coopcloud.tech/abra/pkg/config"
"coopcloud.tech/abra/pkg/recipe"
"coopcloud.tech/abra/pkg/web"
"github.com/sirupsen/logrus"
)
// RecipeCatalogueURL is the only current recipe catalogue available.
@ -76,6 +77,8 @@ func (r RecipeMeta) LatestVersion() string {
version = tag
}
logrus.Debugf("choosing '%s' as latest version of '%s'", version, r.Name)
return version
}
@ -88,9 +91,11 @@ type RecipeCatalogue map[Name]RecipeMeta
// Flatten converts AppCatalogue to slice
func (r RecipeCatalogue) Flatten() []RecipeMeta {
recipes := make([]RecipeMeta, 0, len(r))
for name := range r {
recipes = append(recipes, r[name])
}
return recipes
}
@ -117,9 +122,11 @@ func recipeCatalogueFSIsLatest() (bool, error) {
if err != nil {
return false, err
}
info, err := os.Stat(config.APPS_JSON)
if err != nil {
if os.IsNotExist(err) {
logrus.Debugf("no recipe catalogue found in file system cache")
return false, nil
}
return false, err
@ -129,9 +136,12 @@ func recipeCatalogueFSIsLatest() (bool, error) {
remoteModifiedTime := parsed.Unix()
if localModifiedTime < remoteModifiedTime {
logrus.Debug("file system cached recipe catalogue is out-of-date")
return false, nil
}
logrus.Debug("file system cached recipe catalogue is up-to-date")
return true, nil
}
@ -145,12 +155,14 @@ func ReadRecipeCatalogue() (RecipeCatalogue, error) {
}
if !recipeFSIsLatest {
logrus.Debugf("reading recipe catalogue from web to get latest")
if err := readRecipeCatalogueWeb(&recipes); err != nil {
return nil, err
}
return recipes, nil
}
logrus.Debugf("reading recipe catalogue from file system cache to get latest")
if err := readRecipeCatalogueFS(&recipes); err != nil {
return nil, err
}
@ -164,9 +176,13 @@ func readRecipeCatalogueFS(target interface{}) error {
if err != nil {
return err
}
if err := json.Unmarshal(recipesJSONFS, &target); err != nil {
return err
}
logrus.Debugf("read recipe catalogue from file system cache in '%s'", config.APPS_JSON)
return nil
}
@ -185,6 +201,8 @@ func readRecipeCatalogueWeb(target interface{}) error {
return err
}
logrus.Debugf("read recipe catalogue from web at '%s'", RecipeCatalogueURL)
return nil
}
@ -211,6 +229,8 @@ func VersionsOfService(recipe, serviceName string) ([]string, error) {
}
}
logrus.Debugf("detected versions '%s' for '%s'", strings.Join(versions, ", "), recipe)
return versions, nil
}
@ -226,9 +246,12 @@ func GetRecipeMeta(recipeName string) (RecipeMeta, error) {
err := fmt.Errorf("recipe '%s' does not exist?", recipeName)
return RecipeMeta{}, err
}
if err := recipe.EnsureExists(recipeName); err != nil {
return RecipeMeta{}, err
}
logrus.Debugf("recipe metadata retrieved for '%s'", recipeName)
return recipeMeta, nil
}

2
pkg/client/client.go
View File

@ -48,5 +48,7 @@ func New(contextName string) (*client.Client, error) {
logrus.Fatalf("unable to create Docker client: %s", err)
}
logrus.Debugf("created client for '%s'", contextName)
return cl, nil
}

9
pkg/client/context.go
View File

@ -10,6 +10,7 @@ import (
"github.com/docker/cli/cli/context/docker"
contextStore "github.com/docker/cli/cli/context/store"
"github.com/moby/term"
"github.com/sirupsen/logrus"
)
type Context = contextStore.Metadata
@ -26,6 +27,7 @@ func CreateContext(contextName string, user string, port string) error {
if err := createContext(contextName, host); err != nil {
return err
}
logrus.Debugf("created the '%s' context", contextName)
return nil
}
@ -36,13 +38,16 @@ func createContext(name string, host string) error {
Endpoints: make(map[string]interface{}),
Name: name,
}
contextTLSData := contextStore.ContextTLSData{
Endpoints: make(map[string]contextStore.EndpointTLSData),
}
dockerEP, dockerTLS, err := getDockerEndpointMetadataAndTLS(host)
if err != nil {
return err
}
contextMetadata.Endpoints[docker.DockerEndpoint] = dockerEP
if dockerTLS != nil {
contextTLSData.Endpoints[docker.DockerEndpoint] = *dockerTLS
@ -51,9 +56,11 @@ func createContext(name string, host string) error {
if err := s.CreateOrUpdate(contextMetadata); err != nil {
return err
}
if err := s.ResetTLSMaterial(name, &contextTLSData); err != nil {
return err
}
return nil
}
@ -61,6 +68,7 @@ func DeleteContext(name string) error {
if name == "default" {
return errors.New("context 'default' cannot be removed")
}
if _, err := GetContext(name); err != nil {
return err
}
@ -81,6 +89,7 @@ func GetContext(contextName string) (contextStore.Metadata, error) {
if err != nil {
return contextStore.Metadata{}, err
}
return ctx, nil
}

15
pkg/compose/compose.go
View File

@ -20,6 +20,8 @@ func UpdateTag(pattern, image, tag string) error {
return err
}
logrus.Debugf("considering '%s' config(s) for tag update", strings.Join(composeFiles, ", "))
for _, composeFile := range composeFiles {
opts := stack.Deploy{Composefiles: []string{composeFile}}
emptyEnv := make(map[string]string)
@ -35,7 +37,7 @@ func UpdateTag(pattern, image, tag string) error {
img, _ := reference.ParseNormalizedNamed(service.Image)
if err != nil {
logrus.Fatal(err)
return err
}
composeImage := reference.Path(img)
@ -47,16 +49,20 @@ func UpdateTag(pattern, image, tag string) error {
}
composeTag := img.(reference.NamedTagged).Tag()
logrus.Debugf("parsed '%s' from '%s'", composeTag, service.Image)
if image == composeImage {
bytes, err := ioutil.ReadFile(composeFile)
if err != nil {
logrus.Fatal(err)
return err
}
old := fmt.Sprintf("%s:%s", composeImage, composeTag)
new := fmt.Sprintf("%s:%s", composeImage, tag)
replacedBytes := strings.Replace(string(bytes), old, new, -1)
logrus.Debugf("updating '%s' to '%s' in '%s'", old, new, compose.Filename)
if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
return err
}
@ -74,6 +80,8 @@ func UpdateLabel(pattern, serviceName, label string) error {
return err
}
logrus.Debugf("considering '%s' config(s) for label update", strings.Join(composeFiles, ", "))
for _, composeFile := range composeFiles {
opts := stack.Deploy{Composefiles: []string{composeFile}}
emptyEnv := make(map[string]string)
@ -104,6 +112,9 @@ func UpdateLabel(pattern, serviceName, label string) error {
old := fmt.Sprintf("coop-cloud.${STACK_NAME}.%s.version=%s", service.Name, value)
replacedBytes := strings.Replace(string(bytes), old, label, -1)
logrus.Debugf("updating '%s' to '%s' in '%s'", old, label, compose.Filename)
if err := ioutil.WriteFile(compose.Filename, []byte(replacedBytes), 0644); err != nil {
return err
}

25
pkg/config/app.go
View File

@ -13,6 +13,7 @@ import (
loader "coopcloud.tech/abra/pkg/client/stack"
stack "coopcloud.tech/abra/pkg/client/stack"
composetypes "github.com/docker/cli/cli/compose/types"
"github.com/sirupsen/logrus"
)
// Type aliases to make code hints easier to understand
@ -93,10 +94,14 @@ func readAppEnvFile(appFile AppFile, name AppName) (App, error) {
if err != nil {
return App{}, fmt.Errorf("env file for '%s' couldn't be read: %s", name, err.Error())
}
logrus.Debugf("read env '%s' from '%s'", env, appFile.Path)
app, err := newApp(env, name, appFile)
if err != nil {
return App{}, fmt.Errorf("env file for '%s' has issues: %s", name, err.Error())
}
return app, nil
}
@ -108,6 +113,7 @@ func newApp(env AppEnv, name string, appFile AppFile) (App, error) {
if !ok {
return App{}, errors.New("missing TYPE variable")
}
return App{
Name: name,
Domain: domain,
@ -131,6 +137,9 @@ func LoadAppFiles(servers ...string) (AppFiles, error) {
}
}
}
logrus.Debugf("collecting metadata from '%v' servers: '%s'", len(servers), servers)
for _, server := range servers {
serverDir := path.Join(ABRA_SERVER_FOLDER, server)
files, err := getAllFilesInDirectory(serverDir)
@ -157,16 +166,19 @@ func GetApp(apps AppFiles, name AppName) (App, error) {
if !exists {
return App{}, fmt.Errorf("cannot find app with name '%s'", name)
}
app, err := readAppEnvFile(appFile, name)
if err != nil {
return App{}, err
}
return app, nil
}
// GetApps returns a slice of Apps with their env files read from a given slice of AppFiles
func GetApps(appFiles AppFiles) ([]App, error) {
var apps []App
for name := range appFiles {
app, err := GetApp(appFiles, name)
if err != nil {
@ -174,14 +186,17 @@ func GetApps(appFiles AppFiles) ([]App, error) {
}
apps = append(apps, app)
}
return apps, nil
}
// GetAppNames retrieves a list of app names.
func GetAppNames() ([]string, error) {
appFiles, err := LoadAppFiles("")
if err != nil {
return []string{}, err
}
apps, err := GetApps(appFiles)
if err != nil {
return []string{}, err
@ -213,6 +228,8 @@ func CopyAppEnvSample(appType, appName, server string) error {
return err
}
logrus.Debugf("copied '%s' to '%s'", envSamplePath, appEnvPath)
return nil
}
@ -240,6 +257,8 @@ func GetAppStatuses(appFiles AppFiles) (map[string]string, error) {
}
}
logrus.Debugf("retrieved app statuses: '%s'", statuses)
return statuses, nil
}
@ -261,6 +280,9 @@ func GetAppComposeFiles(recipe string, appEnv AppEnv) ([]string, error) {
path := fmt.Sprintf("%s/%s/%s", APPS_DIR, recipe, file)
composeFiles = append(composeFiles, path)
}
logrus.Debugf("retrieved '%s' configs for '%s'", strings.Join(composeFiles, ", "), recipe)
return composeFiles, nil
}
@ -272,5 +294,8 @@ func GetAppComposeConfig(recipe string, opts stack.Deploy, appEnv AppEnv) (*comp
if err != nil {
return &composetypes.Config{}, err
}
logrus.Debugf("retrieved '%s' for '%s'", compose, recipe)
return compose, nil
}

26
pkg/config/env.go
View File

@ -20,8 +20,10 @@ var APPS_JSON = path.Join(ABRA_DIR, "apps.json")
var APPS_DIR = path.Join(ABRA_DIR, "apps")
var REPOS_BASE_URL = "https://git.coopcloud.tech/coop-cloud"
// GetServers retrieves all servers.
func (a AppFiles) GetServers() []string {
var unique []string
servers := make(map[string]struct{})
for _, appFile := range a {
if _, ok := servers[appFile.Server]; !ok {
@ -29,33 +31,48 @@ func (a AppFiles) GetServers() []string {
unique = append(unique, appFile.Server)
}
}
logrus.Debugf("retrieved servers: '%s'", unique)
return unique
}
// ReadEnv loads an app envivornment into a map.
func ReadEnv(filePath string) (AppEnv, error) {
var envFile AppEnv
envFile, err := godotenv.Read(filePath)
if err != nil {
return nil, err
}
logrus.Debugf("read '%s' from '%s'", envFile, filePath)
return envFile, nil
}
// ReadServerNames retrieves all server names.
func ReadServerNames() ([]string, error) {
serverNames, err := getAllFoldersInDirectory(ABRA_SERVER_FOLDER)
if err != nil {
return nil, err
}
logrus.Debugf("read '%s' from '%s'", strings.Join(serverNames, ","), ABRA_SERVER_FOLDER)
return serverNames, nil
}
// getAllFilesInDirectory returns filenames of all files in directory
func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
var realFiles []fs.FileInfo
files, err := ioutil.ReadDir(directory)
if err != nil {
return nil, err
}
for _, file := range files {
// Follow any symlinks
filePath := path.Join(directory, file.Name())
@ -71,14 +88,15 @@ func getAllFilesInDirectory(directory string) ([]fs.FileInfo, error) {
realFiles = append(realFiles, file)
}
}
}
return realFiles, nil
}
// getAllFoldersInDirectory returns both folder and symlink paths
func getAllFoldersInDirectory(directory string) ([]string, error) {
var folders []string
files, err := ioutil.ReadDir(directory)
if err != nil {
return nil, err
@ -86,6 +104,7 @@ func getAllFoldersInDirectory(directory string) ([]string, error) {
if len(files) == 0 {
return nil, fmt.Errorf("directory is empty: '%s'", directory)
}
for _, file := range files {
// Check if file is directory or symlink
if file.IsDir() || file.Mode()&fs.ModeSymlink != 0 {
@ -99,12 +118,14 @@ func getAllFoldersInDirectory(directory string) ([]string, error) {
}
}
}
return folders, nil
}
// EnsureAbraDirExists checks for the abra config folder and throws error if not
func EnsureAbraDirExists() error {
if _, err := os.Stat(ABRA_DIR); os.IsNotExist(err) {
logrus.Debugf("'%s' does not exist, creating it", ABRA_DIR)
if err := os.Mkdir(ABRA_DIR, 0777); err != nil {
return err
}
@ -112,6 +133,7 @@ func EnsureAbraDirExists() error {
return nil
}
// ReadAbraShEnvVars reads env vars from an abra.sh recipe file.
func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
envVars := make(map[string]string)
@ -137,5 +159,7 @@ func ReadAbraShEnvVars(abraSh string) (map[string]string, error) {
}
}
logrus.Debugf("read '%s' from '%s'", envVars, abraSh)
return envVars, nil
}

33
pkg/git/clone.go Normal file
View File

@ -0,0 +1,33 @@
package git
import (
"os"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/sirupsen/logrus"
)
// Clone runs a git clone which accounts for different default branches.
func Clone(dir, url string) error {
if _, err := os.Stat(dir); os.IsNotExist(err) {
logrus.Debugf("'%s' does not exist, attempting to git clone from '%s'", dir, url)
_, err := git.PlainClone(dir, false, &git.CloneOptions{URL: url, Tags: git.AllTags})
if err != nil {
logrus.Debugf("cloning from default branch failed, attempting from main branch")
// try with main branch because Git is being a Git
_, err := git.PlainClone(dir, false, &git.CloneOptions{
URL: url,
Tags: git.AllTags,
ReferenceName: plumbing.ReferenceName("refs/heads/main"),
})
if err != nil {
return err
}
}
}
logrus.Debugf("'%s' has been git cloned successfully", dir)
return nil
}

26
pkg/recipe/recipe.go
View File

@ -2,7 +2,6 @@ package recipe
import (
"fmt"
"os"
"path"
"path/filepath"
"strings"
@ -11,9 +10,11 @@ import (
loader "coopcloud.tech/abra/pkg/client/stack"
"coopcloud.tech/abra/pkg/compose"
"coopcloud.tech/abra/pkg/config"
gitPkg "coopcloud.tech/abra/pkg/git"
composetypes "github.com/docker/cli/cli/compose/types"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/sirupsen/logrus"
)
// Recipe represents a recipe.
@ -65,23 +66,10 @@ func Get(recipeName string) (Recipe, error) {
// EnsureExists checks whether a recipe has been cloned locally or not.
func EnsureExists(recipe string) error {
recipeDir := path.Join(config.ABRA_DIR, "apps", strings.ToLower(recipe))
if _, err := os.Stat(recipeDir); os.IsNotExist(err) {
url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipe)
_, err := git.PlainClone(recipeDir, false, &git.CloneOptions{URL: url, Tags: git.AllTags})
if err != nil {
// try with main branch because Git is being a Git
_, err := git.PlainClone(recipeDir, false, &git.CloneOptions{
URL: url,
Tags: git.AllTags,
ReferenceName: plumbing.ReferenceName("refs/heads/main"),
})
if err != nil {
return err
}
}
url := fmt.Sprintf("%s/%s.git", config.REPOS_BASE_URL, recipe)
if err := gitPkg.Clone(recipeDir, url); err != nil {
return err
}
return nil
}
@ -99,6 +87,8 @@ func EnsureVersion(recipeName, version string) error {
return nil
}
logrus.Debugf("read '%s' as tags for recipe '%s'", tags, recipeName)
var tagRef plumbing.ReferenceName
if err := tags.ForEach(func(ref *plumbing.Reference) (err error) {
if ref.Name().Short() == version {
@ -123,5 +113,7 @@ func EnsureVersion(recipeName, version string) error {
return err
}
logrus.Debugf("successfully checked '%s' out to '%s' in '%s'", recipeName, tagRef, recipeDir)
return nil
}

4
pkg/secret/pass.go
View File

@ -19,6 +19,8 @@ func PassInsertSecret(secretValue, secretName, appName, server string) error {
secretValue, server, appName, secretName,
)
logrus.Debugf("attempting to run '%s'", cmd)
if err := exec.Command("bash", "-c", cmd).Run(); err != nil {
return err
}
@ -39,6 +41,8 @@ func PassRmSecret(secretName, appName, server string) error {
server, appName, secretName,
)
logrus.Debugf("attempting to run '%s'", cmd)
if err := exec.Command("bash", "-c", cmd).Run(); err != nil {
return err
}

45
pkg/secret/secret.go
View File

@ -12,6 +12,7 @@ import (
"coopcloud.tech/abra/pkg/client"
"coopcloud.tech/abra/pkg/config"
"github.com/schultz-is/passgen"
"github.com/sirupsen/logrus"
)
// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
@ -33,6 +34,8 @@ func GeneratePasswords(count, length uint) ([]string, error) {
return nil, err
}
logrus.Debugf("generated '%s'", strings.Join(passwords, ", "))
return passwords, nil
}
@ -50,17 +53,24 @@ func GeneratePassphrases(count uint) ([]string, error) {
return nil, err
}
logrus.Debugf("generated '%s'", strings.Join(passphrases, ", "))
return passphrases, nil
}
// ReadSecretEnvVars reads secret env vars from an app env var config.
func ReadSecretEnvVars(appEnv config.AppEnv) map[string]string {
secretEnvVars := make(map[string]string)
for envVar := range appEnv {
regex := regexp.MustCompile(`^SECRET.*VERSION.*`)
if string(regex.Find([]byte(envVar))) != "" {
secretEnvVars[envVar] = appEnv[envVar]
}
}
logrus.Debugf("read '%s' as secrets from '%s'", secretEnvVars, appEnv)
return secretEnvVars
}
@ -68,7 +78,9 @@ func ReadSecretEnvVars(appEnv config.AppEnv) map[string]string {
func ParseSecretEnvVarName(secretEnvVar string) string {
withoutPrefix := strings.TrimPrefix(secretEnvVar, "SECRET_")
withoutSuffix := strings.TrimSuffix(withoutPrefix, "_VERSION")
return strings.ToLower(withoutSuffix)
name := strings.ToLower(withoutSuffix)
logrus.Debugf("parsed '%s' as name from '%s'", name, secretEnvVar)
return name
}
// TODO: should probably go in the config/app package?
@ -76,7 +88,9 @@ func ParseGeneratedSecretName(secret string, appEnv config.App) string {
name := fmt.Sprintf("%s_", appEnv.StackName())
withoutAppName := strings.TrimPrefix(secret, name)
idx := strings.LastIndex(withoutAppName, "_")
return withoutAppName[:idx]
parsed := withoutAppName[:idx]
logrus.Debugf("parsed '%s' as name from '%s'", parsed, secret)
return parsed
}
// TODO: should probably go in the config/app package?
@ -85,19 +99,23 @@ func ParseSecretEnvVarValue(secret string) (secretValue, error) {
if len(values) == 0 {
return secretValue{}, fmt.Errorf("unable to parse '%s'", secret)
}
if len(values) == 1 {
return secretValue{Version: values[0], Length: 0}, nil
} else {
split := strings.Split(values[1], "=")
parsed := split[len(split)-1]
stripped := strings.ReplaceAll(parsed, " ", "")
length, err := strconv.Atoi(stripped)
if err != nil {
return secretValue{}, err
}
version := strings.ReplaceAll(values[0], " ", "")
return secretValue{Version: version, Length: length}, nil
}
split := strings.Split(values[1], "=")
parsed := split[len(split)-1]
stripped := strings.ReplaceAll(parsed, " ", "")
length, err := strconv.Atoi(stripped)
if err != nil {
return secretValue{}, err
}
version := strings.ReplaceAll(values[0], " ", "")
logrus.Debugf("parsed version '%s' and length '%s' from '%s'", version, length, secret)
return secretValue{Version: version, Length: length}, nil
}
// GenerateSecrets generates secrets locally and sends them to a remote server for storage.
@ -114,6 +132,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
return
}
secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secretValue.Version)
logrus.Debugf("attempting to generate and store '%s' on '%s'", secretRemoteName, server)
if secretValue.Length > 0 {
passwords, err := GeneratePasswords(1, uint(secretValue.Length))
if err != nil {
@ -147,5 +166,7 @@ func GenerateSecrets(secretEnvVars map[string]string, appName, server string) (m
}
}
logrus.Debugf("generated and stored '%s' on '%s'", secrets, server)
return secrets, nil
}