proper env modifiers support
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone/push Build is passing
Details
This implements proper modifier support in the env file using this new fork of the godotenv library. The modifier implementation is quite basic for but can be improved later if needed. See this commit for the actual implementation. Because we are now using proper modifer parsing, it does not affect the parsing of value, so this is possible again: ``` MY_VAR="#foo" ``` Closes coop-cloud/organising#535
This commit is contained in:
parent
0d83339d80
commit
3957b7c965
|
@ -97,7 +97,7 @@ var appNewCommand = cli.Command{
|
||||||
var secrets AppSecrets
|
var secrets AppSecrets
|
||||||
var secretTable *jsontable.JSONTable
|
var secretTable *jsontable.JSONTable
|
||||||
if internal.Secrets {
|
if internal.Secrets {
|
||||||
sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
|
sampleEnv, err := recipe.SampleEnv()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -168,7 +168,7 @@ var appNewCommand = cli.Command{
|
||||||
type AppSecrets map[string]string
|
type AppSecrets map[string]string
|
||||||
|
|
||||||
// createSecrets creates all secrets for a new app.
|
// createSecrets creates all secrets for a new app.
|
||||||
func createSecrets(cl *dockerClient.Client, secretsConfig map[string]string, sanitisedAppName string) (AppSecrets, error) {
|
func createSecrets(cl *dockerClient.Client, secretsConfig map[string]secret.SecretValue, sanitisedAppName string) (AppSecrets, error) {
|
||||||
// NOTE(d1): trim to match app.StackName() implementation
|
// NOTE(d1): trim to match app.StackName() implementation
|
||||||
if len(sanitisedAppName) > 45 {
|
if len(sanitisedAppName) > 45 {
|
||||||
logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
|
logrus.Debugf("trimming %s to %s to avoid runtime limits", sanitisedAppName, sanitisedAppName[:45])
|
||||||
|
@ -217,7 +217,7 @@ func ensureDomainFlag(recipe recipe.Recipe, server string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// promptForSecrets asks if we should generate secrets for a new app.
|
// promptForSecrets asks if we should generate secrets for a new app.
|
||||||
func promptForSecrets(recipeName string, secretsConfig map[string]string) error {
|
func promptForSecrets(recipeName string, secretsConfig map[string]secret.SecretValue) error {
|
||||||
if len(secretsConfig) == 0 {
|
if len(secretsConfig) == 0 {
|
||||||
logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
|
logrus.Debugf("%s has no secrets to generate, skipping...", recipeName)
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -20,19 +20,23 @@ import (
|
||||||
"github.com/urfave/cli"
|
"github.com/urfave/cli"
|
||||||
)
|
)
|
||||||
|
|
||||||
var allSecrets bool
|
var (
|
||||||
var allSecretsFlag = &cli.BoolFlag{
|
allSecrets bool
|
||||||
Name: "all, a",
|
allSecretsFlag = &cli.BoolFlag{
|
||||||
Destination: &allSecrets,
|
Name: "all, a",
|
||||||
Usage: "Generate all secrets",
|
Destination: &allSecrets,
|
||||||
}
|
Usage: "Generate all secrets",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
var rmAllSecrets bool
|
var (
|
||||||
var rmAllSecretsFlag = &cli.BoolFlag{
|
rmAllSecrets bool
|
||||||
Name: "all, a",
|
rmAllSecretsFlag = &cli.BoolFlag{
|
||||||
Destination: &rmAllSecrets,
|
Name: "all, a",
|
||||||
Usage: "Remove all secrets",
|
Destination: &rmAllSecrets,
|
||||||
}
|
Usage: "Remove all secrets",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
var appSecretGenerateCommand = cli.Command{
|
var appSecretGenerateCommand = cli.Command{
|
||||||
Name: "generate",
|
Name: "generate",
|
||||||
|
@ -87,28 +91,22 @@ var appSecretGenerateCommand = cli.Command{
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
|
secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
secretsToCreate := make(map[string]string)
|
if !allSecrets {
|
||||||
if allSecrets {
|
|
||||||
secretsToCreate = secretsConfig
|
|
||||||
} else {
|
|
||||||
secretName := c.Args().Get(1)
|
secretName := c.Args().Get(1)
|
||||||
secretVersion := c.Args().Get(2)
|
secretVersion := c.Args().Get(2)
|
||||||
matches := false
|
s, ok := secrets[secretName]
|
||||||
for name := range secretsConfig {
|
if !ok {
|
||||||
if secretName == name {
|
|
||||||
secretsToCreate[name] = secretVersion
|
|
||||||
matches = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !matches {
|
|
||||||
logrus.Fatalf("%s doesn't exist in the env config?", secretName)
|
logrus.Fatalf("%s doesn't exist in the env config?", secretName)
|
||||||
}
|
}
|
||||||
|
s.Version = secretVersion
|
||||||
|
secrets = map[string]secret.SecretValue{
|
||||||
|
secretName: s,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
cl, err := client.New(app.Server)
|
cl, err := client.New(app.Server)
|
||||||
|
@ -116,7 +114,7 @@ var appSecretGenerateCommand = cli.Command{
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
secretVals, err := secret.GenerateSecrets(cl, secretsToCreate, app.StackName(), app.Server)
|
secretVals, err := secret.GenerateSecrets(cl, secrets, app.StackName(), app.Server)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -276,7 +274,7 @@ Example:
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
secretsConfig, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
|
secrets, err := secret.ReadSecretsConfig(app.Path, composeFiles, app.Recipe)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.Fatal(err)
|
logrus.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -311,12 +309,7 @@ Example:
|
||||||
|
|
||||||
match := false
|
match := false
|
||||||
secretToRm := c.Args().Get(1)
|
secretToRm := c.Args().Get(1)
|
||||||
for secretName, secretValue := range secretsConfig {
|
for secretName, val := range secrets {
|
||||||
val, err := secret.ParseSecretValue(secretValue)
|
|
||||||
if err != nil {
|
|
||||||
logrus.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
|
secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
|
||||||
if _, ok := remoteSecretNames[secretRemoteName]; ok {
|
if _, ok := remoteSecretNames[secretRemoteName]; ok {
|
||||||
if secretToRm != "" {
|
if secretToRm != "" {
|
||||||
|
|
2
go.mod
2
go.mod
|
@ -4,8 +4,8 @@ go 1.21
|
||||||
|
|
||||||
require (
|
require (
|
||||||
coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
|
coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52
|
||||||
|
git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd
|
||||||
github.com/AlecAivazis/survey/v2 v2.3.7
|
github.com/AlecAivazis/survey/v2 v2.3.7
|
||||||
github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7
|
|
||||||
github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
|
github.com/Gurpartap/logrus-stack v0.0.0-20170710170904-89c00d8a28f4
|
||||||
github.com/docker/cli v24.0.7+incompatible
|
github.com/docker/cli v24.0.7+incompatible
|
||||||
github.com/docker/distribution v2.8.3+incompatible
|
github.com/docker/distribution v2.8.3+incompatible
|
||||||
|
|
4
go.sum
4
go.sum
|
@ -51,12 +51,12 @@ coopcloud.tech/tagcmp v0.0.0-20211103052201-885b22f77d52/go.mod h1:ESVm0wQKcbcFi
|
||||||
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
|
||||||
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
|
||||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||||
|
git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd h1:dctCkMhcsgIWMrkB1Br8S0RJF17eG+LKiqcXXVr3mdU=
|
||||||
|
git.coopcloud.tech/coop-cloud/godotenv v1.5.2-0.20231130100106-7462d91acefd/go.mod h1:Q8V1zbtPAlzYSr/Dvky3wS6x58IQAl3rot2me1oSO2Q=
|
||||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
|
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic=
|
||||||
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
|
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0=
|
||||||
github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
|
github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
|
||||||
github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
|
github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
|
||||||
github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7 h1:asQtdXYbxEYWcwAQqJTVYC/RltB4eqoWKvqWg/LFPOg=
|
|
||||||
github.com/Autonomic-Cooperative/godotenv v1.3.1-0.20210731094149-b031ea1211e7/go.mod h1:oZRCMMRS318l07ei4DTqbZoOawfJlJ4yyo8juk2v4Rk=
|
|
||||||
github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
|
github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
|
||||||
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
|
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
|
||||||
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
|
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
|
||||||
|
|
|
@ -29,7 +29,7 @@ func UpdateTag(pattern, image, tag, recipeName string) (bool, error) {
|
||||||
opts := stack.Deploy{Composefiles: []string{composeFile}}
|
opts := stack.Deploy{Composefiles: []string{composeFile}}
|
||||||
|
|
||||||
envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
|
envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
|
||||||
sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
|
sampleEnv, err := config.ReadEnv(envSamplePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
@ -97,7 +97,7 @@ func UpdateLabel(pattern, serviceName, label, recipeName string) error {
|
||||||
opts := stack.Deploy{Composefiles: []string{composeFile}}
|
opts := stack.Deploy{Composefiles: []string{composeFile}}
|
||||||
|
|
||||||
envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
|
envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
|
||||||
sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
|
sampleEnv, err := config.ReadEnv(envSamplePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,6 +25,9 @@ import (
|
||||||
// AppEnv is a map of the values in an apps env config
|
// AppEnv is a map of the values in an apps env config
|
||||||
type AppEnv = map[string]string
|
type AppEnv = map[string]string
|
||||||
|
|
||||||
|
// AppModifiers is a map of modifiers in an apps env config
|
||||||
|
type AppModifiers = map[string]map[string]string
|
||||||
|
|
||||||
// AppName is AppName
|
// AppName is AppName
|
||||||
type AppName = string
|
type AppName = string
|
||||||
|
|
||||||
|
@ -150,7 +153,7 @@ func (a ByName) Less(i, j int) bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
|
func ReadAppEnvFile(appFile AppFile, name AppName) (App, error) {
|
||||||
env, err := ReadEnv(appFile.Path, ReadEnvOptions{})
|
env, err := ReadEnv(appFile.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
|
return App{}, fmt.Errorf("env file for %s couldn't be read: %s", name, err.Error())
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,7 +12,7 @@ import (
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/Autonomic-Cooperative/godotenv"
|
"git.coopcloud.tech/coop-cloud/godotenv"
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -55,45 +55,34 @@ func GetServers() ([]string, error) {
|
||||||
return servers, nil
|
return servers, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadEnvOptions modifies the ReadEnv processing of env vars.
|
|
||||||
type ReadEnvOptions struct {
|
|
||||||
IncludeModifiers bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// ContainsEnvVarModifier determines if an env var contains a modifier.
|
|
||||||
func ContainsEnvVarModifier(envVar string) bool {
|
|
||||||
for _, mod := range envVarModifiers {
|
|
||||||
if strings.Contains(envVar, fmt.Sprintf("%s=", mod)) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadEnv loads an app envivornment into a map.
|
// ReadEnv loads an app envivornment into a map.
|
||||||
func ReadEnv(filePath string, opts ReadEnvOptions) (AppEnv, error) {
|
func ReadEnv(filePath string) (AppEnv, error) {
|
||||||
var envVars AppEnv
|
var envVars AppEnv
|
||||||
|
|
||||||
envVars, err := godotenv.Read(filePath)
|
envVars, _, err := godotenv.Read(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// for idx, envVar := range envVars {
|
|
||||||
// if strings.Contains(envVar, "#") {
|
|
||||||
// if opts.IncludeModifiers && ContainsEnvVarModifier(envVar) {
|
|
||||||
// continue
|
|
||||||
// }
|
|
||||||
// vals := strings.Split(envVar, "#")
|
|
||||||
// envVars[idx] = strings.TrimSpace(vals[0])
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
|
|
||||||
logrus.Debugf("read %s from %s", envVars, filePath)
|
logrus.Debugf("read %s from %s", envVars, filePath)
|
||||||
|
|
||||||
return envVars, nil
|
return envVars, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReadEnv loads an app envivornment and their modifiers in two different maps.
|
||||||
|
func ReadEnvWithModifiers(filePath string) (AppEnv, AppModifiers, error) {
|
||||||
|
var envVars AppEnv
|
||||||
|
|
||||||
|
envVars, mods, err := godotenv.Read(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mods, err
|
||||||
|
}
|
||||||
|
|
||||||
|
logrus.Debugf("read %s from %s", envVars, filePath)
|
||||||
|
|
||||||
|
return envVars, mods, nil
|
||||||
|
}
|
||||||
|
|
||||||
// ReadServerNames retrieves all server names.
|
// ReadServerNames retrieves all server names.
|
||||||
func ReadServerNames() ([]string, error) {
|
func ReadServerNames() ([]string, error) {
|
||||||
serverNames, err := GetAllFoldersInDirectory(SERVERS_DIR)
|
serverNames, err := GetAllFoldersInDirectory(SERVERS_DIR)
|
||||||
|
@ -227,7 +216,7 @@ func CheckEnv(app App) ([]EnvVar, error) {
|
||||||
return envVars, err
|
return envVars, err
|
||||||
}
|
}
|
||||||
|
|
||||||
envSample, err := ReadEnv(envSamplePath, ReadEnvOptions{})
|
envSample, err := ReadEnv(envSamplePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return envVars, err
|
return envVars, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,15 +13,21 @@ import (
|
||||||
"coopcloud.tech/abra/pkg/recipe"
|
"coopcloud.tech/abra/pkg/recipe"
|
||||||
)
|
)
|
||||||
|
|
||||||
var TestFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
|
var (
|
||||||
var ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
|
TestFolder = os.ExpandEnv("$PWD/../../tests/resources/test_folder")
|
||||||
|
ValidAbraConf = os.ExpandEnv("$PWD/../../tests/resources/valid_abra_config")
|
||||||
|
)
|
||||||
|
|
||||||
// make sure these are in alphabetical order
|
// make sure these are in alphabetical order
|
||||||
var TFolders = []string{"folder1", "folder2"}
|
var (
|
||||||
var TFiles = []string{"bar.env", "foo.env"}
|
TFolders = []string{"folder1", "folder2"}
|
||||||
|
TFiles = []string{"bar.env", "foo.env"}
|
||||||
|
)
|
||||||
|
|
||||||
var AppName = "ecloud"
|
var (
|
||||||
var ServerName = "evil.corp"
|
AppName = "ecloud"
|
||||||
|
ServerName = "evil.corp"
|
||||||
|
)
|
||||||
|
|
||||||
var ExpectedAppEnv = config.AppEnv{
|
var ExpectedAppEnv = config.AppEnv{
|
||||||
"DOMAIN": "ecloud.evil.corp",
|
"DOMAIN": "ecloud.evil.corp",
|
||||||
|
@ -71,7 +77,7 @@ func TestGetAllFilesInDirectory(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestReadEnv(t *testing.T) {
|
func TestReadEnv(t *testing.T) {
|
||||||
env, err := config.ReadEnv(ExpectedAppFile.Path, config.ReadEnvOptions{})
|
env, err := config.ReadEnv(ExpectedAppFile.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -149,7 +155,7 @@ func TestCheckEnv(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
|
envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
|
||||||
envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
|
envSample, err := config.ReadEnv(envSamplePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -183,7 +189,7 @@ func TestCheckEnvError(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
|
envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
|
||||||
envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
|
envSample, err := config.ReadEnv(envSamplePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -211,19 +217,7 @@ func TestCheckEnvError(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestContainsEnvVarModifier(t *testing.T) {
|
|
||||||
if ok := config.ContainsEnvVarModifier("FOO=bar # bing"); ok {
|
|
||||||
t.Fatal("FOO contains no env var modifier")
|
|
||||||
}
|
|
||||||
|
|
||||||
if ok := config.ContainsEnvVarModifier("FOO=bar # length=3"); !ok {
|
|
||||||
t.Fatal("FOO contains an env var modifier (length)")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEnvVarCommentsRemoved(t *testing.T) {
|
func TestEnvVarCommentsRemoved(t *testing.T) {
|
||||||
t.Skip("https://git.coopcloud.tech/coop-cloud/organising/issues/535")
|
|
||||||
|
|
||||||
offline := true
|
offline := true
|
||||||
r, err := recipe.Get("abra-test-recipe", offline)
|
r, err := recipe.Get("abra-test-recipe", offline)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -231,7 +225,7 @@ func TestEnvVarCommentsRemoved(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
|
envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
|
||||||
envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
|
envSample, err := config.ReadEnv(envSamplePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
@ -263,12 +257,19 @@ func TestEnvVarModifiersIncluded(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
|
envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
|
||||||
envSample, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{IncludeModifiers: true})
|
envSample, modifiers, err := config.ReadEnvWithModifiers(envSamplePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "length") {
|
if !strings.Contains(envSample["SECRET_TEST_PASS_TWO_VERSION"], "v1") {
|
||||||
t.Fatal("comment from env var SECRET_TEST_PASS_TWO_VERSION should not be removed")
|
t.Errorf("value should be 'v1', got: '%s'", envSample["SECRET_TEST_PASS_TWO_VERSION"])
|
||||||
|
}
|
||||||
|
if modifiers == nil || modifiers["SECRET_TEST_PASS_TWO_VERSION"] == nil {
|
||||||
|
t.Errorf("no modifiers included")
|
||||||
|
} else {
|
||||||
|
if modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"] != "10" {
|
||||||
|
t.Errorf("length modifier should be '10', got: '%s'", modifiers["SECRET_TEST_PASS_TWO_VERSION"]["length"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -227,7 +227,7 @@ func LintAppService(recipe recipe.Recipe) (bool, error) {
|
||||||
// therefore no matching traefik deploy label will be present.
|
// therefore no matching traefik deploy label will be present.
|
||||||
func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
|
func LintTraefikEnabledSkipCondition(recipe recipe.Recipe) (bool, error) {
|
||||||
envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
|
envSamplePath := path.Join(config.RECIPES_DIR, recipe.Name, ".env.sample")
|
||||||
sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
|
sampleEnv, err := config.ReadEnv(envSamplePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
|
return false, fmt.Errorf("Unable to discover .env.sample for %s", recipe.Name)
|
||||||
}
|
}
|
||||||
|
|
|
@ -227,7 +227,7 @@ func Get(recipeName string, offline bool) (Recipe, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
|
envSamplePath := path.Join(config.RECIPES_DIR, recipeName, ".env.sample")
|
||||||
sampleEnv, err := config.ReadEnv(envSamplePath, config.ReadEnvOptions{})
|
sampleEnv, err := config.ReadEnv(envSamplePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Recipe{}, err
|
return Recipe{}, err
|
||||||
}
|
}
|
||||||
|
@ -255,9 +255,9 @@ func Get(recipeName string, offline bool) (Recipe, error) {
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r Recipe) SampleEnv(opts config.ReadEnvOptions) (map[string]string, error) {
|
func (r Recipe) SampleEnv() (map[string]string, error) {
|
||||||
envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
|
envSamplePath := path.Join(config.RECIPES_DIR, r.Name, ".env.sample")
|
||||||
sampleEnv, err := config.ReadEnv(envSamplePath, opts)
|
sampleEnv, err := config.ReadEnv(envSamplePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name)
|
return sampleEnv, fmt.Errorf("unable to discover .env.sample for %s", r.Name)
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,9 +21,9 @@ import (
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
// secretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
|
// SecretValue represents a parsed `SECRET_FOO=v1 # length=bar` env var config
|
||||||
// secret definition.
|
// secret definition.
|
||||||
type secretValue struct {
|
type SecretValue struct {
|
||||||
Version string
|
Version string
|
||||||
Length int
|
Length int
|
||||||
}
|
}
|
||||||
|
@ -35,7 +35,6 @@ func GeneratePasswords(count, length uint) ([]string, error) {
|
||||||
length,
|
length,
|
||||||
passgen.AlphabetDefault,
|
passgen.AlphabetDefault,
|
||||||
)
|
)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -54,7 +53,6 @@ func GeneratePassphrases(count uint) ([]string, error) {
|
||||||
passgen.PassphraseCasingDefault,
|
passgen.PassphraseCasingDefault,
|
||||||
passgen.WordListDefault,
|
passgen.WordListDefault,
|
||||||
)
|
)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -69,18 +67,20 @@ func GeneratePassphrases(count uint) ([]string, error) {
|
||||||
// and some times you don't (as the caller). We need to be able to handle the
|
// and some times you don't (as the caller). We need to be able to handle the
|
||||||
// "app new" case where we pass in the .env.sample and the "secret generate"
|
// "app new" case where we pass in the .env.sample and the "secret generate"
|
||||||
// case where the app is created.
|
// case where the app is created.
|
||||||
func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]string, error) {
|
func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName string) (map[string]SecretValue, error) {
|
||||||
secretConfigs := make(map[string]string)
|
appEnv, appModifiers, err := config.ReadEnvWithModifiers(appEnvPath)
|
||||||
|
|
||||||
appEnv, err := config.ReadEnv(appEnvPath, config.ReadEnvOptions{IncludeModifiers: true})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return secretConfigs, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
opts := stack.Deploy{Composefiles: composeFiles}
|
opts := stack.Deploy{Composefiles: composeFiles}
|
||||||
config, err := loader.LoadComposefile(opts, appEnv)
|
config, err := loader.LoadComposefile(opts, appEnv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return secretConfigs, err
|
return nil, err
|
||||||
|
}
|
||||||
|
configWithoutEnv, err := loader.LoadComposefile(opts, map[string]string{}, loader.SkipInterpolation)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var enabledSecrets []string
|
var enabledSecrets []string
|
||||||
|
@ -92,12 +92,13 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
|
||||||
|
|
||||||
if len(enabledSecrets) == 0 {
|
if len(enabledSecrets) == 0 {
|
||||||
logrus.Debugf("not generating app secrets, none enabled in recipe config")
|
logrus.Debugf("not generating app secrets, none enabled in recipe config")
|
||||||
return secretConfigs, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
secretValues := map[string]SecretValue{}
|
||||||
for secretId, secretConfig := range config.Secrets {
|
for secretId, secretConfig := range config.Secrets {
|
||||||
if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
|
if string(secretConfig.Name[len(secretConfig.Name)-1]) == "_" {
|
||||||
return secretConfigs, fmt.Errorf("missing version for secret? (%s)", secretId)
|
return nil, fmt.Errorf("missing version for secret? (%s)", secretId)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !(slices.Contains(enabledSecrets, secretId)) {
|
if !(slices.Contains(enabledSecrets, secretId)) {
|
||||||
|
@ -107,60 +108,47 @@ func ReadSecretsConfig(appEnvPath string, composeFiles []string, recipeName stri
|
||||||
|
|
||||||
lastIdx := strings.LastIndex(secretConfig.Name, "_")
|
lastIdx := strings.LastIndex(secretConfig.Name, "_")
|
||||||
secretVersion := secretConfig.Name[lastIdx+1:]
|
secretVersion := secretConfig.Name[lastIdx+1:]
|
||||||
secretConfigs[secretId] = secretVersion
|
value := SecretValue{Version: secretVersion}
|
||||||
|
|
||||||
|
// Check if the length modifier is set for this secret.
|
||||||
|
for k, v := range appModifiers {
|
||||||
|
// configWithoutEnv contains the raw name as defined in the compose.yaml
|
||||||
|
if !strings.Contains(configWithoutEnv.Secrets[secretId].Name, k) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
lengthRaw, ok := v["length"]
|
||||||
|
if ok {
|
||||||
|
length, err := strconv.Atoi(lengthRaw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
value.Length = length
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
secretValues[secretId] = value
|
||||||
}
|
}
|
||||||
|
|
||||||
return secretConfigs, nil
|
return secretValues, nil
|
||||||
}
|
|
||||||
|
|
||||||
func ParseSecretValue(secret string) (secretValue, error) {
|
|
||||||
values := strings.Split(secret, "#")
|
|
||||||
if len(values) == 0 {
|
|
||||||
return secretValue{}, fmt.Errorf("unable to parse %s", secret)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(values) == 1 {
|
|
||||||
return secretValue{Version: values[0], Length: 0}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
split := strings.Split(values[1], "=")
|
|
||||||
parsed := split[len(split)-1]
|
|
||||||
stripped := strings.ReplaceAll(parsed, " ", "")
|
|
||||||
length, err := strconv.Atoi(stripped)
|
|
||||||
if err != nil {
|
|
||||||
return secretValue{}, err
|
|
||||||
}
|
|
||||||
version := strings.ReplaceAll(values[0], " ", "")
|
|
||||||
|
|
||||||
logrus.Debugf("parsed version %s and length '%v' from %s", version, length, secret)
|
|
||||||
|
|
||||||
return secretValue{Version: version, Length: length}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GenerateSecrets generates secrets locally and sends them to a remote server for storage.
|
// GenerateSecrets generates secrets locally and sends them to a remote server for storage.
|
||||||
func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]string, appName, server string) (map[string]string, error) {
|
func GenerateSecrets(cl *dockerClient.Client, secrets map[string]SecretValue, appName, server string) (map[string]string, error) {
|
||||||
secrets := make(map[string]string)
|
secretsGenerated := map[string]string{}
|
||||||
|
|
||||||
var mutex sync.Mutex
|
var mutex sync.Mutex
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
ch := make(chan error, len(secretsFromConfig))
|
ch := make(chan error, len(secrets))
|
||||||
for n, v := range secretsFromConfig {
|
for n, v := range secrets {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
|
|
||||||
go func(secretName, secretValue string) {
|
go func(secretName string, secret SecretValue) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
parsedSecretValue, err := ParseSecretValue(secretValue)
|
secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, secret.Version)
|
||||||
if err != nil {
|
|
||||||
ch <- err
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
secretRemoteName := fmt.Sprintf("%s_%s_%s", appName, secretName, parsedSecretValue.Version)
|
|
||||||
logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
|
logrus.Debugf("attempting to generate and store %s on %s", secretRemoteName, server)
|
||||||
|
|
||||||
if parsedSecretValue.Length > 0 {
|
if secret.Length > 0 {
|
||||||
passwords, err := GeneratePasswords(1, uint(parsedSecretValue.Length))
|
passwords, err := GeneratePasswords(1, uint(secret.Length))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ch <- err
|
ch <- err
|
||||||
return
|
return
|
||||||
|
@ -178,7 +166,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
|
||||||
|
|
||||||
mutex.Lock()
|
mutex.Lock()
|
||||||
defer mutex.Unlock()
|
defer mutex.Unlock()
|
||||||
secrets[secretName] = passwords[0]
|
secretsGenerated[secretName] = passwords[0]
|
||||||
} else {
|
} else {
|
||||||
passphrases, err := GeneratePassphrases(1)
|
passphrases, err := GeneratePassphrases(1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -198,7 +186,7 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
|
||||||
|
|
||||||
mutex.Lock()
|
mutex.Lock()
|
||||||
defer mutex.Unlock()
|
defer mutex.Unlock()
|
||||||
secrets[secretName] = passphrases[0]
|
secretsGenerated[secretName] = passphrases[0]
|
||||||
}
|
}
|
||||||
ch <- nil
|
ch <- nil
|
||||||
}(n, v)
|
}(n, v)
|
||||||
|
@ -206,16 +194,16 @@ func GenerateSecrets(cl *dockerClient.Client, secretsFromConfig map[string]strin
|
||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
for range secretsFromConfig {
|
for range secrets {
|
||||||
err := <-ch
|
err := <-ch
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
logrus.Debugf("generated and stored %s on %s", secrets, server)
|
logrus.Debugf("generated and stored %v on %s", secrets, server)
|
||||||
|
|
||||||
return secrets, nil
|
return secretsGenerated, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type secretStatus struct {
|
type secretStatus struct {
|
||||||
|
@ -257,14 +245,9 @@ func PollSecretsStatus(cl *dockerClient.Client, app config.App) (secretStatuses,
|
||||||
remoteSecretNames[cont.Spec.Annotations.Name] = true
|
remoteSecretNames[cont.Spec.Annotations.Name] = true
|
||||||
}
|
}
|
||||||
|
|
||||||
for secretName, secretValue := range secretsConfig {
|
for secretName, val := range secretsConfig {
|
||||||
createdRemote := false
|
createdRemote := false
|
||||||
|
|
||||||
val, err := ParseSecretValue(secretValue)
|
|
||||||
if err != nil {
|
|
||||||
return secStats, err
|
|
||||||
}
|
|
||||||
|
|
||||||
secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
|
secretRemoteName := fmt.Sprintf("%s_%s_%s", app.StackName(), secretName, val.Version)
|
||||||
if _, ok := remoteSecretNames[secretRemoteName]; ok {
|
if _, ok := remoteSecretNames[secretRemoteName]; ok {
|
||||||
createdRemote = true
|
createdRemote = true
|
||||||
|
|
|
@ -18,7 +18,7 @@ func TestReadSecretsConfig(t *testing.T) {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sampleEnv, err := recipe.SampleEnv(config.ReadEnvOptions{})
|
sampleEnv, err := recipe.SampleEnv()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,15 +18,24 @@ func DontSkipValidation(opts *loader.Options) {
|
||||||
opts.SkipValidation = false
|
opts.SkipValidation = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SkipInterpolation skip interpolating environment variables.
|
||||||
|
func SkipInterpolation(opts *loader.Options) {
|
||||||
|
opts.SkipInterpolation = true
|
||||||
|
}
|
||||||
|
|
||||||
// LoadComposefile parse the composefile specified in the cli and returns its Config and version.
|
// LoadComposefile parse the composefile specified in the cli and returns its Config and version.
|
||||||
func LoadComposefile(opts Deploy, appEnv map[string]string) (*composetypes.Config, error) {
|
func LoadComposefile(opts Deploy, appEnv map[string]string, options ...func(*loader.Options)) (*composetypes.Config, error) {
|
||||||
configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
|
configDetails, err := getConfigDetails(opts.Composefiles, appEnv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if options == nil {
|
||||||
|
options = []func(*loader.Options){DontSkipValidation}
|
||||||
|
}
|
||||||
|
|
||||||
dicts := getDictsFrom(configDetails.ConfigFiles)
|
dicts := getDictsFrom(configDetails.ConfigFiles)
|
||||||
config, err := loader.Load(configDetails, DontSkipValidation)
|
config, err := loader.Load(configDetails, options...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
|
if fpe, ok := err.(*loader.ForbiddenPropertiesError); ok {
|
||||||
return nil, fmt.Errorf("compose file contains unsupported options: %s",
|
return nil, fmt.Errorf("compose file contains unsupported options: %s",
|
||||||
|
|
Loading…
Reference in New Issue