diff --git a/pkg/godotenv/godotenv.go b/pkg/godotenv/godotenv.go index 61b0ebba..1c6779d0 100644 --- a/pkg/godotenv/godotenv.go +++ b/pkg/godotenv/godotenv.go @@ -27,11 +27,11 @@ import ( const doubleQuoteSpecialChars = "\\\n\r\"!$`" // Parse reads an env file from io.Reader, returning a map of keys and values. -func Parse(r io.Reader) (map[string]string, error) { +func Parse(r io.Reader) (map[string]string, map[string]map[string]string, error) { var buf bytes.Buffer _, err := io.Copy(&buf, r) if err != nil { - return nil, err + return nil, nil, err } return UnmarshalBytes(buf.Bytes()) @@ -85,12 +85,13 @@ func Overload(filenames ...string) (err error) { // Read all env (with same file loading semantics as Load) but return values as // a map rather than automatically writing values into env -func Read(filenames ...string) (envMap map[string]string, err error) { +func Read(filenames ...string) (envMap map[string]string, modMap map[string]map[string]string, err error) { filenames = filenamesOrDefault(filenames) envMap = make(map[string]string) + modMap = make(map[string]map[string]string) for _, filename := range filenames { - individualEnvMap, individualErr := readFile(filename) + individualEnvMap, individualModMap, individualErr := readFile(filename) if individualErr != nil { err = individualErr @@ -100,22 +101,27 @@ func Read(filenames ...string) (envMap map[string]string, err error) { for key, value := range individualEnvMap { envMap[key] = value } + + for key, value := range individualModMap { + modMap[key] = value + } } return } // Unmarshal reads an env file from a string, returning a map of keys and values. -func Unmarshal(str string) (envMap map[string]string, err error) { +func Unmarshal(str string) (envMap map[string]string, modifierMap map[string]map[string]string, err error) { return UnmarshalBytes([]byte(str)) } // UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values. -func UnmarshalBytes(src []byte) (map[string]string, error) { - out := make(map[string]string) - err := parseBytes(src, out) +func UnmarshalBytes(src []byte) (map[string]string, map[string]map[string]string, error) { + vars := make(map[string]string) + modifiers := make(map[string]map[string]string) + err := parseBytes(src, vars, modifiers) - return out, err + return vars, modifiers, err } // Exec loads env vars from the specified filenames (empty map falls back to default) @@ -182,7 +188,7 @@ func filenamesOrDefault(filenames []string) []string { } func loadFile(filename string, overload bool) error { - envMap, err := readFile(filename) + envMap, _, err := readFile(filename) if err != nil { return err } @@ -203,7 +209,7 @@ func loadFile(filename string, overload bool) error { return nil } -func readFile(filename string) (envMap map[string]string, err error) { +func readFile(filename string) (envMap map[string]string, modMap map[string]map[string]string, err error) { file, err := os.Open(filename) if err != nil { return diff --git a/pkg/godotenv/godotenv_test.go b/pkg/godotenv/godotenv_test.go index 9b1c99ea..3e76d627 100644 --- a/pkg/godotenv/godotenv_test.go +++ b/pkg/godotenv/godotenv_test.go @@ -12,8 +12,7 @@ import ( var noopPresets = make(map[string]string) func parseAndCompare(t *testing.T, rawEnvLine string, expectedKey string, expectedValue string) { - result, err := Unmarshal(rawEnvLine) - + result, _, err := Unmarshal(rawEnvLine) if err != nil { t.Errorf("Expected %q to parse as %q: %q, errored %q", rawEnvLine, expectedKey, expectedValue, err) return @@ -88,7 +87,7 @@ func TestReadPlainEnv(t *testing.T) { "OPTION_H": "1 2", } - envMap, err := Read(envFileName) + envMap, _, err := Read(envFileName) if err != nil { t.Error("Error reading file") } @@ -105,7 +104,7 @@ func TestReadPlainEnv(t *testing.T) { } func TestParse(t *testing.T) { - envMap, err := Parse(bytes.NewReader([]byte("ONE=1\nTWO='2'\nTHREE = \"3\""))) + envMap, _, err := Parse(bytes.NewReader([]byte("ONE=1\nTWO='2'\nTHREE = \"3\""))) expectedValues := map[string]string{ "ONE": "1", "TWO": "2", @@ -268,7 +267,7 @@ func TestExpanding(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - env, err := Parse(strings.NewReader(tt.input)) + env, _, err := Parse(strings.NewReader(tt.input)) if err != nil { t.Errorf("Error: %s", err.Error()) } @@ -286,7 +285,7 @@ func TestVariableStringValueSeparator(t *testing.T) { want := map[string]string{ "TEST_URLS": "stratum+tcp://stratum.antpool.com:3333\nstratum+tcp://stratum.antpool.com:443", } - got, err := Parse(strings.NewReader(input)) + got, _, err := Parse(strings.NewReader(input)) if err != nil { t.Error(err) } @@ -342,7 +341,7 @@ func TestParsing(t *testing.T) { // parses yaml style options parseAndCompare(t, "OPTION_A: 1", "OPTION_A", "1") - //parses yaml values with equal signs + // parses yaml values with equal signs parseAndCompare(t, "OPTION_A: Foo=bar", "OPTION_A", "Foo=bar") // parses non-yaml options with colons @@ -394,7 +393,7 @@ func TestParsing(t *testing.T) { parseAndCompare(t, `FOO="ba#r"`, "FOO", "ba#r") parseAndCompare(t, "FOO='ba#r'", "FOO", "ba#r") - //newlines and backslashes should be escaped + // newlines and backslashes should be escaped parseAndCompare(t, `FOO="bar\n\ b\az"`, "FOO", "bar\n baz") parseAndCompare(t, `FOO="bar\\\n\ b\az"`, "FOO", "bar\\\n baz") parseAndCompare(t, `FOO="bar\\r\ b\az"`, "FOO", "bar\\r baz") @@ -409,7 +408,7 @@ func TestParsing(t *testing.T) { // it 'throws an error if line format is incorrect' do // expect{env('lol$wut')}.to raise_error(Dotenv::FormatError) badlyFormattedLine := "lol$wut" - _, err := Unmarshal(badlyFormattedLine) + _, _, err := Unmarshal(badlyFormattedLine) if err == nil { t.Errorf("Expected \"%v\" to return error, but it didn't", badlyFormattedLine) } @@ -453,7 +452,7 @@ func TestLinesToIgnore(t *testing.T) { func TestErrorReadDirectory(t *testing.T) { envFileName := "fixtures/" - envMap, err := Read(envFileName) + envMap, _, err := Read(envFileName) if err == nil { t.Errorf("Expected error, got %v", envMap) @@ -462,7 +461,7 @@ func TestErrorReadDirectory(t *testing.T) { func TestErrorParsing(t *testing.T) { envFileName := "fixtures/invalid1.env" - envMap, err := Read(envFileName) + envMap, _, err := Read(envFileName) if err == nil { t.Errorf("Expected error, got %v", envMap) } @@ -481,20 +480,20 @@ func TestComments(t *testing.T) { func TestWrite(t *testing.T) { writeAndCompare := func(env string, expected string) { - envMap, _ := Unmarshal(env) + envMap, _, _ := Unmarshal(env) actual, _ := Marshal(envMap) if expected != actual { t.Errorf("Expected '%v' (%v) to write as '%v', got '%v' instead.", env, envMap, expected, actual) } } - //just test some single lines to show the general idea - //TestRoundtrip makes most of the good assertions + // just test some single lines to show the general idea + // TestRoundtrip makes most of the good assertions - //values are always double-quoted + // values are always double-quoted writeAndCompare(`key=value`, `key="value"`) - //double-quotes are escaped + // double-quotes are escaped writeAndCompare(`key=va"lu"e`, `key="va\"lu\"e"`) - //but single quotes are left alone + // but single quotes are left alone writeAndCompare(`key=va'lu'e`, `key="va'lu'e"`) // newlines, backslashes, and some other special chars are escaped writeAndCompare(`foo="\n\r\\r!"`, `foo="\n\r\\r\!"`) @@ -502,14 +501,13 @@ func TestWrite(t *testing.T) { writeAndCompare("foo=bar\nbaz=buzz", "baz=\"buzz\"\nfoo=\"bar\"") // integers should not be quoted writeAndCompare(`key="10"`, `key=10`) - } func TestRoundtrip(t *testing.T) { fixtures := []string{"equals.env", "exported.env", "plain.env", "quoted.env"} for _, fixture := range fixtures { fixtureFilename := fmt.Sprintf("fixtures/%s", fixture) - env, err := readFile(fixtureFilename) + env, _, err := readFile(fixtureFilename) if err != nil { t.Errorf("Expected '%s' to read without error (%v)", fixtureFilename, err) } @@ -517,7 +515,7 @@ func TestRoundtrip(t *testing.T) { if err != nil { t.Errorf("Expected '%s' to Marshal (%v)", fixtureFilename, err) } - roundtripped, err := Unmarshal(rep) + roundtripped, _, err := Unmarshal(rep) if err != nil { t.Errorf("Expected '%s' to Mashal and Unmarshal (%v)", fixtureFilename, err) } @@ -563,7 +561,7 @@ func TestTrailingNewlines(t *testing.T) { for n, c := range cases { t.Run(n, func(t *testing.T) { - result, err := Unmarshal(c.input) + result, _, err := Unmarshal(c.input) if err != nil { t.Errorf("Input: %q Unexpected error:\t%q", c.input, err) } @@ -624,7 +622,7 @@ func TestWhitespace(t *testing.T) { for n, c := range cases { t.Run(n, func(t *testing.T) { - result, err := Unmarshal(c.input) + result, _, err := Unmarshal(c.input) if err != nil { t.Errorf("Input: %q Unexpected error:\t%q", c.input, err) } @@ -634,3 +632,78 @@ func TestWhitespace(t *testing.T) { }) } } + +func TestModfiers(t *testing.T) { + cases := map[string]struct { + input string + key string + value string + modifiers map[string]string + }{ + // "No Modifier": { + // input: "A=a", + // key: "A", + // value: "a", + // }, + // "With comment": { + // input: "A=a # my comment", + // key: "A", + // value: "a", + // }, + // "With single modifier": { + // input: "A=a # foo=bar", + // key: "A", + // value: "a", + // modifiers: map[string]string{ + // "foo": "bar", + // }, + // }, + // "With multiple modifiers": { + // input: "A=a # foo=bar length=10", + // key: "A", + // value: "a", + // modifiers: map[string]string{ + // "foo": "bar", + // "length": "10", + // }, + // }, + "With quoted var": { + input: "A='a' # foo=bar", + key: "A", + value: "a", + modifiers: map[string]string{ + "foo": "bar", + }, + }, + "With quoted var 2": { + input: "A='a' # foo=bar\nB=b", + key: "A", + value: "a", + modifiers: map[string]string{ + "foo": "bar", + }, + }, + } + + for n, c := range cases { + t.Run(n, func(t *testing.T) { + values, modifiers, err := Unmarshal(c.input) + if err != nil { + t.Errorf("Input: %q Unexpected error:\t%q", c.input, err) + } + if values[c.key] != c.value { + t.Errorf("Input %q Expected:\t %q/%q\nGot:\t %q", c.input, c.key, c.value, values) + } + + if modifiers[c.key] == nil && c.modifiers != nil { + t.Errorf("Input %q Expected modifiers\n Got: none", c.input) + } else { + for k, v := range c.modifiers { + if modifiers[c.key][k] != v { + t.Errorf("Input %q Expected modifier %s=%s\n Got: %s=%s", c.input, k, v, k, modifiers[c.key][k]) + } + } + } + }) + } +} diff --git a/pkg/godotenv/parser.go b/pkg/godotenv/parser.go index cc709af8..9ffec051 100644 --- a/pkg/godotenv/parser.go +++ b/pkg/godotenv/parser.go @@ -17,7 +17,7 @@ const ( exportPrefix = "export" ) -func parseBytes(src []byte, out map[string]string) error { +func parseBytes(src []byte, vars map[string]string, modifiers map[string]map[string]string) error { src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1) cutset := src for { @@ -32,12 +32,13 @@ func parseBytes(src []byte, out map[string]string) error { return err } - value, left, err := extractVarValue(left, out) + value, mods, left, err := extractVarValue(left, vars) if err != nil { return err } - out[key] = value + vars[key] = value + modifiers[key] = mods cutset = left } @@ -117,34 +118,34 @@ loop: } // extractVarValue extracts variable value and returns rest of slice -func extractVarValue(src []byte, vars map[string]string) (value string, rest []byte, err error) { +func extractVarValue(src []byte, vars map[string]string) (value string, modifiers map[string]string, rest []byte, err error) { quote, hasPrefix := hasQuotePrefix(src) - if !hasPrefix { - // unquoted value - read until end of line - endOfLine := bytes.IndexFunc(src, isLineEnd) + // unquoted value - read until end of line + endOfLine := bytes.IndexFunc(src, isLineEnd) + // Hit EOF without a trailing newline + if endOfLine == -1 { + endOfLine = len(src) - // Hit EOF without a trailing newline - if endOfLine == -1 { - endOfLine = len(src) - - if endOfLine == 0 { - return "", nil, nil - } + if endOfLine == 0 { + return "", nil, nil, nil } - + } + if !hasPrefix { // Convert line to rune away to do accurate countback of runes line := []rune(string(src[0:endOfLine])) // Assume end of line is end of var endOfVar := len(line) if endOfVar == 0 { - return "", src[endOfLine:], nil + return "", nil, src[endOfLine:], nil } + comment := "" // Work backwards to check if the line ends in whitespace then // a comment (ie asdasd # some comment) for i := endOfVar - 1; i >= 0; i-- { if line[i] == charComment && i > 0 { + comment = string(line[i+1:]) if isSpace(line[i-1]) { endOfVar = i break @@ -154,7 +155,7 @@ func extractVarValue(src []byte, vars map[string]string) (value string, rest []b trimmed := strings.TrimFunc(string(line[0:endOfVar]), isSpace) - return expandVariables(trimmed, vars), src[endOfLine:], nil + return expandVariables(trimmed, vars), extractModifiers(comment), src[endOfLine:], nil } // lookup quoted string terminator @@ -177,7 +178,7 @@ func extractVarValue(src []byte, vars map[string]string) (value string, rest []b value = expandVariables(expandEscapes(value), vars) } - return value, src[i+1:], nil + return value, extractModifiers(string(src[i+1 : endOfLine])), src[i+1:], nil } // return formatted error if quoted string is not terminated @@ -186,7 +187,24 @@ func extractVarValue(src []byte, vars map[string]string) (value string, rest []b valEndIndex = len(src) } - return "", nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex]) + return "", nil, nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex]) +} + +func extractModifiers(comment string) map[string]string { + if comment == "" { + return nil + } + comment = strings.TrimSpace(comment) + kvpairs := strings.Split(comment, " ") + mods := make(map[string]string) + for _, kv := range kvpairs { + kvsplit := strings.Split(kv, "=") + if len(kvsplit) != 2 { + continue + } + mods[kvsplit[0]] = kvsplit[1] + } + return mods } func expandEscapes(str string) string {