fix: if a line contains multiple # characters, there will be issues w… (#238)

* fix: if a line contains multiple # characters, there will be issues when traversing from back to front

* fix: typo

autoload/autoload.go

@@ -8,7 +8,7 @@ package autoload
 	And bob's your mother's brother
 */
 
-import "git.coopcloud.tech/toolshed/godotenv"
+import "github.com/joho/godotenv"
 
 func init() {
 	godotenv.Load()

cmd/godotenv/cmd.go

@@ -4,10 +4,9 @@ import (
 	"flag"
 	"fmt"
 	"log"
-
 	"strings"
 
-	"git.coopcloud.tech/toolshed/godotenv"
+	"github.com/joho/godotenv"
 )
 
 func main() {

fixtures/comments.env

@@ -1,4 +1,7 @@
 # Full line comment
+qux=thud # fred # other
+thud=fred#qux # other
+fred=qux#baz # other # more
 foo=bar # baz
 bar=foo#baz
 baz="foo"#bar

fixtures/substitutions.env

@@ -3,3 +3,4 @@ OPTION_B=${OPTION_A}
 OPTION_C=$OPTION_B
 OPTION_D=${OPTION_A}${OPTION_B}
 OPTION_E=${OPTION_NOT_DEFINED}
+OPTION_F=${GLOBAL_OPTION}

go.mod

@@ -1,3 +1,3 @@
-module git.coopcloud.tech/toolshed/godotenv
+module github.com/joho/godotenv
 
 go 1.12

godotenv.go

@@ -27,11 +27,11 @@ import (
 const doubleQuoteSpecialChars = "\\\n\r\"!$`"
 
 // Parse reads an env file from io.Reader, returning a map of keys and values.
-func Parse(r io.Reader) (map[string]string, map[string]map[string]string, error) {
+func Parse(r io.Reader) (map[string]string, error) {
 	var buf bytes.Buffer
 	_, err := io.Copy(&buf, r)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	return UnmarshalBytes(buf.Bytes())
@@ -85,13 +85,12 @@ func Overload(filenames ...string) (err error) {
 
 // Read all env (with same file loading semantics as Load) but return values as
 // a map rather than automatically writing values into env
-func Read(filenames ...string) (envMap map[string]string, modMap map[string]map[string]string, err error) {
+func Read(filenames ...string) (envMap map[string]string, err error) {
 	filenames = filenamesOrDefault(filenames)
 	envMap = make(map[string]string)
-	modMap = make(map[string]map[string]string)
 
 	for _, filename := range filenames {
-		individualEnvMap, individualModMap, individualErr := readFile(filename)
+		individualEnvMap, individualErr := readFile(filename)
 
 		if individualErr != nil {
 			err = individualErr
@@ -101,27 +100,22 @@ func Read(filenames ...string) (envMap map[string]string, modMap map[string]map[
 		for key, value := range individualEnvMap {
 			envMap[key] = value
 		}
-
-		for key, value := range individualModMap {
-			modMap[key] = value
-		}
 	}
 
 	return
 }
 
 // Unmarshal reads an env file from a string, returning a map of keys and values.
-func Unmarshal(str string) (envMap map[string]string, modifierMap map[string]map[string]string, err error) {
+func Unmarshal(str string) (envMap map[string]string, err error) {
 	return UnmarshalBytes([]byte(str))
 }
 
 // UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values.
-func UnmarshalBytes(src []byte) (map[string]string, map[string]map[string]string, error) {
-	vars := make(map[string]string)
-	modifiers := make(map[string]map[string]string)
-	err := parseBytes(src, vars, modifiers)
+func UnmarshalBytes(src []byte) (map[string]string, error) {
+	out := make(map[string]string)
+	err := parseBytes(src, out)
 
-	return vars, modifiers, err
+	return out, err
 }
 
 // Exec loads env vars from the specified filenames (empty map falls back to default)
@@ -188,7 +182,7 @@ func filenamesOrDefault(filenames []string) []string {
 }
 
 func loadFile(filename string, overload bool) error {
-	envMap, _, err := readFile(filename)
+	envMap, err := readFile(filename)
 	if err != nil {
 		return err
 	}
@@ -209,7 +203,7 @@ func loadFile(filename string, overload bool) error {
 	return nil
 }
 
-func readFile(filename string) (envMap map[string]string, modMap map[string]map[string]string, err error) {
+func readFile(filename string) (envMap map[string]string, err error) {
 	file, err := os.Open(filename)
 	if err != nil {
 		return

godotenv_test.go

@@ -12,7 +12,8 @@ import (
 var noopPresets = make(map[string]string)
 
 func parseAndCompare(t *testing.T, rawEnvLine string, expectedKey string, expectedValue string) {
-	result, _, err := Unmarshal(rawEnvLine)
+	result, err := Unmarshal(rawEnvLine)
+
 	if err != nil {
 		t.Errorf("Expected %q to parse as %q: %q, errored %q", rawEnvLine, expectedKey, expectedValue, err)
 		return
@@ -87,7 +88,7 @@ func TestReadPlainEnv(t *testing.T) {
 		"OPTION_H": "1 2",
 	}
 
-	envMap, _, err := Read(envFileName)
+	envMap, err := Read(envFileName)
 	if err != nil {
 		t.Error("Error reading file")
 	}
@@ -104,7 +105,7 @@ func TestReadPlainEnv(t *testing.T) {
 }
 
 func TestParse(t *testing.T) {
-	envMap, _, err := Parse(bytes.NewReader([]byte("ONE=1\nTWO='2'\nTHREE = \"3\"")))
+	envMap, err := Parse(bytes.NewReader([]byte("ONE=1\nTWO='2'\nTHREE = \"3\"")))
 	expectedValues := map[string]string{
 		"ONE":   "1",
 		"TWO":   "2",
@@ -206,15 +207,21 @@ func TestLoadQuotedEnv(t *testing.T) {
 
 func TestSubstitutions(t *testing.T) {
 	envFileName := "fixtures/substitutions.env"
+
+	presets := map[string]string{
+		"GLOBAL_OPTION": "global",
+	}
+
 	expectedValues := map[string]string{
 		"OPTION_A": "1",
 		"OPTION_B": "1",
 		"OPTION_C": "1",
 		"OPTION_D": "11",
 		"OPTION_E": "",
+		"OPTION_F": "global",
 	}
 
-	loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets)
+	loadEnvAndCompareValues(t, Load, envFileName, expectedValues, presets)
 }
 
 func TestExpanding(t *testing.T) {
@@ -267,7 +274,7 @@ func TestExpanding(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			env, _, err := Parse(strings.NewReader(tt.input))
+			env, err := Parse(strings.NewReader(tt.input))
 			if err != nil {
 				t.Errorf("Error: %s", err.Error())
 			}
@@ -285,7 +292,7 @@ func TestVariableStringValueSeparator(t *testing.T) {
 	want := map[string]string{
 		"TEST_URLS": "stratum+tcp://stratum.antpool.com:3333\nstratum+tcp://stratum.antpool.com:443",
 	}
-	got, _, err := Parse(strings.NewReader(input))
+	got, err := Parse(strings.NewReader(input))
 	if err != nil {
 		t.Error(err)
 	}
@@ -408,7 +415,7 @@ func TestParsing(t *testing.T) {
 	// it 'throws an error if line format is incorrect' do
 	// expect{env('lol$wut')}.to raise_error(Dotenv::FormatError)
 	badlyFormattedLine := "lol$wut"
-	_, _, err := Unmarshal(badlyFormattedLine)
+	_, err := Unmarshal(badlyFormattedLine)
 	if err == nil {
 		t.Errorf("Expected \"%v\" to return error, but it didn't", badlyFormattedLine)
 	}
@@ -452,7 +459,7 @@ func TestLinesToIgnore(t *testing.T) {
 
 func TestErrorReadDirectory(t *testing.T) {
 	envFileName := "fixtures/"
-	envMap, _, err := Read(envFileName)
+	envMap, err := Read(envFileName)
 
 	if err == nil {
 		t.Errorf("Expected error, got %v", envMap)
@@ -461,7 +468,7 @@ func TestErrorReadDirectory(t *testing.T) {
 
 func TestErrorParsing(t *testing.T) {
 	envFileName := "fixtures/invalid1.env"
-	envMap, _, err := Read(envFileName)
+	envMap, err := Read(envFileName)
 	if err == nil {
 		t.Errorf("Expected error, got %v", envMap)
 	}
@@ -470,6 +477,9 @@ func TestErrorParsing(t *testing.T) {
 func TestComments(t *testing.T) {
 	envFileName := "fixtures/comments.env"
 	expectedValues := map[string]string{
+		"qux":  "thud",
+		"thud": "fred#qux",
+		"fred": "qux#baz",
 		"foo":  "bar",
 		"bar":  "foo#baz",
 		"baz":  "foo",
@@ -480,7 +490,7 @@ func TestComments(t *testing.T) {
 
 func TestWrite(t *testing.T) {
 	writeAndCompare := func(env string, expected string) {
-		envMap, _, _ := Unmarshal(env)
+		envMap, _ := Unmarshal(env)
 		actual, _ := Marshal(envMap)
 		if expected != actual {
 			t.Errorf("Expected '%v' (%v) to write as '%v', got '%v' instead.", env, envMap, expected, actual)
@@ -501,13 +511,14 @@ func TestWrite(t *testing.T) {
 	writeAndCompare("foo=bar\nbaz=buzz", "baz=\"buzz\"\nfoo=\"bar\"")
 	// integers should not be quoted
 	writeAndCompare(`key="10"`, `key=10`)
+
 }
 
 func TestRoundtrip(t *testing.T) {
 	fixtures := []string{"equals.env", "exported.env", "plain.env", "quoted.env"}
 	for _, fixture := range fixtures {
 		fixtureFilename := fmt.Sprintf("fixtures/%s", fixture)
-		env, _, err := readFile(fixtureFilename)
+		env, err := readFile(fixtureFilename)
 		if err != nil {
 			t.Errorf("Expected '%s' to read without error (%v)", fixtureFilename, err)
 		}
@@ -515,7 +526,7 @@ func TestRoundtrip(t *testing.T) {
 		if err != nil {
 			t.Errorf("Expected '%s' to Marshal (%v)", fixtureFilename, err)
 		}
-		roundtripped, _, err := Unmarshal(rep)
+		roundtripped, err := Unmarshal(rep)
 		if err != nil {
 			t.Errorf("Expected '%s' to Mashal and Unmarshal (%v)", fixtureFilename, err)
 		}
@@ -561,7 +572,7 @@ func TestTrailingNewlines(t *testing.T) {
 
 	for n, c := range cases {
 		t.Run(n, func(t *testing.T) {
-			result, _, err := Unmarshal(c.input)
+			result, err := Unmarshal(c.input)
 			if err != nil {
 				t.Errorf("Input: %q Unexpected error:\t%q", c.input, err)
 			}
@@ -622,7 +633,7 @@ func TestWhitespace(t *testing.T) {
 
 	for n, c := range cases {
 		t.Run(n, func(t *testing.T) {
-			result, _, err := Unmarshal(c.input)
+			result, err := Unmarshal(c.input)
 			if err != nil {
 				t.Errorf("Input: %q Unexpected error:\t%q", c.input, err)
 			}
@@ -632,78 +643,3 @@ func TestWhitespace(t *testing.T) {
 		})
 	}
 }
-
-func TestModfiers(t *testing.T) {
-	cases := map[string]struct {
-		input     string
-		key       string
-		value     string
-		modifiers map[string]string
-	}{
-		"No Modifier": {
-			input: "A=a",
-			key:   "A",
-			value: "a",
-		},
-		"With comment": {
-			input: "A=a # my comment",
-			key:   "A",
-			value: "a",
-		},
-		"With single modifier": {
-			input: "A=a # foo=bar",
-			key:   "A",
-			value: "a",
-			modifiers: map[string]string{
-				"foo": "bar",
-			},
-		},
-		"With multiple modifiers": {
-			input: "A=a # foo=bar length=10",
-			key:   "A",
-			value: "a",
-			modifiers: map[string]string{
-				"foo":    "bar",
-				"length": "10",
-			},
-		},
-		"With quoted var": {
-			input: "A='a' # foo=bar",
-			key:   "A",
-			value: "a",
-			modifiers: map[string]string{
-				"foo": "bar",
-			},
-		},
-		"With quoted var 2": {
-			input: "A='a' # foo=bar\nB=b",
-			key:   "A",
-			value: "a",
-			modifiers: map[string]string{
-				"foo": "bar",
-			},
-		},
-	}
-
-	for n, c := range cases {
-		t.Run(n, func(t *testing.T) {
-			values, modifiers, err := Unmarshal(c.input)
-			if err != nil {
-				t.Errorf("Input: %q Unexpected error:\t%q", c.input, err)
-			}
-			if values[c.key] != c.value {
-				t.Errorf("Input %q Expected:\t %q/%q\nGot:\t %q", c.input, c.key, c.value, values)
-			}
-
-			if modifiers[c.key] == nil && c.modifiers != nil {
-				t.Errorf("Input %q Expected modifiers\n Got: none", c.input)
-			} else {
-				for k, v := range c.modifiers {
-					if modifiers[c.key][k] != v {
-						t.Errorf("Input %q Expected modifier %s=%s\n Got: %s=%s", c.input, k, v, k, modifiers[c.key][k])
-					}
-				}
-			}
-		})
-	}
-}

parser.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"os"
 	"regexp"
 	"strings"
 	"unicode"
@@ -17,7 +18,7 @@ const (
 	exportPrefix = "export"
 )
 
-func parseBytes(src []byte, vars map[string]string, modifiers map[string]map[string]string) error {
+func parseBytes(src []byte, out map[string]string) error {
 	src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1)
 	cutset := src
 	for {
@@ -32,13 +33,12 @@ func parseBytes(src []byte, vars map[string]string, modifiers map[string]map[str
 			return err
 		}
 
-		value, mods, left, err := extractVarValue(left, vars)
+		value, left, err := extractVarValue(left, out)
 		if err != nil {
 			return err
 		}
 
-		vars[key] = value
-		modifiers[key] = mods
+		out[key] = value
 		cutset = left
 	}
 
@@ -118,34 +118,34 @@ loop:
 }
 
 // extractVarValue extracts variable value and returns rest of slice
-func extractVarValue(src []byte, vars map[string]string) (value string, modifiers map[string]string, rest []byte, err error) {
+func extractVarValue(src []byte, vars map[string]string) (value string, rest []byte, err error) {
 	quote, hasPrefix := hasQuotePrefix(src)
+	if !hasPrefix {
 		// unquoted value - read until end of line
 		endOfLine := bytes.IndexFunc(src, isLineEnd)
+
 		// Hit EOF without a trailing newline
 		if endOfLine == -1 {
 			endOfLine = len(src)
 
 			if endOfLine == 0 {
-			return "", nil, nil, nil
+				return "", nil, nil
 			}
 		}
-	if !hasPrefix {
+
 		// Convert line to rune away to do accurate countback of runes
 		line := []rune(string(src[0:endOfLine]))
 
 		// Assume end of line is end of var
 		endOfVar := len(line)
 		if endOfVar == 0 {
-			return "", nil, src[endOfLine:], nil
+			return "", src[endOfLine:], nil
 		}
 
-		comment := ""
 		// Work backwards to check if the line ends in whitespace then
-		// a comment (ie asdasd # some comment)
-		for i := endOfVar - 1; i >= 0; i-- {
-			if line[i] == charComment && i > 0 {
-				comment = string(line[i+1:])
+		// a comment, ie: foo=bar # baz # other
+		for i := 0; i < endOfVar; i++ {
+			if line[i] == charComment && i < endOfVar {
 				if isSpace(line[i-1]) {
 					endOfVar = i
 					break
@@ -155,7 +155,7 @@ func extractVarValue(src []byte, vars map[string]string) (value string, modifier
 
 		trimmed := strings.TrimFunc(string(line[0:endOfVar]), isSpace)
 
-		return expandVariables(trimmed, vars), extractModifiers(comment), src[endOfLine:], nil
+		return expandVariables(trimmed, vars), src[endOfLine:], nil
 	}
 
 	// lookup quoted string terminator
@@ -178,11 +178,7 @@ func extractVarValue(src []byte, vars map[string]string) (value string, modifier
 			value = expandVariables(expandEscapes(value), vars)
 		}
 
-		var mods map[string]string
-		if endOfLine > i {
-			mods = extractModifiers(string(src[i+1 : endOfLine]))
-		}
-		return value, mods, src[i+1:], nil
+		return value, src[i+1:], nil
 	}
 
 	// return formatted error if quoted string is not terminated
@@ -191,24 +187,7 @@ func extractVarValue(src []byte, vars map[string]string) (value string, modifier
 		valEndIndex = len(src)
 	}
 
-	return "", nil, nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex])
-}
-
-func extractModifiers(comment string) map[string]string {
-	if comment == "" {
-		return nil
-	}
-	comment = strings.TrimSpace(comment)
-	kvpairs := strings.Split(comment, " ")
-	mods := make(map[string]string)
-	for _, kv := range kvpairs {
-		kvsplit := strings.Split(kv, "=")
-		if len(kvsplit) != 2 {
-			continue
-		}
-		mods[kvsplit[0]] = kvsplit[1]
-	}
-	return mods
+	return "", nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex])
 }
 
 func expandEscapes(str string) string {
@@ -286,6 +265,12 @@ func expandVariables(v string, m map[string]string) string {
 		if submatch[1] == "\\" || submatch[2] == "(" {
 			return submatch[0][1:]
 		} else if submatch[4] != "" {
+			if val, ok := m[submatch[4]]; ok {
+				return val
+			}
+			if val, ok := os.LookupEnv(submatch[4]); ok {
+				return val
+			}
 			return m[submatch[4]]
 		}
 		return s