diff --git a/pkg/godotenv/fixtures/comments.env b/pkg/godotenv/fixtures/comments.env new file mode 100644 index 00000000..af9781fa --- /dev/null +++ b/pkg/godotenv/fixtures/comments.env @@ -0,0 +1,4 @@ +# Full line comment +foo=bar # baz +bar=foo#baz +baz="foo"#bar diff --git a/pkg/godotenv/fixtures/equals.env b/pkg/godotenv/fixtures/equals.env new file mode 100644 index 00000000..00c9809e --- /dev/null +++ b/pkg/godotenv/fixtures/equals.env @@ -0,0 +1 @@ +export OPTION_A='postgres://localhost:5432/database?sslmode=disable' diff --git a/pkg/godotenv/fixtures/exported.env b/pkg/godotenv/fixtures/exported.env new file mode 100644 index 00000000..5821377c --- /dev/null +++ b/pkg/godotenv/fixtures/exported.env @@ -0,0 +1,2 @@ +export OPTION_A=2 +export OPTION_B='\n' diff --git a/pkg/godotenv/fixtures/invalid1.env b/pkg/godotenv/fixtures/invalid1.env new file mode 100644 index 00000000..38f7e0e8 --- /dev/null +++ b/pkg/godotenv/fixtures/invalid1.env @@ -0,0 +1,2 @@ +INVALID LINE +foo=bar diff --git a/pkg/godotenv/fixtures/plain.env b/pkg/godotenv/fixtures/plain.env new file mode 100644 index 00000000..4c341c8a --- /dev/null +++ b/pkg/godotenv/fixtures/plain.env @@ -0,0 +1,8 @@ +OPTION_A=1 +OPTION_B=2 +OPTION_C= 3 +OPTION_D =4 +OPTION_E = 5 +OPTION_F = +OPTION_G= +OPTION_H=1 2 diff --git a/pkg/godotenv/fixtures/quoted.env b/pkg/godotenv/fixtures/quoted.env new file mode 100644 index 00000000..79589337 --- /dev/null +++ b/pkg/godotenv/fixtures/quoted.env @@ -0,0 +1,19 @@ +OPTION_A='1' +OPTION_B='2' +OPTION_C='' +OPTION_D='\n' +OPTION_E="1" +OPTION_F="2" +OPTION_G="" +OPTION_H="\n" +OPTION_I = "echo 'asd'" +OPTION_J='line 1 +line 2' +OPTION_K='line one +this is \'quoted\' +one more line' +OPTION_L="line 1 +line 2" +OPTION_M="line one +this is \"quoted\" +one more line" diff --git a/pkg/godotenv/fixtures/substitutions.env b/pkg/godotenv/fixtures/substitutions.env new file mode 100644 index 00000000..44337a99 --- /dev/null +++ b/pkg/godotenv/fixtures/substitutions.env @@ -0,0 +1,5 @@ +OPTION_A=1 +OPTION_B=${OPTION_A} +OPTION_C=$OPTION_B +OPTION_D=${OPTION_A}${OPTION_B} +OPTION_E=${OPTION_NOT_DEFINED} diff --git a/pkg/godotenv/godotenv.go b/pkg/godotenv/godotenv.go new file mode 100644 index 00000000..61b0ebba --- /dev/null +++ b/pkg/godotenv/godotenv.go @@ -0,0 +1,228 @@ +// Package godotenv is a go port of the ruby dotenv library (https://github.com/bkeepers/dotenv) +// +// Examples/readme can be found on the GitHub page at https://github.com/joho/godotenv +// +// The TL;DR is that you make a .env file that looks something like +// +// SOME_ENV_VAR=somevalue +// +// and then in your go code you can call +// +// godotenv.Load() +// +// and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR") +package godotenv + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "sort" + "strconv" + "strings" +) + +const doubleQuoteSpecialChars = "\\\n\r\"!$`" + +// Parse reads an env file from io.Reader, returning a map of keys and values. +func Parse(r io.Reader) (map[string]string, error) { + var buf bytes.Buffer + _, err := io.Copy(&buf, r) + if err != nil { + return nil, err + } + + return UnmarshalBytes(buf.Bytes()) +} + +// Load will read your env file(s) and load them into ENV for this process. +// +// Call this function as close as possible to the start of your program (ideally in main). +// +// If you call Load without any args it will default to loading .env in the current path. +// +// You can otherwise tell it which files to load (there can be more than one) like: +// +// godotenv.Load("fileone", "filetwo") +// +// It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults. +func Load(filenames ...string) (err error) { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + err = loadFile(filename, false) + if err != nil { + return // return early on a spazout + } + } + return +} + +// Overload will read your env file(s) and load them into ENV for this process. +// +// Call this function as close as possible to the start of your program (ideally in main). +// +// If you call Overload without any args it will default to loading .env in the current path. +// +// You can otherwise tell it which files to load (there can be more than one) like: +// +// godotenv.Overload("fileone", "filetwo") +// +// It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefully set all vars. +func Overload(filenames ...string) (err error) { + filenames = filenamesOrDefault(filenames) + + for _, filename := range filenames { + err = loadFile(filename, true) + if err != nil { + return // return early on a spazout + } + } + return +} + +// Read all env (with same file loading semantics as Load) but return values as +// a map rather than automatically writing values into env +func Read(filenames ...string) (envMap map[string]string, err error) { + filenames = filenamesOrDefault(filenames) + envMap = make(map[string]string) + + for _, filename := range filenames { + individualEnvMap, individualErr := readFile(filename) + + if individualErr != nil { + err = individualErr + return // return early on a spazout + } + + for key, value := range individualEnvMap { + envMap[key] = value + } + } + + return +} + +// Unmarshal reads an env file from a string, returning a map of keys and values. +func Unmarshal(str string) (envMap map[string]string, err error) { + return UnmarshalBytes([]byte(str)) +} + +// UnmarshalBytes parses env file from byte slice of chars, returning a map of keys and values. +func UnmarshalBytes(src []byte) (map[string]string, error) { + out := make(map[string]string) + err := parseBytes(src, out) + + return out, err +} + +// Exec loads env vars from the specified filenames (empty map falls back to default) +// then executes the cmd specified. +// +// Simply hooks up os.Stdin/err/out to the command and calls Run(). +// +// If you want more fine grained control over your command it's recommended +// that you use `Load()`, `Overload()` or `Read()` and the `os/exec` package yourself. +func Exec(filenames []string, cmd string, cmdArgs []string, overload bool) error { + op := Load + if overload { + op = Overload + } + if err := op(filenames...); err != nil { + return err + } + + command := exec.Command(cmd, cmdArgs...) + command.Stdin = os.Stdin + command.Stdout = os.Stdout + command.Stderr = os.Stderr + return command.Run() +} + +// Write serializes the given environment and writes it to a file. +func Write(envMap map[string]string, filename string) error { + content, err := Marshal(envMap) + if err != nil { + return err + } + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + _, err = file.WriteString(content + "\n") + if err != nil { + return err + } + return file.Sync() +} + +// Marshal outputs the given environment as a dotenv-formatted environment file. +// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped. +func Marshal(envMap map[string]string) (string, error) { + lines := make([]string, 0, len(envMap)) + for k, v := range envMap { + if d, err := strconv.Atoi(v); err == nil { + lines = append(lines, fmt.Sprintf(`%s=%d`, k, d)) + } else { + lines = append(lines, fmt.Sprintf(`%s="%s"`, k, doubleQuoteEscape(v))) + } + } + sort.Strings(lines) + return strings.Join(lines, "\n"), nil +} + +func filenamesOrDefault(filenames []string) []string { + if len(filenames) == 0 { + return []string{".env"} + } + return filenames +} + +func loadFile(filename string, overload bool) error { + envMap, err := readFile(filename) + if err != nil { + return err + } + + currentEnv := map[string]bool{} + rawEnv := os.Environ() + for _, rawEnvLine := range rawEnv { + key := strings.Split(rawEnvLine, "=")[0] + currentEnv[key] = true + } + + for key, value := range envMap { + if !currentEnv[key] || overload { + _ = os.Setenv(key, value) + } + } + + return nil +} + +func readFile(filename string) (envMap map[string]string, err error) { + file, err := os.Open(filename) + if err != nil { + return + } + defer file.Close() + + return Parse(file) +} + +func doubleQuoteEscape(line string) string { + for _, c := range doubleQuoteSpecialChars { + toReplace := "\\" + string(c) + if c == '\n' { + toReplace = `\n` + } + if c == '\r' { + toReplace = `\r` + } + line = strings.Replace(line, string(c), toReplace, -1) + } + return line +} diff --git a/pkg/godotenv/godotenv_test.go b/pkg/godotenv/godotenv_test.go new file mode 100644 index 00000000..9b1c99ea --- /dev/null +++ b/pkg/godotenv/godotenv_test.go @@ -0,0 +1,636 @@ +package godotenv + +import ( + "bytes" + "fmt" + "os" + "reflect" + "strings" + "testing" +) + +var noopPresets = make(map[string]string) + +func parseAndCompare(t *testing.T, rawEnvLine string, expectedKey string, expectedValue string) { + result, err := Unmarshal(rawEnvLine) + + if err != nil { + t.Errorf("Expected %q to parse as %q: %q, errored %q", rawEnvLine, expectedKey, expectedValue, err) + return + } + if result[expectedKey] != expectedValue { + t.Errorf("Expected '%v' to parse as '%v' => '%v', got %q instead", rawEnvLine, expectedKey, expectedValue, result) + } +} + +func loadEnvAndCompareValues(t *testing.T, loader func(files ...string) error, envFileName string, expectedValues map[string]string, presets map[string]string) { + // first up, clear the env + os.Clearenv() + + for k, v := range presets { + os.Setenv(k, v) + } + + err := loader(envFileName) + if err != nil { + t.Fatalf("Error loading %v", envFileName) + } + + for k := range expectedValues { + envValue := os.Getenv(k) + v := expectedValues[k] + if envValue != v { + t.Errorf("Mismatch for key '%v': expected '%#v' got '%#v'", k, v, envValue) + } + } +} + +func TestLoadWithNoArgsLoadsDotEnv(t *testing.T) { + err := Load() + pathError := err.(*os.PathError) + if pathError == nil || pathError.Op != "open" || pathError.Path != ".env" { + t.Errorf("Didn't try and open .env by default") + } +} + +func TestOverloadWithNoArgsOverloadsDotEnv(t *testing.T) { + err := Overload() + pathError := err.(*os.PathError) + if pathError == nil || pathError.Op != "open" || pathError.Path != ".env" { + t.Errorf("Didn't try and open .env by default") + } +} + +func TestLoadFileNotFound(t *testing.T) { + err := Load("somefilethatwillneverexistever.env") + if err == nil { + t.Error("File wasn't found but Load didn't return an error") + } +} + +func TestOverloadFileNotFound(t *testing.T) { + err := Overload("somefilethatwillneverexistever.env") + if err == nil { + t.Error("File wasn't found but Overload didn't return an error") + } +} + +func TestReadPlainEnv(t *testing.T) { + envFileName := "fixtures/plain.env" + expectedValues := map[string]string{ + "OPTION_A": "1", + "OPTION_B": "2", + "OPTION_C": "3", + "OPTION_D": "4", + "OPTION_E": "5", + "OPTION_F": "", + "OPTION_G": "", + "OPTION_H": "1 2", + } + + envMap, err := Read(envFileName) + if err != nil { + t.Error("Error reading file") + } + + if len(envMap) != len(expectedValues) { + t.Error("Didn't get the right size map back") + } + + for key, value := range expectedValues { + if envMap[key] != value { + t.Error("Read got one of the keys wrong") + } + } +} + +func TestParse(t *testing.T) { + envMap, err := Parse(bytes.NewReader([]byte("ONE=1\nTWO='2'\nTHREE = \"3\""))) + expectedValues := map[string]string{ + "ONE": "1", + "TWO": "2", + "THREE": "3", + } + if err != nil { + t.Fatalf("error parsing env: %v", err) + } + for key, value := range expectedValues { + if envMap[key] != value { + t.Errorf("expected %s to be %s, got %s", key, value, envMap[key]) + } + } +} + +func TestLoadDoesNotOverride(t *testing.T) { + envFileName := "fixtures/plain.env" + + // ensure NO overload + presets := map[string]string{ + "OPTION_A": "do_not_override", + "OPTION_B": "", + } + + expectedValues := map[string]string{ + "OPTION_A": "do_not_override", + "OPTION_B": "", + } + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, presets) +} + +func TestOverloadDoesOverride(t *testing.T) { + envFileName := "fixtures/plain.env" + + // ensure NO overload + presets := map[string]string{ + "OPTION_A": "do_not_override", + } + + expectedValues := map[string]string{ + "OPTION_A": "1", + } + loadEnvAndCompareValues(t, Overload, envFileName, expectedValues, presets) +} + +func TestLoadPlainEnv(t *testing.T) { + envFileName := "fixtures/plain.env" + expectedValues := map[string]string{ + "OPTION_A": "1", + "OPTION_B": "2", + "OPTION_C": "3", + "OPTION_D": "4", + "OPTION_E": "5", + "OPTION_H": "1 2", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + +func TestLoadExportedEnv(t *testing.T) { + envFileName := "fixtures/exported.env" + expectedValues := map[string]string{ + "OPTION_A": "2", + "OPTION_B": "\\n", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + +func TestLoadEqualsEnv(t *testing.T) { + envFileName := "fixtures/equals.env" + expectedValues := map[string]string{ + "OPTION_A": "postgres://localhost:5432/database?sslmode=disable", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + +func TestLoadQuotedEnv(t *testing.T) { + envFileName := "fixtures/quoted.env" + expectedValues := map[string]string{ + "OPTION_A": "1", + "OPTION_B": "2", + "OPTION_C": "", + "OPTION_D": "\\n", + "OPTION_E": "1", + "OPTION_F": "2", + "OPTION_G": "", + "OPTION_H": "\n", + "OPTION_I": "echo 'asd'", + "OPTION_J": "line 1\nline 2", + "OPTION_K": "line one\nthis is \\'quoted\\'\none more line", + "OPTION_L": "line 1\nline 2", + "OPTION_M": "line one\nthis is \"quoted\"\none more line", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + +func TestSubstitutions(t *testing.T) { + envFileName := "fixtures/substitutions.env" + expectedValues := map[string]string{ + "OPTION_A": "1", + "OPTION_B": "1", + "OPTION_C": "1", + "OPTION_D": "11", + "OPTION_E": "", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + +func TestExpanding(t *testing.T) { + tests := []struct { + name string + input string + expected map[string]string + }{ + { + "expands variables found in values", + "FOO=test\nBAR=$FOO", + map[string]string{"FOO": "test", "BAR": "test"}, + }, + { + "parses variables wrapped in brackets", + "FOO=test\nBAR=${FOO}bar", + map[string]string{"FOO": "test", "BAR": "testbar"}, + }, + { + "expands undefined variables to an empty string", + "BAR=$FOO", + map[string]string{"BAR": ""}, + }, + { + "expands variables in double quoted strings", + "FOO=test\nBAR=\"quote $FOO\"", + map[string]string{"FOO": "test", "BAR": "quote test"}, + }, + { + "does not expand variables in single quoted strings", + "BAR='quote $FOO'", + map[string]string{"BAR": "quote $FOO"}, + }, + { + "does not expand escaped variables", + `FOO="foo\$BAR"`, + map[string]string{"FOO": "foo$BAR"}, + }, + { + "does not expand escaped variables", + `FOO="foo\${BAR}"`, + map[string]string{"FOO": "foo${BAR}"}, + }, + { + "does not expand escaped variables", + "FOO=test\nBAR=\"foo\\${FOO} ${FOO}\"", + map[string]string{"FOO": "test", "BAR": "foo${FOO} test"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + env, err := Parse(strings.NewReader(tt.input)) + if err != nil { + t.Errorf("Error: %s", err.Error()) + } + for k, v := range tt.expected { + if strings.Compare(env[k], v) != 0 { + t.Errorf("Expected: %s, Actual: %s", v, env[k]) + } + } + }) + } +} + +func TestVariableStringValueSeparator(t *testing.T) { + input := "TEST_URLS=\"stratum+tcp://stratum.antpool.com:3333\nstratum+tcp://stratum.antpool.com:443\"" + want := map[string]string{ + "TEST_URLS": "stratum+tcp://stratum.antpool.com:3333\nstratum+tcp://stratum.antpool.com:443", + } + got, err := Parse(strings.NewReader(input)) + if err != nil { + t.Error(err) + } + + if len(got) != len(want) { + t.Fatalf( + "unexpected value:\nwant:\n\t%#v\n\ngot:\n\t%#v", want, got) + } + + for k, wantVal := range want { + gotVal, ok := got[k] + if !ok { + t.Fatalf("key %q doesn't present in result", k) + } + if wantVal != gotVal { + t.Fatalf( + "mismatch in %q value:\nwant:\n\t%s\n\ngot:\n\t%s", k, + wantVal, gotVal) + } + } +} + +func TestActualEnvVarsAreLeftAlone(t *testing.T) { + os.Clearenv() + os.Setenv("OPTION_A", "actualenv") + _ = Load("fixtures/plain.env") + + if os.Getenv("OPTION_A") != "actualenv" { + t.Error("An ENV var set earlier was overwritten") + } +} + +func TestParsing(t *testing.T) { + // unquoted values + parseAndCompare(t, "FOO=bar", "FOO", "bar") + + // parses values with spaces around equal sign + parseAndCompare(t, "FOO =bar", "FOO", "bar") + parseAndCompare(t, "FOO= bar", "FOO", "bar") + + // parses double quoted values + parseAndCompare(t, `FOO="bar"`, "FOO", "bar") + + // parses single quoted values + parseAndCompare(t, "FOO='bar'", "FOO", "bar") + + // parses escaped double quotes + parseAndCompare(t, `FOO="escaped\"bar"`, "FOO", `escaped"bar`) + + // parses single quotes inside double quotes + parseAndCompare(t, `FOO="'d'"`, "FOO", `'d'`) + + // parses yaml style options + parseAndCompare(t, "OPTION_A: 1", "OPTION_A", "1") + + //parses yaml values with equal signs + parseAndCompare(t, "OPTION_A: Foo=bar", "OPTION_A", "Foo=bar") + + // parses non-yaml options with colons + parseAndCompare(t, "OPTION_A=1:B", "OPTION_A", "1:B") + + // parses export keyword + parseAndCompare(t, "export OPTION_A=2", "OPTION_A", "2") + parseAndCompare(t, `export OPTION_B='\n'`, "OPTION_B", "\\n") + parseAndCompare(t, "export exportFoo=2", "exportFoo", "2") + parseAndCompare(t, "exportFOO=2", "exportFOO", "2") + parseAndCompare(t, "export_FOO =2", "export_FOO", "2") + parseAndCompare(t, "export.FOO= 2", "export.FOO", "2") + parseAndCompare(t, "export\tOPTION_A=2", "OPTION_A", "2") + parseAndCompare(t, " export OPTION_A=2", "OPTION_A", "2") + parseAndCompare(t, "\texport OPTION_A=2", "OPTION_A", "2") + + // it 'expands newlines in quoted strings' do + // expect(env('FOO="bar\nbaz"')).to eql('FOO' => "bar\nbaz") + parseAndCompare(t, `FOO="bar\nbaz"`, "FOO", "bar\nbaz") + + // it 'parses variables with "." in the name' do + // expect(env('FOO.BAR=foobar')).to eql('FOO.BAR' => 'foobar') + parseAndCompare(t, "FOO.BAR=foobar", "FOO.BAR", "foobar") + + // it 'parses variables with several "=" in the value' do + // expect(env('FOO=foobar=')).to eql('FOO' => 'foobar=') + parseAndCompare(t, "FOO=foobar=", "FOO", "foobar=") + + // it 'strips unquoted values' do + // expect(env('foo=bar ')).to eql('foo' => 'bar') # not 'bar ' + parseAndCompare(t, "FOO=bar ", "FOO", "bar") + + // unquoted internal whitespace is preserved + parseAndCompare(t, `KEY=value value`, "KEY", "value value") + + // it 'ignores inline comments' do + // expect(env("foo=bar # this is foo")).to eql('foo' => 'bar') + parseAndCompare(t, "FOO=bar # this is foo", "FOO", "bar") + + // it 'allows # in quoted value' do + // expect(env('foo="bar#baz" # comment')).to eql('foo' => 'bar#baz') + parseAndCompare(t, `FOO="bar#baz" # comment`, "FOO", "bar#baz") + parseAndCompare(t, "FOO='bar#baz' # comment", "FOO", "bar#baz") + parseAndCompare(t, `FOO="bar#baz#bang" # comment`, "FOO", "bar#baz#bang") + + // it 'parses # in quoted values' do + // expect(env('foo="ba#r"')).to eql('foo' => 'ba#r') + // expect(env("foo='ba#r'")).to eql('foo' => 'ba#r') + parseAndCompare(t, `FOO="ba#r"`, "FOO", "ba#r") + parseAndCompare(t, "FOO='ba#r'", "FOO", "ba#r") + + //newlines and backslashes should be escaped + parseAndCompare(t, `FOO="bar\n\ b\az"`, "FOO", "bar\n baz") + parseAndCompare(t, `FOO="bar\\\n\ b\az"`, "FOO", "bar\\\n baz") + parseAndCompare(t, `FOO="bar\\r\ b\az"`, "FOO", "bar\\r baz") + + parseAndCompare(t, `="value"`, "", "value") + + // unquoted whitespace around keys should be ignored + parseAndCompare(t, " KEY =value", "KEY", "value") + parseAndCompare(t, " KEY=value", "KEY", "value") + parseAndCompare(t, "\tKEY=value", "KEY", "value") + + // it 'throws an error if line format is incorrect' do + // expect{env('lol$wut')}.to raise_error(Dotenv::FormatError) + badlyFormattedLine := "lol$wut" + _, err := Unmarshal(badlyFormattedLine) + if err == nil { + t.Errorf("Expected \"%v\" to return error, but it didn't", badlyFormattedLine) + } +} + +func TestLinesToIgnore(t *testing.T) { + cases := map[string]struct { + input string + want string + }{ + "Line with nothing but line break": { + input: "\n", + }, + "Line with nothing but windows-style line break": { + input: "\r\n", + }, + "Line full of whitespace": { + input: "\t\t ", + }, + "Comment": { + input: "# Comment", + }, + "Indented comment": { + input: "\t # comment", + }, + "non-ignored value": { + input: `export OPTION_B='\n'`, + want: `export OPTION_B='\n'`, + }, + } + + for n, c := range cases { + t.Run(n, func(t *testing.T) { + got := string(getStatementStart([]byte(c.input))) + if got != c.want { + t.Errorf("Expected:\t %q\nGot:\t %q", c.want, got) + } + }) + } +} + +func TestErrorReadDirectory(t *testing.T) { + envFileName := "fixtures/" + envMap, err := Read(envFileName) + + if err == nil { + t.Errorf("Expected error, got %v", envMap) + } +} + +func TestErrorParsing(t *testing.T) { + envFileName := "fixtures/invalid1.env" + envMap, err := Read(envFileName) + if err == nil { + t.Errorf("Expected error, got %v", envMap) + } +} + +func TestComments(t *testing.T) { + envFileName := "fixtures/comments.env" + expectedValues := map[string]string{ + "foo": "bar", + "bar": "foo#baz", + "baz": "foo", + } + + loadEnvAndCompareValues(t, Load, envFileName, expectedValues, noopPresets) +} + +func TestWrite(t *testing.T) { + writeAndCompare := func(env string, expected string) { + envMap, _ := Unmarshal(env) + actual, _ := Marshal(envMap) + if expected != actual { + t.Errorf("Expected '%v' (%v) to write as '%v', got '%v' instead.", env, envMap, expected, actual) + } + } + //just test some single lines to show the general idea + //TestRoundtrip makes most of the good assertions + + //values are always double-quoted + writeAndCompare(`key=value`, `key="value"`) + //double-quotes are escaped + writeAndCompare(`key=va"lu"e`, `key="va\"lu\"e"`) + //but single quotes are left alone + writeAndCompare(`key=va'lu'e`, `key="va'lu'e"`) + // newlines, backslashes, and some other special chars are escaped + writeAndCompare(`foo="\n\r\\r!"`, `foo="\n\r\\r\!"`) + // lines should be sorted + writeAndCompare("foo=bar\nbaz=buzz", "baz=\"buzz\"\nfoo=\"bar\"") + // integers should not be quoted + writeAndCompare(`key="10"`, `key=10`) + +} + +func TestRoundtrip(t *testing.T) { + fixtures := []string{"equals.env", "exported.env", "plain.env", "quoted.env"} + for _, fixture := range fixtures { + fixtureFilename := fmt.Sprintf("fixtures/%s", fixture) + env, err := readFile(fixtureFilename) + if err != nil { + t.Errorf("Expected '%s' to read without error (%v)", fixtureFilename, err) + } + rep, err := Marshal(env) + if err != nil { + t.Errorf("Expected '%s' to Marshal (%v)", fixtureFilename, err) + } + roundtripped, err := Unmarshal(rep) + if err != nil { + t.Errorf("Expected '%s' to Mashal and Unmarshal (%v)", fixtureFilename, err) + } + if !reflect.DeepEqual(env, roundtripped) { + t.Errorf("Expected '%s' to roundtrip as '%v', got '%v' instead", fixtureFilename, env, roundtripped) + } + + } +} + +func TestTrailingNewlines(t *testing.T) { + cases := map[string]struct { + input string + key string + value string + }{ + "Simple value without trailing newline": { + input: "KEY=value", + key: "KEY", + value: "value", + }, + "Value with internal whitespace without trailing newline": { + input: "KEY=value value", + key: "KEY", + value: "value value", + }, + "Value with internal whitespace with trailing newline": { + input: "KEY=value value\n", + key: "KEY", + value: "value value", + }, + "YAML style - value with internal whitespace without trailing newline": { + input: "KEY: value value", + key: "KEY", + value: "value value", + }, + "YAML style - value with internal whitespace with trailing newline": { + input: "KEY: value value\n", + key: "KEY", + value: "value value", + }, + } + + for n, c := range cases { + t.Run(n, func(t *testing.T) { + result, err := Unmarshal(c.input) + if err != nil { + t.Errorf("Input: %q Unexpected error:\t%q", c.input, err) + } + if result[c.key] != c.value { + t.Errorf("Input %q Expected:\t %q/%q\nGot:\t %q", c.input, c.key, c.value, result) + } + }) + } +} + +func TestWhitespace(t *testing.T) { + cases := map[string]struct { + input string + key string + value string + }{ + "Leading whitespace": { + input: " A=a\n", + key: "A", + value: "a", + }, + "Leading tab": { + input: "\tA=a\n", + key: "A", + value: "a", + }, + "Leading mixed whitespace": { + input: " \t \t\n\t \t A=a\n", + key: "A", + value: "a", + }, + "Leading whitespace before export": { + input: " \t\t export A=a\n", + key: "A", + value: "a", + }, + "Trailing whitespace": { + input: "A=a \t \t\n", + key: "A", + value: "a", + }, + "Trailing whitespace with export": { + input: "export A=a\t \t \n", + key: "A", + value: "a", + }, + "No EOL": { + input: "A=a", + key: "A", + value: "a", + }, + "Trailing whitespace with no EOL": { + input: "A=a ", + key: "A", + value: "a", + }, + } + + for n, c := range cases { + t.Run(n, func(t *testing.T) { + result, err := Unmarshal(c.input) + if err != nil { + t.Errorf("Input: %q Unexpected error:\t%q", c.input, err) + } + if result[c.key] != c.value { + t.Errorf("Input %q Expected:\t %q/%q\nGot:\t %q", c.input, c.key, c.value, result) + } + }) + } +} diff --git a/pkg/godotenv/parser.go b/pkg/godotenv/parser.go new file mode 100644 index 00000000..cc709af8 --- /dev/null +++ b/pkg/godotenv/parser.go @@ -0,0 +1,271 @@ +package godotenv + +import ( + "bytes" + "errors" + "fmt" + "regexp" + "strings" + "unicode" +) + +const ( + charComment = '#' + prefixSingleQuote = '\'' + prefixDoubleQuote = '"' + + exportPrefix = "export" +) + +func parseBytes(src []byte, out map[string]string) error { + src = bytes.Replace(src, []byte("\r\n"), []byte("\n"), -1) + cutset := src + for { + cutset = getStatementStart(cutset) + if cutset == nil { + // reached end of file + break + } + + key, left, err := locateKeyName(cutset) + if err != nil { + return err + } + + value, left, err := extractVarValue(left, out) + if err != nil { + return err + } + + out[key] = value + cutset = left + } + + return nil +} + +// getStatementPosition returns position of statement begin. +// +// It skips any comment line or non-whitespace character. +func getStatementStart(src []byte) []byte { + pos := indexOfNonSpaceChar(src) + if pos == -1 { + return nil + } + + src = src[pos:] + if src[0] != charComment { + return src + } + + // skip comment section + pos = bytes.IndexFunc(src, isCharFunc('\n')) + if pos == -1 { + return nil + } + + return getStatementStart(src[pos:]) +} + +// locateKeyName locates and parses key name and returns rest of slice +func locateKeyName(src []byte) (key string, cutset []byte, err error) { + // trim "export" and space at beginning + src = bytes.TrimLeftFunc(src, isSpace) + if bytes.HasPrefix(src, []byte(exportPrefix)) { + trimmed := bytes.TrimPrefix(src, []byte(exportPrefix)) + if bytes.IndexFunc(trimmed, isSpace) == 0 { + src = bytes.TrimLeftFunc(trimmed, isSpace) + } + } + + // locate key name end and validate it in single loop + offset := 0 +loop: + for i, char := range src { + rchar := rune(char) + if isSpace(rchar) { + continue + } + + switch char { + case '=', ':': + // library also supports yaml-style value declaration + key = string(src[0:i]) + offset = i + 1 + break loop + case '_': + default: + // variable name should match [A-Za-z0-9_.] + if unicode.IsLetter(rchar) || unicode.IsNumber(rchar) || rchar == '.' { + continue + } + + return "", nil, fmt.Errorf( + `unexpected character %q in variable name near %q`, + string(char), string(src)) + } + } + + if len(src) == 0 { + return "", nil, errors.New("zero length string") + } + + // trim whitespace + key = strings.TrimRightFunc(key, unicode.IsSpace) + cutset = bytes.TrimLeftFunc(src[offset:], isSpace) + return key, cutset, nil +} + +// extractVarValue extracts variable value and returns rest of slice +func extractVarValue(src []byte, vars map[string]string) (value string, rest []byte, err error) { + quote, hasPrefix := hasQuotePrefix(src) + if !hasPrefix { + // unquoted value - read until end of line + endOfLine := bytes.IndexFunc(src, isLineEnd) + + // Hit EOF without a trailing newline + if endOfLine == -1 { + endOfLine = len(src) + + if endOfLine == 0 { + return "", nil, nil + } + } + + // Convert line to rune away to do accurate countback of runes + line := []rune(string(src[0:endOfLine])) + + // Assume end of line is end of var + endOfVar := len(line) + if endOfVar == 0 { + return "", src[endOfLine:], nil + } + + // Work backwards to check if the line ends in whitespace then + // a comment (ie asdasd # some comment) + for i := endOfVar - 1; i >= 0; i-- { + if line[i] == charComment && i > 0 { + if isSpace(line[i-1]) { + endOfVar = i + break + } + } + } + + trimmed := strings.TrimFunc(string(line[0:endOfVar]), isSpace) + + return expandVariables(trimmed, vars), src[endOfLine:], nil + } + + // lookup quoted string terminator + for i := 1; i < len(src); i++ { + if char := src[i]; char != quote { + continue + } + + // skip escaped quote symbol (\" or \', depends on quote) + if prevChar := src[i-1]; prevChar == '\\' { + continue + } + + // trim quotes + trimFunc := isCharFunc(rune(quote)) + value = string(bytes.TrimLeftFunc(bytes.TrimRightFunc(src[0:i], trimFunc), trimFunc)) + if quote == prefixDoubleQuote { + // unescape newlines for double quote (this is compat feature) + // and expand environment variables + value = expandVariables(expandEscapes(value), vars) + } + + return value, src[i+1:], nil + } + + // return formatted error if quoted string is not terminated + valEndIndex := bytes.IndexFunc(src, isCharFunc('\n')) + if valEndIndex == -1 { + valEndIndex = len(src) + } + + return "", nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex]) +} + +func expandEscapes(str string) string { + out := escapeRegex.ReplaceAllStringFunc(str, func(match string) string { + c := strings.TrimPrefix(match, `\`) + switch c { + case "n": + return "\n" + case "r": + return "\r" + default: + return match + } + }) + return unescapeCharsRegex.ReplaceAllString(out, "$1") +} + +func indexOfNonSpaceChar(src []byte) int { + return bytes.IndexFunc(src, func(r rune) bool { + return !unicode.IsSpace(r) + }) +} + +// hasQuotePrefix reports whether charset starts with single or double quote and returns quote character +func hasQuotePrefix(src []byte) (prefix byte, isQuored bool) { + if len(src) == 0 { + return 0, false + } + + switch prefix := src[0]; prefix { + case prefixDoubleQuote, prefixSingleQuote: + return prefix, true + default: + return 0, false + } +} + +func isCharFunc(char rune) func(rune) bool { + return func(v rune) bool { + return v == char + } +} + +// isSpace reports whether the rune is a space character but not line break character +// +// this differs from unicode.IsSpace, which also applies line break as space +func isSpace(r rune) bool { + switch r { + case '\t', '\v', '\f', '\r', ' ', 0x85, 0xA0: + return true + } + return false +} + +func isLineEnd(r rune) bool { + if r == '\n' || r == '\r' { + return true + } + return false +} + +var ( + escapeRegex = regexp.MustCompile(`\\.`) + expandVarRegex = regexp.MustCompile(`(\\)?(\$)(\()?\{?([A-Z0-9_]+)?\}?`) + unescapeCharsRegex = regexp.MustCompile(`\\([^$])`) +) + +func expandVariables(v string, m map[string]string) string { + return expandVarRegex.ReplaceAllStringFunc(v, func(s string) string { + submatch := expandVarRegex.FindStringSubmatch(s) + + if submatch == nil { + return s + } + if submatch[1] == "\\" || submatch[2] == "(" { + return submatch[0][1:] + } else if submatch[4] != "" { + return m[submatch[4]] + } + return s + }) +}