Support for passing build-time variables in build context

- The build-time variables are passed as environment-context for command(s)
run as part of the RUN primitve. These variables are not persisted in environment of
intermediate and final images when passed as context for RUN. The build environment
is prepended to the intermediate continer's command string for aiding cache lookups.
It also helps with build traceability. But this also makes the feature less secure from
point of view of passing build time secrets.

- The build-time variables also get used to expand the symbols used in certain
Dockerfile primitves like ADD, COPY, USER etc, without an explicit prior definiton using a
ENV primitive. These variables get persisted in the intermediate and final images
whenever they are expanded.

- The build-time variables are only expanded or passed to the RUN primtive if they
are defined in Dockerfile using the ARG primitive or belong to list of built-in variables.
HTTP_PROXY, HTTPS_PROXY, http_proxy, https_proxy, FTP_PROXY and NO_PROXY are built-in
variables that needn't be explicitly defined in Dockerfile to use this feature.

Signed-off-by: Madhav Puri <madhav.puri@gmail.com>
Upstream-commit: 54240f8da9992880e20a1508e9a6f0e59f2adef1
Component: engine

components/engine/builder/parser/line_parsers.go

@@ -42,15 +42,10 @@ func parseSubCommand(rest string) (*Node, map[string]bool, error) {
 	return &Node{Children: []*Node{child}}, nil, nil
 }
 
-// parse environment like statements. Note that this does *not* handle
-// variable interpolation, which will be handled in the evaluator.
-func parseNameVal(rest string, key string) (*Node, map[string]bool, error) {
-	// This is kind of tricky because we need to support the old
-	// variant:   KEY name value
-	// as well as the new one:    KEY name=value ...
-	// The trigger to know which one is being used will be whether we hit
-	// a space or = first.  space ==> old, "=" ==> new
-
+// helper to parse words (i.e space delimited or quoted strings) in a statement.
+// The quotes are preserved as part of this function and they are stripped later
+// as part of processWords().
+func parseWords(rest string) []string {
 	const (
 		inSpaces = iota // looking for start of a word
 		inWord
@@ -89,15 +84,6 @@ func parseNameVal(rest string, key string) (*Node, map[string]bool, error) {
 				phase = inSpaces
 				if blankOK || len(word) > 0 {
 					words = append(words, word)
-
-					// Look for = and if not there assume
-					// we're doing the old stuff and
-					// just read the rest of the line
-					if !strings.Contains(word, "=") {
-						word = strings.TrimSpace(rest[pos:])
-						words = append(words, word)
-						break
-					}
 				}
 				word = ""
 				blankOK = false
@@ -141,13 +127,26 @@ func parseNameVal(rest string, key string) (*Node, map[string]bool, error) {
 		}
 	}
 
+	return words
+}
+
+// parse environment like statements. Note that this does *not* handle
+// variable interpolation, which will be handled in the evaluator.
+func parseNameVal(rest string, key string) (*Node, map[string]bool, error) {
+	// This is kind of tricky because we need to support the old
+	// variant:   KEY name value
+	// as well as the new one:    KEY name=value ...
+	// The trigger to know which one is being used will be whether we hit
+	// a space or = first.  space ==> old, "=" ==> new
+
+	words := parseWords(rest)
 	if len(words) == 0 {
 		return nil, nil, nil
 	}
 
-	// Old format (KEY name value)
 	var rootnode *Node
 
+	// Old format (KEY name value)
 	if !strings.Contains(words[0], "=") {
 		node := &Node{}
 		rootnode = node
@@ -195,6 +194,38 @@ func parseLabel(rest string) (*Node, map[string]bool, error) {
 	return parseNameVal(rest, "LABEL")
 }
 
+// parses a statement containing one or more keyword definition(s) and/or
+// value assignments, like `name1 name2= name3="" name4=value`.
+// Note that this is a stricter format than the old format of assignment,
+// allowed by parseNameVal(), in a way that this only allows assignment of the
+// form `keyword=[<value>]` like  `name2=`, `name3=""`, and `name4=value` above.
+// In addition, a keyword definition alone is of the form `keyword` like `name1`
+// above. And the assignments `name2=` and `name3=""` are equivalent and
+// assign an empty value to the respective keywords.
+func parseNameOrNameVal(rest string) (*Node, map[string]bool, error) {
+	words := parseWords(rest)
+	if len(words) == 0 {
+		return nil, nil, nil
+	}
+
+	var (
+		rootnode *Node
+		prevNode *Node
+	)
+	for i, word := range words {
+		node := &Node{}
+		node.Value = word
+		if i == 0 {
+			rootnode = node
+		} else {
+			prevNode.Next = node
+		}
+		prevNode = node
+	}
+
+	return rootnode, nil, nil
+}
+
 // parses a whitespace-delimited set of arguments. The result is effectively a
 // linked list of string arguments.
 func parseStringsWhitespaceDelimited(rest string) (*Node, map[string]bool, error) {

components/engine/builder/parser/parser.go

@@ -62,6 +62,7 @@ func init() {
 		command.Expose:     parseStringsWhitespaceDelimited,
 		command.Volume:     parseMaybeJSONToList,
 		command.StopSignal: parseString,
+		command.Arg:        parseNameOrNameVal,
 	}
 }
 

components/engine/builder/parser/parser_test.go

@@ -73,3 +73,40 @@ func TestTestData(t *testing.T) {
 		}
 	}
 }
+
+func TestParseWords(t *testing.T) {
+	tests := []map[string][]string{
+		{
+			"input":  {"foo"},
+			"expect": {"foo"},
+		},
+		{
+			"input":  {"foo bar"},
+			"expect": {"foo", "bar"},
+		},
+		{
+			"input":  {"foo=bar"},
+			"expect": {"foo=bar"},
+		},
+		{
+			"input":  {"foo bar 'abc xyz'"},
+			"expect": {"foo", "bar", "'abc xyz'"},
+		},
+		{
+			"input":  {`foo bar "abc xyz"`},
+			"expect": {"foo", "bar", `"abc xyz"`},
+		},
+	}
+
+	for _, test := range tests {
+		words := parseWords(test["input"][0])
+		if len(words) != len(test["expect"]) {
+			t.Fatalf("length check failed. input: %v, expect: %v, output: %v", test["input"][0], test["expect"], words)
+		}
+		for i, word := range words {
+			if word != test["expect"][i] {
+				t.Fatalf("word check failed for word: %q. input: %v, expect: %v, output: %v", word, test["input"][0], test["expect"], words)
+			}
+		}
+	}
+}