From 28cc28338420aec4814b1093fc85b09c613947b1 Mon Sep 17 00:00:00 2001 From: Brandon Philips Date: Mon, 27 Jan 2014 12:16:45 -0800 Subject: [PATCH 001/403] fix(CONTRIBUTING): Developer Certificate of Origin w/o Grant In 7fb55f77250a9b52b0515a7e655ecc605ae452ca the DCO lost the grant so change the title back to just DCO. Docker-DCO-1.1-Signed-off-by: Brandon Philips (github: philips) Upstream-commit: 8458f99a387ae70cfaa1e43759919fe414c55505 Component: engine --- components/engine/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/CONTRIBUTING.md b/components/engine/CONTRIBUTING.md index f4acb3e716..7bd7ea60d5 100644 --- a/components/engine/CONTRIBUTING.md +++ b/components/engine/CONTRIBUTING.md @@ -113,7 +113,7 @@ pass it on as an open-source patch. The rules are pretty simple: if you can certify the below: ``` -Docker Developer Grant and Certificate of Origin 1.1 +Docker Developer Certificate of Origin 1.1 By making a contribution to the Docker Project ("Project"), I represent and warrant that: From 1ed42315f3d9ef0d7a3e07cce3211a449e028fc8 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 27 Dec 2013 10:47:42 -0700 Subject: [PATCH 002/403] Move UserLookup functionality into a separate pkg/user submodule that implements proper parsing of /etc/passwd and /etc/group, and use that to add support for "docker run -u user:group" and for getting supplementary groups (if ":group" is not specified) Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: ee93f6185be3ae73c16cf41ae397bae3ce2f6c55 Component: engine --- components/engine/execdriver/lxc/init.go | 27 +- .../engine/integration/buildfile_test.go | 80 ++++++ components/engine/pkg/user/MAINTAINERS | 1 + components/engine/pkg/user/user.go | 245 ++++++++++++++++++ components/engine/pkg/user/user_test.go | 94 +++++++ components/engine/utils/utils.go | 31 --- 6 files changed, 430 insertions(+), 48 deletions(-) create mode 100644 components/engine/pkg/user/MAINTAINERS create mode 100644 components/engine/pkg/user/user.go create mode 100644 components/engine/pkg/user/user_test.go diff --git a/components/engine/execdriver/lxc/init.go b/components/engine/execdriver/lxc/init.go index 7c2b039c50..d3af40049f 100644 --- a/components/engine/execdriver/lxc/init.go +++ b/components/engine/execdriver/lxc/init.go @@ -4,11 +4,10 @@ import ( "fmt" "github.com/dotcloud/docker/execdriver" "github.com/dotcloud/docker/pkg/netlink" - "github.com/dotcloud/docker/utils" + "github.com/dotcloud/docker/pkg/user" "github.com/syndtr/gocapability/capability" "net" "os" - "strconv" "strings" "syscall" ) @@ -79,28 +78,22 @@ func setupWorkingDirectory(args *execdriver.InitArgs) error { // Takes care of dropping privileges to the desired user func changeUser(args *execdriver.InitArgs) error { - if args.User == "" { - return nil - } - userent, err := utils.UserLookup(args.User) + uid, gid, suppGids, err := user.GetUserGroupSupplementary( + args.User, + syscall.Getuid(), syscall.Getgid(), + ) if err != nil { - return fmt.Errorf("Unable to find user %v: %v", args.User, err) + return err } - uid, err := strconv.Atoi(userent.Uid) - if err != nil { - return fmt.Errorf("Invalid uid: %v", userent.Uid) + if err := syscall.Setgroups(suppGids); err != nil { + return fmt.Errorf("Setgroups failed: %v", err) } - gid, err := strconv.Atoi(userent.Gid) - if err != nil { - return fmt.Errorf("Invalid gid: %v", userent.Gid) - } - if err := syscall.Setgid(gid); err != nil { - return fmt.Errorf("setgid failed: %v", err) + return fmt.Errorf("Setgid failed: %v", err) } if err := syscall.Setuid(uid); err != nil { - return fmt.Errorf("setuid failed: %v", err) + return fmt.Errorf("Setuid failed: %v", err) } return nil diff --git a/components/engine/integration/buildfile_test.go b/components/engine/integration/buildfile_test.go index 16a51e575d..0a8479e50a 100644 --- a/components/engine/integration/buildfile_test.go +++ b/components/engine/integration/buildfile_test.go @@ -148,6 +148,86 @@ RUN [ "$(/hello.sh)" = "hello world" ] nil, }, + // Users and groups + { + ` +FROM {IMAGE} + +# Make sure our defaults work +RUN [ "$(id -u):$(id -g)" = '0:0' ] +RUN [ "$(id -un):$(id -gn)" = 'root:root' ] + +# TODO decide if "args.user = strconv.Itoa(syscall.Getuid())" is acceptable behavior for changeUser in sysvinit instead of "return nil" when "USER" isn't specified (so that we get the proper group list even if that is the empty list, even in the default case of not supplying an explicit USER to run as, which implies USER 0) +USER root +RUN [ "$(id -G) -- $(id -Gn)" = '0 -- root' ] + +# Setup dockerio user and group +RUN echo 'dockerio:x:1000:1000::/bin:/bin/false' >> /etc/passwd +RUN echo 'dockerio:x:1000:' >> /etc/group + +# Make sure we can switch to our user and all the information is exactly as we expect it to be +USER dockerio +RUN [ "$(id -u):$(id -g)" = '1000:1000' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1000 -- dockerio' ] + +# Switch back to root and double check that worked exactly as we might expect it to +USER root +RUN [ "$(id -u):$(id -g)" = '0:0' ] +RUN [ "$(id -un):$(id -gn)" = 'root:root' ] +RUN [ "$(id -G) -- $(id -Gn)" = '0 -- root' ] + +# Add a "supplementary" group for our dockerio user +RUN echo 'supplementary:x:1001:dockerio' >> /etc/group + +# ... and then go verify that we get it like we expect +USER dockerio +RUN [ "$(id -u):$(id -g)" = '1000:1000' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1000 1001 -- dockerio supplementary' ] +USER 1000 +RUN [ "$(id -u):$(id -g)" = '1000:1000' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1000 1001 -- dockerio supplementary' ] + +# and finally, super test the new "user:group" syntax +USER dockerio:dockerio +RUN [ "$(id -u):$(id -g)" = '1000:1000' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1000 -- dockerio' ] +USER 1000:dockerio +RUN [ "$(id -u):$(id -g)" = '1000:1000' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1000 -- dockerio' ] +USER dockerio:1000 +RUN [ "$(id -u):$(id -g)" = '1000:1000' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1000 -- dockerio' ] +USER 1000:1000 +RUN [ "$(id -u):$(id -g)" = '1000:1000' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1000 -- dockerio' ] +USER dockerio:supplementary +RUN [ "$(id -u):$(id -g)" = '1000:1001' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:supplementary' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1001 -- supplementary' ] +USER dockerio:1001 +RUN [ "$(id -u):$(id -g)" = '1000:1001' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:supplementary' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1001 -- supplementary' ] +USER 1000:supplementary +RUN [ "$(id -u):$(id -g)" = '1000:1001' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:supplementary' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1001 -- supplementary' ] +USER 1000:1001 +RUN [ "$(id -u):$(id -g)" = '1000:1001' ] +RUN [ "$(id -un):$(id -gn)" = 'dockerio:supplementary' ] +RUN [ "$(id -G) -- $(id -Gn)" = '1001 -- supplementary' ] +`, + nil, + nil, + }, + // Environment variable { ` diff --git a/components/engine/pkg/user/MAINTAINERS b/components/engine/pkg/user/MAINTAINERS new file mode 100644 index 0000000000..18e05a3070 --- /dev/null +++ b/components/engine/pkg/user/MAINTAINERS @@ -0,0 +1 @@ +Tianon Gravi (@tianon) diff --git a/components/engine/pkg/user/user.go b/components/engine/pkg/user/user.go new file mode 100644 index 0000000000..30fc90f0e4 --- /dev/null +++ b/components/engine/pkg/user/user.go @@ -0,0 +1,245 @@ +package user + +import ( + "bufio" + "fmt" + "io" + "os" + "reflect" + "strconv" + "strings" +) + +type User struct { + Name string + Pass string + Uid int + Gid int + Gecos string + Home string + Shell string +} + +type Group struct { + Name string + Pass string + Gid int + List []string +} + +func parseLine(line string, v ...interface{}) { + if line == "" { + return + } + + parts := strings.Split(line, ":") + for i, p := range parts { + if len(v) <= i { + // if we have more "parts" than we have places to put them, bail for great "tolerance" of naughty configuration files + break + } + + t := reflect.TypeOf(v[i]) + if t.Kind() != reflect.Ptr { + // panic, because this is a programming/logic error, not a runtime one + panic("parseLine expects only pointers! argument " + strconv.Itoa(i) + " is not a pointer!") + } + + switch t.Elem().Kind() { + case reflect.String: + // "root", "adm", "/bin/bash" + *v[i].(*string) = p + case reflect.Int: + // "0", "4", "1000" + *v[i].(*int), _ = strconv.Atoi(p) + // ignore string to int conversion errors, for great "tolerance" of naughty configuration files + case reflect.Slice, reflect.Array: + // "", "root", "root,adm,daemon" + list := []string{} + if p != "" { + list = strings.Split(p, ",") + } + *v[i].(*[]string) = list + } + } +} + +func ParsePasswd() ([]*User, error) { + return ParsePasswdFilter(nil) +} + +func ParsePasswdFilter(filter func(*User) bool) ([]*User, error) { + f, err := os.Open("/etc/passwd") + if err != nil { + return nil, err + } + defer f.Close() + return parsePasswdFile(f, filter) +} + +func parsePasswdFile(r io.Reader, filter func(*User) bool) ([]*User, error) { + var ( + s = bufio.NewScanner(r) + out = []*User{} + ) + + for s.Scan() { + if err := s.Err(); err != nil { + return nil, err + } + + text := strings.TrimSpace(s.Text()) + if text == "" { + continue + } + + // see: man 5 passwd + // name:password:UID:GID:GECOS:directory:shell + // Name:Pass:Uid:Gid:Gecos:Home:Shell + // root:x:0:0:root:/root:/bin/bash + // adm:x:3:4:adm:/var/adm:/bin/false + p := &User{} + parseLine( + text, + &p.Name, &p.Pass, &p.Uid, &p.Gid, &p.Gecos, &p.Home, &p.Shell, + ) + + if filter == nil || filter(p) { + out = append(out, p) + } + } + + return out, nil +} + +func ParseGroup() ([]*Group, error) { + return ParseGroupFilter(nil) +} + +func ParseGroupFilter(filter func(*Group) bool) ([]*Group, error) { + f, err := os.Open("/etc/group") + if err != nil { + return nil, err + } + defer f.Close() + return parseGroupFile(f, filter) +} + +func parseGroupFile(r io.Reader, filter func(*Group) bool) ([]*Group, error) { + var ( + s = bufio.NewScanner(r) + out = []*Group{} + ) + + for s.Scan() { + if err := s.Err(); err != nil { + return nil, err + } + + text := s.Text() + if text == "" { + continue + } + + // see: man 5 group + // group_name:password:GID:user_list + // Name:Pass:Gid:List + // root:x:0:root + // adm:x:4:root,adm,daemon + p := &Group{} + parseLine( + text, + &p.Name, &p.Pass, &p.Gid, &p.List, + ) + + if filter == nil || filter(p) { + out = append(out, p) + } + } + + return out, nil +} + +// Given a string like "user", "1000", "user:group", "1000:1000", returns the uid, gid, and list of supplementary group IDs, if possible. +func GetUserGroupSupplementary(userSpec string, defaultUid int, defaultGid int) (int, int, []int, error) { + var ( + uid = defaultUid + gid = defaultGid + suppGids = []int{} + + userArg, groupArg string + ) + + // allow for userArg to have either "user" syntax, or optionally "user:group" syntax + parseLine(userSpec, &userArg, &groupArg) + + users, err := ParsePasswdFilter(func(u *User) bool { + if userArg == "" { + return u.Uid == uid + } + return u.Name == userArg || strconv.Itoa(u.Uid) == userArg + }) + if err != nil && !os.IsNotExist(err) { + if userArg == "" { + userArg = strconv.Itoa(uid) + } + return 0, 0, nil, fmt.Errorf("Unable to find user %v: %v", userArg, err) + } + + haveUser := users != nil && len(users) > 0 + if haveUser { + // if we found any user entries that matched our filter, let's take the first one as "correct" + uid = users[0].Uid + gid = users[0].Gid + } else if userArg != "" { + // we asked for a user but didn't find them... let's check to see if we wanted a numeric user + uid, err = strconv.Atoi(userArg) + if err != nil { + // not numeric - we have to bail + return 0, 0, nil, fmt.Errorf("Unable to find user %v", userArg) + } + + // if userArg couldn't be found in /etc/passwd but is numeric, just roll with it - this is legit + } + + if groupArg != "" || (haveUser && users[0].Name != "") { + groups, err := ParseGroupFilter(func(g *Group) bool { + if groupArg != "" { + return g.Name == groupArg || strconv.Itoa(g.Gid) == groupArg + } + for _, u := range g.List { + if u == users[0].Name { + return true + } + } + return false + }) + if err != nil && !os.IsNotExist(err) { + return 0, 0, nil, fmt.Errorf("Unable to find groups for user %v: %v", users[0].Name, err) + } + + haveGroup := groups != nil && len(groups) > 0 + if groupArg != "" { + if haveGroup { + // if we found any group entries that matched our filter, let's take the first one as "correct" + gid = groups[0].Gid + } else { + // we asked for a group but didn't find id... let's check to see if we wanted a numeric group + gid, err = strconv.Atoi(groupArg) + if err != nil { + // not numeric - we have to bail + return 0, 0, nil, fmt.Errorf("Unable to find group %v", groupArg) + } + + // if groupArg couldn't be found in /etc/group but is numeric, just roll with it - this is legit + } + } else if haveGroup { + suppGids = make([]int, len(groups)) + for i, group := range groups { + suppGids[i] = group.Gid + } + } + } + + return uid, gid, suppGids, nil +} diff --git a/components/engine/pkg/user/user_test.go b/components/engine/pkg/user/user_test.go new file mode 100644 index 0000000000..136632c27e --- /dev/null +++ b/components/engine/pkg/user/user_test.go @@ -0,0 +1,94 @@ +package user + +import ( + "strings" + "testing" +) + +func TestUserParseLine(t *testing.T) { + var ( + a, b string + c []string + d int + ) + + parseLine("", &a, &b) + if a != "" || b != "" { + t.Fatalf("a and b should be empty ('%v', '%v')", a, b) + } + + parseLine("a", &a, &b) + if a != "a" || b != "" { + t.Fatalf("a should be 'a' and b should be empty ('%v', '%v')", a, b) + } + + parseLine("bad boys:corny cows", &a, &b) + if a != "bad boys" || b != "corny cows" { + t.Fatalf("a should be 'bad boys' and b should be 'corny cows' ('%v', '%v')", a, b) + } + + parseLine("", &c) + if len(c) != 0 { + t.Fatalf("c should be empty (%#v)", c) + } + + parseLine("d,e,f:g:h:i,j,k", &c, &a, &b, &c) + if a != "g" || b != "h" || len(c) != 3 || c[0] != "i" || c[1] != "j" || c[2] != "k" { + t.Fatalf("a should be 'g', b should be 'h', and c should be ['i','j','k'] ('%v', '%v', '%#v')", a, b, c) + } + + parseLine("::::::::::", &a, &b, &c) + if a != "" || b != "" || len(c) != 0 { + t.Fatalf("a, b, and c should all be empty ('%v', '%v', '%#v')", a, b, c) + } + + parseLine("not a number", &d) + if d != 0 { + t.Fatalf("d should be 0 (%v)", d) + } + + parseLine("b:12:c", &a, &d, &b) + if a != "b" || b != "c" || d != 12 { + t.Fatalf("a should be 'b' and b should be 'c', and d should be 12 ('%v', '%v', %v)", a, b, d) + } +} + +func TestUserParsePasswd(t *testing.T) { + users, err := parsePasswdFile(strings.NewReader(` +root:x:0:0:root:/root:/bin/bash +adm:x:3:4:adm:/var/adm:/bin/false +this is just some garbage data +`), nil) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if len(users) != 3 { + t.Fatalf("Expected 3 users, got %v", len(users)) + } + if users[0].Uid != 0 || users[0].Name != "root" { + t.Fatalf("Expected users[0] to be 0 - root, got %v - %v", users[0].Uid, users[0].Name) + } + if users[1].Uid != 3 || users[1].Name != "adm" { + t.Fatalf("Expected users[1] to be 3 - adm, got %v - %v", users[1].Uid, users[1].Name) + } +} + +func TestUserParseGroup(t *testing.T) { + groups, err := parseGroupFile(strings.NewReader(` +root:x:0:root +adm:x:4:root,adm,daemon +this is just some garbage data +`), nil) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if len(groups) != 3 { + t.Fatalf("Expected 3 groups, got %v", len(groups)) + } + if groups[0].Gid != 0 || groups[0].Name != "root" || len(groups[0].List) != 1 { + t.Fatalf("Expected groups[0] to be 0 - root - 1 member, got %v - %v - %v", groups[0].Gid, groups[0].Name, len(groups[0].List)) + } + if groups[1].Gid != 4 || groups[1].Name != "adm" || len(groups[1].List) != 3 { + t.Fatalf("Expected groups[1] to be 4 - adm - 3 members, got %v - %v - %v", groups[1].Gid, groups[1].Name, len(groups[1].List)) + } +} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 542ab49702..5caf792549 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -836,37 +836,6 @@ func ParseRepositoryTag(repos string) (string, string) { return repos, "" } -type User struct { - Uid string // user id - Gid string // primary group id - Username string - Name string - HomeDir string -} - -// UserLookup check if the given username or uid is present in /etc/passwd -// and returns the user struct. -// If the username is not found, an error is returned. -func UserLookup(uid string) (*User, error) { - file, err := ioutil.ReadFile("/etc/passwd") - if err != nil { - return nil, err - } - for _, line := range strings.Split(string(file), "\n") { - data := strings.Split(line, ":") - if len(data) > 5 && (data[0] == uid || data[2] == uid) { - return &User{ - Uid: data[2], - Gid: data[3], - Username: data[0], - Name: data[4], - HomeDir: data[5], - }, nil - } - } - return nil, fmt.Errorf("User not found in /etc/passwd") -} - // An StatusError reports an unsuccessful exit by a command. type StatusError struct { Status string From 57b57cdf8c66e4f55d194bd0a1e9dd1ca891fc5b Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 17 Jan 2014 13:41:38 -0800 Subject: [PATCH 003/403] Use type switch instead of reflection Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 4ffc52385cec66cbd930bc6b5ab828ed65f43dbb Component: engine --- components/engine/pkg/user/user.go | 36 +++++++++++++----------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/components/engine/pkg/user/user.go b/components/engine/pkg/user/user.go index 30fc90f0e4..1672f7e679 100644 --- a/components/engine/pkg/user/user.go +++ b/components/engine/pkg/user/user.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "os" - "reflect" "strconv" "strings" ) @@ -39,28 +38,25 @@ func parseLine(line string, v ...interface{}) { break } - t := reflect.TypeOf(v[i]) - if t.Kind() != reflect.Ptr { + switch e := v[i].(type) { + case *string: + // "root", "adm", "/bin/bash" + *e = p + case *int: + // "0", "4", "1000" + // ignore string to int conversion errors, for great "tolerance" of naughty configuration files + *e, _ = strconv.Atoi(p) + case *[]string: + // "", "root", "root,adm,daemon" + if p != "" { + *e = strings.Split(p, ",") + } else { + *e = []string{} + } + default: // panic, because this is a programming/logic error, not a runtime one panic("parseLine expects only pointers! argument " + strconv.Itoa(i) + " is not a pointer!") } - - switch t.Elem().Kind() { - case reflect.String: - // "root", "adm", "/bin/bash" - *v[i].(*string) = p - case reflect.Int: - // "0", "4", "1000" - *v[i].(*int), _ = strconv.Atoi(p) - // ignore string to int conversion errors, for great "tolerance" of naughty configuration files - case reflect.Slice, reflect.Array: - // "", "root", "root,adm,daemon" - list := []string{} - if p != "" { - list = strings.Split(p, ",") - } - *v[i].(*[]string) = list - } } } From 0e1ea1b20489d82a005aa4ba5d177728f36f1741 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 27 Jan 2014 23:14:30 -0700 Subject: [PATCH 004/403] Consolidate a bunch of the supplementary-groups tests to cut down on the number of RUN lines in a single Dockerfile within TestBuild Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: e8963748fca208e6015e3cb7981fc0bf07929e3e Component: engine --- .../engine/integration/buildfile_test.go | 59 ++++++------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/components/engine/integration/buildfile_test.go b/components/engine/integration/buildfile_test.go index 0a8479e50a..2165fe3b06 100644 --- a/components/engine/integration/buildfile_test.go +++ b/components/engine/integration/buildfile_test.go @@ -154,12 +154,11 @@ RUN [ "$(/hello.sh)" = "hello world" ] FROM {IMAGE} # Make sure our defaults work -RUN [ "$(id -u):$(id -g)" = '0:0' ] -RUN [ "$(id -un):$(id -gn)" = 'root:root' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)" = '0:0/root:root' ] # TODO decide if "args.user = strconv.Itoa(syscall.Getuid())" is acceptable behavior for changeUser in sysvinit instead of "return nil" when "USER" isn't specified (so that we get the proper group list even if that is the empty list, even in the default case of not supplying an explicit USER to run as, which implies USER 0) USER root -RUN [ "$(id -G) -- $(id -Gn)" = '0 -- root' ] +RUN [ "$(id -G):$(id -Gn)" = '0:root' ] # Setup dockerio user and group RUN echo 'dockerio:x:1000:1000::/bin:/bin/false' >> /etc/passwd @@ -167,62 +166,42 @@ RUN echo 'dockerio:x:1000:' >> /etc/group # Make sure we can switch to our user and all the information is exactly as we expect it to be USER dockerio -RUN [ "$(id -u):$(id -g)" = '1000:1000' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1000 -- dockerio' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] # Switch back to root and double check that worked exactly as we might expect it to USER root -RUN [ "$(id -u):$(id -g)" = '0:0' ] -RUN [ "$(id -un):$(id -gn)" = 'root:root' ] -RUN [ "$(id -G) -- $(id -Gn)" = '0 -- root' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '0:0/root:root/0:root' ] # Add a "supplementary" group for our dockerio user RUN echo 'supplementary:x:1001:dockerio' >> /etc/group # ... and then go verify that we get it like we expect USER dockerio -RUN [ "$(id -u):$(id -g)" = '1000:1000' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1000 1001 -- dockerio supplementary' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000 1001:dockerio supplementary' ] USER 1000 -RUN [ "$(id -u):$(id -g)" = '1000:1000' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1000 1001 -- dockerio supplementary' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000 1001:dockerio supplementary' ] -# and finally, super test the new "user:group" syntax +# super test the new "user:group" syntax USER dockerio:dockerio -RUN [ "$(id -u):$(id -g)" = '1000:1000' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1000 -- dockerio' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] USER 1000:dockerio -RUN [ "$(id -u):$(id -g)" = '1000:1000' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1000 -- dockerio' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] USER dockerio:1000 -RUN [ "$(id -u):$(id -g)" = '1000:1000' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1000 -- dockerio' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] USER 1000:1000 -RUN [ "$(id -u):$(id -g)" = '1000:1000' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:dockerio' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1000 -- dockerio' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] USER dockerio:supplementary -RUN [ "$(id -u):$(id -g)" = '1000:1001' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:supplementary' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1001 -- supplementary' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] USER dockerio:1001 -RUN [ "$(id -u):$(id -g)" = '1000:1001' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:supplementary' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1001 -- supplementary' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] USER 1000:supplementary -RUN [ "$(id -u):$(id -g)" = '1000:1001' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:supplementary' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1001 -- supplementary' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] USER 1000:1001 -RUN [ "$(id -u):$(id -g)" = '1000:1001' ] -RUN [ "$(id -un):$(id -gn)" = 'dockerio:supplementary' ] -RUN [ "$(id -G) -- $(id -Gn)" = '1001 -- supplementary' ] +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] + +# make sure unknown uid/gid still works properly +USER 1042:1043 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1042:1043/1042:1043/1043:1043' ] `, nil, nil, From c7f0c458dfaf043b1e30c021121dab43c058cbfa Mon Sep 17 00:00:00 2001 From: James DeFelice Date: Fri, 31 Jan 2014 12:44:20 -0500 Subject: [PATCH 005/403] Update lxc_template.go If networking is disabled, but then pipework is used later to add nics, the network still doesn't function. Using flags=up for empty networking fixes this. Docker-DCO-1.1-Signed-off-by: James DeFelice (github: jdef) Upstream-commit: 17c124baf6a0f3725615b22aa52ca96ac5b95326 Component: engine --- components/engine/execdriver/lxc/lxc_template.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/execdriver/lxc/lxc_template.go b/components/engine/execdriver/lxc/lxc_template.go index 705bdf5363..a89365f989 100644 --- a/components/engine/execdriver/lxc/lxc_template.go +++ b/components/engine/execdriver/lxc/lxc_template.go @@ -15,6 +15,7 @@ lxc.network.name = eth0 {{else}} # network is disabled (-n=false) lxc.network.type = empty +lxc.network.flags = up {{end}} # root filesystem From 5a4e6a3400fb5e275c67ed436b2819598baacafe Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 5 Feb 2014 10:15:22 +1000 Subject: [PATCH 006/403] remove mention of #docker-meeting Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: d43c85961f3378607ace3e05dbab43ab2481699c Component: engine --- components/engine/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/CONTRIBUTING.md b/components/engine/CONTRIBUTING.md index 93c1d11b1f..ba8c3d1a67 100644 --- a/components/engine/CONTRIBUTING.md +++ b/components/engine/CONTRIBUTING.md @@ -163,7 +163,7 @@ If you have any questions, please refer to the FAQ in the [docs](http://docs.doc * Step 1: learn the component inside out * Step 2: make yourself useful by contributing code, bugfixes, support etc. * Step 3: volunteer on the irc channel (#docker@freenode) -* Step 4: propose yourself at a scheduled #docker-meeting +* Step 4: propose yourself at a scheduled docker meeting in #docker-dev Don't forget: being a maintainer is a time investment. Make sure you will have time to make yourself available. You don't have to be a maintainer to make a difference on the project! From 3ce6127194c14c5ee091aa39410e36fbf5b390b5 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 4 Feb 2014 17:13:00 +0100 Subject: [PATCH 007/403] devmapper: Rename DeviceSet.RemoveDevice to DeleteDevice "Remove" is a bit overloaded, as it is also what deactivating a device mapper device is called. Using "delete" is more clear here. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 63d4b293e7c91b8e2ed8da56f338f1bd0a4e1464 Component: engine --- components/engine/graphdriver/devmapper/deviceset.go | 8 ++++---- components/engine/graphdriver/devmapper/driver.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/graphdriver/devmapper/deviceset.go b/components/engine/graphdriver/devmapper/deviceset.go index 8432d92a4e..b6f21fa386 100644 --- a/components/engine/graphdriver/devmapper/deviceset.go +++ b/components/engine/graphdriver/devmapper/deviceset.go @@ -290,7 +290,7 @@ func (devices *DeviceSet) setupBaseImage() error { if oldInfo != nil && !oldInfo.Initialized { utils.Debugf("Removing uninitialized base image") - if err := devices.removeDevice(""); err != nil { + if err := devices.deleteDevice(""); err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } @@ -562,7 +562,7 @@ func (devices *DeviceSet) AddDevice(hash, baseHash string) error { return nil } -func (devices *DeviceSet) removeDevice(hash string) error { +func (devices *DeviceSet) deleteDevice(hash string) error { info := devices.Devices[hash] if info == nil { return fmt.Errorf("hash %s doesn't exists", hash) @@ -610,11 +610,11 @@ func (devices *DeviceSet) removeDevice(hash string) error { return nil } -func (devices *DeviceSet) RemoveDevice(hash string) error { +func (devices *DeviceSet) DeleteDevice(hash string) error { devices.Lock() defer devices.Unlock() - return devices.removeDevice(hash) + return devices.deleteDevice(hash) } func (devices *DeviceSet) deactivateDevice(hash string) error { diff --git a/components/engine/graphdriver/devmapper/driver.go b/components/engine/graphdriver/devmapper/driver.go index 664899cfbf..9b6cb109d2 100644 --- a/components/engine/graphdriver/devmapper/driver.go +++ b/components/engine/graphdriver/devmapper/driver.go @@ -99,7 +99,7 @@ func (d *Driver) Remove(id string) error { if err := d.unmount(id, mp); err != nil { return err } - return d.DeviceSet.RemoveDevice(id) + return d.DeviceSet.DeleteDevice(id) } func (d *Driver) Get(id string) (string, error) { From 9d46aecf3b33f8573681674052f5031a3bfd339a Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 4 Feb 2014 16:06:54 +0100 Subject: [PATCH 008/403] devmapper: add and use removeDeviceAndWait helper This adds a function that calls the lowlevel removeDevice and then waits for it to finish. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: dca21dfac77de2bbd69dd5a52b8a2b7816a54367 Component: engine --- .../engine/graphdriver/devmapper/deviceset.go | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/components/engine/graphdriver/devmapper/deviceset.go b/components/engine/graphdriver/devmapper/deviceset.go index b6f21fa386..e4c835a3de 100644 --- a/components/engine/graphdriver/devmapper/deviceset.go +++ b/components/engine/graphdriver/devmapper/deviceset.go @@ -632,28 +632,33 @@ func (devices *DeviceSet) deactivateDevice(hash string) error { return err } if devinfo.Exists != 0 { - if err := removeDevice(devname); err != nil { + if err := devices.removeDeviceAndWait(devname); err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } - if err := devices.waitRemove(hash); err != nil { - return err - } } return nil } +// Issues the underlying dm remove operation and then waits +// for it to finish. +func (devices *DeviceSet) removeDeviceAndWait(devname string) error { + if err := removeDevice(devname); err != nil { + return err + } + if err := devices.waitRemove(devname); err != nil { + return err + } + return nil +} + // waitRemove blocks until either: // a) the device registered at - is removed, // or b) the 1 second timeout expires. -func (devices *DeviceSet) waitRemove(hash string) error { - utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, hash) - defer utils.Debugf("[deviceset %s] waitRemove(%) END", devices.devicePrefix, hash) - devname, err := devices.byHash(hash) - if err != nil { - return err - } +func (devices *DeviceSet) waitRemove(devname string) error { + utils.Debugf("[deviceset %s] waitRemove(%s)", devices.devicePrefix, devname) + defer utils.Debugf("[deviceset %s] waitRemove(%s) END", devices.devicePrefix, devname) i := 0 for ; i < 1000; i += 1 { devinfo, err := getInfo(devname) From d2595404a2f6c4c9e43025a708be188ff50eb364 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 4 Feb 2014 18:19:09 +0100 Subject: [PATCH 009/403] devmapper: Move refcounting to DeviceSet We already have some kind of refcounting in DeviceSet, this fleshes it out to allow it to completely subsume the refcounting in devmapper.Driver. This allows us to drop the double refcounting, and the locking inside devmapper.Driver. This, in particular the locking simplification will make it easier in the future to parallelize the device mapper. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: b95c560fdda4813319a2377e240592a3261e30ef Component: engine --- .../engine/graphdriver/devmapper/deviceset.go | 108 +++++++++++++----- .../engine/graphdriver/devmapper/driver.go | 70 +++--------- .../graphdriver/devmapper/driver_test.go | 3 - 3 files changed, 93 insertions(+), 88 deletions(-) diff --git a/components/engine/graphdriver/devmapper/deviceset.go b/components/engine/graphdriver/devmapper/deviceset.go index e4c835a3de..4d708c840a 100644 --- a/components/engine/graphdriver/devmapper/deviceset.go +++ b/components/engine/graphdriver/devmapper/deviceset.go @@ -29,6 +29,15 @@ type DevInfo struct { TransactionId uint64 `json:"transaction_id"` Initialized bool `json:"initialized"` devices *DeviceSet `json:"-"` + + mountCount int `json:"-"` + mountPath string `json:"-"` + // A floating mount means one reference is not owned and + // will be stolen by the next mount. This allows us to + // avoid unmounting directly after creation before the + // first get (since we need to mount to set up the device + // a bit first). + floating bool `json:"-"` } type MetaData struct { @@ -43,7 +52,6 @@ type DeviceSet struct { TransactionId uint64 NewTransactionId uint64 nextFreeDevice int - activeMounts map[string]int } type DiskUsage struct { @@ -69,6 +77,14 @@ type DevStatus struct { HighestMappedSector uint64 } +type UnmountMode int + +const ( + UnmountRegular UnmountMode = iota + UnmountFloat + UnmountSink +) + func getDevName(name string) string { return "/dev/mapper/" + name } @@ -733,13 +749,12 @@ func (devices *DeviceSet) Shutdown() error { utils.Debugf("[devmapper] Shutting down DeviceSet: %s", devices.root) defer utils.Debugf("[deviceset %s] shutdown END", devices.devicePrefix) - for path, count := range devices.activeMounts { - for i := count; i > 0; i-- { - if err := sysUnmount(path, 0); err != nil { - utils.Debugf("Shutdown unmounting %s, error: %s\n", path, err) + for _, info := range devices.Devices { + if info.mountCount > 0 { + if err := sysUnmount(info.mountPath, 0); err != nil { + utils.Debugf("Shutdown unmounting %s, error: %s\n", info.mountPath, err) } } - delete(devices.activeMounts, path) } for _, d := range devices.Devices { @@ -761,22 +776,32 @@ func (devices *DeviceSet) Shutdown() error { return nil } -func (devices *DeviceSet) MountDevice(hash, path string, readOnly bool) error { +func (devices *DeviceSet) MountDevice(hash, path string) error { devices.Lock() defer devices.Unlock() + info := devices.Devices[hash] + + if info.mountCount > 0 { + if path != info.mountPath { + return fmt.Errorf("Trying to mount devmapper device in multple places (%s, %s)", info.mountPath, path) + } + + if info.floating { + // Steal floating ref + info.floating = false + } else { + info.mountCount++ + } + return nil + } + if err := devices.activateDeviceIfNeeded(hash); err != nil { return fmt.Errorf("Error activating devmapper device for '%s': %s", hash, err) } - info := devices.Devices[hash] - var flags uintptr = sysMsMgcVal - if readOnly { - flags = flags | sysMsRdOnly - } - err := sysMount(info.DevName(), path, "ext4", flags, "discard") if err != nil && err == sysEInval { err = sysMount(info.DevName(), path, "ext4", flags, "") @@ -785,20 +810,50 @@ func (devices *DeviceSet) MountDevice(hash, path string, readOnly bool) error { return fmt.Errorf("Error mounting '%s' on '%s': %s", info.DevName(), path, err) } - count := devices.activeMounts[path] - devices.activeMounts[path] = count + 1 + info.mountCount = 1 + info.mountPath = path + info.floating = false return devices.setInitialized(hash) } -func (devices *DeviceSet) UnmountDevice(hash, path string, deactivate bool) error { - utils.Debugf("[devmapper] UnmountDevice(hash=%s path=%s)", hash, path) +func (devices *DeviceSet) UnmountDevice(hash string, mode UnmountMode) error { + utils.Debugf("[devmapper] UnmountDevice(hash=%s, mode=%d)", hash, mode) defer utils.Debugf("[devmapper] UnmountDevice END") devices.Lock() defer devices.Unlock() - utils.Debugf("[devmapper] Unmount(%s)", path) - if err := sysUnmount(path, 0); err != nil { + info := devices.Devices[hash] + + if mode == UnmountFloat { + if info.floating { + return fmt.Errorf("UnmountDevice: can't float floating reference %s\n", hash) + } + + // Leave this reference floating + info.floating = true + return nil + } + + if mode == UnmountSink { + if !info.floating { + // Someone already sunk this + return nil + } + // Otherwise, treat this as a regular unmount + } + + if info.mountCount == 0 { + return fmt.Errorf("UnmountDevice: device not-mounted id %s\n", hash) + } + + info.mountCount-- + if info.mountCount > 0 { + return nil + } + + utils.Debugf("[devmapper] Unmount(%s)", info.mountPath) + if err := sysUnmount(info.mountPath, 0); err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } @@ -809,15 +864,9 @@ func (devices *DeviceSet) UnmountDevice(hash, path string, deactivate bool) erro return err } - if count := devices.activeMounts[path]; count > 1 { - devices.activeMounts[path] = count - 1 - } else { - delete(devices.activeMounts, path) - } + devices.deactivateDevice(hash) - if deactivate { - devices.deactivateDevice(hash) - } + info.mountPath = "" return nil } @@ -960,9 +1009,8 @@ func NewDeviceSet(root string, doInit bool) (*DeviceSet, error) { SetDevDir("/dev") devices := &DeviceSet{ - root: root, - MetaData: MetaData{Devices: make(map[string]*DevInfo)}, - activeMounts: make(map[string]int), + root: root, + MetaData: MetaData{Devices: make(map[string]*DevInfo)}, } if err := devices.initDevmapper(doInit); err != nil { diff --git a/components/engine/graphdriver/devmapper/driver.go b/components/engine/graphdriver/devmapper/driver.go index 9b6cb109d2..25b998e59f 100644 --- a/components/engine/graphdriver/devmapper/driver.go +++ b/components/engine/graphdriver/devmapper/driver.go @@ -8,7 +8,6 @@ import ( "github.com/dotcloud/docker/utils" "io/ioutil" "path" - "sync" ) func init() { @@ -22,9 +21,7 @@ func init() { type Driver struct { *DeviceSet - home string - sync.Mutex // Protects concurrent modification to active - active map[string]int + home string } var Init = func(home string) (graphdriver.Driver, error) { @@ -35,7 +32,6 @@ var Init = func(home string) (graphdriver.Driver, error) { d := &Driver{ DeviceSet: deviceSet, home: home, - active: make(map[string]int), } return d, nil } @@ -83,55 +79,36 @@ func (d *Driver) Create(id, parent string) error { return err } + // We float this reference so that the next Get call can + // steal it, so we don't have to unmount + if err := d.DeviceSet.UnmountDevice(id, UnmountFloat); err != nil { + return err + } + return nil } func (d *Driver) Remove(id string) error { - // Protect the d.active from concurrent access - d.Lock() - defer d.Unlock() - - if d.active[id] != 0 { - utils.Errorf("Warning: removing active id %s\n", id) - } - - mp := path.Join(d.home, "mnt", id) - if err := d.unmount(id, mp); err != nil { + // Sink the float from create in case no Get() call was made + if err := d.DeviceSet.UnmountDevice(id, UnmountSink); err != nil { return err } + // This assumes the device has been properly Get/Put:ed and thus is unmounted return d.DeviceSet.DeleteDevice(id) } func (d *Driver) Get(id string) (string, error) { - // Protect the d.active from concurrent access - d.Lock() - defer d.Unlock() - - count := d.active[id] - mp := path.Join(d.home, "mnt", id) - if count == 0 { - if err := d.mount(id, mp); err != nil { - return "", err - } + if err := d.mount(id, mp); err != nil { + return "", err } - d.active[id] = count + 1 - return path.Join(mp, "rootfs"), nil } func (d *Driver) Put(id string) { - // Protect the d.active from concurrent access - d.Lock() - defer d.Unlock() - - if count := d.active[id]; count > 1 { - d.active[id] = count - 1 - } else { - mp := path.Join(d.home, "mnt", id) - d.unmount(id, mp) - delete(d.active, id) + if err := d.DeviceSet.UnmountDevice(id, UnmountRegular); err != nil { + utils.Errorf("Warning: error unmounting device %s: %s\n", id, err) } } @@ -140,25 +117,8 @@ func (d *Driver) mount(id, mountPoint string) error { if err := osMkdirAll(mountPoint, 0755); err != nil && !osIsExist(err) { return err } - // If mountpoint is already mounted, do nothing - if mounted, err := Mounted(mountPoint); err != nil { - return fmt.Errorf("Error checking mountpoint: %s", err) - } else if mounted { - return nil - } // Mount the device - return d.DeviceSet.MountDevice(id, mountPoint, false) -} - -func (d *Driver) unmount(id, mountPoint string) error { - // If mountpoint is not mounted, do nothing - if mounted, err := Mounted(mountPoint); err != nil { - return fmt.Errorf("Error checking mountpoint: %s", err) - } else if !mounted { - return nil - } - // Unmount the device - return d.DeviceSet.UnmountDevice(id, mountPoint, true) + return d.DeviceSet.MountDevice(id, mountPoint) } func (d *Driver) Exists(id string) bool { diff --git a/components/engine/graphdriver/devmapper/driver_test.go b/components/engine/graphdriver/devmapper/driver_test.go index 785845ce6e..ff4ec45e5d 100644 --- a/components/engine/graphdriver/devmapper/driver_test.go +++ b/components/engine/graphdriver/devmapper/driver_test.go @@ -495,7 +495,6 @@ func TestDriverCreate(t *testing.T) { "DmTaskCreate", "DmTaskGetInfo", "sysMount", - "Mounted", "DmTaskRun", "DmTaskSetTarget", "DmTaskSetSector", @@ -614,7 +613,6 @@ func TestDriverRemove(t *testing.T) { "DmTaskCreate", "DmTaskGetInfo", "sysMount", - "Mounted", "DmTaskRun", "DmTaskSetTarget", "DmTaskSetSector", @@ -645,7 +643,6 @@ func TestDriverRemove(t *testing.T) { "DmTaskSetTarget", "DmTaskSetAddNode", "DmUdevWait", - "Mounted", "sysUnmount", ) }() From 931e825d565d33a2da6801d8e50bc625ca5a50e9 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 4 Feb 2014 16:08:45 +0100 Subject: [PATCH 010/403] devmapper: Use removeDeviceAndWait in DeviceSet.removeDevice() This makes sure the device is removed just like in deactivateDevice. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 7e25cd5891f84c2fab656c364002290e4b87f934 Component: engine --- components/engine/graphdriver/devmapper/deviceset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/graphdriver/devmapper/deviceset.go b/components/engine/graphdriver/devmapper/deviceset.go index 4d708c840a..4955fb5cf0 100644 --- a/components/engine/graphdriver/devmapper/deviceset.go +++ b/components/engine/graphdriver/devmapper/deviceset.go @@ -595,7 +595,7 @@ func (devices *DeviceSet) deleteDevice(hash string) error { devinfo, _ := getInfo(info.Name()) if devinfo != nil && devinfo.Exists != 0 { - if err := removeDevice(info.Name()); err != nil { + if err := devices.removeDeviceAndWait(info.Name()); err != nil { utils.Debugf("Error removing device: %s\n", err) return err } From 8a70fd74d7ddfdda9f4d33c32694aa76af173dd7 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 5 Feb 2014 21:37:53 +0100 Subject: [PATCH 011/403] devmapper: Handle EBUSY while removing For some reason we seem to get transient EBUSY when removing thinp devices, which prohibit removing containers. When this happens we retry a few times which seems to fix the issue for me. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 2c82fd93d8a01cc1f53fe861378e6d2dca0486c6 Component: engine --- .../engine/graphdriver/devmapper/deviceset.go | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/components/engine/graphdriver/devmapper/deviceset.go b/components/engine/graphdriver/devmapper/deviceset.go index 4955fb5cf0..3dedfd75b5 100644 --- a/components/engine/graphdriver/devmapper/deviceset.go +++ b/components/engine/graphdriver/devmapper/deviceset.go @@ -12,6 +12,7 @@ import ( "path" "path/filepath" "strconv" + "strings" "sync" "time" ) @@ -52,6 +53,7 @@ type DeviceSet struct { TransactionId uint64 NewTransactionId uint64 nextFreeDevice int + sawBusy bool } type DiskUsage struct { @@ -371,6 +373,10 @@ func (devices *DeviceSet) log(level int, file string, line int, dmError int, mes return // Ignore _LOG_DEBUG } + if strings.Contains(message, "busy") { + devices.sawBusy = true + } + utils.Debugf("libdevmapper(%d): %s:%d (%d) %s", level, file, line, dmError, message) } @@ -660,9 +666,26 @@ func (devices *DeviceSet) deactivateDevice(hash string) error { // Issues the underlying dm remove operation and then waits // for it to finish. func (devices *DeviceSet) removeDeviceAndWait(devname string) error { - if err := removeDevice(devname); err != nil { + var err error + + for i := 0; i < 10; i++ { + devices.sawBusy = false + err = removeDevice(devname) + if err == nil { + break + } + if !devices.sawBusy { + return err + } + + // If we see EBUSY it may be a transient error, + // sleep a bit a retry a few times. + time.Sleep(5 * time.Millisecond) + } + if err != nil { return err } + if err := devices.waitRemove(devname); err != nil { return err } From bd67bce0231112871be2740a088f5f096f5efd3c Mon Sep 17 00:00:00 2001 From: Brandon Philips Date: Thu, 6 Feb 2014 11:34:25 -0800 Subject: [PATCH 012/403] pkg: systemd: add initial MAINTAINERS I volunteered for pkg/systemd MAINTAINER and there were no objections during the #docker-dev meeting. For context I wrote most of the stuff in here and wrote the dependent calls in api.go. Plus, I actively test the code via CoreOS. Docker-DCO-1.1-Signed-off-by: Brandon Philips (github: philips) Upstream-commit: 15711ed670e8ab59611030ea1191189ad495ea63 Component: engine --- components/engine/pkg/systemd/MAINTAINERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 components/engine/pkg/systemd/MAINTAINERS diff --git a/components/engine/pkg/systemd/MAINTAINERS b/components/engine/pkg/systemd/MAINTAINERS new file mode 100644 index 0000000000..51228b368a --- /dev/null +++ b/components/engine/pkg/systemd/MAINTAINERS @@ -0,0 +1 @@ +Brandon Philips (@philips) From 4c4ba56ea42af6aa191a8a20f17f425052d79767 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 6 Feb 2014 22:26:09 +0100 Subject: [PATCH 013/403] devmapper: Fix UnmountDevice for non-existing device Properly error out if passed an id that doesn't exist. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 304e33a2fe2d006cc1063c8060c6e22c9ade3d59 Component: engine --- components/engine/graphdriver/devmapper/deviceset.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/graphdriver/devmapper/deviceset.go b/components/engine/graphdriver/devmapper/deviceset.go index 3dedfd75b5..c1f2f558a0 100644 --- a/components/engine/graphdriver/devmapper/deviceset.go +++ b/components/engine/graphdriver/devmapper/deviceset.go @@ -847,6 +847,9 @@ func (devices *DeviceSet) UnmountDevice(hash string, mode UnmountMode) error { defer devices.Unlock() info := devices.Devices[hash] + if info == nil { + return fmt.Errorf("UnmountDevice: no such device %s\n", hash) + } if mode == UnmountFloat { if info.floating { From 3959ecf0a97738a02c5c3f8263c02a37f2687e42 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 6 Feb 2014 23:08:17 +0100 Subject: [PATCH 014/403] devmapper: Fix MountDevice for non-existing Device Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 693d3f8c6ab4c9fed5fe4c5ac5fcb44568fe1638 Component: engine --- components/engine/graphdriver/devmapper/deviceset.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/graphdriver/devmapper/deviceset.go b/components/engine/graphdriver/devmapper/deviceset.go index c1f2f558a0..637192dbcb 100644 --- a/components/engine/graphdriver/devmapper/deviceset.go +++ b/components/engine/graphdriver/devmapper/deviceset.go @@ -804,6 +804,9 @@ func (devices *DeviceSet) MountDevice(hash, path string) error { defer devices.Unlock() info := devices.Devices[hash] + if info == nil { + return fmt.Errorf("Unknown device %s", hash) + } if info.mountCount > 0 { if path != info.mountPath { From f006ee4708ab7b5fb9b5e4c9cc9b1f21f055f88f Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Fri, 7 Feb 2014 01:42:11 +0100 Subject: [PATCH 015/403] Added Security FAQ security Docker-DCO-1.1-Signed-off-by: James Turnbull (github: jamtur01) Upstream-commit: 55b74bfe469115f25a52c6ad48b13eb9c7e6197c Component: engine --- components/engine/docs/sources/faq.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/engine/docs/sources/faq.rst b/components/engine/docs/sources/faq.rst index cf072f34a7..5c8e69703f 100644 --- a/components/engine/docs/sources/faq.rst +++ b/components/engine/docs/sources/faq.rst @@ -183,6 +183,12 @@ Cloud: - Google Compute Engine - Rackspace +How do I report a security issue with Docker? +............................................. + +You can learn about the project's security policy `here `_ +and report security issues to this `mailbox `_. + Can I help by adding some questions and answers? ................................................ From 9378ca24df3afaeee9bc0300d1c6cae22118a107 Mon Sep 17 00:00:00 2001 From: Dafydd Crosby Date: Fri, 7 Feb 2014 00:10:47 -0700 Subject: [PATCH 016/403] Fix RST formatting Upstream-commit: b2945f27767f35a273feb6e255945f3e1f6108d2 Component: engine --- components/engine/docs/sources/reference/run.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/docs/sources/reference/run.rst b/components/engine/docs/sources/reference/run.rst index 307edace00..d8de280671 100644 --- a/components/engine/docs/sources/reference/run.rst +++ b/components/engine/docs/sources/reference/run.rst @@ -143,6 +143,7 @@ Network Settings ---------------- :: + -n=true : Enable networking for this container -dns=[] : Set custom dns servers for the container From 4f66485afb9dcdad76c7833228614613f1858107 Mon Sep 17 00:00:00 2001 From: German DZ Date: Fri, 7 Feb 2014 09:22:10 +0100 Subject: [PATCH 017/403] New folder structure to support TextMate2 bundles format Docker-DCO-1.1-Signed-off-by: German Del Zotto (github: GermanDZ) Upstream-commit: f289b3a19e03b99a6618009320abaec2c0cf1ec1 Component: engine --- .../Preferences}/Dockerfile.tmPreferences | 0 .../Syntaxes}/Dockerfile.tmLanguage | 0 .../syntax/textmate/Docker.tmbundle/info.plist | 16 ++++++++++++++++ 3 files changed, 16 insertions(+) rename components/engine/contrib/syntax/textmate/{ => Docker.tmbundle/Preferences}/Dockerfile.tmPreferences (100%) rename components/engine/contrib/syntax/textmate/{ => Docker.tmbundle/Syntaxes}/Dockerfile.tmLanguage (100%) create mode 100644 components/engine/contrib/syntax/textmate/Docker.tmbundle/info.plist diff --git a/components/engine/contrib/syntax/textmate/Dockerfile.tmPreferences b/components/engine/contrib/syntax/textmate/Docker.tmbundle/Preferences/Dockerfile.tmPreferences similarity index 100% rename from components/engine/contrib/syntax/textmate/Dockerfile.tmPreferences rename to components/engine/contrib/syntax/textmate/Docker.tmbundle/Preferences/Dockerfile.tmPreferences diff --git a/components/engine/contrib/syntax/textmate/Dockerfile.tmLanguage b/components/engine/contrib/syntax/textmate/Docker.tmbundle/Syntaxes/Dockerfile.tmLanguage similarity index 100% rename from components/engine/contrib/syntax/textmate/Dockerfile.tmLanguage rename to components/engine/contrib/syntax/textmate/Docker.tmbundle/Syntaxes/Dockerfile.tmLanguage diff --git a/components/engine/contrib/syntax/textmate/Docker.tmbundle/info.plist b/components/engine/contrib/syntax/textmate/Docker.tmbundle/info.plist new file mode 100644 index 0000000000..239f4b0a9b --- /dev/null +++ b/components/engine/contrib/syntax/textmate/Docker.tmbundle/info.plist @@ -0,0 +1,16 @@ + + + + + contactEmailRot13 + germ@andz.com.ar + contactName + GermanDZ + description + Helpers for Docker. + name + Docker + uuid + 8B9DDBAF-E65C-4E12-FFA7-467D4AA535B1 + + From 1097b506cc8c026455b85620abfeba97d2713142 Mon Sep 17 00:00:00 2001 From: German DZ Date: Fri, 7 Feb 2014 09:30:11 +0100 Subject: [PATCH 018/403] Instructions for Textmate's bundle installation Docker-DCO-1.1-Signed-off-by: German Del Zotto (github: GermanDZ) Upstream-commit: ac06646b1fb7172937c52d5f1217dcac5a86f0f9 Component: engine --- components/engine/contrib/syntax/textmate/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/engine/contrib/syntax/textmate/README.md b/components/engine/contrib/syntax/textmate/README.md index 8f3ec693b7..e78b76af45 100644 --- a/components/engine/contrib/syntax/textmate/README.md +++ b/components/engine/contrib/syntax/textmate/README.md @@ -1,4 +1,4 @@ -# Dockerfile.tmLanguage +# Docker.tmbundle Dockerfile syntaxt highlighting for TextMate and Sublime Text. @@ -6,11 +6,11 @@ Dockerfile syntaxt highlighting for TextMate and Sublime Text. ### Sublime Text -Available for Sublime Text under [package control](https://sublime.wbond.net/packages/Dockerfile%20Syntax%20Highlighting). +Available for Sublime Text under [package control](https://sublime.wbond.net/packages/Dockerfile%20Syntax%20Highlighting). Search for *Dockerfile Syntax Highlighting* -### TextMate +### TextMate 2 -*...unknown. Probably put it somewhere smart.* +Copy the directory `Docker.tmbundle` (showed as a Package in OSX) to `~/Library/Application Support/TextMate/Managed/Bundles` enjoy. From 9e9837fce8088c70e7a56020363b57c2a590d294 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 7 Feb 2014 15:21:33 +1000 Subject: [PATCH 019/403] please, for the love of Docker, do. not. use. symlinks Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: f787bec4a6853b46b1ba755d4f8a2ee07d9cbd3e Component: engine --- .../engine/docs/sources/reference/commandline/cli.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index ae77080309..5f4f821e12 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -102,12 +102,17 @@ the ``-H`` flag for the client. docker ps # both are equal - To run the daemon with `systemd socket activation `_, use ``docker -d -H fd://``. Using ``fd://`` will work perfectly for most setups but you can also specify individual sockets too ``docker -d -H fd://3``. If the specified socket activated files aren't found then docker will exit. You can find examples of using systemd socket activation with docker and systemd in the `docker source tree `_. +.. warning:: + Docker and LXC do not support the use of softlinks for either the Docker data directory (``/var/lib/docker``) or for ``/tmp``. + If your system is likely to be set up in that way, you can use ``readlink -f`` to canonicalise the links: + + ``TMPDIR=$(readlink -f /tmp) /usr/local/bin/docker -d -D -g $(readlink -f /var/lib/docker) -H unix:// $EXPOSE_ALL > /var/lib/boot2docker/docker.log 2>&1`` + .. _cli_attach: ``attach`` From 29b4f1c1236b902888c489bcc1e80eb5851d3ad8 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Fri, 7 Feb 2014 19:09:52 -0800 Subject: [PATCH 020/403] Remove panic in lxc driver. Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 7c06d5e34e2ebf5006ce3a34438f18c071153e97 Component: engine --- components/engine/execdriver/lxc/driver.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/engine/execdriver/lxc/driver.go b/components/engine/execdriver/lxc/driver.go index 4c3979e718..ee4d02a6b6 100644 --- a/components/engine/execdriver/lxc/driver.go +++ b/components/engine/execdriver/lxc/driver.go @@ -279,7 +279,8 @@ func (i *info) IsRunning() bool { output, err := i.driver.getInfo(i.ID) if err != nil { - panic(err) + utils.Errorf("Error getting info for lxc container %s: %s (%s)", i.ID, err, output) + return false } if strings.Contains(string(output), "RUNNING") { running = true From 7920ed3ee7b4c95f169bc46c69b623dce50ad6ab Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sat, 8 Feb 2014 09:53:04 -0800 Subject: [PATCH 021/403] Add set master for interface Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: a6c791e8a92f29a2ae2c6bc81e4ab873d3ceb41f Component: engine --- .../engine/pkg/netlink/netlink_linux.go | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 0ea5b4dbac..01b3ff08fc 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -386,6 +386,39 @@ func NetworkSetMTU(iface *net.Interface, mtu int) error { return s.HandleAck(wb.Seq) } +// same as ip link set $name master $master +func NetworkSetMaster(iface, master *net.Interface) error { + s, err := getNetlinkSocket() + if err != nil { + return err + } + defer s.Close() + + wb := newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := newIfInfomsg(syscall.AF_UNSPEC) + msg.Type = syscall.RTM_SETLINK + msg.Flags = syscall.NLM_F_REQUEST + msg.Index = int32(iface.Index) + msg.Change = 0xFFFFFFFF + wb.AddData(msg) + + var ( + b = make([]byte, 4) + native = nativeEndian() + ) + native.PutUint32(b, uint32(master.Index)) + + data := newRtAttr(syscall.IFLA_MASTER, b) + wb.AddData(data) + + if err := s.Send(wb); err != nil { + return err + } + + return s.HandleAck(wb.Seq) +} + // Add an Ip address to an interface. This is identical to: // ip addr add $ip/$ipNet dev $iface func NetworkLinkAddIp(iface *net.Interface, ip net.IP, ipNet *net.IPNet) error { From e66ed158d681c87038e9435019c2126cbdeffcf8 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sat, 8 Feb 2014 10:03:16 -0800 Subject: [PATCH 022/403] Add network set interface in namespace by pid Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: e3762e8d6937741a5654b1443f95beb784f018f1 Component: engine --- .../engine/pkg/netlink/netlink_linux.go | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 01b3ff08fc..e69635e2f6 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -419,6 +419,38 @@ func NetworkSetMaster(iface, master *net.Interface) error { return s.HandleAck(wb.Seq) } +func NetworkSetNsPid(iface *net.Interface, nspid int) error { + s, err := getNetlinkSocket() + if err != nil { + return err + } + defer s.Close() + + wb := newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := newIfInfomsg(syscall.AF_UNSPEC) + msg.Type = syscall.RTM_SETLINK + msg.Flags = syscall.NLM_F_REQUEST + msg.Index = int32(iface.Index) + msg.Change = 0xFFFFFFFF + wb.AddData(msg) + + var ( + b = make([]byte, 4) + native = nativeEndian() + ) + native.PutUint32(b, uint32(nspid)) + + data := newRtAttr(syscall.IFLA_NET_NS_PID, b) + wb.AddData(data) + + if err := s.Send(wb); err != nil { + return err + } + + return s.HandleAck(wb.Seq) +} + // Add an Ip address to an interface. This is identical to: // ip addr add $ip/$ipNet dev $iface func NetworkLinkAddIp(iface *net.Interface, ip net.IP, ipNet *net.IPNet) error { From 31b1c10e097f3cdb6dc389a566290e977ef42c10 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sat, 8 Feb 2014 11:34:11 -0700 Subject: [PATCH 023/403] Swap Firefox to Iceweasel so that the Desktop Integration example is simplified and easier to understand Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 72c72f91c621d9dd2431eee01a28a4ca490588bc Component: engine --- .../{README.txt => README.md} | 2 +- .../desktop-integration/data/Dockerfile | 20 ++++---- .../desktop-integration/firefox/Dockerfile | 49 ------------------- .../desktop-integration/iceweasel/Dockerfile | 41 ++++++++++++++++ 4 files changed, 52 insertions(+), 60 deletions(-) rename components/engine/contrib/desktop-integration/{README.txt => README.md} (71%) delete mode 100644 components/engine/contrib/desktop-integration/firefox/Dockerfile create mode 100644 components/engine/contrib/desktop-integration/iceweasel/Dockerfile diff --git a/components/engine/contrib/desktop-integration/README.txt b/components/engine/contrib/desktop-integration/README.md similarity index 71% rename from components/engine/contrib/desktop-integration/README.txt rename to components/engine/contrib/desktop-integration/README.md index 2f55c979e3..02181a5f75 100644 --- a/components/engine/contrib/desktop-integration/README.txt +++ b/components/engine/contrib/desktop-integration/README.md @@ -8,4 +8,4 @@ Examples ======== * Data container: ./data/Dockerfile creates a data image sharing /data volume -* Firefox: ./firefox/Dockerfile shows a way to dockerize a common multimedia application +* Iceweasel: ./iceweasel/Dockerfile shows a way to dockerize a common multimedia application diff --git a/components/engine/contrib/desktop-integration/data/Dockerfile b/components/engine/contrib/desktop-integration/data/Dockerfile index 453afdd3d6..a9843a52ad 100644 --- a/components/engine/contrib/desktop-integration/data/Dockerfile +++ b/components/engine/contrib/desktop-integration/data/Dockerfile @@ -11,28 +11,28 @@ # # Build data image # docker build -t data -rm . # -# # Create a data container. (eg: firefox-data) -# docker run -name firefox-data data true +# # Create a data container. (eg: iceweasel-data) +# docker run -name iceweasel-data data true # # # List data from it -# docker run -volumes-from firefox-data busybox ls -al /data +# docker run -volumes-from iceweasel-data busybox ls -al /data docker-version 0.6.5 # Smallest base image, just to launch a container -from busybox -maintainer Daniel Mizyrycki +FROM busybox +MAINTAINER Daniel Mizyrycki # Create a regular user -run echo 'sysadmin:x:1000:1000::/data:/bin/sh' >> /etc/passwd -run echo 'sysadmin:x:1000:' >> /etc/group +RUN echo 'sysadmin:x:1000:1000::/data:/bin/sh' >> /etc/passwd +RUN echo 'sysadmin:x:1000:' >> /etc/group # Create directory for that user -run mkdir /data -run chown sysadmin.sysadmin /data +RUN mkdir /data +RUN chown sysadmin.sysadmin /data # Add content to /data. This will keep sysadmin ownership -run touch /data/init_volume +RUN touch /data/init_volume # Create /data volume VOLUME /data diff --git a/components/engine/contrib/desktop-integration/firefox/Dockerfile b/components/engine/contrib/desktop-integration/firefox/Dockerfile deleted file mode 100644 index f8924f4b4a..0000000000 --- a/components/engine/contrib/desktop-integration/firefox/Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# VERSION: 0.7 -# DESCRIPTION: Create firefox container with its dependencies -# AUTHOR: Daniel Mizyrycki -# COMMENTS: -# This file describes how to build a Firefox container with all -# dependencies installed. It uses native X11 unix socket and alsa -# sound devices. Tested on Debian 7.2 -# USAGE: -# # Download Firefox Dockerfile -# wget http://raw.github.com/dotcloud/docker/master/contrib/desktop-integration/firefox/Dockerfile -# -# # Build firefox image -# docker build -t firefox -rm . -# -# # Run stateful data-on-host firefox. For ephemeral, remove -v /data/firefox:/data -# docker run -v /data/firefox:/data -v /tmp/.X11-unix:/tmp/.X11-unix \ -# -v /dev/snd:/dev/snd -lxc-conf='lxc.cgroup.devices.allow = c 116:* rwm' \ -# -e DISPLAY=unix$DISPLAY firefox -# -# # To run stateful dockerized data containers -# docker run -volumes-from firefox-data -v /tmp/.X11-unix:/tmp/.X11-unix \ -# -v /dev/snd:/dev/snd -lxc-conf='lxc.cgroup.devices.allow = c 116:* rwm' \ -# -e DISPLAY=unix$DISPLAY firefox - -docker-version 0.6.5 - -# Base docker image -from tianon/debian:wheezy -maintainer Daniel Mizyrycki - -# Install firefox dependencies -run echo "deb http://ftp.debian.org/debian/ wheezy main contrib" > /etc/apt/sources.list -run apt-get update -run DEBIAN_FRONTEND=noninteractive apt-get install -y libXrender1 libasound2 \ - libdbus-glib-1-2 libgtk2.0-0 libpango1.0-0 libxt6 wget bzip2 sudo - -# Install Firefox -run mkdir /application -run cd /application; wget -O - \ - http://ftp.mozilla.org/pub/mozilla.org/firefox/releases/25.0/linux-x86_64/en-US/firefox-25.0.tar.bz2 | tar jx - -# create sysadmin account -run useradd -m -d /data -p saIVpsc0EVTwA sysadmin -run sed -Ei 's/sudo:x:27:/sudo:x:27:sysadmin/' /etc/group -run sed -Ei 's/(\%sudo\s+ALL=\(ALL\:ALL\) )ALL/\1 NOPASSWD:ALL/' /etc/sudoers - -# Autorun firefox. -no-remote is necessary to create a new container, as firefox -# appears to communicate with itself through X11. -cmd ["/bin/sh", "-c", "/usr/bin/sudo -u sysadmin -H -E /application/firefox/firefox -no-remote"] diff --git a/components/engine/contrib/desktop-integration/iceweasel/Dockerfile b/components/engine/contrib/desktop-integration/iceweasel/Dockerfile new file mode 100644 index 0000000000..721cc6d2cf --- /dev/null +++ b/components/engine/contrib/desktop-integration/iceweasel/Dockerfile @@ -0,0 +1,41 @@ +# VERSION: 0.7 +# DESCRIPTION: Create iceweasel container with its dependencies +# AUTHOR: Daniel Mizyrycki +# COMMENTS: +# This file describes how to build a Iceweasel container with all +# dependencies installed. It uses native X11 unix socket and alsa +# sound devices. Tested on Debian 7.2 +# USAGE: +# # Download Iceweasel Dockerfile +# wget http://raw.github.com/dotcloud/docker/master/contrib/desktop-integration/iceweasel/Dockerfile +# +# # Build iceweasel image +# docker build -t iceweasel -rm . +# +# # Run stateful data-on-host iceweasel. For ephemeral, remove -v /data/iceweasel:/data +# docker run -v /data/iceweasel:/data -v /tmp/.X11-unix:/tmp/.X11-unix \ +# -v /dev/snd:/dev/snd -lxc-conf='lxc.cgroup.devices.allow = c 116:* rwm' \ +# -e DISPLAY=unix$DISPLAY iceweasel +# +# # To run stateful dockerized data containers +# docker run -volumes-from iceweasel-data -v /tmp/.X11-unix:/tmp/.X11-unix \ +# -v /dev/snd:/dev/snd -lxc-conf='lxc.cgroup.devices.allow = c 116:* rwm' \ +# -e DISPLAY=unix$DISPLAY iceweasel + +docker-version 0.6.5 + +# Base docker image +FROM debian:wheezy +MAINTAINER Daniel Mizyrycki + +# Install Iceweasel and "sudo" +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq iceweasel sudo + +# create sysadmin account +RUN useradd -m -d /data -p saIVpsc0EVTwA sysadmin +RUN sed -Ei 's/sudo:x:27:/sudo:x:27:sysadmin/' /etc/group +RUN sed -Ei 's/(\%sudo\s+ALL=\(ALL\:ALL\) )ALL/\1 NOPASSWD:ALL/' /etc/sudoers + +# Autorun iceweasel. -no-remote is necessary to create a new container, as +# iceweasel appears to communicate with itself through X11. +CMD ["/usr/bin/sudo", "-u", "sysadmin", "-H", "-E", "/usr/bin/iceweasel", "-no-remote"] From 347c7fca0d09592fa110dd4ea8cf3ab28b5e3e54 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sat, 8 Feb 2014 20:44:04 -0800 Subject: [PATCH 024/403] Use c to change interface name Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: ee39033073ece35e91c6c5a8cb66d23246511fb0 Component: engine --- .../engine/pkg/netlink/netlink_linux.go | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index e69635e2f6..77ebfcd46a 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -2,6 +2,60 @@ package netlink +/* +#include +#include +#include +#include + +static int get_socket(void) { + int s_errno; + int fd; + + fd = socket(PF_INET, SOCK_DGRAM, 0); + if (fd >= 0) { + return fd; + } + s_errno = errno; + + fd = socket(PF_PACKET, SOCK_DGRAM, 0); + if (fd >= 0) { + return fd; + } + + fd = socket(PF_INET6, SOCK_DGRAM, 0); + if (fd >= 0) { + return fd; + } + errno = s_errno; + return -1; +} + + +static int change_name(const char *old_name, const char *new_name) { + struct ifreq ifr; + int err; + int fd; + + fd = get_socket(); + if (fd < 0) { + return -1; + } + + strncpy(ifr.ifr_name, old_name, IFNAMSIZ); + strncpy(ifr.ifr_newname, new_name, IFNAMSIZ); + + err = ioctl(fd, SIOCSIFNAME, &ifr); + if (err) { + close(fd); + return -1; + } + close(fd); + return err; +} +*/ +import "C" + import ( "encoding/binary" "fmt" @@ -641,3 +695,15 @@ done: return res, nil } + +func NetworkChangeName(oldName, newName string) error { + var ( + cold = C.CString(oldName) + cnew = C.CString(newName) + ) + + if errno := int(C.change_name(cold, cnew)); errno != 0 { + return fmt.Errorf("unable to change name %d", errno) + } + return nil +} From 81f4bd236b25d06becafb4b46b5cf6efaa95d4fa Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Sun, 9 Feb 2014 07:25:34 +0100 Subject: [PATCH 025/403] devmapper: Enable skip_block_zeroing This makes the device mapper not zero out blocks allocated on the thinp device. This is safe in our use case, as we access the device via a filesystem that doesn't leak any uninitialized data to userspace. This partially helps with https://github.com/dotcloud/docker/issues/3280 and should generally improve preformance on the devicemapper backend. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 695719b29a6311a56faae0c6ed3c985b7a75add0 Component: engine --- components/engine/graphdriver/devmapper/devmapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/graphdriver/devmapper/devmapper.go b/components/engine/graphdriver/devmapper/devmapper.go index 7f83a09df9..7317118dcf 100644 --- a/components/engine/graphdriver/devmapper/devmapper.go +++ b/components/engine/graphdriver/devmapper/devmapper.go @@ -324,7 +324,7 @@ func createPool(poolName string, dataFile, metadataFile *osFile) error { return fmt.Errorf("Can't get data size") } - params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768" + params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768 1 skip_block_zeroing" if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil { return fmt.Errorf("Can't add target") } From 64b4ce5805a246950a0be1bd761cafc13229f2db Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sun, 9 Feb 2014 01:43:46 -0800 Subject: [PATCH 026/403] Add bidirectional iptables rule back to links Fixes #4014 Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 0cba91e26447e14260f111bef6d3745d2dd55444 Component: engine --- components/engine/networkdriver/lxc/driver.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/components/engine/networkdriver/lxc/driver.go b/components/engine/networkdriver/lxc/driver.go index 3f9c0af011..0bc24b91b6 100644 --- a/components/engine/networkdriver/lxc/driver.go +++ b/components/engine/networkdriver/lxc/driver.go @@ -172,7 +172,6 @@ func setupIPTables(addr net.Addr, icc bool) error { iptables.Raw(append([]string{"-D"}, acceptArgs...)...) if !iptables.Exists(dropArgs...) { - utils.Debugf("Disable inter-container communication") if output, err := iptables.Raw(append([]string{"-I"}, dropArgs...)...); err != nil { return fmt.Errorf("Unable to prevent intercontainer communication: %s", err) @@ -470,6 +469,20 @@ func LinkContainers(job *engine.Job) engine.Status { job.Errorf("Error toggle iptables forward: %s", output) return engine.StatusErr } + + if output, err := iptables.Raw(action, "FORWARD", + "-i", bridgeIface, "-o", bridgeIface, + "-p", proto, + "-s", childIP, + "--sport", port, + "-d", parentIP, + "-j", "ACCEPT"); !ignoreErrors && err != nil { + job.Error(err) + return engine.StatusErr + } else if len(output) != 0 { + job.Errorf("Error toggle iptables forward: %s", output) + return engine.StatusErr + } } return engine.StatusOK } From de9b1efd2f62cdd335380752743a9e8f91ee7f3e Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 6 Feb 2014 14:13:03 -0800 Subject: [PATCH 027/403] Remove linux specific calls Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 3dfc910d7774d57c533b067fbe59d6b24dd803cd Component: engine --- components/engine/archive/archive.go | 25 +++++++++++-------- components/engine/archive/stat_linux.go | 7 ++++++ .../{stat_darwin.go => stat_unsupported.go} | 8 ++++-- 3 files changed, 28 insertions(+), 12 deletions(-) rename components/engine/archive/{stat_darwin.go => stat_unsupported.go} (67%) diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index b1400c2210..3a1c111ea2 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/bzip2" "compress/gzip" + "errors" "fmt" "github.com/dotcloud/docker/utils" "io" @@ -17,14 +18,18 @@ import ( "syscall" ) -type Archive io.Reader +type ( + Archive io.Reader + Compression int + TarOptions struct { + Includes []string + Compression Compression + } +) -type Compression int - -type TarOptions struct { - Includes []string - Compression Compression -} +var ( + ErrNotImplemented = errors.New("Function not implemented") +) const ( Uncompressed Compression = iota @@ -236,14 +241,14 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader *tar.Reader) return fmt.Errorf("Unhandled tar header type %d\n", hdr.Typeflag) } - if err := syscall.Lchown(path, hdr.Uid, hdr.Gid); err != nil { + if err := os.Lchown(path, hdr.Uid, hdr.Gid); err != nil { return err } // There is no LChmod, so ignore mode for symlink. Also, this // must happen after chown, as that can modify the file mode if hdr.Typeflag != tar.TypeSymlink { - if err := syscall.Chmod(path, uint32(hdr.Mode&07777)); err != nil { + if err := os.Chmod(path, os.FileMode(hdr.Mode&07777)); err != nil { return err } } @@ -251,7 +256,7 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader *tar.Reader) ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} // syscall.UtimesNano doesn't support a NOFOLLOW flag atm, and if hdr.Typeflag != tar.TypeSymlink { - if err := syscall.UtimesNano(path, ts); err != nil { + if err := UtimesNano(path, ts); err != nil { return err } } else { diff --git a/components/engine/archive/stat_linux.go b/components/engine/archive/stat_linux.go index 2f7a520ccd..f87a99c55a 100644 --- a/components/engine/archive/stat_linux.go +++ b/components/engine/archive/stat_linux.go @@ -30,3 +30,10 @@ func LUtimesNano(path string, ts []syscall.Timespec) error { return nil } + +func UtimesNano(path string, ts []syscall.Timespec) error { + if err := syscall.UtimesNano(path, ts); err != nil { + return err + } + return nil +} diff --git a/components/engine/archive/stat_darwin.go b/components/engine/archive/stat_unsupported.go similarity index 67% rename from components/engine/archive/stat_darwin.go rename to components/engine/archive/stat_unsupported.go index 32203299dd..50ca461867 100644 --- a/components/engine/archive/stat_darwin.go +++ b/components/engine/archive/stat_unsupported.go @@ -1,4 +1,4 @@ -// +build !linux !amd64 +// +build !linux package archive @@ -13,5 +13,9 @@ func getLastModification(stat *syscall.Stat_t) syscall.Timespec { } func LUtimesNano(path string, ts []syscall.Timespec) error { - return nil + return ErrNotImplemented +} + +func UtimesNano(path string, ts []syscall.Timespec) error { + return ErrNotImplemented } From c36542fc7c5dee450e3a8a386253f628690a210a Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sun, 9 Feb 2014 05:54:13 -0800 Subject: [PATCH 028/403] Replace my C code with tianons Go code Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 524416560a4624d30023db32101c9fe5ebffc895 Component: engine --- .../engine/pkg/netlink/netlink_linux.go | 97 +++++++------------ 1 file changed, 35 insertions(+), 62 deletions(-) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 77ebfcd46a..46bd3d8373 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -2,60 +2,6 @@ package netlink -/* -#include -#include -#include -#include - -static int get_socket(void) { - int s_errno; - int fd; - - fd = socket(PF_INET, SOCK_DGRAM, 0); - if (fd >= 0) { - return fd; - } - s_errno = errno; - - fd = socket(PF_PACKET, SOCK_DGRAM, 0); - if (fd >= 0) { - return fd; - } - - fd = socket(PF_INET6, SOCK_DGRAM, 0); - if (fd >= 0) { - return fd; - } - errno = s_errno; - return -1; -} - - -static int change_name(const char *old_name, const char *new_name) { - struct ifreq ifr; - int err; - int fd; - - fd = get_socket(); - if (fd < 0) { - return -1; - } - - strncpy(ifr.ifr_name, old_name, IFNAMSIZ); - strncpy(ifr.ifr_newname, new_name, IFNAMSIZ); - - err = ioctl(fd, SIOCSIFNAME, &ifr); - if (err) { - close(fd); - return -1; - } - close(fd); - return err; -} -*/ -import "C" - import ( "encoding/binary" "fmt" @@ -696,14 +642,41 @@ done: return res, nil } -func NetworkChangeName(oldName, newName string) error { - var ( - cold = C.CString(oldName) - cnew = C.CString(newName) - ) - - if errno := int(C.change_name(cold, cnew)); errno != 0 { - return fmt.Errorf("unable to change name %d", errno) +func getIfSocket() (int, error) { + fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, 0) + if err == nil { + return fd, err } + sErr := err + + fd, err = syscall.Socket(syscall.AF_PACKET, syscall.SOCK_DGRAM, 0) + if err == nil { + return fd, err + } + + fd, err = syscall.Socket(syscall.AF_INET6, syscall.SOCK_DGRAM, 0) + if err == nil { + return fd, err + } + + return -1, sErr +} + +func NetworkChangeName(oldName, newName string) error { + fd, err := getIfSocket() + if err != nil { + return err + } + defer syscall.Close(fd) + IFNAMSIZ := 16 + + data := [32]byte{} + copy(data[:IFNAMSIZ-1], oldName) + copy(data[IFNAMSIZ:IFNAMSIZ*2-1], newName) + + if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, uintptr(fd), syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&data[0]))); errno != 0 { + return errno + } + return nil } From a53586948286f9ff6b960c384d5f81c0ab252dc1 Mon Sep 17 00:00:00 2001 From: Andrew Williams Date: Sun, 9 Feb 2014 15:36:04 -0600 Subject: [PATCH 029/403] Adjust test to match its comment Docker-DCO-1.1-Signed-off-by: Andrew Williams (github: TheDude05) Upstream-commit: 95bcb8924ac2cbe0fb625a604d2c6d5a8e2fa06a Component: engine --- components/engine/networkdriver/network_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/networkdriver/network_test.go b/components/engine/networkdriver/network_test.go index c15f8b1cf5..6224c2dffb 100644 --- a/components/engine/networkdriver/network_test.go +++ b/components/engine/networkdriver/network_test.go @@ -105,7 +105,7 @@ func TestNetworkOverlaps(t *testing.T) { //netY starts before and ends at same IP of netX AssertOverlap("172.16.1.1/24", "172.16.0.1/23", t) //netY starts before and ends outside of netX - AssertOverlap("172.16.1.1/24", "172.16.0.1/23", t) + AssertOverlap("172.16.1.1/24", "172.16.0.1/22", t) //netY starts and ends before netX AssertNoOverlap("172.16.1.1/25", "172.16.0.1/24", t) //netX starts and ends before netY From aaca90f4b6ea71d0d77e2679206d2f094fe94824 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 9 Feb 2014 18:12:43 -0700 Subject: [PATCH 030/403] Update NetworkChangeName to be more similar to my original (moving IFNAMSIZ constant outside the function like it should've been) Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 3a7c144e9992591f32daf2d4f1b35b7b6520a07e Component: engine --- components/engine/pkg/netlink/netlink_linux.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 46bd3d8373..b94cbf6822 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -662,15 +662,17 @@ func getIfSocket() (int, error) { return -1, sErr } +// from +const IFNAMSIZ = 16 + func NetworkChangeName(oldName, newName string) error { fd, err := getIfSocket() if err != nil { return err } defer syscall.Close(fd) - IFNAMSIZ := 16 - data := [32]byte{} + data := [IFNAMSIZ * 2]byte{} copy(data[:IFNAMSIZ-1], oldName) copy(data[IFNAMSIZ:IFNAMSIZ*2-1], newName) From aa80369d9ad65869acb93f6a9ed1c8463ee16b54 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sun, 9 Feb 2014 18:21:01 -0700 Subject: [PATCH 031/403] Add slightly better GOPATH detection/handling This also adds a new "AUTO_GOPATH" environment variable that will create an appropriate GOPATH as part of the build process. Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: d3d85d38fb5bdd0327fdc9fe8b637cb0f4e1dcf2 Component: engine --- components/engine/hack/make.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index ef13c1a283..d73b33a4fa 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -68,6 +68,19 @@ else exit 1 fi +if [ "$AUTO_GOPATH" ]; then + rm -rf .gopath + mkdir -p .gopath/src/github.com/dotcloud + ln -sf ../../../.. .gopath/src/github.com/dotcloud/docker + export GOPATH="$(pwd)/.gopath:$(pwd)/vendor" +fi + +if [ ! "$GOPATH" ]; then + echo >&2 'error: missing GOPATH; please see http://golang.org/doc/code.html#GOPATH' + echo >&2 ' alternatively, set AUTO_GOPATH=1' + exit 1 +fi + # Use these flags when compiling the tests and final binary LDFLAGS='-X main.GITCOMMIT "'$GITCOMMIT'" -X main.VERSION "'$VERSION'" -w' LDFLAGS_STATIC='-X github.com/dotcloud/docker/utils.IAMSTATIC true -linkmode external -extldflags "-lpthread -static -Wl,--unresolved-symbols=ignore-in-object-files"' From 4ce6d81f0dd510dabacc05da1c85b5c933454055 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 10 Feb 2014 11:04:24 -0800 Subject: [PATCH 032/403] Improve no command handling Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 44821158409d59024173336188e087c605e1da1a Component: engine --- components/engine/runtime.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 7e4ae79b40..a16238e200 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -378,9 +378,7 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin } } - if len(config.Entrypoint) != 0 && config.Cmd == nil { - config.Cmd = []string{} - } else if config.Cmd == nil || len(config.Cmd) == 0 { + if len(config.Entrypoint) == 0 && len(config.Cmd) == 0 { return nil, nil, fmt.Errorf("No command specified") } From 5fb9dcf8436d627b209cd9705d741e17affe9f4d Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 10 Feb 2014 11:36:23 -0800 Subject: [PATCH 033/403] Improve get if socket loop Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 27df18ff11dbfc22ca539462710abf07d507c0d9 Component: engine --- .../engine/pkg/netlink/netlink_linux.go | 43 +++++++++---------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index b94cbf6822..4e091ef103 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -10,6 +10,12 @@ import ( "unsafe" ) +const ( + IFNAMSIZ = 16 + DEFAULT_CHANGE = 0xFFFFFFFF + IFLA_INFO_KIND = 1 +) + var nextSeqNr int func nativeEndian() binary.ByteOrder { @@ -368,7 +374,7 @@ func NetworkSetMTU(iface *net.Interface, mtu int) error { msg.Type = syscall.RTM_SETLINK msg.Flags = syscall.NLM_F_REQUEST msg.Index = int32(iface.Index) - msg.Change = 0xFFFFFFFF + msg.Change = DEFAULT_CHANGE wb.AddData(msg) var ( @@ -400,7 +406,7 @@ func NetworkSetMaster(iface, master *net.Interface) error { msg.Type = syscall.RTM_SETLINK msg.Flags = syscall.NLM_F_REQUEST msg.Index = int32(iface.Index) - msg.Change = 0xFFFFFFFF + msg.Change = DEFAULT_CHANGE wb.AddData(msg) var ( @@ -432,7 +438,7 @@ func NetworkSetNsPid(iface *net.Interface, nspid int) error { msg.Type = syscall.RTM_SETLINK msg.Flags = syscall.NLM_F_REQUEST msg.Index = int32(iface.Index) - msg.Change = 0xFFFFFFFF + msg.Change = DEFAULT_CHANGE wb.AddData(msg) var ( @@ -524,8 +530,6 @@ func NetworkLinkAdd(name string, linkType string) error { nameData := newRtAttr(syscall.IFLA_IFNAME, zeroTerminated(name)) wb.AddData(nameData) - IFLA_INFO_KIND := 1 - kindData := newRtAttr(IFLA_INFO_KIND, nonZeroTerminated(linkType)) infoData := newRtAttr(syscall.IFLA_LINKINFO, kindData.ToWireFormat()) @@ -642,29 +646,22 @@ done: return res, nil } -func getIfSocket() (int, error) { - fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, 0) - if err == nil { - return fd, err +func getIfSocket() (fd int, err error) { + for _, socket := range []int{ + syscall.AF_INET, + syscall.AF_PACKET, + syscall.AF_INET6, + } { + if fd, err = syscall.Socket(socket, syscall.SOCK_DGRAM, 0); err == nil { + break + } } - sErr := err - - fd, err = syscall.Socket(syscall.AF_PACKET, syscall.SOCK_DGRAM, 0) if err == nil { - return fd, err + return fd, nil } - - fd, err = syscall.Socket(syscall.AF_INET6, syscall.SOCK_DGRAM, 0) - if err == nil { - return fd, err - } - - return -1, sErr + return -1, err } -// from -const IFNAMSIZ = 16 - func NetworkChangeName(oldName, newName string) error { fd, err := getIfSocket() if err != nil { From f694361fe52fead4488c67c8121af06ba8f2c6dc Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 10 Feb 2014 13:44:34 -0700 Subject: [PATCH 034/403] Add better ".git" detection and use This way, packagers can set GIT_DIR appropriately if they'd prefer to not have ".git" inside their working directory. Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 0a0406450fb83f4ff7d08235cd3e94ea8fba752f Component: engine --- components/engine/hack/make.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index d73b33a4fa..cebb9064e6 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -53,9 +53,9 @@ DEFAULT_BUNDLES=( ) VERSION=$(cat ./VERSION) -if [ -d .git ] && command -v git &> /dev/null; then +if command -v git &> /dev/null && git rev-parse &> /dev/null; then GITCOMMIT=$(git rev-parse --short HEAD) - if [ -n "$(git status --porcelain)" ]; then + if [ -n "$(git status --porcelain --untracked-files=no)" ]; then GITCOMMIT="$GITCOMMIT-dirty" fi elif [ "$DOCKER_GITCOMMIT" ]; then From 2095048f8cccbd9e784b16b9ed989c0f9bb07737 Mon Sep 17 00:00:00 2001 From: "O.S.Tezer" Date: Sun, 9 Feb 2014 14:40:31 +0200 Subject: [PATCH 035/403] docs: Installation Instruction Title & Description Fixes Docker-DCO-1.1-Signed-off-by: O.S. Tezer (github: ostezer) 1. All titles are listed by simple platform names apart from Windows' "Installing Docker On Windows". Changed this to "Windows" to match the rest. 2. Some articles' description (and title) information does not match the majority. Modified them to match the rest (i.e. Please note this project is currently under heavy development. It should not be used in production.) 3. Removed "Linux" from Gentoo & Ubuntu descriptions. Upstream-commit: 10d57b648fe77fe34642d0af6c5c010ff1180c85 Component: engine --- components/engine/docs/sources/installation/amazon.rst | 2 +- components/engine/docs/sources/installation/archlinux.rst | 2 +- components/engine/docs/sources/installation/fedora.rst | 2 +- .../engine/docs/sources/installation/frugalware.rst | 2 +- .../engine/docs/sources/installation/gentoolinux.rst | 4 ++-- components/engine/docs/sources/installation/mac.rst | 2 +- components/engine/docs/sources/installation/openSUSE.rst | 2 +- components/engine/docs/sources/installation/rackspace.rst | 4 ++-- components/engine/docs/sources/installation/rhel.rst | 2 +- .../engine/docs/sources/installation/ubuntulinux.rst | 2 +- components/engine/docs/sources/installation/windows.rst | 8 ++++---- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/components/engine/docs/sources/installation/amazon.rst b/components/engine/docs/sources/installation/amazon.rst index e8fdc2c1ca..31090a070c 100644 --- a/components/engine/docs/sources/installation/amazon.rst +++ b/components/engine/docs/sources/installation/amazon.rst @@ -1,5 +1,5 @@ :title: Installation on Amazon EC2 -:description: Docker installation on Amazon EC2 +:description: Please note this project is currently under heavy development. It should not be used in production. :keywords: amazon ec2, virtualization, cloud, docker, documentation, installation Amazon EC2 diff --git a/components/engine/docs/sources/installation/archlinux.rst b/components/engine/docs/sources/installation/archlinux.rst index 2d823bfd46..c9b4c1d2c5 100644 --- a/components/engine/docs/sources/installation/archlinux.rst +++ b/components/engine/docs/sources/installation/archlinux.rst @@ -1,5 +1,5 @@ :title: Installation on Arch Linux -:description: Docker installation on Arch Linux. +:description: Please note this project is currently under heavy development. It should not be used in production. :keywords: arch linux, virtualization, docker, documentation, installation .. _arch_linux: diff --git a/components/engine/docs/sources/installation/fedora.rst b/components/engine/docs/sources/installation/fedora.rst index 6dd2bf91d9..7e0aee78fd 100644 --- a/components/engine/docs/sources/installation/fedora.rst +++ b/components/engine/docs/sources/installation/fedora.rst @@ -1,4 +1,4 @@ -:title: Requirements and Installation on Fedora +:title: Installation on Fedora :description: Please note this project is currently under heavy development. It should not be used in production. :keywords: Docker, Docker documentation, Fedora, requirements, virtualbox, vagrant, git, ssh, putty, cygwin, linux diff --git a/components/engine/docs/sources/installation/frugalware.rst b/components/engine/docs/sources/installation/frugalware.rst index de2b92ae10..ed9bb2bfaa 100644 --- a/components/engine/docs/sources/installation/frugalware.rst +++ b/components/engine/docs/sources/installation/frugalware.rst @@ -1,5 +1,5 @@ :title: Installation on FrugalWare -:description: Docker installation on FrugalWare. +:description: Please note this project is currently under heavy development. It should not be used in production. :keywords: frugalware linux, virtualization, docker, documentation, installation .. _frugalware: diff --git a/components/engine/docs/sources/installation/gentoolinux.rst b/components/engine/docs/sources/installation/gentoolinux.rst index 421af0a1e7..5abfddeb91 100644 --- a/components/engine/docs/sources/installation/gentoolinux.rst +++ b/components/engine/docs/sources/installation/gentoolinux.rst @@ -1,5 +1,5 @@ -:title: Installation on Gentoo Linux -:description: Docker installation instructions and nuances for Gentoo Linux. +:title: Installation on Gentoo +:description: Please note this project is currently under heavy development. It should not be used in production. :keywords: gentoo linux, virtualization, docker, documentation, installation .. _gentoo_linux: diff --git a/components/engine/docs/sources/installation/mac.rst b/components/engine/docs/sources/installation/mac.rst index c247c4aba2..3cf1c93a15 100644 --- a/components/engine/docs/sources/installation/mac.rst +++ b/components/engine/docs/sources/installation/mac.rst @@ -1,4 +1,4 @@ -:title: Requirements and Installation on Mac OS X 10.6 Snow Leopard +:title: Installation on Mac OS X 10.6 Snow Leopard :description: Please note this project is currently under heavy development. It should not be used in production. :keywords: Docker, Docker documentation, requirements, virtualbox, ssh, linux, os x, osx, mac diff --git a/components/engine/docs/sources/installation/openSUSE.rst b/components/engine/docs/sources/installation/openSUSE.rst index ded5de44a4..c791beacbf 100644 --- a/components/engine/docs/sources/installation/openSUSE.rst +++ b/components/engine/docs/sources/installation/openSUSE.rst @@ -1,5 +1,5 @@ :title: Installation on openSUSE -:description: Docker installation on openSUSE. +:description: Please note this project is currently under heavy development. It should not be used in production. :keywords: openSUSE, virtualbox, docker, documentation, installation .. _openSUSE: diff --git a/components/engine/docs/sources/installation/rackspace.rst b/components/engine/docs/sources/installation/rackspace.rst index d0005a14bc..687131a413 100644 --- a/components/engine/docs/sources/installation/rackspace.rst +++ b/components/engine/docs/sources/installation/rackspace.rst @@ -1,5 +1,5 @@ -:title: Rackspace Cloud Installation -:description: Installing Docker on Ubuntu proviced by Rackspace +:title: Installation on Rackspace Cloud +:description: Please note this project is currently under heavy development. It should not be used in production. :keywords: Rackspace Cloud, installation, docker, linux, ubuntu Rackspace Cloud diff --git a/components/engine/docs/sources/installation/rhel.rst b/components/engine/docs/sources/installation/rhel.rst index 9036fb79ea..7930da6309 100644 --- a/components/engine/docs/sources/installation/rhel.rst +++ b/components/engine/docs/sources/installation/rhel.rst @@ -1,4 +1,4 @@ -:title: Requirements and Installation on Red Hat Enterprise Linux +:title: Installation on Red Hat Enterprise Linux :description: Please note this project is currently under heavy development. It should not be used in production. :keywords: Docker, Docker documentation, requirements, linux, rhel, centos diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index 3d6ee6415d..f37be90d7d 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -1,4 +1,4 @@ -:title: Requirements and Installation on Ubuntu Linux +:title: Installation on Ubuntu :description: Please note this project is currently under heavy development. It should not be used in production. :keywords: Docker, Docker documentation, requirements, virtualbox, vagrant, git, ssh, putty, cygwin, linux diff --git a/components/engine/docs/sources/installation/windows.rst b/components/engine/docs/sources/installation/windows.rst index c980a32df9..9d965ea3fe 100644 --- a/components/engine/docs/sources/installation/windows.rst +++ b/components/engine/docs/sources/installation/windows.rst @@ -1,11 +1,11 @@ -:title: Requirements and Installation on Windows -:description: Docker's tutorial to run docker on Windows +:title: Installation on Windows +:description: Please note this project is currently under heavy development. It should not be used in production. :keywords: Docker, Docker documentation, Windows, requirements, virtualbox, vagrant, git, ssh, putty, cygwin .. _windows: -Installing Docker on Windows -============================ +Windows +======= Docker can run on Windows using a VM like VirtualBox. You then run Linux within the VM. From 6da22355911c2ddf5e8eb0389b6435c6e7698bac Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 10 Feb 2014 13:37:16 -0800 Subject: [PATCH 036/403] Create veth pair via netlink Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 38eabfa65c5be78a08a8287b600e230dbe0bfac5 Component: engine --- .../engine/pkg/netlink/netlink_linux.go | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 4e091ef103..52c77a26a4 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -14,6 +14,8 @@ const ( IFNAMSIZ = 16 DEFAULT_CHANGE = 0xFFFFFFFF IFLA_INFO_KIND = 1 + IFLA_INFO_DATA = 2 + VETH_PEER = 1 ) var nextSeqNr int @@ -197,7 +199,9 @@ func (rr *NetlinkRequest) ToWireFormat() []byte { } func (rr *NetlinkRequest) AddData(data NetlinkRequestData) { - rr.Data = append(rr.Data, data) + if data != nil { + rr.Data = append(rr.Data, data) + } } func newNetlinkRequest(proto, flags int) *NetlinkRequest { @@ -676,6 +680,41 @@ func NetworkChangeName(oldName, newName string) error { if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, uintptr(fd), syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&data[0]))); errno != 0 { return errno } - return nil } + +func NetworkCreateVethPair(name1, name2 string) error { + s, err := getNetlinkSocket() + if err != nil { + return err + } + defer s.Close() + + wb := newNetlinkRequest(syscall.RTM_NEWLINK, syscall.NLM_F_CREATE|syscall.NLM_F_EXCL|syscall.NLM_F_ACK) + + msg := newIfInfomsg(syscall.AF_UNSPEC) + wb.AddData(msg) + + kindData := newRtAttr(IFLA_INFO_KIND, nonZeroTerminated("veth")) + info := newRtAttr(syscall.IFLA_LINKINFO, kindData.ToWireFormat()) + // wb.AddData(info) + + peerName := newRtAttr(syscall.IFLA_IFNAME, nonZeroTerminated(name2)) + peer := newRtAttr(VETH_PEER, peerName.ToWireFormat()) + // wb.AddData(peer) + + b := []byte{} + b = append(b, peer.ToWireFormat()...) + b = append(b, info.ToWireFormat()...) + + infoData := newRtAttr(IFLA_INFO_DATA, b) + wb.AddData(infoData) + + nameData := newRtAttr(syscall.IFLA_IFNAME, zeroTerminated(name1)) + wb.AddData(nameData) + + if err := s.Send(wb); err != nil { + return err + } + return s.HandleAck(wb.Seq) +} From 7dcdfb0d44d2c1a991494f75c27a27548361bde9 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Mon, 10 Feb 2014 23:52:15 +0000 Subject: [PATCH 037/403] fix --run in docker commit Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) Upstream-commit: 5d29749e9dba4d918fc5ed5d6049f397e98967cf Component: engine --- components/engine/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index 7c229eead0..3bc8a8b504 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -367,7 +367,7 @@ func postCommit(eng *engine.Engine, version float64, w http.ResponseWriter, r *h env engine.Env job = eng.Job("commit", r.Form.Get("container")) ) - if err := config.Import(r.Body); err != nil { + if err := config.Decode(r.Body); err != nil { utils.Errorf("%s", err) } From 06def9a48b934fae9b5e92088baf96be21e9e4c3 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Tue, 11 Feb 2014 10:10:50 +1000 Subject: [PATCH 038/403] bring back the explaination of our LGTM process Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 5051b8610d7802db2d479cfba14009552a95789f Component: engine --- components/engine/CONTRIBUTING.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/engine/CONTRIBUTING.md b/components/engine/CONTRIBUTING.md index ee40b1d3b2..3c5a57e131 100644 --- a/components/engine/CONTRIBUTING.md +++ b/components/engine/CONTRIBUTING.md @@ -109,6 +109,18 @@ name and email address match your git configuration. The AUTHORS file is regenerated occasionally from the git commit history, so a mismatch may result in your changes being overwritten. +### Merge approval + +Docker maintainers use LGTM (looks good to me) in comments on the code review +to indicate acceptance. + +A change requires LGTMs from an absolute majority of the maintainers of each +component affected. For example, if a change affects docs/ and registry/, it +needs an absolute majority from the maintainers of docs/ AND, separately, an +absolute majority of the maintainers of registry + +For more details see [MAINTAINERS.md](hack/MAINTAINERS.md) + ### Sign your work The sign-off is a simple line at the end of the explanation for the From 1093e871da328f465db19fc02cf45a566da456a4 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 10 Feb 2014 16:41:16 -0800 Subject: [PATCH 039/403] Allow add of empty name Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 4dec36d1ee8cfd8b396210daa5367d146bbb34bc Component: engine --- components/engine/pkg/netlink/netlink_linux.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 52c77a26a4..23dba0c884 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -531,8 +531,10 @@ func NetworkLinkAdd(name string, linkType string) error { msg := newIfInfomsg(syscall.AF_UNSPEC) wb.AddData(msg) - nameData := newRtAttr(syscall.IFLA_IFNAME, zeroTerminated(name)) - wb.AddData(nameData) + if name != "" { + nameData := newRtAttr(syscall.IFLA_IFNAME, zeroTerminated(name)) + wb.AddData(nameData) + } kindData := newRtAttr(IFLA_INFO_KIND, nonZeroTerminated(linkType)) From 11beb9de931c94290a2f55328a6ecb87aa9da51f Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Mon, 10 Feb 2014 16:10:15 +1000 Subject: [PATCH 040/403] point out that ENV DEBIAN_FRONTEND will persist, so its not recommended Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: c4b9e1c9add2ab3c97cc89c8f6b12b83750cea1d Component: engine --- components/engine/docs/sources/reference/builder.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/components/engine/docs/sources/reference/builder.rst b/components/engine/docs/sources/reference/builder.rst index 571824c36c..4b6a151006 100644 --- a/components/engine/docs/sources/reference/builder.rst +++ b/components/engine/docs/sources/reference/builder.rst @@ -251,9 +251,14 @@ value ````. This value will be passed to all future ``RUN`` instructions. This is functionally equivalent to prefixing the command with ``=`` +The environment variables set using ``ENV`` will persist when a container is run +from the resulting image. You can view the values using ``docker inspect``, and change them using ``docker run --env =``. + .. note:: - The environment variables will persist when a container is run - from the resulting image. + One example where this can cause unexpected consequenses, is setting + ``ENV DEBIAN_FRONTEND noninteractive``. + Which will persist when the container is run interactively; for example: + ``docker run -t -i image bash`` .. _dockerfile_add: From 71b22701b5877e7bbf00e08f44e4176387c64753 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Tue, 11 Feb 2014 14:58:42 +1000 Subject: [PATCH 041/403] add a little info on upgrading Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 82863aecab824118e223b53fed905ed5fd9182bb Component: engine --- components/engine/docs/sources/installation/mac.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/engine/docs/sources/installation/mac.rst b/components/engine/docs/sources/installation/mac.rst index 4f0550ad3d..5381842432 100644 --- a/components/engine/docs/sources/installation/mac.rst +++ b/components/engine/docs/sources/installation/mac.rst @@ -147,6 +147,18 @@ If SSH complains about keys: ssh-keygen -R '[localhost]:2022' +Upgrading to a newer release of boot2docker +------------------------------------------- + +To upgrade an initialised VM, you can use the following 3 commands. Your persistence +disk will not be changed, so you won't lose your images and containers: + +.. code-block:: bash + + ./boot2docker stop + ./boot2docker download + ./boot2docker start + About the way Docker works on Mac OS X: --------------------------------------- From c0caaa92a354d45f99a70088a91fd1e3b97da4ca Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 10 Feb 2014 22:32:07 -0800 Subject: [PATCH 042/403] Add more netlink functions for set ns by fd and bring iface down Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: f9cd1be6ffbda85f20cc7926274fab7484a19823 Component: engine --- .../engine/pkg/netlink/netlink_linux.go | 61 ++++++++++++++++++- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 23dba0c884..3b21a38d06 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -15,7 +15,8 @@ const ( DEFAULT_CHANGE = 0xFFFFFFFF IFLA_INFO_KIND = 1 IFLA_INFO_DATA = 2 - VETH_PEER = 1 + VETH_INFO_PEER = 1 + IFLA_NET_NS_FD = 28 ) var nextSeqNr int @@ -365,6 +366,28 @@ func NetworkLinkUp(iface *net.Interface) error { return s.HandleAck(wb.Seq) } +func NetworkLinkDown(iface *net.Interface) error { + s, err := getNetlinkSocket() + if err != nil { + return err + } + defer s.Close() + + wb := newNetlinkRequest(syscall.RTM_NEWLINK, syscall.NLM_F_ACK) + + msg := newIfInfomsg(syscall.AF_UNSPEC) + msg.Change = syscall.IFF_UP + msg.Flags = 0 & ^syscall.IFF_UP + msg.Index = int32(iface.Index) + wb.AddData(msg) + + if err := s.Send(wb); err != nil { + return err + } + + return s.HandleAck(wb.Seq) +} + func NetworkSetMTU(iface *net.Interface, mtu int) error { s, err := getNetlinkSocket() if err != nil { @@ -461,6 +484,38 @@ func NetworkSetNsPid(iface *net.Interface, nspid int) error { return s.HandleAck(wb.Seq) } +func NetworkSetNsFd(iface *net.Interface, fd int) error { + s, err := getNetlinkSocket() + if err != nil { + return err + } + defer s.Close() + + wb := newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := newIfInfomsg(syscall.AF_UNSPEC) + msg.Type = syscall.RTM_SETLINK + msg.Flags = syscall.NLM_F_REQUEST + msg.Index = int32(iface.Index) + msg.Change = DEFAULT_CHANGE + wb.AddData(msg) + + var ( + b = make([]byte, 4) + native = nativeEndian() + ) + native.PutUint32(b, uint32(fd)) + + data := newRtAttr(IFLA_NET_NS_FD, b) + wb.AddData(data) + + if err := s.Send(wb); err != nil { + return err + } + + return s.HandleAck(wb.Seq) +} + // Add an Ip address to an interface. This is identical to: // ip addr add $ip/$ipNet dev $iface func NetworkLinkAddIp(iface *net.Interface, ip net.IP, ipNet *net.IPNet) error { @@ -668,7 +723,7 @@ func getIfSocket() (fd int, err error) { return -1, err } -func NetworkChangeName(oldName, newName string) error { +func NetworkChangeName(iface *net.Interface, newName string) error { fd, err := getIfSocket() if err != nil { return err @@ -676,7 +731,7 @@ func NetworkChangeName(oldName, newName string) error { defer syscall.Close(fd) data := [IFNAMSIZ * 2]byte{} - copy(data[:IFNAMSIZ-1], oldName) + copy(data[:IFNAMSIZ-1], iface.Name) copy(data[IFNAMSIZ:IFNAMSIZ*2-1], newName) if _, _, errno := syscall.Syscall(syscall.SYS_IOCTL, uintptr(fd), syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&data[0]))); errno != 0 { From abb4de85469c76a53fd8a2106d41560a00ed2b78 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 11 Feb 2014 09:40:13 +0100 Subject: [PATCH 043/403] devmapper: Remove directory when removing devicemapper device We're currently leaving around lots of empty directories in /var/lib/docker/devicemapper/mnt/ for removed images and containers. Fix this by removing the directory when the device is removed. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 2343fe44533f19ebae5e6127f4a2a19d1d8773fa Component: engine --- components/engine/graphdriver/devmapper/driver.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/components/engine/graphdriver/devmapper/driver.go b/components/engine/graphdriver/devmapper/driver.go index 25b998e59f..4d414f9a75 100644 --- a/components/engine/graphdriver/devmapper/driver.go +++ b/components/engine/graphdriver/devmapper/driver.go @@ -7,6 +7,7 @@ import ( "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/utils" "io/ioutil" + "os" "path" ) @@ -94,7 +95,16 @@ func (d *Driver) Remove(id string) error { return err } // This assumes the device has been properly Get/Put:ed and thus is unmounted - return d.DeviceSet.DeleteDevice(id) + if err := d.DeviceSet.DeleteDevice(id); err != nil { + return err + } + + mp := path.Join(d.home, "mnt", id) + if err := os.RemoveAll(mp); err != nil && !os.IsNotExist(err) { + return err + } + + return nil } func (d *Driver) Get(id string) (string, error) { From 9ca9fdc33f1da57898214159ca003b89ae10b370 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 11 Feb 2014 09:42:34 +0100 Subject: [PATCH 044/403] devicemapper: Fix tests on fedora If /dev/loop-control exists on the system running the test then ioctlLoopCtlGetFree() will be called, but if not it won't. It does not exist in the standard docker build environment, so the tests currently require this to not be called. This makes it instead optional, allowing the tests to run on e.g. Fedora. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: eb4578daee98561b16d11d2978b5f5e297d903e8 Component: engine --- components/engine/graphdriver/devmapper/driver_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/components/engine/graphdriver/devmapper/driver_test.go b/components/engine/graphdriver/devmapper/driver_test.go index ff4ec45e5d..7939241987 100644 --- a/components/engine/graphdriver/devmapper/driver_test.go +++ b/components/engine/graphdriver/devmapper/driver_test.go @@ -136,7 +136,12 @@ type Set map[string]bool func (r Set) Assert(t *testing.T, names ...string) { for _, key := range names { - if _, exists := r[key]; !exists { + required := true + if strings.HasPrefix(key, "?") { + key = key[1:] + required = false + } + if _, exists := r[key]; !exists && required { t.Fatalf("Key not set: %s", key) } delete(r, key) @@ -486,6 +491,7 @@ func TestDriverCreate(t *testing.T) { "ioctl.blkgetsize", "ioctl.loopsetfd", "ioctl.loopsetstatus", + "?ioctl.loopctlgetfree", ) if err := d.Create("1", ""); err != nil { @@ -603,6 +609,7 @@ func TestDriverRemove(t *testing.T) { "ioctl.blkgetsize", "ioctl.loopsetfd", "ioctl.loopsetstatus", + "?ioctl.loopctlgetfree", ) if err := d.Create("1", ""); err != nil { From 96a42d5055d3d6f9b5081414fcf7da48f32313ba Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 11 Feb 2014 11:10:55 +0100 Subject: [PATCH 045/403] lxc: Drop NET_ADMIN capability in non-privileged containers With this capability set the container can e.g. change the ip address of his devices to that of another container on the docker0 bridge. In a quick test I was able to listen to a port on a different ip than the one docker assigned me, but was not able to hijack an open port redirection that another container had open. Maybe its possible with some more knowledge of networking though. Anyway, network setup is meant to be handled by docker, not the apps, so I believe denying this is generally in the spirit of docker, and it closes down potential security issues. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 02fddffd51da782f912e2709ea814c330269515b Component: engine --- components/engine/execdriver/lxc/init.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/execdriver/lxc/init.go b/components/engine/execdriver/lxc/init.go index 7c2b039c50..0a7746dd15 100644 --- a/components/engine/execdriver/lxc/init.go +++ b/components/engine/execdriver/lxc/init.go @@ -127,6 +127,7 @@ func setupCapabilities(args *execdriver.InitArgs) error { capability.CAP_AUDIT_CONTROL, capability.CAP_MAC_OVERRIDE, capability.CAP_MAC_ADMIN, + capability.CAP_NET_ADMIN, } c, err := capability.NewPid(os.Getpid()) From d33fce79f6a0ab67e56a246c90cef8c23ea7185f Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Tue, 11 Feb 2014 19:53:45 +1000 Subject: [PATCH 046/403] add a little more information about the docker run -P option Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: b2146cb0e7a23553d1520ed87837a1b5455c9130 Component: engine --- .../engine/docs/sources/reference/commandline/cli.rst | 4 ++++ components/engine/docs/sources/use/port_redirection.rst | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index 5f4f821e12..18cef685aa 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -1088,6 +1088,10 @@ is, ``docker run`` is equivalent to the API ``/containers/create`` then The ``docker run`` command can be used in combination with ``docker commit`` to :ref:`change the command that a container runs `. +See :ref:`port_redirection` for more detailed information about the ``--expose``, +``-p``, ``-P`` and ``--link`` parameters, and :ref:`working_with_links_names` for +specific examples using ``--link``. + Known Issues (run -volumes-from) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/components/engine/docs/sources/use/port_redirection.rst b/components/engine/docs/sources/use/port_redirection.rst index 5cddb238e4..2f4fd37c76 100644 --- a/components/engine/docs/sources/use/port_redirection.rst +++ b/components/engine/docs/sources/use/port_redirection.rst @@ -31,6 +31,15 @@ container, Docker provide ways to bind the container port to an interface of the host system. To simplify communication between containers, Docker provides the linking mechanism. +Auto map all exposed ports on the host +------------------------------------- + +To bind all the exposed container ports to the host automatically, use +``docker run -P ``. The mapped host ports will be auto-selected +from a pool of unused ports (49000..49900), and you will need to use +``docker ps``, ``docker inspect `` or +``docker port `` to determine what they are. + Binding a port to a host interface ----------------------------------- From 6d92748c6f7c8960aeb386905ac59cee8b656f85 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 11 Feb 2014 03:32:35 -0800 Subject: [PATCH 047/403] Exec out to ip right now for creating the veth pair Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 27ed9a9f98750ab666d6221553b6f4ea59d396b8 Component: engine --- .../engine/pkg/netlink/netlink_linux.go | 36 +++---------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 3b21a38d06..b9e04a339a 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "fmt" "net" + "os/exec" "syscall" "unsafe" ) @@ -741,37 +742,8 @@ func NetworkChangeName(iface *net.Interface, newName string) error { } func NetworkCreateVethPair(name1, name2 string) error { - s, err := getNetlinkSocket() - if err != nil { - return err + if data, err := exec.Command("ip", "link", "add", name1, "type", "veth", "peer", "name", name2).Output(); err != nil { + return fmt.Errorf("%s %s", data, err) } - defer s.Close() - - wb := newNetlinkRequest(syscall.RTM_NEWLINK, syscall.NLM_F_CREATE|syscall.NLM_F_EXCL|syscall.NLM_F_ACK) - - msg := newIfInfomsg(syscall.AF_UNSPEC) - wb.AddData(msg) - - kindData := newRtAttr(IFLA_INFO_KIND, nonZeroTerminated("veth")) - info := newRtAttr(syscall.IFLA_LINKINFO, kindData.ToWireFormat()) - // wb.AddData(info) - - peerName := newRtAttr(syscall.IFLA_IFNAME, nonZeroTerminated(name2)) - peer := newRtAttr(VETH_PEER, peerName.ToWireFormat()) - // wb.AddData(peer) - - b := []byte{} - b = append(b, peer.ToWireFormat()...) - b = append(b, info.ToWireFormat()...) - - infoData := newRtAttr(IFLA_INFO_DATA, b) - wb.AddData(infoData) - - nameData := newRtAttr(syscall.IFLA_IFNAME, zeroTerminated(name1)) - wb.AddData(nameData) - - if err := s.Send(wb); err != nil { - return err - } - return s.HandleAck(wb.Seq) + return nil } From 3f55cc6f4171cf1abf21b46736513e7d4ba51109 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 10 Feb 2014 15:11:17 -0700 Subject: [PATCH 048/403] Add comment clarifying null termination Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: c626349f651a660302f64101055d65dc6e990307 Component: engine --- components/engine/pkg/netlink/netlink_linux.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 3b21a38d06..85dac51ea4 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -731,6 +731,8 @@ func NetworkChangeName(iface *net.Interface, newName string) error { defer syscall.Close(fd) data := [IFNAMSIZ * 2]byte{} + // the "-1"s here are very important for ensuring we get proper null + // termination of our new C strings copy(data[:IFNAMSIZ-1], iface.Name) copy(data[IFNAMSIZ:IFNAMSIZ*2-1], newName) From d3d8c35b4150df6c5565f6cc3432ea9d1289b3e5 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 11 Feb 2014 12:47:59 +0100 Subject: [PATCH 049/403] devmapper: Fix shutdown warnings Shutdown contains debug warnings like: [debug] deviceset.go:699 [deviceset docker-0:33-17945897] waitRemove(/dev/mapper/docker-0:33-17945897-pool) [debug] deviceset.go:380 libdevmapper(3): libdm-common.c:552 (-1) Device /dev/mapper/docker-0:33-17945897-pool not found This is because shutdown is using removeDeviceAndWait() to remove the pool device and the wait part fails because the pool is gone. We fix this by adding a pool specific removal function which avoids all the trickiness of the normal remove. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: eab270395e5b47b16a41c54ec6e1427f8144bffc Component: engine --- .../engine/graphdriver/devmapper/deviceset.go | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/components/engine/graphdriver/devmapper/deviceset.go b/components/engine/graphdriver/devmapper/deviceset.go index 637192dbcb..9b5ae62b67 100644 --- a/components/engine/graphdriver/devmapper/deviceset.go +++ b/components/engine/graphdriver/devmapper/deviceset.go @@ -639,6 +639,22 @@ func (devices *DeviceSet) DeleteDevice(hash string) error { return devices.deleteDevice(hash) } +func (devices *DeviceSet) deactivatePool() error { + utils.Debugf("[devmapper] deactivatePool()") + defer utils.Debugf("[devmapper] deactivatePool END") + devname := devices.getPoolDevName() + devinfo, err := getInfo(devname) + if err != nil { + utils.Debugf("\n--->Err: %s\n", err) + return err + } + if devinfo.Exists != 0 { + return removeDevice(devname) + } + + return nil +} + func (devices *DeviceSet) deactivateDevice(hash string) error { utils.Debugf("[devmapper] deactivateDevice(%s)", hash) defer utils.Debugf("[devmapper] deactivateDevice END") @@ -789,11 +805,8 @@ func (devices *DeviceSet) Shutdown() error { } } - pool := devices.getPoolDevName() - if devinfo, err := getInfo(pool); err == nil && devinfo.Exists != 0 { - if err := devices.deactivateDevice("pool"); err != nil { - utils.Debugf("Shutdown deactivate %s , error: %s\n", pool, err) - } + if err := devices.deactivatePool(); err != nil { + utils.Debugf("Shutdown deactivate pool , error: %s\n", err) } return nil From 78f5f41e8eade8831716809d05cf7bb7cfe322d6 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 11 Feb 2014 12:55:40 +0100 Subject: [PATCH 050/403] devmapper: Remove byHash hack We no longer pass "pool" anywhere that uses byHash() per the last commit, so we can now remove this hack. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 6128dcea4a9bbe808baba4e18c9c4fee3a265532 Component: engine --- .../engine/graphdriver/devmapper/deviceset.go | 40 ++++++------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/components/engine/graphdriver/devmapper/deviceset.go b/components/engine/graphdriver/devmapper/deviceset.go index 9b5ae62b67..303e363e92 100644 --- a/components/engine/graphdriver/devmapper/deviceset.go +++ b/components/engine/graphdriver/devmapper/deviceset.go @@ -658,19 +658,18 @@ func (devices *DeviceSet) deactivatePool() error { func (devices *DeviceSet) deactivateDevice(hash string) error { utils.Debugf("[devmapper] deactivateDevice(%s)", hash) defer utils.Debugf("[devmapper] deactivateDevice END") - var devname string - // FIXME: shouldn't we just register the pool into devices? - devname, err := devices.byHash(hash) - if err != nil { - return err + + info := devices.Devices[hash] + if info == nil { + return fmt.Errorf("Unknown device %s", hash) } - devinfo, err := getInfo(devname) + devinfo, err := getInfo(info.Name()) if err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } if devinfo.Exists != 0 { - if err := devices.removeDeviceAndWait(devname); err != nil { + if err := devices.removeDeviceAndWait(info.Name()); err != nil { utils.Debugf("\n--->Err: %s\n", err) return err } @@ -741,18 +740,18 @@ func (devices *DeviceSet) waitRemove(devname string) error { // a) the device registered at - is closed, // or b) the 1 second timeout expires. func (devices *DeviceSet) waitClose(hash string) error { - devname, err := devices.byHash(hash) - if err != nil { - return err + info := devices.Devices[hash] + if info == nil { + return fmt.Errorf("Unknown device %s", hash) } i := 0 for ; i < 1000; i += 1 { - devinfo, err := getInfo(devname) + devinfo, err := getInfo(info.Name()) if err != nil { return err } if i%100 == 0 { - utils.Debugf("Waiting for unmount of %s: opencount=%d", devname, devinfo.OpenCount) + utils.Debugf("Waiting for unmount of %s: opencount=%d", hash, devinfo.OpenCount) } if devinfo.OpenCount == 0 { break @@ -760,26 +759,11 @@ func (devices *DeviceSet) waitClose(hash string) error { time.Sleep(1 * time.Millisecond) } if i == 1000 { - return fmt.Errorf("Timeout while waiting for device %s to close", devname) + return fmt.Errorf("Timeout while waiting for device %s to close", hash) } return nil } -// byHash is a hack to allow looking up the deviceset's pool by the hash "pool". -// FIXME: it seems probably cleaner to register the pool in devices.Devices, -// but I am afraid of arcane implications deep in the devicemapper code, -// so this will do. -func (devices *DeviceSet) byHash(hash string) (devname string, err error) { - if hash == "pool" { - return devices.getPoolDevName(), nil - } - info := devices.Devices[hash] - if info == nil { - return "", fmt.Errorf("hash %s doesn't exists", hash) - } - return info.Name(), nil -} - func (devices *DeviceSet) Shutdown() error { devices.Lock() defer devices.Unlock() From a0e7d3fc71b49f9b0bf6e37bf9e9b129b764742e Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 11 Feb 2014 06:21:33 -0800 Subject: [PATCH 051/403] Ensure docker cp stream is closes properly Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 35821ad78f4ad2b20cf839ecc5a73b44c568c5eb Component: engine --- components/engine/container.go | 4 ++-- components/engine/server.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index 81e8749d2a..0bfa8befe6 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -1562,7 +1562,7 @@ func (container *Container) GetSize() (int64, int64) { return sizeRw, sizeRootfs } -func (container *Container) Copy(resource string) (archive.Archive, error) { +func (container *Container) Copy(resource string) (io.ReadCloser, error) { if err := container.Mount(); err != nil { return nil, err } @@ -1589,7 +1589,7 @@ func (container *Container) Copy(resource string) (archive.Archive, error) { if err != nil { return nil, err } - return EofReader(archive, func() { container.Unmount() }), nil + return utils.NewReadCloserWrapper(archive, container.Unmount), nil } // Returns true if the container exposes a certain port diff --git a/components/engine/server.go b/components/engine/server.go index f108f61740..20b45f65aa 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -2339,6 +2339,7 @@ func (srv *Server) ContainerCopy(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } + defer data.Close() if _, err := io.Copy(job.Stdout, data); err != nil { return job.Error(err) From c19931ad4a73697d8f63aac72e7ea08342816b87 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 11 Feb 2014 18:02:30 +0100 Subject: [PATCH 052/403] Avoid extra mount/unmount during container registration Runtime.Register() called driver.Get()/Put() in order to read back the basefs of the container. However, this is not needed, as the basefs is read during container.Mount() anyway, and basefs is only valid while mounted (and all current calls satisfy this). This seems minor, but this is actually problematic, as the Get/Put pair will create a spurious mount/unmount cycle that is not needed and slows things down. Additionally it will create a supurious devicemapper activate/deactivate cycle that causes races with udev as seen in https://github.com/dotcloud/docker/issues/4036. With this change devicemapper is now race-free, and container startup is slightly faster. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 0c71015dcf788c5342bb9422a447b7f9ad12d43d Component: engine --- components/engine/runtime.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/components/engine/runtime.go b/components/engine/runtime.go index a16238e200..176f51b0b1 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -133,14 +133,6 @@ func (runtime *Runtime) Register(container *Container) error { return err } - // Get the root filesystem from the driver - basefs, err := runtime.driver.Get(container.ID) - if err != nil { - return fmt.Errorf("Error getting container filesystem %s from driver %s: %s", container.ID, runtime.driver, err) - } - defer runtime.driver.Put(container.ID) - container.basefs = basefs - container.runtime = runtime // Attach to stdout and stderr From 33338e981dfdd73baec9d6ad791e22d9c70e3071 Mon Sep 17 00:00:00 2001 From: Vladimir Bulyga Date: Wed, 12 Feb 2014 00:26:19 +0200 Subject: [PATCH 053/403] Update remote_api_client_libraries.rst hello, add please my interface Docker-DCO-1.1-Signed-off-by: Vladimir (github: 13w) Upstream-commit: d35b03ef8db9d773937133277cc16364a551ab74 Component: engine --- .../docs/sources/reference/api/remote_api_client_libraries.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/docs/sources/reference/api/remote_api_client_libraries.rst b/components/engine/docs/sources/reference/api/remote_api_client_libraries.rst index 45ce8ff9d1..c7ced0055e 100644 --- a/components/engine/docs/sources/reference/api/remote_api_client_libraries.rst +++ b/components/engine/docs/sources/reference/api/remote_api_client_libraries.rst @@ -26,6 +26,9 @@ and we will add the libraries here. +----------------------+----------------+--------------------------------------------+----------+ | Javascript | docker-js | https://github.com/dgoujard/docker-js | Active | +----------------------+----------------+--------------------------------------------+----------+ +| Javascript (Angular) | docker-cp | https://github.com/13W/docker-cp | Active | +| **WebUI** | | | | ++----------------------+----------------+--------------------------------------------+----------+ | Javascript (Angular) | dockerui | https://github.com/crosbymichael/dockerui | Active | | **WebUI** | | | | +----------------------+----------------+--------------------------------------------+----------+ From 231362cb9e6bd53dddbabcc9aa1fd636e1cb77fd Mon Sep 17 00:00:00 2001 From: Cameron Boehmer Date: Sun, 9 Feb 2014 05:05:19 -0800 Subject: [PATCH 054/403] add port forwarding notes for mac/boot2docker docs Docker-DCO-1.1-Signed-off-by: Cameron Boehmer (github: cameronboehmer) Upstream-commit: 49b928c0b05852b58881ce8ebdb164db92befd3a Component: engine --- .../engine/docs/sources/installation/mac.rst | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/components/engine/docs/sources/installation/mac.rst b/components/engine/docs/sources/installation/mac.rst index 4f0550ad3d..5d57c258de 100644 --- a/components/engine/docs/sources/installation/mac.rst +++ b/components/engine/docs/sources/installation/mac.rst @@ -49,10 +49,10 @@ Run the following commands to get boot2docker: # Enter the installation directory cd ~/bin - + # Get the file curl https://raw.github.com/steeve/boot2docker/master/boot2docker > boot2docker - + # Mark it executable chmod +x boot2docker @@ -67,13 +67,13 @@ Run the following commands to get it downloaded and set up: # Get the file curl -o docker https://get.docker.io/builds/Darwin/x86_64/docker-latest - + # Mark it executable chmod +x docker # Set the environment variable for the docker daemon export DOCKER_HOST=tcp:// - + # Copy the executable file sudo cp docker /usr/local/bin/ @@ -94,7 +94,7 @@ Inside the ``~/bin`` directory, run the following commands: # Run the VM (the docker daemon) ./boot2docker up - + # To see all available commands: ./boot2docker @@ -116,6 +116,21 @@ client just like any other application. # Git commit (server): c348c04 # Go version (server): go1.2 +Forwarding VM Port Range to Host +------------------------------- + +If we take the port range that docker uses by default with the -P option +(49000-49900), and forward same range from host to vm, we'll be able to interact +with our containers as if they were running locally: + +.. code-block:: bash + + # vm must be powered off + for i in {4900..49900}; do + VBoxManage modifyvm "boot2docker-vm" --natpf1 "tcp-port$i,tcp,,$i,,$i"; + VBoxManage modifyvm "boot2docker-vm" --natpf1 "udp-port$i,udp,,$i,,$i"; + done + SSH-ing The VM -------------- From ef486c66301274afc749d61f6b7f08da68230a04 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 11 Feb 2014 16:05:45 -0800 Subject: [PATCH 055/403] Move docker version introspection to a sub-package. This facilitates the refactoring of commands.go. Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: ae3c7dec3b075a263da8d1e1cc510275b281e799 Component: engine --- components/engine/commands.go | 5 ----- components/engine/docker/docker.go | 12 +++--------- components/engine/dockerversion/dockerversion.go | 11 +++++++++++ components/engine/hack/make.sh | 2 +- components/engine/version.go | 9 +++++++++ 5 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 components/engine/dockerversion/dockerversion.go diff --git a/components/engine/commands.go b/components/engine/commands.go index cc019f8c10..55cb3e21c7 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -38,11 +38,6 @@ import ( "time" ) -var ( - GITCOMMIT string - VERSION string -) - var ( ErrConnectionRefused = errors.New("Can't connect to docker daemon. Is 'docker -d' running on this host?") ) diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index d92f4d98ea..df99a20450 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -8,17 +8,13 @@ import ( "github.com/dotcloud/docker" "github.com/dotcloud/docker/api" + "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" flag "github.com/dotcloud/docker/pkg/mflag" "github.com/dotcloud/docker/sysinit" "github.com/dotcloud/docker/utils" ) -var ( - GITCOMMIT string - VERSION string -) - func main() { if selfPath := utils.SelfPath(); selfPath == "/sbin/init" || selfPath == "/.dockerinit" { // Running in init mode @@ -71,8 +67,6 @@ func main() { if *flDebug { os.Setenv("DEBUG", "1") } - docker.GITCOMMIT = GITCOMMIT - docker.VERSION = VERSION if *flDaemon { if flag.NArg() != 0 { flag.Usage() @@ -104,7 +98,7 @@ func main() { job = eng.Job("serveapi", flHosts.GetAll()...) job.SetenvBool("Logging", true) job.SetenvBool("EnableCors", *flEnableCors) - job.Setenv("Version", VERSION) + job.Setenv("Version", dockerversion.VERSION) if err := job.Run(); err != nil { log.Fatal(err) } @@ -126,5 +120,5 @@ func main() { } func showVersion() { - fmt.Printf("Docker version %s, build %s\n", VERSION, GITCOMMIT) + fmt.Printf("Docker version %s, build %s\n", dockerversion.VERSION, dockerversion.GITCOMMIT) } diff --git a/components/engine/dockerversion/dockerversion.go b/components/engine/dockerversion/dockerversion.go new file mode 100644 index 0000000000..1872691ed2 --- /dev/null +++ b/components/engine/dockerversion/dockerversion.go @@ -0,0 +1,11 @@ +package dockerversion + +// FIXME: this should be embedded in the docker/docker.go, +// but we can't because distro policy requires us to +// package a separate dockerinit binary, and that binary needs +// to know its version too. + +var ( + GITCOMMIT string + VERSION string +) diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index d73b33a4fa..011b796201 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -82,7 +82,7 @@ if [ ! "$GOPATH" ]; then fi # Use these flags when compiling the tests and final binary -LDFLAGS='-X main.GITCOMMIT "'$GITCOMMIT'" -X main.VERSION "'$VERSION'" -w' +LDFLAGS='-X github.com/dotcloud/docker/dockerversion.GITCOMMIT "'$GITCOMMIT'" -X github.com/dotcloud/docker/dockerversion.VERSION "'$VERSION'" -w' LDFLAGS_STATIC='-X github.com/dotcloud/docker/utils.IAMSTATIC true -linkmode external -extldflags "-lpthread -static -Wl,--unresolved-symbols=ignore-in-object-files"' BUILDFLAGS='-tags netgo -a' diff --git a/components/engine/version.go b/components/engine/version.go index a4288245f7..2eee68c2f2 100644 --- a/components/engine/version.go +++ b/components/engine/version.go @@ -1,11 +1,20 @@ package docker import ( + "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/utils" "runtime" ) +var ( + // FIXME: this is a convenience indirection to preserve legacy + // code. It can be removed by using dockerversion.VERSION and + // dockerversion.GITCOMMIT directly + GITCOMMIT string = dockerversion.GITCOMMIT + VERSION string = dockerversion.VERSION +) + func init() { engine.Register("version", jobVersion) } From deaadc8b5665cccdef88bccfd66a04286ed2c844 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Tue, 11 Feb 2014 16:19:42 -0800 Subject: [PATCH 056/403] fix underline/heading Upstream-commit: c76a4e34896fd0785cf982372a3713c4ef70754b Component: engine --- components/engine/docs/sources/installation/mac.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/installation/mac.rst b/components/engine/docs/sources/installation/mac.rst index 86a01f6bb6..7d2a0b19e8 100644 --- a/components/engine/docs/sources/installation/mac.rst +++ b/components/engine/docs/sources/installation/mac.rst @@ -117,7 +117,7 @@ client just like any other application. # Go version (server): go1.2 Forwarding VM Port Range to Host -------------------------------- +-------------------------------- If we take the port range that docker uses by default with the -P option (49000-49900), and forward same range from host to vm, we'll be able to interact From 9ed97b673b0a73d0703c8b999d0a780c117b0be7 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 11 Feb 2014 17:26:54 -0700 Subject: [PATCH 057/403] Move even more stuff into dockerversion Also, use it in all the places. :) Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: da04f49b383c02ee28c32f948048b9e9a402bb4f Component: engine --- components/engine/commands.go | 17 +++++++++-------- components/engine/dockerinit/dockerinit.go | 5 ----- .../engine/dockerversion/dockerversion.go | 4 ++++ components/engine/graph.go | 3 ++- components/engine/hack/make/dynbinary | 2 +- components/engine/runtime.go | 3 ++- components/engine/server.go | 3 ++- components/engine/utils/utils.go | 13 ++++--------- components/engine/version.go | 12 ++---------- 9 files changed, 26 insertions(+), 36 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 55cb3e21c7..651173555e 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -11,6 +11,7 @@ import ( "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" flag "github.com/dotcloud/docker/pkg/mflag" "github.com/dotcloud/docker/pkg/sysinfo" @@ -383,12 +384,12 @@ func (cli *DockerCli) CmdVersion(args ...string) error { cmd.Usage() return nil } - if VERSION != "" { - fmt.Fprintf(cli.out, "Client version: %s\n", VERSION) + if dockerversion.VERSION != "" { + fmt.Fprintf(cli.out, "Client version: %s\n", dockerversion.VERSION) } fmt.Fprintf(cli.out, "Go version (client): %s\n", runtime.Version()) - if GITCOMMIT != "" { - fmt.Fprintf(cli.out, "Git commit (client): %s\n", GITCOMMIT) + if dockerversion.GITCOMMIT != "" { + fmt.Fprintf(cli.out, "Git commit (client): %s\n", dockerversion.GITCOMMIT) } body, _, err := readBody(cli.call("GET", "/version", nil, false)) @@ -413,7 +414,7 @@ func (cli *DockerCli) CmdVersion(args ...string) error { release := utils.GetReleaseVersion() if release != "" { fmt.Fprintf(cli.out, "Last stable version: %s", release) - if (VERSION != "" || remoteVersion.Exists("Version")) && (strings.Trim(VERSION, "-dev") != release || strings.Trim(remoteVersion.Get("Version"), "-dev") != release) { + if (dockerversion.VERSION != "" || remoteVersion.Exists("Version")) && (strings.Trim(dockerversion.VERSION, "-dev") != release || strings.Trim(remoteVersion.Get("Version"), "-dev") != release) { fmt.Fprintf(cli.out, ", please update docker") } fmt.Fprintf(cli.out, "\n") @@ -2298,7 +2299,7 @@ func (cli *DockerCli) call(method, path string, data interface{}, passAuthInfo b } } } - req.Header.Set("User-Agent", "Docker-Client/"+VERSION) + req.Header.Set("User-Agent", "Docker-Client/"+dockerversion.VERSION) req.Host = cli.addr if data != nil { req.Header.Set("Content-Type", "application/json") @@ -2355,7 +2356,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, h if err != nil { return err } - req.Header.Set("User-Agent", "Docker-Client/"+VERSION) + req.Header.Set("User-Agent", "Docker-Client/"+dockerversion.VERSION) req.Host = cli.addr if method == "POST" { req.Header.Set("Content-Type", "plain/text") @@ -2419,7 +2420,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea if err != nil { return err } - req.Header.Set("User-Agent", "Docker-Client/"+VERSION) + req.Header.Set("User-Agent", "Docker-Client/"+dockerversion.VERSION) req.Header.Set("Content-Type", "plain/text") req.Host = cli.addr diff --git a/components/engine/dockerinit/dockerinit.go b/components/engine/dockerinit/dockerinit.go index 0c363f4ac3..1d0689387a 100644 --- a/components/engine/dockerinit/dockerinit.go +++ b/components/engine/dockerinit/dockerinit.go @@ -4,11 +4,6 @@ import ( "github.com/dotcloud/docker/sysinit" ) -var ( - GITCOMMIT string - VERSION string -) - func main() { // Running in init mode sysinit.SysInit() diff --git a/components/engine/dockerversion/dockerversion.go b/components/engine/dockerversion/dockerversion.go index 1872691ed2..c130ac2810 100644 --- a/components/engine/dockerversion/dockerversion.go +++ b/components/engine/dockerversion/dockerversion.go @@ -8,4 +8,8 @@ package dockerversion var ( GITCOMMIT string VERSION string + + IAMSTATIC bool // whether or not Docker itself was compiled statically via ./hack/make.sh binary + INITSHA1 string // sha1sum of separate static dockerinit, if Docker itself was compiled dynamically via ./hack/make.sh dynbinary + INITPATH string // custom location to search for a valid dockerinit binary (available for packagers as a last resort escape hatch) ) diff --git a/components/engine/graph.go b/components/engine/graph.go index 42da42c8af..138c7b8613 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -3,6 +3,7 @@ package docker import ( "fmt" "github.com/dotcloud/docker/archive" + "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/utils" "io" @@ -130,7 +131,7 @@ func (graph *Graph) Create(layerData archive.Archive, container *Container, comm ID: GenerateID(), Comment: comment, Created: time.Now().UTC(), - DockerVersion: VERSION, + DockerVersion: dockerversion.VERSION, Author: author, Config: config, Architecture: runtime.GOARCH, diff --git a/components/engine/hack/make/dynbinary b/components/engine/hack/make/dynbinary index c02094c0c5..7de3a6cb59 100644 --- a/components/engine/hack/make/dynbinary +++ b/components/engine/hack/make/dynbinary @@ -12,6 +12,6 @@ export DOCKER_INITSHA1="$(sha1sum $DEST/dockerinit-$VERSION | cut -d' ' -f1)" # exported so that "dyntest" can easily access it later without recalculating it ( - export LDFLAGS_STATIC="-X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\" -X github.com/dotcloud/docker/utils.INITPATH \"$DOCKER_INITPATH\"" + export LDFLAGS_STATIC="-X github.com/dotcloud/docker/dockerversion.INITSHA1 \"$DOCKER_INITSHA1\" -X github.com/dotcloud/docker/dockerversion.INITPATH \"$DOCKER_INITPATH\"" source "$(dirname "$BASH_SOURCE")/binary" ) diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 176f51b0b1..cec5444090 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -4,6 +4,7 @@ import ( "container/list" "fmt" "github.com/dotcloud/docker/archive" + "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/execdriver" "github.com/dotcloud/docker/execdriver/chroot" @@ -678,7 +679,7 @@ func NewRuntimeFromDirectory(config *DaemonConfig, eng *engine.Engine) (*Runtime return nil, err } - localCopy := path.Join(config.Root, "init", fmt.Sprintf("dockerinit-%s", VERSION)) + localCopy := path.Join(config.Root, "init", fmt.Sprintf("dockerinit-%s", dockerversion.VERSION)) sysInitPath := utils.DockerInitPath(localCopy) if sysInitPath == "" { return nil, fmt.Errorf("Could not locate dockerinit: This usually means docker was built incorrectly. See http://docs.docker.io/en/latest/contributing/devenvironment for official build instructions.") diff --git a/components/engine/server.go b/components/engine/server.go index f108f61740..cb677266e5 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/registry" @@ -827,7 +828,7 @@ func (srv *Server) DockerInfo(job *engine.Job) engine.Status { v.SetInt("NEventsListener", len(srv.events)) v.Set("KernelVersion", kernelVersion) v.Set("IndexServerAddress", auth.IndexServerAddress()) - v.Set("InitSha1", utils.INITSHA1) + v.Set("InitSha1", dockerversion.INITSHA1) v.Set("InitPath", initPath) if _, err := v.WriteTo(job.Stdout); err != nil { return job.Error(err) diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 5caf792549..60952606d5 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/dotcloud/docker/dockerversion" "index/suffixarray" "io" "io/ioutil" @@ -23,12 +24,6 @@ import ( "time" ) -var ( - IAMSTATIC bool // whether or not Docker itself was compiled statically via ./hack/make.sh binary - INITSHA1 string // sha1sum of separate static dockerinit, if Docker itself was compiled dynamically via ./hack/make.sh dynbinary - INITPATH string // custom location to search for a valid dockerinit binary (available for packagers as a last resort escape hatch) -) - // A common interface to access the Fatal method of // both testing.B and testing.T. type Fataler interface { @@ -201,7 +196,7 @@ func isValidDockerInitPath(target string, selfPath string) bool { // target and if target == "" { return false } - if IAMSTATIC { + if dockerversion.IAMSTATIC { if selfPath == "" { return false } @@ -218,7 +213,7 @@ func isValidDockerInitPath(target string, selfPath string) bool { // target and } return os.SameFile(targetFileInfo, selfPathFileInfo) } - return INITSHA1 != "" && dockerInitSha1(target) == INITSHA1 + return dockerversion.INITSHA1 != "" && dockerInitSha1(target) == dockerversion.INITSHA1 } // Figure out the path of our dockerinit (which may be SelfPath()) @@ -230,7 +225,7 @@ func DockerInitPath(localCopy string) string { } var possibleInits = []string{ localCopy, - INITPATH, + dockerversion.INITPATH, filepath.Join(filepath.Dir(selfPath), "dockerinit"), // FHS 3.0 Draft: "/usr/libexec includes internal binaries that are not intended to be executed directly by users or shell scripts. Applications may use a single subdirectory under /usr/libexec." diff --git a/components/engine/version.go b/components/engine/version.go index 2eee68c2f2..88298a16cb 100644 --- a/components/engine/version.go +++ b/components/engine/version.go @@ -7,14 +7,6 @@ import ( "runtime" ) -var ( - // FIXME: this is a convenience indirection to preserve legacy - // code. It can be removed by using dockerversion.VERSION and - // dockerversion.GITCOMMIT directly - GITCOMMIT string = dockerversion.GITCOMMIT - VERSION string = dockerversion.VERSION -) - func init() { engine.Register("version", jobVersion) } @@ -31,8 +23,8 @@ func jobVersion(job *engine.Job) engine.Status { // environment. func dockerVersion() *engine.Env { v := &engine.Env{} - v.Set("Version", VERSION) - v.Set("GitCommit", GITCOMMIT) + v.Set("Version", dockerversion.VERSION) + v.Set("GitCommit", dockerversion.GITCOMMIT) v.Set("GoVersion", runtime.Version()) v.Set("Os", runtime.GOOS) v.Set("Arch", runtime.GOARCH) From afe2170b4f040ea721f5eeb51bc0e7141782225f Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 7 Feb 2014 09:37:10 +1000 Subject: [PATCH 058/403] rewrite the PostgreSQL example using a Dockerfile, and add details to it Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 2ed2ba4e8caa2d70d57d124a2f3509f400d8187e Component: engine --- .../examples/postgresql_service.Dockerfile | 53 ++++++ .../sources/examples/postgresql_service.rst | 179 +++++++----------- 2 files changed, 121 insertions(+), 111 deletions(-) create mode 100644 components/engine/docs/sources/examples/postgresql_service.Dockerfile diff --git a/components/engine/docs/sources/examples/postgresql_service.Dockerfile b/components/engine/docs/sources/examples/postgresql_service.Dockerfile new file mode 100644 index 0000000000..af1423f258 --- /dev/null +++ b/components/engine/docs/sources/examples/postgresql_service.Dockerfile @@ -0,0 +1,53 @@ +# +# example Dockerfile for http://docs.docker.io/en/latest/examples/postgresql_service/ +# + +FROM ubuntu +MAINTAINER SvenDowideit@docker.com + +# Add the PostgreSQL PGP key to verify their Debian packages. +# It should be the same key as https://www.postgresql.org/media/keys/ACCC4CF8.asc +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 + +# Add PostgreSQL's repository. It contains the most recent stable release +# of PostgreSQL, ``9.3``. +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main" > /etc/apt/sources.list.d/pgdg.list + +# Update the Ubuntu and PostgreSQL repository indexes +RUN apt-get update + +# Install ``python-software-properties``, ``software-properties-common`` and PostgreSQL 9.3 +# There are some warnings (in red) that show up during the build. You can hide +# them by prefixing each apt-get statement with DEBIAN_FRONTEND=noninteractive +RUN apt-get -y -q install python-software-properties software-properties-common +RUN apt-get -y -q install postgresql-9.3 postgresql-client-9.3 postgresql-contrib-9.3 + +# Note: The official Debian and Ubuntu images automatically ``apt-get clean`` +# after each ``apt-get`` + +# Run the rest of the commands as the ``postgres`` user created by the ``postgres-9.3`` package when it was ``apt-get installed`` +USER postgres + +# Create a PostgreSQL role named ``docker`` with ``docker`` as the password and +# then create a database `docker` owned by the ``docker`` role. +# Note: here we use ``&&\`` to run commands one after the other - the ``\`` +# allows the RUN command to span multiple lines. +RUN /etc/init.d/postgresql start &&\ + psql --command "CREATE USER docker WITH SUPERUSER PASSWORD 'docker';" &&\ + createdb -O docker docker + +# Adjust PostgreSQL configuration so that remote connections to the +# database are possible. +RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/9.3/main/pg_hba.conf + +# And add ``listen_addresses`` to ``/etc/postgresql/9.3/main/postgresql.conf`` +RUN echo "listen_addresses='*'" >> /etc/postgresql/9.3/main/postgresql.conf + +# Expose the PostgreSQL port +EXPOSE 5432 + +# Add VOLUMEs to allow backup of config, logs and databases +VOLUME ["/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"] + +# Set the default command to run when starting the container +CMD ["/usr/lib/postgresql/9.3/bin/postgres", "-D", "/var/lib/postgresql/9.3/main", "-c", "config_file=/etc/postgresql/9.3/main/postgresql.conf"] diff --git a/components/engine/docs/sources/examples/postgresql_service.rst b/components/engine/docs/sources/examples/postgresql_service.rst index 1c427563e7..5a2323471b 100644 --- a/components/engine/docs/sources/examples/postgresql_service.rst +++ b/components/engine/docs/sources/examples/postgresql_service.rst @@ -9,152 +9,109 @@ PostgreSQL Service .. include:: example_header.inc -.. note:: - - A shorter version of `this blog post`_. - -.. _this blog post: http://zaiste.net/2013/08/docker_postgresql_how_to/ - Installing PostgreSQL on Docker ------------------------------- -Run an interactive shell in a Docker container. +Assuming there is no Docker image that suits your needs in `the index`_, you +can create one yourself. -.. code-block:: bash +.. _the index: http://index.docker.io - sudo docker run -i -t ubuntu /bin/bash - -Update its dependencies. - -.. code-block:: bash - - apt-get update - -Install ``python-software-properties``, ``software-properties-common``, ``wget`` and ``vim``. - -.. code-block:: bash - - apt-get -y install python-software-properties software-properties-common wget vim - -Add PostgreSQL's repository. It contains the most recent stable release -of PostgreSQL, ``9.3``. - -.. code-block:: bash - - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main" > /etc/apt/sources.list.d/pgdg.list - apt-get update - -Finally, install PostgreSQL 9.3 - -.. code-block:: bash - - apt-get -y install postgresql-9.3 postgresql-client-9.3 postgresql-contrib-9.3 - -Now, create a PostgreSQL superuser role that can create databases and -other roles. Following Vagrant's convention the role will be named -``docker`` with ``docker`` password assigned to it. - -.. code-block:: bash - - su postgres -c "createuser -P -d -r -s docker" - -Create a test database also named ``docker`` owned by previously created ``docker`` -role. - -.. code-block:: bash - - su postgres -c "createdb -O docker docker" - -Adjust PostgreSQL configuration so that remote connections to the -database are possible. Make sure that inside -``/etc/postgresql/9.3/main/pg_hba.conf`` you have following line: - -.. code-block:: bash - - host all all 0.0.0.0/0 md5 - -Additionaly, inside ``/etc/postgresql/9.3/main/postgresql.conf`` -uncomment ``listen_addresses`` like so: - -.. code-block:: bash - - listen_addresses='*' +Start by creating a new Dockerfile: .. note:: This PostgreSQL setup is for development only purposes. Refer - to PostgreSQL documentation how to fine-tune these settings so that it - is secure enough. + to the PostgreSQL documentation to fine-tune these settings so that it + is suitably secure. -Exit. +.. literalinclude:: postgresql_service.Dockerfile + +Build an image from the Dockerfile assign it a name. .. code-block:: bash - exit + $ sudo docker build -t eg_postgresql . -Create an image from our container and assign it a name. The ```` -is in the Bash prompt; you can also locate it using ``docker ps -a``. +And run the PostgreSQL server container (in the foreground): .. code-block:: bash - sudo docker commit /postgresql + $ sudo docker run -rm -P -name pg_test eg_postgresql -Finally, run the PostgreSQL server via ``docker``. +There are 2 ways to connect to the PostgreSQL server. We can use +:ref:`working_with_links_names`, or we can access it from our host (or the network). + +.. note:: The ``-rm`` removes the container and its image when the container + exists successfully. + +Using container linking +^^^^^^^^^^^^^^^^^^^^^^^ + +Containers can be linked to another container's ports directly using +``-link remote_name:local_alias`` in the client's ``docker run``. This will +set a number of environment variables that can then be used to connect: .. code-block:: bash - CONTAINER=$(sudo docker run -d -p 5432 \ - -t /postgresql \ - /bin/su postgres -c '/usr/lib/postgresql/9.3/bin/postgres \ - -D /var/lib/postgresql/9.3/main \ - -c config_file=/etc/postgresql/9.3/main/postgresql.conf') + $ sudo docker run -rm -t -i -link pg_test:pg eg_postgresql bash -Connect the PostgreSQL server using ``psql`` (You will need the -postgresql client installed on the machine. For ubuntu, use something -like ``sudo apt-get install postgresql-client``). + postgres@7ef98b1b7243:/$ psql -h $PG_PORT_5432_TCP_ADDR -p $PG_PORT_5432_TCP_PORT -d docker -U docker --password + +Connecting from your host system +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Assuming you have the postgresql-client installed, you can use the host-mapped port +to test as well. You need to use ``docker ps`` to find out what local host port the +container is mapped to first: .. code-block:: bash - CONTAINER_IP=$(sudo docker inspect -format='{{.NetworkSettings.IPAddress}}' $CONTAINER) - psql -h $CONTAINER_IP -p 5432 -d docker -U docker -W + $ docker ps + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + 5e24362f27f6 eg_postgresql:latest /usr/lib/postgresql/ About an hour ago Up About an hour 0.0.0.0:49153->5432/tcp pg_test + $ psql -h localhost -p 49153 -d docker -U docker --password -As before, create roles or databases if needed. +Testing the database +^^^^^^^^^^^^^^^^^^^^ + +Once you have authenticated and have a ``docker =#`` prompt, you can +create a table and populate it. .. code-block:: bash psql (9.3.1) Type "help" for help. - docker=# CREATE DATABASE foo OWNER=docker; - CREATE DATABASE + docker=# CREATE TABLE cities ( + docker(# name varchar(80), + docker(# location point + docker(# ); + CREATE TABLE + docker=# INSERT INTO cities VALUES ('San Francisco', '(-194.0, 53.0)'); + INSERT 0 1 + docker=# select * from cities; + name | location + ---------------+----------- + San Francisco | (-194,53) + (1 row) -Additionally, publish your newly created image on the Docker Index. +Using the container volumes +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can use the defined volumes to inspect the PostgreSQL log files and to backup your +configuration and data: .. code-block:: bash - sudo docker login - Username: - [...] + docker run -rm --volumes-from pg_test -t -i busybox sh -.. code-block:: bash + / # ls + bin etc lib linuxrc mnt proc run sys usr + dev home lib64 media opt root sbin tmp var + / # ls /etc/postgresql/9.3/main/ + environment pg_hba.conf postgresql.conf + pg_ctl.conf pg_ident.conf start.conf + /tmp # ls /var/log + ldconfig postgresql - sudo docker push /postgresql - -PostgreSQL service auto-launch ------------------------------- - -Running our image seems complicated. We have to specify the whole command with -``docker run``. Let's simplify it so the service starts automatically when the -container starts. - -.. code-block:: bash - - sudo docker commit -run='{"Cmd": \ - ["/bin/su", "postgres", "-c", "/usr/lib/postgresql/9.3/bin/postgres -D \ - /var/lib/postgresql/9.3/main -c \ - config_file=/etc/postgresql/9.3/main/postgresql.conf"], "PortSpecs": ["5432"]}' \ - /postgresql - -From now on, just type ``docker run /postgresql`` and -PostgreSQL should automatically start. From 944b4ea1c7e6621b95eea01367ce65976bba0daf Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 11 Feb 2014 16:48:44 -0800 Subject: [PATCH 059/403] New package `nat`: utilities for manipulating the text description of network ports. This facilitates the refactoring of commands.go Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: 3ecd8ff0c80c593e1874cbfa0cc2abf946eeaf66 Component: engine --- components/engine/commands.go | 9 +- components/engine/config_test.go | 3 +- components/engine/container.go | 52 ++----- components/engine/container_unit_test.go | 7 +- components/engine/integration/runtime_test.go | 11 +- components/engine/links.go | 11 +- components/engine/links_test.go | 11 +- components/engine/nat/nat.go | 133 ++++++++++++++++++ components/engine/nat/sort.go | 28 ++++ .../{sorter_unit_test.go => nat/sort_test.go} | 6 +- components/engine/sorter.go | 25 ---- components/engine/utils.go | 103 ++------------ 12 files changed, 212 insertions(+), 187 deletions(-) create mode 100644 components/engine/nat/nat.go create mode 100644 components/engine/nat/sort.go rename components/engine/{sorter_unit_test.go => nat/sort_test.go} (86%) diff --git a/components/engine/commands.go b/components/engine/commands.go index cc019f8c10..db8cc1d2f5 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -12,6 +12,7 @@ import ( "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/nat" flag "github.com/dotcloud/docker/pkg/mflag" "github.com/dotcloud/docker/pkg/sysinfo" "github.com/dotcloud/docker/pkg/term" @@ -799,7 +800,7 @@ func (cli *DockerCli) CmdPort(args ...string) error { return err } - if frontends, exists := out.NetworkSettings.Ports[Port(port+"/"+proto)]; exists && frontends != nil { + if frontends, exists := out.NetworkSettings.Ports[nat.Port(port+"/"+proto)]; exists && frontends != nil { for _, frontend := range frontends { fmt.Fprintf(cli.out, "%s:%s\n", frontend.HostIp, frontend.HostPort) } @@ -1792,7 +1793,7 @@ func parseRun(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Conf cmd.Var(&flLinks, []string{"#link", "-link"}, "Add link to another container (name:alias)") cmd.Var(&flEnv, []string{"e", "-env"}, "Set environment variables") - cmd.Var(&flPublish, []string{"p", "-publish"}, fmt.Sprintf("Publish a container's port to the host (format: %s) (use 'docker port' to see the actual mapping)", PortSpecTemplateFormat)) + cmd.Var(&flPublish, []string{"p", "-publish"}, fmt.Sprintf("Publish a container's port to the host (format: %s) (use 'docker port' to see the actual mapping)", nat.PortSpecTemplateFormat)) cmd.Var(&flExpose, []string{"#expose", "-expose"}, "Expose a port from the container without publishing it to your host") cmd.Var(&flDns, []string{"#dns", "-dns"}, "Set custom dns servers") cmd.Var(&flVolumesFrom, []string{"#volumes-from", "-volumes-from"}, "Mount volumes from the specified container(s)") @@ -1885,7 +1886,7 @@ func parseRun(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Conf domainname = parts[1] } - ports, portBindings, err := parsePortSpecs(flPublish.GetAll()) + ports, portBindings, err := nat.ParsePortSpecs(flPublish.GetAll()) if err != nil { return nil, nil, cmd, err } @@ -1895,7 +1896,7 @@ func parseRun(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Conf if strings.Contains(e, ":") { return nil, nil, cmd, fmt.Errorf("Invalid port format for --expose: %s", e) } - p := NewPort(splitProtoPort(e)) + p := nat.NewPort(nat.SplitProtoPort(e)) if _, exists := ports[p]; !exists { ports[p] = struct{}{} } diff --git a/components/engine/config_test.go b/components/engine/config_test.go index 31c961135a..1c808163e2 100644 --- a/components/engine/config_test.go +++ b/components/engine/config_test.go @@ -1,6 +1,7 @@ package docker import ( + "github.com/dotcloud/docker/nat" "testing" ) @@ -125,7 +126,7 @@ func TestMergeConfig(t *testing.T) { t.Fatalf("Expected VolumesFrom to be 1111, found %s", configUser.VolumesFrom) } - ports, _, err := parsePortSpecs([]string{"0000"}) + ports, _, err := nat.ParsePortSpecs([]string{"0000"}) if err != nil { t.Error(err) } diff --git a/components/engine/container.go b/components/engine/container.go index 81e8749d2a..991c2aeee7 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -8,6 +8,7 @@ import ( "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/execdriver" "github.com/dotcloud/docker/graphdriver" + "github.com/dotcloud/docker/nat" "github.com/dotcloud/docker/pkg/mount" "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/utils" @@ -86,7 +87,7 @@ type Config struct { AttachStdout bool AttachStderr bool PortSpecs []string // Deprecated - Can be in the format of 8080/tcp - ExposedPorts map[Port]struct{} + ExposedPorts map[nat.Port]struct{} Tty bool // Attach standard streams to a tty, including stdin if it is not closed. OpenStdin bool // Open stdin StdinOnce bool // If true, close stdin after the 1 attached client disconnects. @@ -147,7 +148,7 @@ type HostConfig struct { ContainerIDFile string LxcConf []KeyValuePair Privileged bool - PortBindings map[Port][]PortBinding + PortBindings nat.PortMap Links []string PublishAllPorts bool } @@ -189,38 +190,7 @@ type KeyValuePair struct { Value string } -type PortBinding struct { - HostIp string - HostPort string -} - -// 80/tcp -type Port string - -func (p Port) Proto() string { - parts := strings.Split(string(p), "/") - if len(parts) == 1 { - return "tcp" - } - return parts[1] -} - -func (p Port) Port() string { - return strings.Split(string(p), "/")[0] -} - -func (p Port) Int() int { - i, err := parsePort(p.Port()) - if err != nil { - panic(err) - } - return i -} - -func NewPort(proto, port string) Port { - return Port(fmt.Sprintf("%s/%s", port, proto)) -} - +// FIXME: move deprecated port stuff to nat to clean up the core. type PortMapping map[string]string // Deprecated type NetworkSettings struct { @@ -229,13 +199,13 @@ type NetworkSettings struct { Gateway string Bridge string PortMapping map[string]PortMapping // Deprecated - Ports map[Port][]PortBinding + Ports nat.PortMap } func (settings *NetworkSettings) PortMappingAPI() *engine.Table { var outs = engine.NewTable("", 0) for port, bindings := range settings.Ports { - p, _ := parsePort(port.Port()) + p, _ := nat.ParsePort(port.Port()) if len(bindings) == 0 { out := &engine.Env{} out.SetInt("PublicPort", p) @@ -245,7 +215,7 @@ func (settings *NetworkSettings) PortMappingAPI() *engine.Table { } for _, binding := range bindings { out := &engine.Env{} - h, _ := parsePort(binding.HostPort) + h, _ := nat.ParsePort(binding.HostPort) out.SetInt("PrivatePort", p) out.SetInt("PublicPort", h) out.Set("Type", port.Proto()) @@ -1152,8 +1122,8 @@ func (container *Container) allocateNetwork() error { } var ( - portSpecs = make(map[Port]struct{}) - bindings = make(map[Port][]PortBinding) + portSpecs = make(nat.PortSet) + bindings = make(nat.PortMap) ) if !container.State.IsGhost() { @@ -1177,7 +1147,7 @@ func (container *Container) allocateNetwork() error { for port := range portSpecs { binding := bindings[port] if container.hostConfig.PublishAllPorts && len(binding) == 0 { - binding = append(binding, PortBinding{}) + binding = append(binding, nat.PortBinding{}) } for i := 0; i < len(binding); i++ { @@ -1593,7 +1563,7 @@ func (container *Container) Copy(resource string) (archive.Archive, error) { } // Returns true if the container exposes a certain port -func (container *Container) Exposes(p Port) bool { +func (container *Container) Exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] return exists } diff --git a/components/engine/container_unit_test.go b/components/engine/container_unit_test.go index 679ff57e73..dd915ad2e4 100644 --- a/components/engine/container_unit_test.go +++ b/components/engine/container_unit_test.go @@ -1,6 +1,7 @@ package docker import ( + "github.com/dotcloud/docker/nat" "testing" ) @@ -22,7 +23,7 @@ func TestParseLxcConfOpt(t *testing.T) { } func TestParseNetworkOptsPrivateOnly(t *testing.T) { - ports, bindings, err := parsePortSpecs([]string{"192.168.1.100::80"}) + ports, bindings, err := nat.ParsePortSpecs([]string{"192.168.1.100::80"}) if err != nil { t.Fatal(err) } @@ -64,7 +65,7 @@ func TestParseNetworkOptsPrivateOnly(t *testing.T) { } func TestParseNetworkOptsPublic(t *testing.T) { - ports, bindings, err := parsePortSpecs([]string{"192.168.1.100:8080:80"}) + ports, bindings, err := nat.ParsePortSpecs([]string{"192.168.1.100:8080:80"}) if err != nil { t.Fatal(err) } @@ -106,7 +107,7 @@ func TestParseNetworkOptsPublic(t *testing.T) { } func TestParseNetworkOptsUdp(t *testing.T) { - ports, bindings, err := parsePortSpecs([]string{"192.168.1.100::6000/udp"}) + ports, bindings, err := nat.ParsePortSpecs([]string{"192.168.1.100::6000/udp"}) if err != nil { t.Fatal(err) } diff --git a/components/engine/integration/runtime_test.go b/components/engine/integration/runtime_test.go index da95967a30..70dde8f497 100644 --- a/components/engine/integration/runtime_test.go +++ b/components/engine/integration/runtime_test.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/dotcloud/docker" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/nat" "github.com/dotcloud/docker/sysinit" "github.com/dotcloud/docker/utils" "io" @@ -368,7 +369,7 @@ func startEchoServerContainer(t *testing.T, proto string) (*docker.Runtime, *doc eng = NewTestEngine(t) runtime = mkRuntimeFromEngine(eng, t) port = 5554 - p docker.Port + p nat.Port ) defer func() { if err != nil { @@ -387,8 +388,8 @@ func startEchoServerContainer(t *testing.T, proto string) (*docker.Runtime, *doc } else { t.Fatal(fmt.Errorf("Unknown protocol %v", proto)) } - ep := make(map[docker.Port]struct{}, 1) - p = docker.Port(fmt.Sprintf("%s/%s", strPort, proto)) + ep := make(map[nat.Port]struct{}, 1) + p = nat.Port(fmt.Sprintf("%s/%s", strPort, proto)) ep[p] = struct{}{} jobCreate := eng.Job("create") @@ -411,8 +412,8 @@ func startEchoServerContainer(t *testing.T, proto string) (*docker.Runtime, *doc } jobStart := eng.Job("start", id) - portBindings := make(map[docker.Port][]docker.PortBinding) - portBindings[p] = []docker.PortBinding{ + portBindings := make(map[nat.Port][]nat.PortBinding) + portBindings[p] = []nat.PortBinding{ {}, } if err := jobStart.SetenvJson("PortsBindings", portBindings); err != nil { diff --git a/components/engine/links.go b/components/engine/links.go index aa1c08374b..ff39947a0d 100644 --- a/components/engine/links.go +++ b/components/engine/links.go @@ -3,6 +3,7 @@ package docker import ( "fmt" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/nat" "path" "strings" ) @@ -12,7 +13,7 @@ type Link struct { ChildIP string Name string ChildEnvironment []string - Ports []Port + Ports []nat.Port IsEnabled bool eng *engine.Engine } @@ -25,7 +26,7 @@ func NewLink(parent, child *Container, name string, eng *engine.Engine) (*Link, return nil, fmt.Errorf("Cannot link to a non running container: %s AS %s", child.Name, name) } - ports := make([]Port, len(child.Config.ExposedPorts)) + ports := make([]nat.Port, len(child.Config.ExposedPorts)) var i int for p := range child.Config.ExposedPorts { ports[i] = p @@ -85,14 +86,14 @@ func (l *Link) ToEnv() []string { } // Default port rules -func (l *Link) getDefaultPort() *Port { - var p Port +func (l *Link) getDefaultPort() *nat.Port { + var p nat.Port i := len(l.Ports) if i == 0 { return nil } else if i > 1 { - sortPorts(l.Ports, func(ip, jp Port) bool { + nat.Sort(l.Ports, func(ip, jp nat.Port) bool { // If the two ports have the same number, tcp takes priority // Sort in desc order return ip.Int() < jp.Int() || (ip.Int() == jp.Int() && strings.ToLower(ip.Proto()) == "tcp") diff --git a/components/engine/links_test.go b/components/engine/links_test.go index 8a266a9a3d..0286d2395b 100644 --- a/components/engine/links_test.go +++ b/components/engine/links_test.go @@ -1,6 +1,7 @@ package docker import ( + "github.com/dotcloud/docker/nat" "strings" "testing" ) @@ -22,9 +23,9 @@ func TestLinkNew(t *testing.T) { from := newMockLinkContainer(fromID, "172.0.17.2") from.Config.Env = []string{} from.State = State{Running: true} - ports := make(map[Port]struct{}) + ports := make(nat.PortSet) - ports[Port("6379/tcp")] = struct{}{} + ports[nat.Port("6379/tcp")] = struct{}{} from.Config.ExposedPorts = ports @@ -51,7 +52,7 @@ func TestLinkNew(t *testing.T) { t.Fail() } for _, p := range link.Ports { - if p != Port("6379/tcp") { + if p != nat.Port("6379/tcp") { t.Fail() } } @@ -64,9 +65,9 @@ func TestLinkEnv(t *testing.T) { from := newMockLinkContainer(fromID, "172.0.17.2") from.Config.Env = []string{"PASSWORD=gordon"} from.State = State{Running: true} - ports := make(map[Port]struct{}) + ports := make(nat.PortSet) - ports[Port("6379/tcp")] = struct{}{} + ports[nat.Port("6379/tcp")] = struct{}{} from.Config.ExposedPorts = ports diff --git a/components/engine/nat/nat.go b/components/engine/nat/nat.go new file mode 100644 index 0000000000..f3af362f8b --- /dev/null +++ b/components/engine/nat/nat.go @@ -0,0 +1,133 @@ +package nat + +// nat is a convenience package for docker's manipulation of strings describing +// network ports. + +import ( + "fmt" + "github.com/dotcloud/docker/utils" + "strconv" + "strings" +) + +const ( + PortSpecTemplate = "ip:hostPort:containerPort" + PortSpecTemplateFormat = "ip:hostPort:containerPort | ip::containerPort | hostPort:containerPort" +) + +type PortBinding struct { + HostIp string + HostPort string +} + +type PortMap map[Port][]PortBinding + +type PortSet map[Port]struct{} + +// 80/tcp +type Port string + +func NewPort(proto, port string) Port { + return Port(fmt.Sprintf("%s/%s", port, proto)) +} + +func ParsePort(rawPort string) (int, error) { + port, err := strconv.ParseUint(rawPort, 10, 16) + if err != nil { + return 0, err + } + return int(port), nil +} + +func (p Port) Proto() string { + parts := strings.Split(string(p), "/") + if len(parts) == 1 { + return "tcp" + } + return parts[1] +} + +func (p Port) Port() string { + return strings.Split(string(p), "/")[0] +} + +func (p Port) Int() int { + i, err := ParsePort(p.Port()) + if err != nil { + panic(err) + } + return i +} + +// Splits a port in the format of port/proto +func SplitProtoPort(rawPort string) (string, string) { + parts := strings.Split(rawPort, "/") + l := len(parts) + if l == 0 { + return "", "" + } + if l == 1 { + return "tcp", rawPort + } + return parts[0], parts[1] +} + +// We will receive port specs in the format of ip:public:private/proto and these need to be +// parsed in the internal types +func ParsePortSpecs(ports []string) (map[Port]struct{}, map[Port][]PortBinding, error) { + var ( + exposedPorts = make(map[Port]struct{}, len(ports)) + bindings = make(map[Port][]PortBinding) + ) + + for _, rawPort := range ports { + proto := "tcp" + + if i := strings.LastIndex(rawPort, "/"); i != -1 { + proto = rawPort[i+1:] + rawPort = rawPort[:i] + } + if !strings.Contains(rawPort, ":") { + rawPort = fmt.Sprintf("::%s", rawPort) + } else if len(strings.Split(rawPort, ":")) == 2 { + rawPort = fmt.Sprintf(":%s", rawPort) + } + + parts, err := utils.PartParser(PortSpecTemplate, rawPort) + if err != nil { + return nil, nil, err + } + + var ( + containerPort = parts["containerPort"] + rawIp = parts["ip"] + hostPort = parts["hostPort"] + ) + + if containerPort == "" { + return nil, nil, fmt.Errorf("No port specified: %s", rawPort) + } + if _, err := strconv.ParseUint(containerPort, 10, 16); err != nil { + return nil, nil, fmt.Errorf("Invalid containerPort: %s", containerPort) + } + if _, err := strconv.ParseUint(hostPort, 10, 16); hostPort != "" && err != nil { + return nil, nil, fmt.Errorf("Invalid hostPort: %s", hostPort) + } + + port := NewPort(proto, containerPort) + if _, exists := exposedPorts[port]; !exists { + exposedPorts[port] = struct{}{} + } + + binding := PortBinding{ + HostIp: rawIp, + HostPort: hostPort, + } + bslice, exists := bindings[port] + if !exists { + bslice = []PortBinding{} + } + bindings[port] = append(bslice, binding) + } + return exposedPorts, bindings, nil +} diff --git a/components/engine/nat/sort.go b/components/engine/nat/sort.go new file mode 100644 index 0000000000..f36c12f7bb --- /dev/null +++ b/components/engine/nat/sort.go @@ -0,0 +1,28 @@ +package nat + +import "sort" + +type portSorter struct { + ports []Port + by func(i, j Port) bool +} + +func (s *portSorter) Len() int { + return len(s.ports) +} + +func (s *portSorter) Swap(i, j int) { + s.ports[i], s.ports[j] = s.ports[j], s.ports[i] +} + +func (s *portSorter) Less(i, j int) bool { + ip := s.ports[i] + jp := s.ports[j] + + return s.by(ip, jp) +} + +func Sort(ports []Port, predicate func(i, j Port) bool) { + s := &portSorter{ports, predicate} + sort.Sort(s) +} diff --git a/components/engine/sorter_unit_test.go b/components/engine/nat/sort_test.go similarity index 86% rename from components/engine/sorter_unit_test.go rename to components/engine/nat/sort_test.go index 0669feedb3..5d490e321b 100644 --- a/components/engine/sorter_unit_test.go +++ b/components/engine/nat/sort_test.go @@ -1,4 +1,4 @@ -package docker +package nat import ( "fmt" @@ -11,7 +11,7 @@ func TestSortUniquePorts(t *testing.T) { Port("22/tcp"), } - sortPorts(ports, func(ip, jp Port) bool { + Sort(ports, func(ip, jp Port) bool { return ip.Int() < jp.Int() || (ip.Int() == jp.Int() && ip.Proto() == "tcp") }) @@ -30,7 +30,7 @@ func TestSortSamePortWithDifferentProto(t *testing.T) { Port("6379/udp"), } - sortPorts(ports, func(ip, jp Port) bool { + Sort(ports, func(ip, jp Port) bool { return ip.Int() < jp.Int() || (ip.Int() == jp.Int() && ip.Proto() == "tcp") }) diff --git a/components/engine/sorter.go b/components/engine/sorter.go index 9b3e1a9486..b49ac58c24 100644 --- a/components/engine/sorter.go +++ b/components/engine/sorter.go @@ -2,31 +2,6 @@ package docker import "sort" -type portSorter struct { - ports []Port - by func(i, j Port) bool -} - -func (s *portSorter) Len() int { - return len(s.ports) -} - -func (s *portSorter) Swap(i, j int) { - s.ports[i], s.ports[j] = s.ports[j], s.ports[i] -} - -func (s *portSorter) Less(i, j int) bool { - ip := s.ports[i] - jp := s.ports[j] - - return s.by(ip, jp) -} - -func sortPorts(ports []Port, predicate func(i, j Port) bool) { - s := &portSorter{ports, predicate} - sort.Sort(s) -} - type containerSorter struct { containers []*Container by func(i, j *Container) bool diff --git a/components/engine/utils.go b/components/engine/utils.go index e3ba08d51c..d4718954da 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -3,10 +3,10 @@ package docker import ( "fmt" "github.com/dotcloud/docker/archive" + "github.com/dotcloud/docker/nat" "github.com/dotcloud/docker/pkg/namesgenerator" "github.com/dotcloud/docker/utils" "io" - "strconv" "strings" "sync/atomic" ) @@ -98,7 +98,7 @@ func MergeConfig(userConf, imageConf *Config) error { userConf.ExposedPorts = imageConf.ExposedPorts } else if imageConf.ExposedPorts != nil { if userConf.ExposedPorts == nil { - userConf.ExposedPorts = make(map[Port]struct{}) + userConf.ExposedPorts = make(nat.PortSet) } for port := range imageConf.ExposedPorts { if _, exists := userConf.ExposedPorts[port]; !exists { @@ -109,9 +109,9 @@ func MergeConfig(userConf, imageConf *Config) error { if userConf.PortSpecs != nil && len(userConf.PortSpecs) > 0 { if userConf.ExposedPorts == nil { - userConf.ExposedPorts = make(map[Port]struct{}) + userConf.ExposedPorts = make(nat.PortSet) } - ports, _, err := parsePortSpecs(userConf.PortSpecs) + ports, _, err := nat.ParsePortSpecs(userConf.PortSpecs) if err != nil { return err } @@ -125,10 +125,10 @@ func MergeConfig(userConf, imageConf *Config) error { if imageConf.PortSpecs != nil && len(imageConf.PortSpecs) > 0 { utils.Debugf("Migrating image port specs to containter: %s", strings.Join(imageConf.PortSpecs, ", ")) if userConf.ExposedPorts == nil { - userConf.ExposedPorts = make(map[Port]struct{}) + userConf.ExposedPorts = make(nat.PortSet) } - ports, _, err := parsePortSpecs(imageConf.PortSpecs) + ports, _, err := nat.ParsePortSpecs(imageConf.PortSpecs) if err != nil { return err } @@ -212,96 +212,9 @@ func parseLxcOpt(opt string) (string, string, error) { return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), nil } -// FIXME: network related stuff (including parsing) should be grouped in network file -const ( - PortSpecTemplate = "ip:hostPort:containerPort" - PortSpecTemplateFormat = "ip:hostPort:containerPort | ip::containerPort | hostPort:containerPort" -) - -// We will receive port specs in the format of ip:public:private/proto and these need to be -// parsed in the internal types -func parsePortSpecs(ports []string) (map[Port]struct{}, map[Port][]PortBinding, error) { - var ( - exposedPorts = make(map[Port]struct{}, len(ports)) - bindings = make(map[Port][]PortBinding) - ) - - for _, rawPort := range ports { - proto := "tcp" - - if i := strings.LastIndex(rawPort, "/"); i != -1 { - proto = rawPort[i+1:] - rawPort = rawPort[:i] - } - if !strings.Contains(rawPort, ":") { - rawPort = fmt.Sprintf("::%s", rawPort) - } else if len(strings.Split(rawPort, ":")) == 2 { - rawPort = fmt.Sprintf(":%s", rawPort) - } - - parts, err := utils.PartParser(PortSpecTemplate, rawPort) - if err != nil { - return nil, nil, err - } - - var ( - containerPort = parts["containerPort"] - rawIp = parts["ip"] - hostPort = parts["hostPort"] - ) - - if containerPort == "" { - return nil, nil, fmt.Errorf("No port specified: %s", rawPort) - } - if _, err := strconv.ParseUint(containerPort, 10, 16); err != nil { - return nil, nil, fmt.Errorf("Invalid containerPort: %s", containerPort) - } - if _, err := strconv.ParseUint(hostPort, 10, 16); hostPort != "" && err != nil { - return nil, nil, fmt.Errorf("Invalid hostPort: %s", hostPort) - } - - port := NewPort(proto, containerPort) - if _, exists := exposedPorts[port]; !exists { - exposedPorts[port] = struct{}{} - } - - binding := PortBinding{ - HostIp: rawIp, - HostPort: hostPort, - } - bslice, exists := bindings[port] - if !exists { - bslice = []PortBinding{} - } - bindings[port] = append(bslice, binding) - } - return exposedPorts, bindings, nil -} - -// Splits a port in the format of port/proto -func splitProtoPort(rawPort string) (string, string) { - parts := strings.Split(rawPort, "/") - l := len(parts) - if l == 0 { - return "", "" - } - if l == 1 { - return "tcp", rawPort - } - return parts[0], parts[1] -} - -func parsePort(rawPort string) (int, error) { - port, err := strconv.ParseUint(rawPort, 10, 16) - if err != nil { - return 0, err - } - return int(port), nil -} - func migratePortMappings(config *Config, hostConfig *HostConfig) error { if config.PortSpecs != nil { - ports, bindings, err := parsePortSpecs(config.PortSpecs) + ports, bindings, err := nat.ParsePortSpecs(config.PortSpecs) if err != nil { return err } @@ -314,7 +227,7 @@ func migratePortMappings(config *Config, hostConfig *HostConfig) error { } if config.ExposedPorts == nil { - config.ExposedPorts = make(map[Port]struct{}, len(ports)) + config.ExposedPorts = make(nat.PortSet, len(ports)) } for k, v := range ports { config.ExposedPorts[k] = v From 1cb1e79554b88ca7de83f616ad4e88050b54cbd8 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Tue, 11 Feb 2014 16:53:08 -0800 Subject: [PATCH 060/403] fix underline/heading Docker-DCO-1.1-Signed-off-by: Andy Rothfusz (github: metalivedev) Upstream-commit: 6fd8e5d976e872447447d8c25d20d525305f281d Component: engine --- components/engine/docs/sources/use/port_redirection.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/use/port_redirection.rst b/components/engine/docs/sources/use/port_redirection.rst index 2f4fd37c76..2612d63aec 100644 --- a/components/engine/docs/sources/use/port_redirection.rst +++ b/components/engine/docs/sources/use/port_redirection.rst @@ -32,7 +32,7 @@ interface of the host system. To simplify communication between containers, Docker provides the linking mechanism. Auto map all exposed ports on the host -------------------------------------- +-------------------------------------- To bind all the exposed container ports to the host automatically, use ``docker run -P ``. The mapped host ports will be auto-selected From 2b6a0d82a2ad45cccee2e2c0c642aaf0159d85bb Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 11 Feb 2014 18:23:17 -0700 Subject: [PATCH 061/403] Fix the one spot I missed dockerversion Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 7ea725fdc530e989516fae60f5dafecb139d75b8 Component: engine --- components/engine/hack/make.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index 011b796201..55409c55fc 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -83,7 +83,7 @@ fi # Use these flags when compiling the tests and final binary LDFLAGS='-X github.com/dotcloud/docker/dockerversion.GITCOMMIT "'$GITCOMMIT'" -X github.com/dotcloud/docker/dockerversion.VERSION "'$VERSION'" -w' -LDFLAGS_STATIC='-X github.com/dotcloud/docker/utils.IAMSTATIC true -linkmode external -extldflags "-lpthread -static -Wl,--unresolved-symbols=ignore-in-object-files"' +LDFLAGS_STATIC='-X github.com/dotcloud/docker/dockerversion.IAMSTATIC true -linkmode external -extldflags "-lpthread -static -Wl,--unresolved-symbols=ignore-in-object-files"' BUILDFLAGS='-tags netgo -a' HAVE_GO_TEST_COVER= From e8ca8bfeb882447056f7c568e3813cec2be5e068 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 11 Feb 2014 19:10:23 -0700 Subject: [PATCH 062/403] Fix the tests, too Seriously. There's not much codebase left we haven't touched. Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 0d871840b202fc31418990bbcbe0df1c4ad689fb Component: engine --- components/engine/integration/api_test.go | 3 ++- components/engine/integration/graph_test.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/engine/integration/api_test.go b/components/engine/integration/api_test.go index 82de56a8ba..d3efb75969 100644 --- a/components/engine/integration/api_test.go +++ b/components/engine/integration/api_test.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/dotcloud/docker" "github.com/dotcloud/docker/api" + "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/utils" "io" @@ -45,7 +46,7 @@ func TestGetVersion(t *testing.T) { t.Fatal(err) } out.Close() - expected := docker.VERSION + expected := dockerversion.VERSION if result := v.Get("Version"); result != expected { t.Errorf("Expected version %s, %s found", expected, result) } diff --git a/components/engine/integration/graph_test.go b/components/engine/integration/graph_test.go index eec4c5c7dc..ff1c0d9361 100644 --- a/components/engine/integration/graph_test.go +++ b/components/engine/integration/graph_test.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/dotcloud/docker" "github.com/dotcloud/docker/archive" + "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/utils" "io" @@ -105,8 +106,8 @@ func TestGraphCreate(t *testing.T) { if image.Comment != "Testing" { t.Fatalf("Wrong comment: should be '%s', not '%s'", "Testing", image.Comment) } - if image.DockerVersion != docker.VERSION { - t.Fatalf("Wrong docker_version: should be '%s', not '%s'", docker.VERSION, image.DockerVersion) + if image.DockerVersion != dockerversion.VERSION { + t.Fatalf("Wrong docker_version: should be '%s', not '%s'", dockerversion.VERSION, image.DockerVersion) } images, err := graph.Map() if err != nil { From cbe391a3ed8f867bfaa8f594248fd35cee2a88d7 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 11 Feb 2014 17:48:50 -0800 Subject: [PATCH 063/403] Move api-specific code to the api package This facilitates the refactoring of commands.go. Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: e08a1c53aa5a19a17d40d41accbab040611c4411 Component: engine --- components/engine/api/api.go | 9 +++++++++ components/engine/docker/docker.go | 2 +- components/engine/opts.go | 10 ---------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index 3bc8a8b504..e0077a94e3 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -27,6 +27,7 @@ import ( "syscall" ) +// FIXME: move code common to client and server to common.go const ( APIVERSION = 1.9 DEFAULTHTTPHOST = "127.0.0.1" @@ -34,6 +35,14 @@ const ( DEFAULTUNIXSOCKET = "/var/run/docker.sock" ) +func ValidateHost(val string) (string, error) { + host, err := utils.ParseHost(DEFAULTHTTPHOST, DEFAULTHTTPPORT, DEFAULTUNIXSOCKET, val) + if err != nil { + return val, err + } + return host, nil +} + type HttpApiFunc func(eng *engine.Engine, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error func init() { diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index d92f4d98ea..d2fe7c2596 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -42,7 +42,7 @@ func main() { flDefaultIp = flag.String([]string{"#ip", "-ip"}, "0.0.0.0", "Default IP address to use when binding container ports") flInterContainerComm = flag.Bool([]string{"#icc", "-icc"}, true, "Enable inter-container communication") flGraphDriver = flag.String([]string{"s", "-storage-driver"}, "", "Force the docker runtime to use a specific storage driver") - flHosts = docker.NewListOpts(docker.ValidateHost) + flHosts = docker.NewListOpts(api.ValidateHost) flMtu = flag.Int([]string{"#mtu", "-mtu"}, 0, "Set the containers network MTU; if no value is provided: default to the default route MTU or 1500 if not default route is available") ) flag.Var(&flDns, []string{"#dns", "-dns"}, "Force docker to use specific DNS servers") diff --git a/components/engine/opts.go b/components/engine/opts.go index b1d71c491d..dffbcb404e 100644 --- a/components/engine/opts.go +++ b/components/engine/opts.go @@ -2,8 +2,6 @@ package docker import ( "fmt" - "github.com/dotcloud/docker/api" - "github.com/dotcloud/docker/utils" "os" "path/filepath" "regexp" @@ -129,14 +127,6 @@ func ValidateEnv(val string) (string, error) { return fmt.Sprintf("%s=%s", val, os.Getenv(val)), nil } -func ValidateHost(val string) (string, error) { - host, err := utils.ParseHost(api.DEFAULTHTTPHOST, api.DEFAULTHTTPPORT, api.DEFAULTUNIXSOCKET, val) - if err != nil { - return val, err - } - return host, nil -} - func ValidateIp4Address(val string) (string, error) { re := regexp.MustCompile(`^(([0-9]+\.){3}([0-9]+))\s*$`) var ns = re.FindSubmatch([]byte(val)) From faccead7fb03dd012b9528525fcc89e2866de5ed Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 11 Feb 2014 18:46:55 -0800 Subject: [PATCH 064/403] pkg/opts: a collection of custom value parsers implementing flag.Value This facilitates the refactoring of commands.go. Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: e6e320acc757f22e113da755293583402c6c1e7c Component: engine --- components/engine/commands.go | 19 ++++++++++--------- components/engine/docker/docker.go | 5 +++-- components/engine/{ => pkg/opts}/opts.go | 13 ++++++++++++- .../opts/opts_test.go} | 2 +- components/engine/utils.go | 3 ++- 5 files changed, 28 insertions(+), 14 deletions(-) rename components/engine/{ => pkg/opts}/opts.go (88%) rename components/engine/{opts_unit_test.go => pkg/opts/opts_test.go} (97%) diff --git a/components/engine/commands.go b/components/engine/commands.go index db8cc1d2f5..973a41d86a 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -14,6 +14,7 @@ import ( "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/nat" flag "github.com/dotcloud/docker/pkg/mflag" + "github.com/dotcloud/docker/pkg/opts" "github.com/dotcloud/docker/pkg/sysinfo" "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/registry" @@ -1757,16 +1758,16 @@ func ParseRun(args []string, sysInfo *sysinfo.SysInfo) (*Config, *HostConfig, *f func parseRun(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Config, *HostConfig, *flag.FlagSet, error) { var ( // FIXME: use utils.ListOpts for attach and volumes? - flAttach = NewListOpts(ValidateAttach) - flVolumes = NewListOpts(ValidatePath) - flLinks = NewListOpts(ValidateLink) - flEnv = NewListOpts(ValidateEnv) + flAttach = opts.NewListOpts(opts.ValidateAttach) + flVolumes = opts.NewListOpts(opts.ValidatePath) + flLinks = opts.NewListOpts(opts.ValidateLink) + flEnv = opts.NewListOpts(opts.ValidateEnv) - flPublish ListOpts - flExpose ListOpts - flDns ListOpts - flVolumesFrom ListOpts - flLxcOpts ListOpts + flPublish opts.ListOpts + flExpose opts.ListOpts + flDns opts.ListOpts + flVolumesFrom opts.ListOpts + flLxcOpts opts.ListOpts flAutoRemove = cmd.Bool([]string{"#rm", "-rm"}, false, "Automatically remove the container when it exits (incompatible with -d)") flDetach = cmd.Bool([]string{"d", "-detach"}, false, "Detached mode: Run container in the background, print new container id") diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index d2fe7c2596..775dfd3533 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -10,6 +10,7 @@ import ( "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/engine" flag "github.com/dotcloud/docker/pkg/mflag" + "github.com/dotcloud/docker/pkg/opts" "github.com/dotcloud/docker/sysinit" "github.com/dotcloud/docker/utils" ) @@ -36,13 +37,13 @@ func main() { pidfile = flag.String([]string{"p", "-pidfile"}, "/var/run/docker.pid", "Path to use for daemon PID file") flRoot = flag.String([]string{"g", "-graph"}, "/var/lib/docker", "Path to use as the root of the docker runtime") flEnableCors = flag.Bool([]string{"#api-enable-cors", "-api-enable-cors"}, false, "Enable CORS headers in the remote API") - flDns = docker.NewListOpts(docker.ValidateIp4Address) + flDns = opts.NewListOpts(opts.ValidateIp4Address) flEnableIptables = flag.Bool([]string{"#iptables", "-iptables"}, true, "Disable docker's addition of iptables rules") flEnableIpForward = flag.Bool([]string{"#ip-forward", "-ip-forward"}, true, "Disable enabling of net.ipv4.ip_forward") flDefaultIp = flag.String([]string{"#ip", "-ip"}, "0.0.0.0", "Default IP address to use when binding container ports") flInterContainerComm = flag.Bool([]string{"#icc", "-icc"}, true, "Enable inter-container communication") flGraphDriver = flag.String([]string{"s", "-storage-driver"}, "", "Force the docker runtime to use a specific storage driver") - flHosts = docker.NewListOpts(api.ValidateHost) + flHosts = opts.NewListOpts(api.ValidateHost) flMtu = flag.Int([]string{"#mtu", "-mtu"}, 0, "Set the containers network MTU; if no value is provided: default to the default route MTU or 1500 if not default route is available") ) flag.Var(&flDns, []string{"#dns", "-dns"}, "Force docker to use specific DNS servers") diff --git a/components/engine/opts.go b/components/engine/pkg/opts/opts.go similarity index 88% rename from components/engine/opts.go rename to components/engine/pkg/opts/opts.go index dffbcb404e..a1b8752bad 100644 --- a/components/engine/opts.go +++ b/components/engine/pkg/opts/opts.go @@ -1,7 +1,8 @@ -package docker +package opts import ( "fmt" + "github.com/dotcloud/docker/utils" "os" "path/filepath" "regexp" @@ -97,6 +98,16 @@ func ValidateLink(val string) (string, error) { return val, nil } +// FIXME: this is a duplicate of docker.utils.parseLink. +// it can't be moved to a separate links/ package because +// links depends on Container which is defined in the core. +// +// Links come in the format of +// name:alias +func parseLink(rawLink string) (map[string]string, error) { + return utils.PartParser("name:alias", rawLink) +} + func ValidatePath(val string) (string, error) { var containerPath string diff --git a/components/engine/opts_unit_test.go b/components/engine/pkg/opts/opts_test.go similarity index 97% rename from components/engine/opts_unit_test.go rename to components/engine/pkg/opts/opts_test.go index 67b061771b..a5c1fac9ca 100644 --- a/components/engine/opts_unit_test.go +++ b/components/engine/pkg/opts/opts_test.go @@ -1,4 +1,4 @@ -package docker +package opts import ( "testing" diff --git a/components/engine/utils.go b/components/engine/utils.go index d4718954da..e3a58cc67e 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -5,6 +5,7 @@ import ( "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/nat" "github.com/dotcloud/docker/pkg/namesgenerator" + "github.com/dotcloud/docker/pkg/opts" "github.com/dotcloud/docker/utils" "io" "strings" @@ -192,7 +193,7 @@ func MergeConfig(userConf, imageConf *Config) error { return nil } -func parseLxcConfOpts(opts ListOpts) ([]KeyValuePair, error) { +func parseLxcConfOpts(opts opts.ListOpts) ([]KeyValuePair, error) { out := make([]KeyValuePair, opts.Len()) for i, o := range opts.GetAll() { k, v, err := parseLxcOpt(o) From 15bba51638ecc595f321b15685df2a418644d991 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 11 Feb 2014 20:16:58 -0800 Subject: [PATCH 065/403] Remove useless code in client implementation of 'run'. This facilitates refactoring commands.go. Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: e3140e1e69b24f32ffcc2dd94b7d82ad9305a7bc Component: engine --- components/engine/commands.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 973a41d86a..331e519c8d 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1993,12 +1993,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { stream, statusCode, err := cli.call("POST", "/containers/create?"+containerValues.Encode(), config, false) //if image not found try to pull it if statusCode == 404 { - _, tag := utils.ParseRepositoryTag(config.Image) - if tag == "" { - tag = DEFAULTTAG - } - - fmt.Fprintf(cli.err, "Unable to find image '%s' (tag: %s) locally\n", config.Image, tag) + fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", config.Image) v := url.Values{} repos, tag := utils.ParseRepositoryTag(config.Image) From e3789229811fa99d41508ffcc5afb95db2876a97 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 11 Feb 2014 20:04:39 -0800 Subject: [PATCH 066/403] Move the canonical run configuration objects to a sub-package * Config is now runconfig.Config * HostConfig is now runconfig.HostConfig * MergeConfig is now runconfig.Merge * CompareConfig is now runconfig.Compare * ParseRun is now runconfig.Parse * ContainerConfigFromJob is now runconfig.ContainerConfigFromJob * ContainerHostConfigFromJob is now runconfig.ContainerHostConfigFromJob This facilitates refactoring commands.go and shrinks the core. Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: 6393c38339e11b4a099a460ecf46bb5cafc4283b Component: engine --- components/engine/buildfile.go | 11 +- components/engine/commands.go | 212 +-------------- components/engine/commands_unit_test.go | 7 +- components/engine/container.go | 117 +-------- components/engine/container_unit_test.go | 17 -- components/engine/graph.go | 3 +- components/engine/image.go | 23 +- components/engine/integration/api_test.go | 39 +-- .../engine/integration/container_test.go | 78 +++--- components/engine/integration/runtime_test.go | 33 +-- components/engine/integration/server_test.go | 21 +- components/engine/integration/utils_test.go | 9 +- components/engine/links_test.go | 3 +- components/engine/runconfig/compare.go | 67 +++++ components/engine/runconfig/config.go | 76 ++++++ .../engine/{ => runconfig}/config_test.go | 30 +-- components/engine/runconfig/hostconfig.go | 39 +++ components/engine/runconfig/merge.go | 119 +++++++++ components/engine/runconfig/parse.go | 246 ++++++++++++++++++ components/engine/runconfig/parse_test.go | 22 ++ components/engine/runtime.go | 11 +- components/engine/server.go | 17 +- components/engine/utils.go | 205 +-------------- 23 files changed, 731 insertions(+), 674 deletions(-) create mode 100644 components/engine/runconfig/compare.go create mode 100644 components/engine/runconfig/config.go rename components/engine/{ => runconfig}/config_test.go (83%) create mode 100644 components/engine/runconfig/hostconfig.go create mode 100644 components/engine/runconfig/merge.go create mode 100644 components/engine/runconfig/parse.go create mode 100644 components/engine/runconfig/parse_test.go diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 2a4b163bec..a121276c21 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -9,6 +9,7 @@ import ( "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/registry" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -38,7 +39,7 @@ type buildFile struct { image string maintainer string - config *Config + config *runconfig.Config contextPath string context *utils.TarSum @@ -101,7 +102,7 @@ func (b *buildFile) CmdFrom(name string) error { } } b.image = image.ID - b.config = &Config{} + b.config = &runconfig.Config{} if image.Config != nil { b.config = image.Config } @@ -158,14 +159,14 @@ func (b *buildFile) CmdRun(args string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to run") } - config, _, _, err := ParseRun(append([]string{b.image}, b.buildCmdFromJson(args)...), nil) + config, _, _, err := runconfig.Parse(append([]string{b.image}, b.buildCmdFromJson(args)...), nil) if err != nil { return err } cmd := b.config.Cmd b.config.Cmd = nil - MergeConfig(b.config, config) + runconfig.Merge(b.config, config) defer func(cmd []string) { b.config.Cmd = cmd }(cmd) @@ -742,7 +743,7 @@ func NewBuildFile(srv *Server, outStream, errStream io.Writer, verbose, utilizeC return &buildFile{ runtime: srv.runtime, srv: srv, - config: &Config{}, + config: &runconfig.Config{}, outStream: outStream, errStream: errStream, tmpContainers: make(map[string]struct{}), diff --git a/components/engine/commands.go b/components/engine/commands.go index ac93a32638..285b7a57b4 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -15,10 +15,9 @@ import ( "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/nat" flag "github.com/dotcloud/docker/pkg/mflag" - "github.com/dotcloud/docker/pkg/opts" - "github.com/dotcloud/docker/pkg/sysinfo" "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/registry" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -1449,11 +1448,11 @@ func (cli *DockerCli) CmdCommit(args ...string) error { v.Set("comment", *flComment) v.Set("author", *flAuthor) var ( - config *Config + config *runconfig.Config env engine.Env ) if *flConfig != "" { - config = &Config{} + config = &runconfig.Config{} if err := json.Unmarshal([]byte(*flConfig), config); err != nil { return err } @@ -1743,210 +1742,9 @@ func (cli *DockerCli) CmdTag(args ...string) error { return nil } -//FIXME Only used in tests -func ParseRun(args []string, sysInfo *sysinfo.SysInfo) (*Config, *HostConfig, *flag.FlagSet, error) { - cmd := flag.NewFlagSet("run", flag.ContinueOnError) - cmd.SetOutput(ioutil.Discard) - cmd.Usage = nil - return parseRun(cmd, args, sysInfo) -} - -func parseRun(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Config, *HostConfig, *flag.FlagSet, error) { - var ( - // FIXME: use utils.ListOpts for attach and volumes? - flAttach = opts.NewListOpts(opts.ValidateAttach) - flVolumes = opts.NewListOpts(opts.ValidatePath) - flLinks = opts.NewListOpts(opts.ValidateLink) - flEnv = opts.NewListOpts(opts.ValidateEnv) - - flPublish opts.ListOpts - flExpose opts.ListOpts - flDns opts.ListOpts - flVolumesFrom opts.ListOpts - flLxcOpts opts.ListOpts - - flAutoRemove = cmd.Bool([]string{"#rm", "-rm"}, false, "Automatically remove the container when it exits (incompatible with -d)") - flDetach = cmd.Bool([]string{"d", "-detach"}, false, "Detached mode: Run container in the background, print new container id") - flNetwork = cmd.Bool([]string{"n", "-networking"}, true, "Enable networking for this container") - flPrivileged = cmd.Bool([]string{"#privileged", "-privileged"}, false, "Give extended privileges to this container") - flPublishAll = cmd.Bool([]string{"P", "-publish-all"}, false, "Publish all exposed ports to the host interfaces") - flStdin = cmd.Bool([]string{"i", "-interactive"}, false, "Keep stdin open even if not attached") - flTty = cmd.Bool([]string{"t", "-tty"}, false, "Allocate a pseudo-tty") - flContainerIDFile = cmd.String([]string{"#cidfile", "-cidfile"}, "", "Write the container ID to the file") - flEntrypoint = cmd.String([]string{"#entrypoint", "-entrypoint"}, "", "Overwrite the default entrypoint of the image") - flHostname = cmd.String([]string{"h", "-hostname"}, "", "Container host name") - flMemoryString = cmd.String([]string{"m", "-memory"}, "", "Memory limit (format: , where unit = b, k, m or g)") - flUser = cmd.String([]string{"u", "-user"}, "", "Username or UID") - flWorkingDir = cmd.String([]string{"w", "-workdir"}, "", "Working directory inside the container") - flCpuShares = cmd.Int64([]string{"c", "-cpu-shares"}, 0, "CPU shares (relative weight)") - - // For documentation purpose - _ = cmd.Bool([]string{"#sig-proxy", "-sig-proxy"}, true, "Proxify all received signal to the process (even in non-tty mode)") - _ = cmd.String([]string{"#name", "-name"}, "", "Assign a name to the container") - ) - - cmd.Var(&flAttach, []string{"a", "-attach"}, "Attach to stdin, stdout or stderr.") - cmd.Var(&flVolumes, []string{"v", "-volume"}, "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)") - cmd.Var(&flLinks, []string{"#link", "-link"}, "Add link to another container (name:alias)") - cmd.Var(&flEnv, []string{"e", "-env"}, "Set environment variables") - - cmd.Var(&flPublish, []string{"p", "-publish"}, fmt.Sprintf("Publish a container's port to the host (format: %s) (use 'docker port' to see the actual mapping)", nat.PortSpecTemplateFormat)) - cmd.Var(&flExpose, []string{"#expose", "-expose"}, "Expose a port from the container without publishing it to your host") - cmd.Var(&flDns, []string{"#dns", "-dns"}, "Set custom dns servers") - cmd.Var(&flVolumesFrom, []string{"#volumes-from", "-volumes-from"}, "Mount volumes from the specified container(s)") - cmd.Var(&flLxcOpts, []string{"#lxc-conf", "-lxc-conf"}, "Add custom lxc options -lxc-conf=\"lxc.cgroup.cpuset.cpus = 0,1\"") - - if err := cmd.Parse(args); err != nil { - return nil, nil, cmd, err - } - - // Check if the kernel supports memory limit cgroup. - if sysInfo != nil && *flMemoryString != "" && !sysInfo.MemoryLimit { - *flMemoryString = "" - } - - // Validate input params - if *flDetach && flAttach.Len() > 0 { - return nil, nil, cmd, ErrConflictAttachDetach - } - if *flWorkingDir != "" && !path.IsAbs(*flWorkingDir) { - return nil, nil, cmd, ErrInvalidWorikingDirectory - } - if *flDetach && *flAutoRemove { - return nil, nil, cmd, ErrConflictDetachAutoRemove - } - - // If neither -d or -a are set, attach to everything by default - if flAttach.Len() == 0 && !*flDetach { - if !*flDetach { - flAttach.Set("stdout") - flAttach.Set("stderr") - if *flStdin { - flAttach.Set("stdin") - } - } - } - - var flMemory int64 - if *flMemoryString != "" { - parsedMemory, err := utils.RAMInBytes(*flMemoryString) - if err != nil { - return nil, nil, cmd, err - } - flMemory = parsedMemory - } - - var binds []string - // add any bind targets to the list of container volumes - for bind := range flVolumes.GetMap() { - if arr := strings.Split(bind, ":"); len(arr) > 1 { - if arr[0] == "/" { - return nil, nil, cmd, fmt.Errorf("Invalid bind mount: source can't be '/'") - } - dstDir := arr[1] - flVolumes.Set(dstDir) - binds = append(binds, bind) - flVolumes.Delete(bind) - } else if bind == "/" { - return nil, nil, cmd, fmt.Errorf("Invalid volume: path can't be '/'") - } - } - - var ( - parsedArgs = cmd.Args() - runCmd []string - entrypoint []string - image string - ) - if len(parsedArgs) >= 1 { - image = cmd.Arg(0) - } - if len(parsedArgs) > 1 { - runCmd = parsedArgs[1:] - } - if *flEntrypoint != "" { - entrypoint = []string{*flEntrypoint} - } - - lxcConf, err := parseLxcConfOpts(flLxcOpts) - if err != nil { - return nil, nil, cmd, err - } - - var ( - domainname string - hostname = *flHostname - parts = strings.SplitN(hostname, ".", 2) - ) - if len(parts) > 1 { - hostname = parts[0] - domainname = parts[1] - } - - ports, portBindings, err := nat.ParsePortSpecs(flPublish.GetAll()) - if err != nil { - return nil, nil, cmd, err - } - - // Merge in exposed ports to the map of published ports - for _, e := range flExpose.GetAll() { - if strings.Contains(e, ":") { - return nil, nil, cmd, fmt.Errorf("Invalid port format for --expose: %s", e) - } - p := nat.NewPort(nat.SplitProtoPort(e)) - if _, exists := ports[p]; !exists { - ports[p] = struct{}{} - } - } - - config := &Config{ - Hostname: hostname, - Domainname: domainname, - PortSpecs: nil, // Deprecated - ExposedPorts: ports, - User: *flUser, - Tty: *flTty, - NetworkDisabled: !*flNetwork, - OpenStdin: *flStdin, - Memory: flMemory, - CpuShares: *flCpuShares, - AttachStdin: flAttach.Get("stdin"), - AttachStdout: flAttach.Get("stdout"), - AttachStderr: flAttach.Get("stderr"), - Env: flEnv.GetAll(), - Cmd: runCmd, - Dns: flDns.GetAll(), - Image: image, - Volumes: flVolumes.GetMap(), - VolumesFrom: strings.Join(flVolumesFrom.GetAll(), ","), - Entrypoint: entrypoint, - WorkingDir: *flWorkingDir, - } - - hostConfig := &HostConfig{ - Binds: binds, - ContainerIDFile: *flContainerIDFile, - LxcConf: lxcConf, - Privileged: *flPrivileged, - PortBindings: portBindings, - Links: flLinks.GetAll(), - PublishAllPorts: *flPublishAll, - } - - if sysInfo != nil && flMemory > 0 && !sysInfo.SwapLimit { - //fmt.Fprintf(stdout, "WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n") - config.MemorySwap = -1 - } - - // When allocating stdin in attached mode, close stdin at client disconnect - if config.OpenStdin && config.AttachStdin { - config.StdinOnce = true - } - return config, hostConfig, cmd, nil -} - func (cli *DockerCli) CmdRun(args ...string) error { - config, hostConfig, cmd, err := parseRun(cli.Subcmd("run", "[OPTIONS] IMAGE [COMMAND] [ARG...]", "Run a command in a new container"), args, nil) + // FIXME: just use runconfig.Parse already + config, hostConfig, cmd, err := runconfig.ParseSubcommand(cli.Subcmd("run", "[OPTIONS] IMAGE [COMMAND] [ARG...]", "Run a command in a new container"), args, nil) if err != nil { return err } diff --git a/components/engine/commands_unit_test.go b/components/engine/commands_unit_test.go index e44d9a1854..60d8d60398 100644 --- a/components/engine/commands_unit_test.go +++ b/components/engine/commands_unit_test.go @@ -1,16 +1,17 @@ package docker import ( + "github.com/dotcloud/docker/runconfig" "strings" "testing" ) -func parse(t *testing.T, args string) (*Config, *HostConfig, error) { - config, hostConfig, _, err := ParseRun(strings.Split(args+" ubuntu bash", " "), nil) +func parse(t *testing.T, args string) (*runconfig.Config, *runconfig.HostConfig, error) { + config, hostConfig, _, err := runconfig.Parse(strings.Split(args+" ubuntu bash", " "), nil) return config, hostConfig, err } -func mustParse(t *testing.T, args string) (*Config, *HostConfig) { +func mustParse(t *testing.T, args string) (*runconfig.Config, *runconfig.HostConfig) { config, hostConfig, err := parse(t, args) if err != nil { t.Fatal(err) diff --git a/components/engine/container.go b/components/engine/container.go index 991c2aeee7..f068d00375 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -11,6 +11,7 @@ import ( "github.com/dotcloud/docker/nat" "github.com/dotcloud/docker/pkg/mount" "github.com/dotcloud/docker/pkg/term" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" "github.com/kr/pty" "io" @@ -42,7 +43,7 @@ type Container struct { Path string Args []string - Config *Config + Config *runconfig.Config State State Image string @@ -68,109 +69,11 @@ type Container struct { // Store rw/ro in a separate structure to preserve reverse-compatibility on-disk. // Easier than migrating older container configs :) VolumesRW map[string]bool - hostConfig *HostConfig + hostConfig *runconfig.HostConfig activeLinks map[string]*Link } -// Note: the Config structure should hold only portable information about the container. -// Here, "portable" means "independent from the host we are running on". -// Non-portable information *should* appear in HostConfig. -type Config struct { - Hostname string - Domainname string - User string - Memory int64 // Memory limit (in bytes) - MemorySwap int64 // Total memory usage (memory + swap); set `-1' to disable swap - CpuShares int64 // CPU shares (relative weight vs. other containers) - AttachStdin bool - AttachStdout bool - AttachStderr bool - PortSpecs []string // Deprecated - Can be in the format of 8080/tcp - ExposedPorts map[nat.Port]struct{} - Tty bool // Attach standard streams to a tty, including stdin if it is not closed. - OpenStdin bool // Open stdin - StdinOnce bool // If true, close stdin after the 1 attached client disconnects. - Env []string - Cmd []string - Dns []string - Image string // Name of the image as it was passed by the operator (eg. could be symbolic) - Volumes map[string]struct{} - VolumesFrom string - WorkingDir string - Entrypoint []string - NetworkDisabled bool - OnBuild []string -} - -func ContainerConfigFromJob(job *engine.Job) *Config { - config := &Config{ - Hostname: job.Getenv("Hostname"), - Domainname: job.Getenv("Domainname"), - User: job.Getenv("User"), - Memory: job.GetenvInt64("Memory"), - MemorySwap: job.GetenvInt64("MemorySwap"), - CpuShares: job.GetenvInt64("CpuShares"), - AttachStdin: job.GetenvBool("AttachStdin"), - AttachStdout: job.GetenvBool("AttachStdout"), - AttachStderr: job.GetenvBool("AttachStderr"), - Tty: job.GetenvBool("Tty"), - OpenStdin: job.GetenvBool("OpenStdin"), - StdinOnce: job.GetenvBool("StdinOnce"), - Image: job.Getenv("Image"), - VolumesFrom: job.Getenv("VolumesFrom"), - WorkingDir: job.Getenv("WorkingDir"), - NetworkDisabled: job.GetenvBool("NetworkDisabled"), - } - job.GetenvJson("ExposedPorts", &config.ExposedPorts) - job.GetenvJson("Volumes", &config.Volumes) - if PortSpecs := job.GetenvList("PortSpecs"); PortSpecs != nil { - config.PortSpecs = PortSpecs - } - if Env := job.GetenvList("Env"); Env != nil { - config.Env = Env - } - if Cmd := job.GetenvList("Cmd"); Cmd != nil { - config.Cmd = Cmd - } - if Dns := job.GetenvList("Dns"); Dns != nil { - config.Dns = Dns - } - if Entrypoint := job.GetenvList("Entrypoint"); Entrypoint != nil { - config.Entrypoint = Entrypoint - } - - return config -} - -type HostConfig struct { - Binds []string - ContainerIDFile string - LxcConf []KeyValuePair - Privileged bool - PortBindings nat.PortMap - Links []string - PublishAllPorts bool -} - -func ContainerHostConfigFromJob(job *engine.Job) *HostConfig { - hostConfig := &HostConfig{ - ContainerIDFile: job.Getenv("ContainerIDFile"), - Privileged: job.GetenvBool("Privileged"), - PublishAllPorts: job.GetenvBool("PublishAllPorts"), - } - job.GetenvJson("LxcConf", &hostConfig.LxcConf) - job.GetenvJson("PortBindings", &hostConfig.PortBindings) - if Binds := job.GetenvList("Binds"); Binds != nil { - hostConfig.Binds = Binds - } - if Links := job.GetenvList("Links"); Links != nil { - hostConfig.Links = Links - } - - return hostConfig -} - type BindMap struct { SrcPath string DstPath string @@ -178,18 +81,10 @@ type BindMap struct { } var ( - ErrContainerStart = errors.New("The container failed to start. Unknown error") - ErrContainerStartTimeout = errors.New("The container failed to start due to timed out.") - ErrInvalidWorikingDirectory = errors.New("The working directory is invalid. It needs to be an absolute path.") - ErrConflictAttachDetach = errors.New("Conflicting options: -a and -d") - ErrConflictDetachAutoRemove = errors.New("Conflicting options: -rm and -d") + ErrContainerStart = errors.New("The container failed to start. Unknown error") + ErrContainerStartTimeout = errors.New("The container failed to start due to timed out.") ) -type KeyValuePair struct { - Key string - Value string -} - // FIXME: move deprecated port stuff to nat to clean up the core. type PortMapping map[string]string // Deprecated @@ -292,7 +187,7 @@ func (container *Container) ToDisk() (err error) { } func (container *Container) readHostConfig() error { - container.hostConfig = &HostConfig{} + container.hostConfig = &runconfig.HostConfig{} // If the hostconfig file does not exist, do not read it. // (We still have to initialize container.hostConfig, // but that's OK, since we just did that above.) diff --git a/components/engine/container_unit_test.go b/components/engine/container_unit_test.go index dd915ad2e4..3877b7f0da 100644 --- a/components/engine/container_unit_test.go +++ b/components/engine/container_unit_test.go @@ -5,23 +5,6 @@ import ( "testing" ) -func TestParseLxcConfOpt(t *testing.T) { - opts := []string{"lxc.utsname=docker", "lxc.utsname = docker "} - - for _, o := range opts { - k, v, err := parseLxcOpt(o) - if err != nil { - t.FailNow() - } - if k != "lxc.utsname" { - t.Fail() - } - if v != "docker" { - t.Fail() - } - } -} - func TestParseNetworkOptsPrivateOnly(t *testing.T) { ports, bindings, err := nat.ParsePortSpecs([]string{"192.168.1.100::80"}) if err != nil { diff --git a/components/engine/graph.go b/components/engine/graph.go index 138c7b8613..01cd50f4f0 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -5,6 +5,7 @@ import ( "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/graphdriver" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -126,7 +127,7 @@ func (graph *Graph) Get(name string) (*Image, error) { } // Create creates a new image and registers it in the graph. -func (graph *Graph) Create(layerData archive.Archive, container *Container, comment, author string, config *Config) (*Image, error) { +func (graph *Graph) Create(layerData archive.Archive, container *Container, comment, author string, config *runconfig.Config) (*Image, error) { img := &Image{ ID: GenerateID(), Comment: comment, diff --git a/components/engine/image.go b/components/engine/image.go index dbd2173597..593dc14f00 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/graphdriver" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -18,17 +19,17 @@ import ( ) type Image struct { - ID string `json:"id"` - Parent string `json:"parent,omitempty"` - Comment string `json:"comment,omitempty"` - Created time.Time `json:"created"` - Container string `json:"container,omitempty"` - ContainerConfig Config `json:"container_config,omitempty"` - DockerVersion string `json:"docker_version,omitempty"` - Author string `json:"author,omitempty"` - Config *Config `json:"config,omitempty"` - Architecture string `json:"architecture,omitempty"` - OS string `json:"os,omitempty"` + ID string `json:"id"` + Parent string `json:"parent,omitempty"` + Comment string `json:"comment,omitempty"` + Created time.Time `json:"created"` + Container string `json:"container,omitempty"` + ContainerConfig runconfig.Config `json:"container_config,omitempty"` + DockerVersion string `json:"docker_version,omitempty"` + Author string `json:"author,omitempty"` + Config *runconfig.Config `json:"config,omitempty"` + Architecture string `json:"architecture,omitempty"` + OS string `json:"os,omitempty"` graph *Graph Size int64 } diff --git a/components/engine/integration/api_test.go b/components/engine/integration/api_test.go index d3efb75969..c587f111a2 100644 --- a/components/engine/integration/api_test.go +++ b/components/engine/integration/api_test.go @@ -10,6 +10,7 @@ import ( "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" "io" "net" @@ -309,7 +310,7 @@ func TestGetContainersJSON(t *testing.T) { } beginLen := len(outs.Data) - containerID := createTestContainer(eng, &docker.Config{ + containerID := createTestContainer(eng, &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"echo", "test"}, }, t) @@ -346,7 +347,7 @@ func TestGetContainersExport(t *testing.T) { // Create a container and remove a file containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"touch", "/test"}, }, @@ -394,7 +395,7 @@ func TestGetContainersChanges(t *testing.T) { // Create a container and remove a file containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/rm", "/etc/passwd"}, }, @@ -433,7 +434,7 @@ func TestGetContainersTop(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/sh", "-c", "cat"}, OpenStdin: true, @@ -510,7 +511,7 @@ func TestGetContainersByName(t *testing.T) { // Create a container and remove a file containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"echo", "test"}, }, @@ -542,7 +543,7 @@ func TestPostCommit(t *testing.T) { // Create a container and remove a file containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"touch", "/test"}, }, @@ -578,7 +579,7 @@ func TestPostContainersCreate(t *testing.T) { eng := NewTestEngine(t) defer mkRuntimeFromEngine(eng, t).Nuke() - configJSON, err := json.Marshal(&docker.Config{ + configJSON, err := json.Marshal(&runconfig.Config{ Image: unitTestImageID, Memory: 33554432, Cmd: []string{"touch", "/test"}, @@ -620,7 +621,7 @@ func TestPostContainersKill(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/cat"}, OpenStdin: true, @@ -659,7 +660,7 @@ func TestPostContainersRestart(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/top"}, OpenStdin: true, @@ -705,7 +706,7 @@ func TestPostContainersStart(t *testing.T) { containerID := createTestContainer( eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/cat"}, OpenStdin: true, @@ -713,7 +714,7 @@ func TestPostContainersStart(t *testing.T) { t, ) - hostConfigJSON, err := json.Marshal(&docker.HostConfig{}) + hostConfigJSON, err := json.Marshal(&runconfig.HostConfig{}) req, err := http.NewRequest("POST", "/containers/"+containerID+"/start", bytes.NewReader(hostConfigJSON)) if err != nil { @@ -758,7 +759,7 @@ func TestRunErrorBindMountRootSource(t *testing.T) { containerID := createTestContainer( eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/cat"}, OpenStdin: true, @@ -766,7 +767,7 @@ func TestRunErrorBindMountRootSource(t *testing.T) { t, ) - hostConfigJSON, err := json.Marshal(&docker.HostConfig{ + hostConfigJSON, err := json.Marshal(&runconfig.HostConfig{ Binds: []string{"/:/tmp"}, }) @@ -792,7 +793,7 @@ func TestPostContainersStop(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/top"}, OpenStdin: true, @@ -832,7 +833,7 @@ func TestPostContainersWait(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/sleep", "1"}, OpenStdin: true, @@ -870,7 +871,7 @@ func TestPostContainersAttach(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/cat"}, OpenStdin: true, @@ -948,7 +949,7 @@ func TestPostContainersAttachStderr(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/sh", "-c", "/bin/cat >&2"}, OpenStdin: true, @@ -1029,7 +1030,7 @@ func TestDeleteContainers(t *testing.T) { defer mkRuntimeFromEngine(eng, t).Nuke() containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"touch", "/test"}, }, @@ -1164,7 +1165,7 @@ func TestPostContainersCopy(t *testing.T) { // Create a container and remove a file containerID := createTestContainer(eng, - &docker.Config{ + &runconfig.Config{ Image: unitTestImageID, Cmd: []string{"touch", "/test.txt"}, }, diff --git a/components/engine/integration/container_test.go b/components/engine/integration/container_test.go index 97f4cd282f..b961e1d147 100644 --- a/components/engine/integration/container_test.go +++ b/components/engine/integration/container_test.go @@ -3,7 +3,7 @@ package docker import ( "bufio" "fmt" - "github.com/dotcloud/docker" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -20,7 +20,7 @@ func TestIDFormat(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) container1, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"/bin/sh", "-c", "echo hello world"}, }, @@ -234,7 +234,7 @@ func TestCommitAutoRun(t *testing.T) { t.Errorf("Container shouldn't be running") } - img, err := runtime.Commit(container1, "", "", "unit test commited image", "", &docker.Config{Cmd: []string{"cat", "/world"}}) + img, err := runtime.Commit(container1, "", "", "unit test commited image", "", &runconfig.Config{Cmd: []string{"cat", "/world"}}) if err != nil { t.Error(err) } @@ -415,7 +415,7 @@ func TestOutput(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) container, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"echo", "-n", "foobar"}, }, @@ -438,7 +438,7 @@ func TestContainerNetwork(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) container, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"ping", "-c", "1", "127.0.0.1"}, }, @@ -460,7 +460,7 @@ func TestKillDifferentUser(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"cat"}, OpenStdin: true, @@ -520,7 +520,7 @@ func TestCreateVolume(t *testing.T) { runtime := mkRuntimeFromEngine(eng, t) defer nuke(runtime) - config, hc, _, err := docker.ParseRun([]string{"-v", "/var/lib/data", unitTestImageID, "echo", "hello", "world"}, nil) + config, hc, _, err := runconfig.Parse([]string{"-v", "/var/lib/data", unitTestImageID, "echo", "hello", "world"}, nil) if err != nil { t.Fatal(err) } @@ -552,7 +552,7 @@ func TestCreateVolume(t *testing.T) { func TestKill(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"sleep", "2"}, }, @@ -596,7 +596,7 @@ func TestExitCode(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - trueContainer, _, err := runtime.Create(&docker.Config{ + trueContainer, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"/bin/true"}, }, "") @@ -611,7 +611,7 @@ func TestExitCode(t *testing.T) { t.Fatalf("Unexpected exit code %d (expected 0)", code) } - falseContainer, _, err := runtime.Create(&docker.Config{ + falseContainer, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"/bin/false"}, }, "") @@ -630,7 +630,7 @@ func TestExitCode(t *testing.T) { func TestRestart(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"echo", "-n", "foobar"}, }, @@ -661,7 +661,7 @@ func TestRestart(t *testing.T) { func TestRestartStdin(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"cat"}, @@ -739,7 +739,7 @@ func TestUser(t *testing.T) { defer nuke(runtime) // Default user must be root - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"id"}, }, @@ -758,7 +758,7 @@ func TestUser(t *testing.T) { } // Set a username - container, _, err = runtime.Create(&docker.Config{ + container, _, err = runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"id"}, @@ -779,7 +779,7 @@ func TestUser(t *testing.T) { } // Set a UID - container, _, err = runtime.Create(&docker.Config{ + container, _, err = runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"id"}, @@ -800,7 +800,7 @@ func TestUser(t *testing.T) { } // Set a different user by uid - container, _, err = runtime.Create(&docker.Config{ + container, _, err = runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"id"}, @@ -823,7 +823,7 @@ func TestUser(t *testing.T) { } // Set a different user by username - container, _, err = runtime.Create(&docker.Config{ + container, _, err = runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"id"}, @@ -844,7 +844,7 @@ func TestUser(t *testing.T) { } // Test an wrong username - container, _, err = runtime.Create(&docker.Config{ + container, _, err = runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"id"}, @@ -866,7 +866,7 @@ func TestMultipleContainers(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container1, _, err := runtime.Create(&docker.Config{ + container1, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"sleep", "2"}, }, @@ -877,7 +877,7 @@ func TestMultipleContainers(t *testing.T) { } defer runtime.Destroy(container1) - container2, _, err := runtime.Create(&docker.Config{ + container2, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"sleep", "2"}, }, @@ -921,7 +921,7 @@ func TestMultipleContainers(t *testing.T) { func TestStdin(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"cat"}, @@ -966,7 +966,7 @@ func TestStdin(t *testing.T) { func TestTty(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"cat"}, @@ -1013,7 +1013,7 @@ func TestEnv(t *testing.T) { os.Setenv("TRICKY", "tri\ncky\n") runtime := mkRuntime(t) defer nuke(runtime) - config, _, _, err := docker.ParseRun([]string{"-e=FALSE=true", "-e=TRUE", "-e=TRICKY", GetTestImage(runtime).ID, "env"}, nil) + config, _, _, err := runconfig.Parse([]string{"-e=FALSE=true", "-e=TRUE", "-e=TRICKY", GetTestImage(runtime).ID, "env"}, nil) if err != nil { t.Fatal(err) } @@ -1067,7 +1067,7 @@ func TestEntrypoint(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) container, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Entrypoint: []string{"/bin/echo"}, Cmd: []string{"-n", "foobar"}, @@ -1091,7 +1091,7 @@ func TestEntrypointNoCmd(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) container, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Entrypoint: []string{"/bin/echo", "foobar"}, }, @@ -1114,7 +1114,7 @@ func BenchmarkRunSequencial(b *testing.B) { runtime := mkRuntime(b) defer nuke(runtime) for i := 0; i < b.N; i++ { - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"echo", "-n", "foo"}, }, @@ -1147,7 +1147,7 @@ func BenchmarkRunParallel(b *testing.B) { complete := make(chan error) tasks = append(tasks, complete) go func(i int, complete chan error) { - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"echo", "-n", "foo"}, }, @@ -1301,7 +1301,7 @@ func TestFromVolumesInReadonlyMode(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) container, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"/bin/echo", "-n", "foobar"}, Volumes: map[string]struct{}{"/test": {}}, @@ -1321,7 +1321,7 @@ func TestFromVolumesInReadonlyMode(t *testing.T) { } container2, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"/bin/echo", "-n", "foobar"}, VolumesFrom: container.ID + ":ro", @@ -1362,7 +1362,7 @@ func TestVolumesFromReadonlyMount(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) container, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"/bin/echo", "-n", "foobar"}, Volumes: map[string]struct{}{"/test": {}}, @@ -1382,7 +1382,7 @@ func TestVolumesFromReadonlyMount(t *testing.T) { } container2, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"/bin/echo", "-n", "foobar"}, VolumesFrom: container.ID, @@ -1418,7 +1418,7 @@ func TestRestartWithVolumes(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"echo", "-n", "foobar"}, Volumes: map[string]struct{}{"/test": {}}, @@ -1462,7 +1462,7 @@ func TestVolumesFromWithVolumes(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"sh", "-c", "echo -n bar > /test/foo"}, Volumes: map[string]struct{}{"/test": {}}, @@ -1491,7 +1491,7 @@ func TestVolumesFromWithVolumes(t *testing.T) { } container2, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"cat", "/test/foo"}, VolumesFrom: container.ID, @@ -1529,7 +1529,7 @@ func TestOnlyLoopbackExistsWhenUsingDisableNetworkOption(t *testing.T) { runtime := mkRuntimeFromEngine(eng, t) defer nuke(runtime) - config, hc, _, err := docker.ParseRun([]string{"-n=false", GetTestImage(runtime).ID, "ip", "addr", "show"}, nil) + config, hc, _, err := runconfig.Parse([]string{"-n=false", GetTestImage(runtime).ID, "ip", "addr", "show"}, nil) if err != nil { t.Fatal(err) } @@ -1617,7 +1617,7 @@ func TestMultipleVolumesFrom(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"sh", "-c", "echo -n bar > /test/foo"}, Volumes: map[string]struct{}{"/test": {}}, @@ -1646,7 +1646,7 @@ func TestMultipleVolumesFrom(t *testing.T) { } container2, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"sh", "-c", "echo -n bar > /other/foo"}, Volumes: map[string]struct{}{"/other": {}}, @@ -1668,7 +1668,7 @@ func TestMultipleVolumesFrom(t *testing.T) { } container3, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"/bin/echo", "-n", "foobar"}, VolumesFrom: strings.Join([]string{container.ID, container2.ID}, ","), @@ -1696,7 +1696,7 @@ func TestRestartGhost(t *testing.T) { defer nuke(runtime) container, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"sh", "-c", "echo -n bar > /test/foo"}, Volumes: map[string]struct{}{"/test": {}}, diff --git a/components/engine/integration/runtime_test.go b/components/engine/integration/runtime_test.go index 70dde8f497..170f4c9638 100644 --- a/components/engine/integration/runtime_test.go +++ b/components/engine/integration/runtime_test.go @@ -6,6 +6,7 @@ import ( "github.com/dotcloud/docker" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/nat" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/sysinit" "github.com/dotcloud/docker/utils" "io" @@ -200,7 +201,7 @@ func TestRuntimeCreate(t *testing.T) { t.Errorf("Expected 0 containers, %v found", len(runtime.List())) } - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"ls", "-al"}, }, @@ -243,23 +244,23 @@ func TestRuntimeCreate(t *testing.T) { // Test that conflict error displays correct details testContainer, _, _ := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"ls", "-al"}, }, "conflictname", ) - if _, _, err := runtime.Create(&docker.Config{Image: GetTestImage(runtime).ID, Cmd: []string{"ls", "-al"}}, testContainer.Name); err == nil || !strings.Contains(err.Error(), utils.TruncateID(testContainer.ID)) { + if _, _, err := runtime.Create(&runconfig.Config{Image: GetTestImage(runtime).ID, Cmd: []string{"ls", "-al"}}, testContainer.Name); err == nil || !strings.Contains(err.Error(), utils.TruncateID(testContainer.ID)) { t.Fatalf("Name conflict error doesn't include the correct short id. Message was: %s", err.Error()) } // Make sure create with bad parameters returns an error - if _, _, err = runtime.Create(&docker.Config{Image: GetTestImage(runtime).ID}, ""); err == nil { + if _, _, err = runtime.Create(&runconfig.Config{Image: GetTestImage(runtime).ID}, ""); err == nil { t.Fatal("Builder.Create should throw an error when Cmd is missing") } if _, _, err := runtime.Create( - &docker.Config{ + &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{}, }, @@ -268,7 +269,7 @@ func TestRuntimeCreate(t *testing.T) { t.Fatal("Builder.Create should throw an error when Cmd is empty") } - config := &docker.Config{ + config := &runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"/bin/ls"}, PortSpecs: []string{"80"}, @@ -281,7 +282,7 @@ func TestRuntimeCreate(t *testing.T) { } // test expose 80:8000 - container, warnings, err := runtime.Create(&docker.Config{ + container, warnings, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"ls", "-al"}, PortSpecs: []string{"80:8000"}, @@ -300,7 +301,7 @@ func TestDestroy(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"ls", "-al"}, }, "") @@ -712,7 +713,7 @@ func TestDefaultContainerName(t *testing.T) { runtime := mkRuntimeFromEngine(eng, t) defer nuke(runtime) - config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil) + config, _, _, err := runconfig.Parse([]string{unitTestImageID, "echo test"}, nil) if err != nil { t.Fatal(err) } @@ -736,7 +737,7 @@ func TestRandomContainerName(t *testing.T) { runtime := mkRuntimeFromEngine(eng, t) defer nuke(runtime) - config, _, _, err := docker.ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) + config, _, _, err := runconfig.Parse([]string{GetTestImage(runtime).ID, "echo test"}, nil) if err != nil { t.Fatal(err) } @@ -767,7 +768,7 @@ func TestContainerNameValidation(t *testing.T) { {"abc-123_AAA.1", true}, {"\000asdf", false}, } { - config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil) + config, _, _, err := runconfig.Parse([]string{unitTestImageID, "echo test"}, nil) if err != nil { if !test.Valid { continue @@ -808,7 +809,7 @@ func TestLinkChildContainer(t *testing.T) { runtime := mkRuntimeFromEngine(eng, t) defer nuke(runtime) - config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil) + config, _, _, err := runconfig.Parse([]string{unitTestImageID, "echo test"}, nil) if err != nil { t.Fatal(err) } @@ -824,7 +825,7 @@ func TestLinkChildContainer(t *testing.T) { t.Fatalf("Expect webapp id to match container id: %s != %s", webapp.ID, container.ID) } - config, _, _, err = docker.ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) + config, _, _, err = runconfig.Parse([]string{GetTestImage(runtime).ID, "echo test"}, nil) if err != nil { t.Fatal(err) } @@ -850,7 +851,7 @@ func TestGetAllChildren(t *testing.T) { runtime := mkRuntimeFromEngine(eng, t) defer nuke(runtime) - config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil) + config, _, _, err := runconfig.Parse([]string{unitTestImageID, "echo test"}, nil) if err != nil { t.Fatal(err) } @@ -866,7 +867,7 @@ func TestGetAllChildren(t *testing.T) { t.Fatalf("Expect webapp id to match container id: %s != %s", webapp.ID, container.ID) } - config, _, _, err = docker.ParseRun([]string{unitTestImageID, "echo test"}, nil) + config, _, _, err = runconfig.Parse([]string{unitTestImageID, "echo test"}, nil) if err != nil { t.Fatal(err) } @@ -903,7 +904,7 @@ func TestDestroyWithInitLayer(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) - container, _, err := runtime.Create(&docker.Config{ + container, _, err := runtime.Create(&runconfig.Config{ Image: GetTestImage(runtime).ID, Cmd: []string{"ls", "-al"}, }, "") diff --git a/components/engine/integration/server_test.go b/components/engine/integration/server_test.go index 45d4930ad7..2b7ef13cbd 100644 --- a/components/engine/integration/server_test.go +++ b/components/engine/integration/server_test.go @@ -3,6 +3,7 @@ package docker import ( "github.com/dotcloud/docker" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/runconfig" "strings" "testing" "time" @@ -71,7 +72,7 @@ func TestCreateRm(t *testing.T) { eng := NewTestEngine(t) defer mkRuntimeFromEngine(eng, t).Nuke() - config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil) + config, _, _, err := runconfig.Parse([]string{unitTestImageID, "echo test"}, nil) if err != nil { t.Fatal(err) } @@ -118,7 +119,7 @@ func TestCreateNumberHostname(t *testing.T) { eng := NewTestEngine(t) defer mkRuntimeFromEngine(eng, t).Nuke() - config, _, _, err := docker.ParseRun([]string{"-h", "web.0", unitTestImageID, "echo test"}, nil) + config, _, _, err := runconfig.Parse([]string{"-h", "web.0", unitTestImageID, "echo test"}, nil) if err != nil { t.Fatal(err) } @@ -130,7 +131,7 @@ func TestCreateNumberUsername(t *testing.T) { eng := NewTestEngine(t) defer mkRuntimeFromEngine(eng, t).Nuke() - config, _, _, err := docker.ParseRun([]string{"-u", "1002", unitTestImageID, "echo test"}, nil) + config, _, _, err := runconfig.Parse([]string{"-u", "1002", unitTestImageID, "echo test"}, nil) if err != nil { t.Fatal(err) } @@ -142,7 +143,7 @@ func TestCreateRmVolumes(t *testing.T) { eng := NewTestEngine(t) defer mkRuntimeFromEngine(eng, t).Nuke() - config, hostConfig, _, err := docker.ParseRun([]string{"-v", "/srv", unitTestImageID, "echo", "test"}, nil) + config, hostConfig, _, err := runconfig.Parse([]string{"-v", "/srv", unitTestImageID, "echo", "test"}, nil) if err != nil { t.Fatal(err) } @@ -202,7 +203,7 @@ func TestCommit(t *testing.T) { eng := NewTestEngine(t) defer mkRuntimeFromEngine(eng, t).Nuke() - config, _, _, err := docker.ParseRun([]string{unitTestImageID, "/bin/cat"}, nil) + config, _, _, err := runconfig.Parse([]string{unitTestImageID, "/bin/cat"}, nil) if err != nil { t.Fatal(err) } @@ -224,7 +225,7 @@ func TestRestartKillWait(t *testing.T) { runtime := mkRuntimeFromEngine(eng, t) defer runtime.Nuke() - config, hostConfig, _, err := docker.ParseRun([]string{"-i", unitTestImageID, "/bin/cat"}, nil) + config, hostConfig, _, err := runconfig.Parse([]string{"-i", unitTestImageID, "/bin/cat"}, nil) if err != nil { t.Fatal(err) } @@ -302,7 +303,7 @@ func TestCreateStartRestartStopStartKillRm(t *testing.T) { srv := mkServerFromEngine(eng, t) defer mkRuntimeFromEngine(eng, t).Nuke() - config, hostConfig, _, err := docker.ParseRun([]string{"-i", unitTestImageID, "/bin/cat"}, nil) + config, hostConfig, _, err := runconfig.Parse([]string{"-i", unitTestImageID, "/bin/cat"}, nil) if err != nil { t.Fatal(err) } @@ -401,7 +402,7 @@ func TestRmi(t *testing.T) { initialImages := getAllImages(eng, t) - config, hostConfig, _, err := docker.ParseRun([]string{unitTestImageID, "echo", "test"}, nil) + config, hostConfig, _, err := runconfig.Parse([]string{unitTestImageID, "echo", "test"}, nil) if err != nil { t.Fatal(err) } @@ -548,7 +549,7 @@ func TestListContainers(t *testing.T) { srv := mkServerFromEngine(eng, t) defer mkRuntimeFromEngine(eng, t).Nuke() - config := docker.Config{ + config := runconfig.Config{ Image: unitTestImageID, Cmd: []string{"/bin/sh", "-c", "cat"}, OpenStdin: true, @@ -671,7 +672,7 @@ func TestDeleteTagWithExistingContainers(t *testing.T) { } // Create a container from the image - config, _, _, err := docker.ParseRun([]string{unitTestImageID, "echo test"}, nil) + config, _, _, err := runconfig.Parse([]string{unitTestImageID, "echo test"}, nil) if err != nil { t.Fatal(err) } diff --git a/components/engine/integration/utils_test.go b/components/engine/integration/utils_test.go index 450cb7527f..6b0f458564 100644 --- a/components/engine/integration/utils_test.go +++ b/components/engine/integration/utils_test.go @@ -16,6 +16,7 @@ import ( "github.com/dotcloud/docker" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" ) @@ -48,7 +49,7 @@ func mkRuntime(f utils.Fataler) *docker.Runtime { return r } -func createNamedTestContainer(eng *engine.Engine, config *docker.Config, f utils.Fataler, name string) (shortId string) { +func createNamedTestContainer(eng *engine.Engine, config *runconfig.Config, f utils.Fataler, name string) (shortId string) { job := eng.Job("create", name) if err := job.ImportEnv(config); err != nil { f.Fatal(err) @@ -60,7 +61,7 @@ func createNamedTestContainer(eng *engine.Engine, config *docker.Config, f utils return } -func createTestContainer(eng *engine.Engine, config *docker.Config, f utils.Fataler) (shortId string) { +func createTestContainer(eng *engine.Engine, config *runconfig.Config, f utils.Fataler) (shortId string) { return createNamedTestContainer(eng, config, f, "") } @@ -252,8 +253,8 @@ func readFile(src string, t *testing.T) (content string) { // dynamically replaced by the current test image. // The caller is responsible for destroying the container. // Call t.Fatal() at the first error. -func mkContainer(r *docker.Runtime, args []string, t *testing.T) (*docker.Container, *docker.HostConfig, error) { - config, hc, _, err := docker.ParseRun(args, nil) +func mkContainer(r *docker.Runtime, args []string, t *testing.T) (*docker.Container, *runconfig.HostConfig, error) { + config, hc, _, err := runconfig.Parse(args, nil) defer func() { if err != nil && t != nil { t.Fatal(err) diff --git a/components/engine/links_test.go b/components/engine/links_test.go index 0286d2395b..7b85a8e86d 100644 --- a/components/engine/links_test.go +++ b/components/engine/links_test.go @@ -2,13 +2,14 @@ package docker import ( "github.com/dotcloud/docker/nat" + "github.com/dotcloud/docker/runconfig" "strings" "testing" ) func newMockLinkContainer(id string, ip string) *Container { return &Container{ - Config: &Config{}, + Config: &runconfig.Config{}, ID: id, NetworkSettings: &NetworkSettings{ IPAddress: ip, diff --git a/components/engine/runconfig/compare.go b/components/engine/runconfig/compare.go new file mode 100644 index 0000000000..c09f897716 --- /dev/null +++ b/components/engine/runconfig/compare.go @@ -0,0 +1,67 @@ +package runconfig + +// Compare two Config struct. Do not compare the "Image" nor "Hostname" fields +// If OpenStdin is set, then it differs +func Compare(a, b *Config) bool { + if a == nil || b == nil || + a.OpenStdin || b.OpenStdin { + return false + } + if a.AttachStdout != b.AttachStdout || + a.AttachStderr != b.AttachStderr || + a.User != b.User || + a.Memory != b.Memory || + a.MemorySwap != b.MemorySwap || + a.CpuShares != b.CpuShares || + a.OpenStdin != b.OpenStdin || + a.Tty != b.Tty || + a.VolumesFrom != b.VolumesFrom { + return false + } + if len(a.Cmd) != len(b.Cmd) || + len(a.Dns) != len(b.Dns) || + len(a.Env) != len(b.Env) || + len(a.PortSpecs) != len(b.PortSpecs) || + len(a.ExposedPorts) != len(b.ExposedPorts) || + len(a.Entrypoint) != len(b.Entrypoint) || + len(a.Volumes) != len(b.Volumes) { + return false + } + + for i := 0; i < len(a.Cmd); i++ { + if a.Cmd[i] != b.Cmd[i] { + return false + } + } + for i := 0; i < len(a.Dns); i++ { + if a.Dns[i] != b.Dns[i] { + return false + } + } + for i := 0; i < len(a.Env); i++ { + if a.Env[i] != b.Env[i] { + return false + } + } + for i := 0; i < len(a.PortSpecs); i++ { + if a.PortSpecs[i] != b.PortSpecs[i] { + return false + } + } + for k := range a.ExposedPorts { + if _, exists := b.ExposedPorts[k]; !exists { + return false + } + } + for i := 0; i < len(a.Entrypoint); i++ { + if a.Entrypoint[i] != b.Entrypoint[i] { + return false + } + } + for key := range a.Volumes { + if _, exists := b.Volumes[key]; !exists { + return false + } + } + return true +} diff --git a/components/engine/runconfig/config.go b/components/engine/runconfig/config.go new file mode 100644 index 0000000000..9faa823a57 --- /dev/null +++ b/components/engine/runconfig/config.go @@ -0,0 +1,76 @@ +package runconfig + +import ( + "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/nat" +) + +// Note: the Config structure should hold only portable information about the container. +// Here, "portable" means "independent from the host we are running on". +// Non-portable information *should* appear in HostConfig. +type Config struct { + Hostname string + Domainname string + User string + Memory int64 // Memory limit (in bytes) + MemorySwap int64 // Total memory usage (memory + swap); set `-1' to disable swap + CpuShares int64 // CPU shares (relative weight vs. other containers) + AttachStdin bool + AttachStdout bool + AttachStderr bool + PortSpecs []string // Deprecated - Can be in the format of 8080/tcp + ExposedPorts map[nat.Port]struct{} + Tty bool // Attach standard streams to a tty, including stdin if it is not closed. + OpenStdin bool // Open stdin + StdinOnce bool // If true, close stdin after the 1 attached client disconnects. + Env []string + Cmd []string + Dns []string + Image string // Name of the image as it was passed by the operator (eg. could be symbolic) + Volumes map[string]struct{} + VolumesFrom string + WorkingDir string + Entrypoint []string + NetworkDisabled bool + OnBuild []string +} + +func ContainerConfigFromJob(job *engine.Job) *Config { + config := &Config{ + Hostname: job.Getenv("Hostname"), + Domainname: job.Getenv("Domainname"), + User: job.Getenv("User"), + Memory: job.GetenvInt64("Memory"), + MemorySwap: job.GetenvInt64("MemorySwap"), + CpuShares: job.GetenvInt64("CpuShares"), + AttachStdin: job.GetenvBool("AttachStdin"), + AttachStdout: job.GetenvBool("AttachStdout"), + AttachStderr: job.GetenvBool("AttachStderr"), + Tty: job.GetenvBool("Tty"), + OpenStdin: job.GetenvBool("OpenStdin"), + StdinOnce: job.GetenvBool("StdinOnce"), + Image: job.Getenv("Image"), + VolumesFrom: job.Getenv("VolumesFrom"), + WorkingDir: job.Getenv("WorkingDir"), + NetworkDisabled: job.GetenvBool("NetworkDisabled"), + } + job.GetenvJson("ExposedPorts", &config.ExposedPorts) + job.GetenvJson("Volumes", &config.Volumes) + if PortSpecs := job.GetenvList("PortSpecs"); PortSpecs != nil { + config.PortSpecs = PortSpecs + } + if Env := job.GetenvList("Env"); Env != nil { + config.Env = Env + } + if Cmd := job.GetenvList("Cmd"); Cmd != nil { + config.Cmd = Cmd + } + if Dns := job.GetenvList("Dns"); Dns != nil { + config.Dns = Dns + } + if Entrypoint := job.GetenvList("Entrypoint"); Entrypoint != nil { + config.Entrypoint = Entrypoint + } + + return config +} diff --git a/components/engine/config_test.go b/components/engine/runconfig/config_test.go similarity index 83% rename from components/engine/config_test.go rename to components/engine/runconfig/config_test.go index 1c808163e2..3ef31491fc 100644 --- a/components/engine/config_test.go +++ b/components/engine/runconfig/config_test.go @@ -1,11 +1,11 @@ -package docker +package runconfig import ( "github.com/dotcloud/docker/nat" "testing" ) -func TestCompareConfig(t *testing.T) { +func TestCompare(t *testing.T) { volumes1 := make(map[string]struct{}) volumes1["/test1"] = struct{}{} config1 := Config{ @@ -45,24 +45,24 @@ func TestCompareConfig(t *testing.T) { VolumesFrom: "11111111", Volumes: volumes2, } - if CompareConfig(&config1, &config2) { - t.Fatalf("CompareConfig should return false, Dns are different") + if Compare(&config1, &config2) { + t.Fatalf("Compare should return false, Dns are different") } - if CompareConfig(&config1, &config3) { - t.Fatalf("CompareConfig should return false, PortSpecs are different") + if Compare(&config1, &config3) { + t.Fatalf("Compare should return false, PortSpecs are different") } - if CompareConfig(&config1, &config4) { - t.Fatalf("CompareConfig should return false, VolumesFrom are different") + if Compare(&config1, &config4) { + t.Fatalf("Compare should return false, VolumesFrom are different") } - if CompareConfig(&config1, &config5) { - t.Fatalf("CompareConfig should return false, Volumes are different") + if Compare(&config1, &config5) { + t.Fatalf("Compare should return false, Volumes are different") } - if !CompareConfig(&config1, &config1) { - t.Fatalf("CompareConfig should return true") + if !Compare(&config1, &config1) { + t.Fatalf("Compare should return true") } } -func TestMergeConfig(t *testing.T) { +func TestMerge(t *testing.T) { volumesImage := make(map[string]struct{}) volumesImage["/test1"] = struct{}{} volumesImage["/test2"] = struct{}{} @@ -83,7 +83,7 @@ func TestMergeConfig(t *testing.T) { Volumes: volumesUser, } - if err := MergeConfig(configUser, configImage); err != nil { + if err := Merge(configUser, configImage); err != nil { t.Error(err) } @@ -134,7 +134,7 @@ func TestMergeConfig(t *testing.T) { ExposedPorts: ports, } - if err := MergeConfig(configUser, configImage2); err != nil { + if err := Merge(configUser, configImage2); err != nil { t.Error(err) } diff --git a/components/engine/runconfig/hostconfig.go b/components/engine/runconfig/hostconfig.go new file mode 100644 index 0000000000..6c8618ee81 --- /dev/null +++ b/components/engine/runconfig/hostconfig.go @@ -0,0 +1,39 @@ +package runconfig + +import ( + "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/nat" +) + +type HostConfig struct { + Binds []string + ContainerIDFile string + LxcConf []KeyValuePair + Privileged bool + PortBindings nat.PortMap + Links []string + PublishAllPorts bool +} + +type KeyValuePair struct { + Key string + Value string +} + +func ContainerHostConfigFromJob(job *engine.Job) *HostConfig { + hostConfig := &HostConfig{ + ContainerIDFile: job.Getenv("ContainerIDFile"), + Privileged: job.GetenvBool("Privileged"), + PublishAllPorts: job.GetenvBool("PublishAllPorts"), + } + job.GetenvJson("LxcConf", &hostConfig.LxcConf) + job.GetenvJson("PortBindings", &hostConfig.PortBindings) + if Binds := job.GetenvList("Binds"); Binds != nil { + hostConfig.Binds = Binds + } + if Links := job.GetenvList("Links"); Links != nil { + hostConfig.Links = Links + } + + return hostConfig +} diff --git a/components/engine/runconfig/merge.go b/components/engine/runconfig/merge.go new file mode 100644 index 0000000000..a8d677baa8 --- /dev/null +++ b/components/engine/runconfig/merge.go @@ -0,0 +1,119 @@ +package runconfig + +import ( + "github.com/dotcloud/docker/nat" + "github.com/dotcloud/docker/utils" + "strings" +) + +func Merge(userConf, imageConf *Config) error { + if userConf.User == "" { + userConf.User = imageConf.User + } + if userConf.Memory == 0 { + userConf.Memory = imageConf.Memory + } + if userConf.MemorySwap == 0 { + userConf.MemorySwap = imageConf.MemorySwap + } + if userConf.CpuShares == 0 { + userConf.CpuShares = imageConf.CpuShares + } + if userConf.ExposedPorts == nil || len(userConf.ExposedPorts) == 0 { + userConf.ExposedPorts = imageConf.ExposedPorts + } else if imageConf.ExposedPorts != nil { + if userConf.ExposedPorts == nil { + userConf.ExposedPorts = make(nat.PortSet) + } + for port := range imageConf.ExposedPorts { + if _, exists := userConf.ExposedPorts[port]; !exists { + userConf.ExposedPorts[port] = struct{}{} + } + } + } + + if userConf.PortSpecs != nil && len(userConf.PortSpecs) > 0 { + if userConf.ExposedPorts == nil { + userConf.ExposedPorts = make(nat.PortSet) + } + ports, _, err := nat.ParsePortSpecs(userConf.PortSpecs) + if err != nil { + return err + } + for port := range ports { + if _, exists := userConf.ExposedPorts[port]; !exists { + userConf.ExposedPorts[port] = struct{}{} + } + } + userConf.PortSpecs = nil + } + if imageConf.PortSpecs != nil && len(imageConf.PortSpecs) > 0 { + // FIXME: I think we can safely remove this. Leaving it for now for the sake of reverse-compat paranoia. + utils.Debugf("Migrating image port specs to containter: %s", strings.Join(imageConf.PortSpecs, ", ")) + if userConf.ExposedPorts == nil { + userConf.ExposedPorts = make(nat.PortSet) + } + + ports, _, err := nat.ParsePortSpecs(imageConf.PortSpecs) + if err != nil { + return err + } + for port := range ports { + if _, exists := userConf.ExposedPorts[port]; !exists { + userConf.ExposedPorts[port] = struct{}{} + } + } + } + if !userConf.Tty { + userConf.Tty = imageConf.Tty + } + if !userConf.OpenStdin { + userConf.OpenStdin = imageConf.OpenStdin + } + if !userConf.StdinOnce { + userConf.StdinOnce = imageConf.StdinOnce + } + if userConf.Env == nil || len(userConf.Env) == 0 { + userConf.Env = imageConf.Env + } else { + for _, imageEnv := range imageConf.Env { + found := false + imageEnvKey := strings.Split(imageEnv, "=")[0] + for _, userEnv := range userConf.Env { + userEnvKey := strings.Split(userEnv, "=")[0] + if imageEnvKey == userEnvKey { + found = true + } + } + if !found { + userConf.Env = append(userConf.Env, imageEnv) + } + } + } + if userConf.Cmd == nil || len(userConf.Cmd) == 0 { + userConf.Cmd = imageConf.Cmd + } + if userConf.Dns == nil || len(userConf.Dns) == 0 { + userConf.Dns = imageConf.Dns + } else { + //duplicates aren't an issue here + userConf.Dns = append(userConf.Dns, imageConf.Dns...) + } + if userConf.Entrypoint == nil || len(userConf.Entrypoint) == 0 { + userConf.Entrypoint = imageConf.Entrypoint + } + if userConf.WorkingDir == "" { + userConf.WorkingDir = imageConf.WorkingDir + } + if userConf.VolumesFrom == "" { + userConf.VolumesFrom = imageConf.VolumesFrom + } + if userConf.Volumes == nil || len(userConf.Volumes) == 0 { + userConf.Volumes = imageConf.Volumes + } else { + for k, v := range imageConf.Volumes { + userConf.Volumes[k] = v + } + } + return nil +} diff --git a/components/engine/runconfig/parse.go b/components/engine/runconfig/parse.go new file mode 100644 index 0000000000..fb08c068b2 --- /dev/null +++ b/components/engine/runconfig/parse.go @@ -0,0 +1,246 @@ +package runconfig + +import ( + "fmt" + "github.com/dotcloud/docker/nat" + flag "github.com/dotcloud/docker/pkg/mflag" + "github.com/dotcloud/docker/pkg/opts" + "github.com/dotcloud/docker/pkg/sysinfo" + "github.com/dotcloud/docker/utils" + "io/ioutil" + "path" + "strings" +) + +var ( + ErrInvalidWorikingDirectory = fmt.Errorf("The working directory is invalid. It needs to be an absolute path.") + ErrConflictAttachDetach = fmt.Errorf("Conflicting options: -a and -d") + ErrConflictDetachAutoRemove = fmt.Errorf("Conflicting options: -rm and -d") +) + +//FIXME Only used in tests +func Parse(args []string, sysInfo *sysinfo.SysInfo) (*Config, *HostConfig, *flag.FlagSet, error) { + cmd := flag.NewFlagSet("run", flag.ContinueOnError) + cmd.SetOutput(ioutil.Discard) + cmd.Usage = nil + return parseRun(cmd, args, sysInfo) +} + +// FIXME: this maps the legacy commands.go code. It should be merged with Parse to only expose a single parse function. +func ParseSubcommand(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Config, *HostConfig, *flag.FlagSet, error) { + return parseRun(cmd, args, sysInfo) +} + +func parseRun(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Config, *HostConfig, *flag.FlagSet, error) { + var ( + // FIXME: use utils.ListOpts for attach and volumes? + flAttach = opts.NewListOpts(opts.ValidateAttach) + flVolumes = opts.NewListOpts(opts.ValidatePath) + flLinks = opts.NewListOpts(opts.ValidateLink) + flEnv = opts.NewListOpts(opts.ValidateEnv) + + flPublish opts.ListOpts + flExpose opts.ListOpts + flDns opts.ListOpts + flVolumesFrom opts.ListOpts + flLxcOpts opts.ListOpts + + flAutoRemove = cmd.Bool([]string{"#rm", "-rm"}, false, "Automatically remove the container when it exits (incompatible with -d)") + flDetach = cmd.Bool([]string{"d", "-detach"}, false, "Detached mode: Run container in the background, print new container id") + flNetwork = cmd.Bool([]string{"n", "-networking"}, true, "Enable networking for this container") + flPrivileged = cmd.Bool([]string{"#privileged", "-privileged"}, false, "Give extended privileges to this container") + flPublishAll = cmd.Bool([]string{"P", "-publish-all"}, false, "Publish all exposed ports to the host interfaces") + flStdin = cmd.Bool([]string{"i", "-interactive"}, false, "Keep stdin open even if not attached") + flTty = cmd.Bool([]string{"t", "-tty"}, false, "Allocate a pseudo-tty") + flContainerIDFile = cmd.String([]string{"#cidfile", "-cidfile"}, "", "Write the container ID to the file") + flEntrypoint = cmd.String([]string{"#entrypoint", "-entrypoint"}, "", "Overwrite the default entrypoint of the image") + flHostname = cmd.String([]string{"h", "-hostname"}, "", "Container host name") + flMemoryString = cmd.String([]string{"m", "-memory"}, "", "Memory limit (format: , where unit = b, k, m or g)") + flUser = cmd.String([]string{"u", "-user"}, "", "Username or UID") + flWorkingDir = cmd.String([]string{"w", "-workdir"}, "", "Working directory inside the container") + flCpuShares = cmd.Int64([]string{"c", "-cpu-shares"}, 0, "CPU shares (relative weight)") + + // For documentation purpose + _ = cmd.Bool([]string{"#sig-proxy", "-sig-proxy"}, true, "Proxify all received signal to the process (even in non-tty mode)") + _ = cmd.String([]string{"#name", "-name"}, "", "Assign a name to the container") + ) + + cmd.Var(&flAttach, []string{"a", "-attach"}, "Attach to stdin, stdout or stderr.") + cmd.Var(&flVolumes, []string{"v", "-volume"}, "Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)") + cmd.Var(&flLinks, []string{"#link", "-link"}, "Add link to another container (name:alias)") + cmd.Var(&flEnv, []string{"e", "-env"}, "Set environment variables") + + cmd.Var(&flPublish, []string{"p", "-publish"}, fmt.Sprintf("Publish a container's port to the host (format: %s) (use 'docker port' to see the actual mapping)", nat.PortSpecTemplateFormat)) + cmd.Var(&flExpose, []string{"#expose", "-expose"}, "Expose a port from the container without publishing it to your host") + cmd.Var(&flDns, []string{"#dns", "-dns"}, "Set custom dns servers") + cmd.Var(&flVolumesFrom, []string{"#volumes-from", "-volumes-from"}, "Mount volumes from the specified container(s)") + cmd.Var(&flLxcOpts, []string{"#lxc-conf", "-lxc-conf"}, "Add custom lxc options -lxc-conf=\"lxc.cgroup.cpuset.cpus = 0,1\"") + + if err := cmd.Parse(args); err != nil { + return nil, nil, cmd, err + } + + // Check if the kernel supports memory limit cgroup. + if sysInfo != nil && *flMemoryString != "" && !sysInfo.MemoryLimit { + *flMemoryString = "" + } + + // Validate input params + if *flDetach && flAttach.Len() > 0 { + return nil, nil, cmd, ErrConflictAttachDetach + } + if *flWorkingDir != "" && !path.IsAbs(*flWorkingDir) { + return nil, nil, cmd, ErrInvalidWorikingDirectory + } + if *flDetach && *flAutoRemove { + return nil, nil, cmd, ErrConflictDetachAutoRemove + } + + // If neither -d or -a are set, attach to everything by default + if flAttach.Len() == 0 && !*flDetach { + if !*flDetach { + flAttach.Set("stdout") + flAttach.Set("stderr") + if *flStdin { + flAttach.Set("stdin") + } + } + } + + var flMemory int64 + if *flMemoryString != "" { + parsedMemory, err := utils.RAMInBytes(*flMemoryString) + if err != nil { + return nil, nil, cmd, err + } + flMemory = parsedMemory + } + + var binds []string + // add any bind targets to the list of container volumes + for bind := range flVolumes.GetMap() { + if arr := strings.Split(bind, ":"); len(arr) > 1 { + if arr[0] == "/" { + return nil, nil, cmd, fmt.Errorf("Invalid bind mount: source can't be '/'") + } + dstDir := arr[1] + flVolumes.Set(dstDir) + binds = append(binds, bind) + flVolumes.Delete(bind) + } else if bind == "/" { + return nil, nil, cmd, fmt.Errorf("Invalid volume: path can't be '/'") + } + } + + var ( + parsedArgs = cmd.Args() + runCmd []string + entrypoint []string + image string + ) + if len(parsedArgs) >= 1 { + image = cmd.Arg(0) + } + if len(parsedArgs) > 1 { + runCmd = parsedArgs[1:] + } + if *flEntrypoint != "" { + entrypoint = []string{*flEntrypoint} + } + + lxcConf, err := parseLxcConfOpts(flLxcOpts) + if err != nil { + return nil, nil, cmd, err + } + + var ( + domainname string + hostname = *flHostname + parts = strings.SplitN(hostname, ".", 2) + ) + if len(parts) > 1 { + hostname = parts[0] + domainname = parts[1] + } + + ports, portBindings, err := nat.ParsePortSpecs(flPublish.GetAll()) + if err != nil { + return nil, nil, cmd, err + } + + // Merge in exposed ports to the map of published ports + for _, e := range flExpose.GetAll() { + if strings.Contains(e, ":") { + return nil, nil, cmd, fmt.Errorf("Invalid port format for --expose: %s", e) + } + p := nat.NewPort(nat.SplitProtoPort(e)) + if _, exists := ports[p]; !exists { + ports[p] = struct{}{} + } + } + + config := &Config{ + Hostname: hostname, + Domainname: domainname, + PortSpecs: nil, // Deprecated + ExposedPorts: ports, + User: *flUser, + Tty: *flTty, + NetworkDisabled: !*flNetwork, + OpenStdin: *flStdin, + Memory: flMemory, + CpuShares: *flCpuShares, + AttachStdin: flAttach.Get("stdin"), + AttachStdout: flAttach.Get("stdout"), + AttachStderr: flAttach.Get("stderr"), + Env: flEnv.GetAll(), + Cmd: runCmd, + Dns: flDns.GetAll(), + Image: image, + Volumes: flVolumes.GetMap(), + VolumesFrom: strings.Join(flVolumesFrom.GetAll(), ","), + Entrypoint: entrypoint, + WorkingDir: *flWorkingDir, + } + + hostConfig := &HostConfig{ + Binds: binds, + ContainerIDFile: *flContainerIDFile, + LxcConf: lxcConf, + Privileged: *flPrivileged, + PortBindings: portBindings, + Links: flLinks.GetAll(), + PublishAllPorts: *flPublishAll, + } + + if sysInfo != nil && flMemory > 0 && !sysInfo.SwapLimit { + //fmt.Fprintf(stdout, "WARNING: Your kernel does not support swap limit capabilities. Limitation discarded.\n") + config.MemorySwap = -1 + } + + // When allocating stdin in attached mode, close stdin at client disconnect + if config.OpenStdin && config.AttachStdin { + config.StdinOnce = true + } + return config, hostConfig, cmd, nil +} + +func parseLxcConfOpts(opts opts.ListOpts) ([]KeyValuePair, error) { + out := make([]KeyValuePair, opts.Len()) + for i, o := range opts.GetAll() { + k, v, err := parseLxcOpt(o) + if err != nil { + return nil, err + } + out[i] = KeyValuePair{Key: k, Value: v} + } + return out, nil +} + +func parseLxcOpt(opt string) (string, string, error) { + parts := strings.SplitN(opt, "=", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("Unable to parse lxc conf option: %s", opt) + } + return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), nil +} diff --git a/components/engine/runconfig/parse_test.go b/components/engine/runconfig/parse_test.go new file mode 100644 index 0000000000..2b89e88ec3 --- /dev/null +++ b/components/engine/runconfig/parse_test.go @@ -0,0 +1,22 @@ +package runconfig + +import ( + "testing" +) + +func TestParseLxcConfOpt(t *testing.T) { + opts := []string{"lxc.utsname=docker", "lxc.utsname = docker "} + + for _, o := range opts { + k, v, err := parseLxcOpt(o) + if err != nil { + t.FailNow() + } + if k != "lxc.utsname" { + t.Fail() + } + if v != "docker" { + t.Fail() + } + } +} diff --git a/components/engine/runtime.go b/components/engine/runtime.go index cec5444090..828d3f0e66 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -18,6 +18,7 @@ import ( "github.com/dotcloud/docker/networkdriver/portallocator" "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/pkg/sysinfo" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -329,7 +330,7 @@ func (runtime *Runtime) restore() error { } // Create creates a new container from the given configuration with a given name. -func (runtime *Runtime) Create(config *Config, name string) (*Container, []string, error) { +func (runtime *Runtime) Create(config *runconfig.Config, name string) (*Container, []string, error) { // Lookup image img, err := runtime.repositories.LookupImage(config.Image) if err != nil { @@ -347,7 +348,7 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin return nil, nil, fmt.Errorf("Cannot create container with more than %d parents", MaxImageDepth) } - checkDeprecatedExpose := func(config *Config) bool { + checkDeprecatedExpose := func(config *runconfig.Config) bool { if config != nil { if config.PortSpecs != nil { for _, p := range config.PortSpecs { @@ -366,7 +367,7 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin } if img.Config != nil { - if err := MergeConfig(config, img.Config); err != nil { + if err := runconfig.Merge(config, img.Config); err != nil { return nil, nil, err } } @@ -441,7 +442,7 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin Path: entrypoint, Args: args, //FIXME: de-duplicate from config Config: config, - hostConfig: &HostConfig{}, + hostConfig: &runconfig.HostConfig{}, Image: img.ID, // Always use the resolved image id NetworkSettings: &NetworkSettings{}, Name: name, @@ -518,7 +519,7 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin // Commit creates a new filesystem image from the current state of a container. // The image can optionally be tagged into a repository -func (runtime *Runtime) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { +func (runtime *Runtime) Commit(container *Container, repository, tag, comment, author string, config *runconfig.Config) (*Image, error) { // FIXME: freeze the container before copying it to avoid data corruption? // FIXME: this shouldn't be in commands. if err := container.Mount(); err != nil { diff --git a/components/engine/server.go b/components/engine/server.go index cb677266e5..7c60f29378 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -10,6 +10,7 @@ import ( "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/registry" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -662,7 +663,7 @@ func (srv *Server) ImageInsert(job *engine.Job) engine.Status { } defer file.Body.Close() - config, _, _, err := ParseRun([]string{img.ID, "echo", "insert", url, path}, srv.runtime.sysInfo) + config, _, _, err := runconfig.Parse([]string{img.ID, "echo", "insert", url, path}, srv.runtime.sysInfo) if err != nil { return job.Error(err) } @@ -1043,7 +1044,7 @@ func (srv *Server) ContainerCommit(job *engine.Job) engine.Status { if container == nil { return job.Errorf("No such container: %s", name) } - var config Config + var config runconfig.Config if err := job.GetenvJson("config", &config); err != nil { return job.Error(err) } @@ -1623,7 +1624,7 @@ func (srv *Server) ContainerCreate(job *engine.Job) engine.Status { } else if len(job.Args) > 1 { return job.Errorf("Usage: %s", job.Name) } - config := ContainerConfigFromJob(job) + config := runconfig.ContainerConfigFromJob(job) if config.Memory != 0 && config.Memory < 524288 { return job.Errorf("Minimum memory limit allowed is 512k") } @@ -1989,7 +1990,7 @@ func (srv *Server) canDeleteImage(imgID string) error { return nil } -func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error) { +func (srv *Server) ImageGetCached(imgID string, config *runconfig.Config) (*Image, error) { // Retrieve all images images, err := srv.runtime.graph.Map() @@ -2013,7 +2014,7 @@ func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error) if err != nil { return nil, err } - if CompareConfig(&img.ContainerConfig, config) { + if runconfig.Compare(&img.ContainerConfig, config) { if match == nil || match.Created.Before(img.Created) { match = img } @@ -2022,7 +2023,7 @@ func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error) return match, nil } -func (srv *Server) RegisterLinks(container *Container, hostConfig *HostConfig) error { +func (srv *Server) RegisterLinks(container *Container, hostConfig *runconfig.HostConfig) error { runtime := srv.runtime if hostConfig != nil && hostConfig.Links != nil { @@ -2066,7 +2067,7 @@ func (srv *Server) ContainerStart(job *engine.Job) engine.Status { } // If no environment was set, then no hostconfig was passed. if len(job.Environ()) > 0 { - hostConfig := ContainerHostConfigFromJob(job) + hostConfig := runconfig.ContainerHostConfigFromJob(job) // Validate the HostConfig binds. Make sure that: // 1) the source of a bind mount isn't / // The bind mount "/:/foo" isn't allowed. @@ -2310,7 +2311,7 @@ func (srv *Server) JobInspect(job *engine.Job) engine.Status { } object = &struct { *Container - HostConfig *HostConfig + HostConfig *runconfig.HostConfig }{container, container.hostConfig} default: return job.Errorf("Unknown kind: %s", kind) diff --git a/components/engine/utils.go b/components/engine/utils.go index e3a58cc67e..68cd2f24e5 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -1,14 +1,12 @@ package docker import ( - "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/nat" "github.com/dotcloud/docker/pkg/namesgenerator" - "github.com/dotcloud/docker/pkg/opts" + "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" "io" - "strings" "sync/atomic" ) @@ -16,204 +14,7 @@ type Change struct { archive.Change } -// Compare two Config struct. Do not compare the "Image" nor "Hostname" fields -// If OpenStdin is set, then it differs -func CompareConfig(a, b *Config) bool { - if a == nil || b == nil || - a.OpenStdin || b.OpenStdin { - return false - } - if a.AttachStdout != b.AttachStdout || - a.AttachStderr != b.AttachStderr || - a.User != b.User || - a.Memory != b.Memory || - a.MemorySwap != b.MemorySwap || - a.CpuShares != b.CpuShares || - a.OpenStdin != b.OpenStdin || - a.Tty != b.Tty || - a.VolumesFrom != b.VolumesFrom { - return false - } - if len(a.Cmd) != len(b.Cmd) || - len(a.Dns) != len(b.Dns) || - len(a.Env) != len(b.Env) || - len(a.PortSpecs) != len(b.PortSpecs) || - len(a.ExposedPorts) != len(b.ExposedPorts) || - len(a.Entrypoint) != len(b.Entrypoint) || - len(a.Volumes) != len(b.Volumes) { - return false - } - - for i := 0; i < len(a.Cmd); i++ { - if a.Cmd[i] != b.Cmd[i] { - return false - } - } - for i := 0; i < len(a.Dns); i++ { - if a.Dns[i] != b.Dns[i] { - return false - } - } - for i := 0; i < len(a.Env); i++ { - if a.Env[i] != b.Env[i] { - return false - } - } - for i := 0; i < len(a.PortSpecs); i++ { - if a.PortSpecs[i] != b.PortSpecs[i] { - return false - } - } - for k := range a.ExposedPorts { - if _, exists := b.ExposedPorts[k]; !exists { - return false - } - } - for i := 0; i < len(a.Entrypoint); i++ { - if a.Entrypoint[i] != b.Entrypoint[i] { - return false - } - } - for key := range a.Volumes { - if _, exists := b.Volumes[key]; !exists { - return false - } - } - return true -} - -func MergeConfig(userConf, imageConf *Config) error { - if userConf.User == "" { - userConf.User = imageConf.User - } - if userConf.Memory == 0 { - userConf.Memory = imageConf.Memory - } - if userConf.MemorySwap == 0 { - userConf.MemorySwap = imageConf.MemorySwap - } - if userConf.CpuShares == 0 { - userConf.CpuShares = imageConf.CpuShares - } - if userConf.ExposedPorts == nil || len(userConf.ExposedPorts) == 0 { - userConf.ExposedPorts = imageConf.ExposedPorts - } else if imageConf.ExposedPorts != nil { - if userConf.ExposedPorts == nil { - userConf.ExposedPorts = make(nat.PortSet) - } - for port := range imageConf.ExposedPorts { - if _, exists := userConf.ExposedPorts[port]; !exists { - userConf.ExposedPorts[port] = struct{}{} - } - } - } - - if userConf.PortSpecs != nil && len(userConf.PortSpecs) > 0 { - if userConf.ExposedPorts == nil { - userConf.ExposedPorts = make(nat.PortSet) - } - ports, _, err := nat.ParsePortSpecs(userConf.PortSpecs) - if err != nil { - return err - } - for port := range ports { - if _, exists := userConf.ExposedPorts[port]; !exists { - userConf.ExposedPorts[port] = struct{}{} - } - } - userConf.PortSpecs = nil - } - if imageConf.PortSpecs != nil && len(imageConf.PortSpecs) > 0 { - utils.Debugf("Migrating image port specs to containter: %s", strings.Join(imageConf.PortSpecs, ", ")) - if userConf.ExposedPorts == nil { - userConf.ExposedPorts = make(nat.PortSet) - } - - ports, _, err := nat.ParsePortSpecs(imageConf.PortSpecs) - if err != nil { - return err - } - for port := range ports { - if _, exists := userConf.ExposedPorts[port]; !exists { - userConf.ExposedPorts[port] = struct{}{} - } - } - } - if !userConf.Tty { - userConf.Tty = imageConf.Tty - } - if !userConf.OpenStdin { - userConf.OpenStdin = imageConf.OpenStdin - } - if !userConf.StdinOnce { - userConf.StdinOnce = imageConf.StdinOnce - } - if userConf.Env == nil || len(userConf.Env) == 0 { - userConf.Env = imageConf.Env - } else { - for _, imageEnv := range imageConf.Env { - found := false - imageEnvKey := strings.Split(imageEnv, "=")[0] - for _, userEnv := range userConf.Env { - userEnvKey := strings.Split(userEnv, "=")[0] - if imageEnvKey == userEnvKey { - found = true - } - } - if !found { - userConf.Env = append(userConf.Env, imageEnv) - } - } - } - if userConf.Cmd == nil || len(userConf.Cmd) == 0 { - userConf.Cmd = imageConf.Cmd - } - if userConf.Dns == nil || len(userConf.Dns) == 0 { - userConf.Dns = imageConf.Dns - } else { - //duplicates aren't an issue here - userConf.Dns = append(userConf.Dns, imageConf.Dns...) - } - if userConf.Entrypoint == nil || len(userConf.Entrypoint) == 0 { - userConf.Entrypoint = imageConf.Entrypoint - } - if userConf.WorkingDir == "" { - userConf.WorkingDir = imageConf.WorkingDir - } - if userConf.VolumesFrom == "" { - userConf.VolumesFrom = imageConf.VolumesFrom - } - if userConf.Volumes == nil || len(userConf.Volumes) == 0 { - userConf.Volumes = imageConf.Volumes - } else { - for k, v := range imageConf.Volumes { - userConf.Volumes[k] = v - } - } - return nil -} - -func parseLxcConfOpts(opts opts.ListOpts) ([]KeyValuePair, error) { - out := make([]KeyValuePair, opts.Len()) - for i, o := range opts.GetAll() { - k, v, err := parseLxcOpt(o) - if err != nil { - return nil, err - } - out[i] = KeyValuePair{Key: k, Value: v} - } - return out, nil -} - -func parseLxcOpt(opt string) (string, string, error) { - parts := strings.SplitN(opt, "=", 2) - if len(parts) != 2 { - return "", "", fmt.Errorf("Unable to parse lxc conf option: %s", opt) - } - return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), nil -} - -func migratePortMappings(config *Config, hostConfig *HostConfig) error { +func migratePortMappings(config *runconfig.Config, hostConfig *runconfig.HostConfig) error { if config.PortSpecs != nil { ports, bindings, err := nat.ParsePortSpecs(config.PortSpecs) if err != nil { @@ -222,7 +23,7 @@ func migratePortMappings(config *Config, hostConfig *HostConfig) error { config.PortSpecs = nil if len(bindings) > 0 { if hostConfig == nil { - hostConfig = &HostConfig{} + hostConfig = &runconfig.HostConfig{} } hostConfig.PortBindings = bindings } From 3648d21c65db28b2bdade852e34cf677f6b773cf Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 12 Feb 2014 16:30:18 +1000 Subject: [PATCH 067/403] tell the reader that they need git and make for this tutorial Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: d9f215eb2a48e6e836af9cc4015c73f17dc5be9e Component: engine --- .../docs/sources/contributing/devenvironment.rst | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/contributing/devenvironment.rst b/components/engine/docs/sources/contributing/devenvironment.rst index 6675173584..cab4c71afb 100644 --- a/components/engine/docs/sources/contributing/devenvironment.rst +++ b/components/engine/docs/sources/contributing/devenvironment.rst @@ -24,7 +24,17 @@ a working, up-to-date docker installation, then continue to the next step. -Step 2: Check out the Source +Step 2: Install tools used for this tutorial +-------------------------------------------- + +Install ``git``; honest, it's very good. You can use other ways to get the Docker +source, but they're not anywhere near as easy. + +Install ``make``. This tutorial uses our base Makefile to kick off the docker +containers in a repeatable and consistent way. Again, you can do it in other ways +but you need to do more work. + +Step 3: Check out the Source ---------------------------- .. code-block:: bash @@ -35,7 +45,7 @@ Step 2: Check out the Source To checkout a different revision just use ``git checkout`` with the name of branch or revision number. -Step 3: Build the Environment +Step 4: Build the Environment ----------------------------- This following command will build a development environment using the Dockerfile in the current directory. Essentially, it will install all the build and runtime dependencies necessary to build and test Docker. This command will take some time to complete when you first execute it. @@ -48,7 +58,7 @@ If the build is successful, congratulations! You have produced a clean build of docker, neatly encapsulated in a standard build environment. -Step 4: Build the Docker Binary +Step 5: Build the Docker Binary ------------------------------- To create the Docker binary, run this command: From bcb95a48949644bad3fbe3ed201894bf4cf2d429 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 12 Feb 2014 04:09:56 -0800 Subject: [PATCH 068/403] Implement create veth Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 2d2c237f50b7954993f6cd1db67c6f8c6d06f881 Component: engine --- .../engine/pkg/netlink/netlink_linux.go | 84 ++++++++++++------- 1 file changed, 53 insertions(+), 31 deletions(-) diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 23dba0c884..b16dec0612 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -142,29 +142,61 @@ func rtaAlignOf(attrlen int) int { type RtAttr struct { syscall.RtAttr - Data []byte + Data []byte + children []*RtAttr + prefix int } func newRtAttr(attrType int, data []byte) *RtAttr { - attr := &RtAttr{} + attr := &RtAttr{ + children: []*RtAttr{}, + } attr.Type = uint16(attrType) attr.Data = data return attr } -func (attr *RtAttr) ToWireFormat() []byte { +func newRtAttrChild(parent *RtAttr, attrType int, data []byte) *RtAttr { + attr := newRtAttr(attrType, data) + parent.children = append(parent.children, attr) + return attr +} + +func (a *RtAttr) length() int { + l := 0 + for _, child := range a.children { + l += child.length() + syscall.SizeofRtAttr + child.prefix + } + if l == 0 { + l++ + } + return rtaAlignOf(l + len(a.Data)) +} + +func (a *RtAttr) ToWireFormat() []byte { native := nativeEndian() - len := syscall.SizeofRtAttr + len(attr.Data) - b := make([]byte, rtaAlignOf(len)) - native.PutUint16(b[0:2], uint16(len)) - native.PutUint16(b[2:4], attr.Type) - for i, d := range attr.Data { - b[4+i] = d + length := a.length() + buf := make([]byte, rtaAlignOf(length+syscall.SizeofRtAttr)) + + if a.Data != nil { + copy(buf[4:], a.Data) + } else { + next := 4 + for _, child := range a.children { + childBuf := child.ToWireFormat() + copy(buf[next+child.prefix:], childBuf) + next += rtaAlignOf(len(childBuf)) + } } - return b + if l := uint16(rtaAlignOf(length)); l != 0 { + native.PutUint16(buf[0:2], l+1) + } + native.PutUint16(buf[2:4], a.Type) + + return buf } type NetlinkRequest struct { @@ -501,12 +533,7 @@ func NetworkLinkAddIp(iface *net.Interface, ip net.IP, ipNet *net.IPNet) error { } func zeroTerminated(s string) []byte { - bytes := make([]byte, len(s)+1) - for i := 0; i < len(s); i++ { - bytes[i] = s[i] - } - bytes[len(s)] = 0 - return bytes + return []byte(s + "\000") } func nonZeroTerminated(s string) []byte { @@ -697,24 +724,19 @@ func NetworkCreateVethPair(name1, name2 string) error { msg := newIfInfomsg(syscall.AF_UNSPEC) wb.AddData(msg) - kindData := newRtAttr(IFLA_INFO_KIND, nonZeroTerminated("veth")) - info := newRtAttr(syscall.IFLA_LINKINFO, kindData.ToWireFormat()) - // wb.AddData(info) - - peerName := newRtAttr(syscall.IFLA_IFNAME, nonZeroTerminated(name2)) - peer := newRtAttr(VETH_PEER, peerName.ToWireFormat()) - // wb.AddData(peer) - - b := []byte{} - b = append(b, peer.ToWireFormat()...) - b = append(b, info.ToWireFormat()...) - - infoData := newRtAttr(IFLA_INFO_DATA, b) - wb.AddData(infoData) - nameData := newRtAttr(syscall.IFLA_IFNAME, zeroTerminated(name1)) wb.AddData(nameData) + nest1 := newRtAttr(syscall.IFLA_LINKINFO, nil) + newRtAttrChild(nest1, IFLA_INFO_KIND, zeroTerminated("veth")) + nest2 := newRtAttrChild(nest1, IFLA_INFO_DATA, nil) + nest3 := newRtAttrChild(nest2, VETH_PEER, nil) + + last := newRtAttrChild(nest3, syscall.IFLA_IFNAME, zeroTerminated(name2)) + last.prefix = syscall.SizeofIfInfomsg + + wb.AddData(nest1) + if err := s.Send(wb); err != nil { return err } From 8f6a38f17aadef2cb5a5a0346a4eda9a47facb5d Mon Sep 17 00:00:00 2001 From: "Chris St. Pierre" Date: Wed, 12 Feb 2014 08:02:36 -0500 Subject: [PATCH 069/403] Fix bogus variable reference in mkimage-yum.sh Fixes this: https://github.com/dotcloud/docker/commit/d419da7227826e84e9375ece4fd9d4978a42cbf7#commitcomment-5344982 Docker-DCO-1.1-Signed-off-by: Chris St. Pierre (github: stpierre) Upstream-commit: 85263cdee894d1f1e3bca21a8d18264ef530c4d5 Component: engine --- components/engine/contrib/mkimage-yum.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/contrib/mkimage-yum.sh b/components/engine/contrib/mkimage-yum.sh index 54e99f1f04..d7ffdc1902 100755 --- a/components/engine/contrib/mkimage-yum.sh +++ b/components/engine/contrib/mkimage-yum.sh @@ -51,7 +51,7 @@ done yum -c "$yum_config" --installroot="$target" --setopt=tsflags=nodocs \ --setopt=group_package_types=mandatory -y groupinstall Core -yum -c "$yum_config" --installroot="$mount" -y clean all +yum -c "$yum_config" --installroot="$target" -y clean all cat > "$target"/etc/sysconfig/network < Date: Wed, 12 Feb 2014 16:02:53 +0100 Subject: [PATCH 070/403] Avoid extra mount/unmount during build CmdRun() calls first run() and then wait() to wait for it to exit, then it runs commit(). The run command will mount the container and the container exiting will unmount it. Then the commit will immediately mount it again to do a diff. This seems minor, but this is actually problematic, as the Get/Put pair will create a spurious mount/unmount cycle that is not needed and slows things down. Additionally it will create a supurious devicemapper activate/deactivate cycle that causes races with udev as seen in https://github.com/dotcloud/docker/issues/4036. To ensure that we only unmount once we split up run() into create() and run() and reference the mount until after the commit(). With this change docker build on devicemapper is now race-free, and slightly faster. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 59347fa66dfdd059c270a5f72e24320b4d0203ea Component: engine --- components/engine/buildfile.go | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index a121276c21..3279fd6e25 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -180,11 +180,20 @@ func (b *buildFile) CmdRun(args string) error { return nil } - cid, err := b.run() + c, err := b.create() if err != nil { return err } - if err := b.commit(cid, cmd, "run"); err != nil { + // Ensure that we keep the container mounted until the commit + // to avoid unmounting and then mounting directly again + c.Mount() + defer c.Unmount() + + err = b.run(c) + if err != nil { + return err + } + if err := b.commit(c.ID, cmd, "run"); err != nil { return err } @@ -555,16 +564,16 @@ func (sf *StderrFormater) Write(buf []byte) (int, error) { return len(buf), err } -func (b *buildFile) run() (string, error) { +func (b *buildFile) create() (*Container, error) { if b.image == "" { - return "", fmt.Errorf("Please provide a source image with `from` prior to run") + return nil, fmt.Errorf("Please provide a source image with `from` prior to run") } b.config.Image = b.image // Create the container and start it c, _, err := b.runtime.Create(b.config, "") if err != nil { - return "", err + return nil, err } b.tmpContainers[c.ID] = struct{}{} fmt.Fprintf(b.outStream, " ---> Running in %s\n", utils.TruncateID(c.ID)) @@ -573,6 +582,10 @@ func (b *buildFile) run() (string, error) { c.Path = b.config.Cmd[0] c.Args = b.config.Cmd[1:] + return c, nil +} + +func (b *buildFile) run(c *Container) error { var errCh chan error if b.verbose { @@ -583,12 +596,12 @@ func (b *buildFile) run() (string, error) { //start the container if err := c.Start(); err != nil { - return "", err + return err } if errCh != nil { if err := <-errCh; err != nil { - return "", err + return err } } @@ -598,10 +611,10 @@ func (b *buildFile) run() (string, error) { Message: fmt.Sprintf("The command %v returned a non-zero code: %d", b.config.Cmd, ret), Code: ret, } - return "", err + return err } - return c.ID, nil + return nil } // Commit the container with the autorun command From 68a1243dca3995e8c9966347bfda3c3f1854ed37 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 12 Feb 2014 09:29:06 -0800 Subject: [PATCH 071/403] Simplify code + Allow more generic attr children + remove prefix Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 38e5b4e70fb6b93aa2b86e4d625a0032b97b991a Component: engine --- components/engine/pkg/netlink/netlink.go | 10 +- .../engine/pkg/netlink/netlink_linux.go | 131 ++++++++++-------- 2 files changed, 79 insertions(+), 62 deletions(-) diff --git a/components/engine/pkg/netlink/netlink.go b/components/engine/pkg/netlink/netlink.go index 5098b4b816..5cc756256d 100644 --- a/components/engine/pkg/netlink/netlink.go +++ b/components/engine/pkg/netlink/netlink.go @@ -5,7 +5,15 @@ // netlink_darwin.go package netlink -import "net" +import ( + "errors" + "net" +) + +var ( + ErrWrongSockType = errors.New("Wrong socket type") + ErrShortResponse = errors.New("Got short response from netlink") +) // A Route is a subnet associated with the interface to reach it. type Route struct { diff --git a/components/engine/pkg/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go index 1f48a6425c..f8bb6bac3c 100644 --- a/components/engine/pkg/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -45,6 +45,7 @@ func getIpFamily(ip net.IP) int { } type NetlinkRequestData interface { + Len() int ToWireFormat() []byte } @@ -53,21 +54,24 @@ type IfInfomsg struct { } func newIfInfomsg(family int) *IfInfomsg { - msg := &IfInfomsg{} - msg.Family = uint8(family) - msg.Type = uint16(0) - msg.Index = int32(0) - msg.Flags = uint32(0) - msg.Change = uint32(0) + return &IfInfomsg{ + IfInfomsg: syscall.IfInfomsg{ + Family: uint8(family), + }, + } +} +func newIfInfomsgChild(parent *RtAttr, family int) *IfInfomsg { + msg := newIfInfomsg(family) + parent.children = append(parent.children, msg) return msg } func (msg *IfInfomsg) ToWireFormat() []byte { native := nativeEndian() - len := syscall.SizeofIfInfomsg - b := make([]byte, len) + length := syscall.SizeofIfInfomsg + b := make([]byte, length) b[0] = msg.Family b[1] = 0 native.PutUint16(b[2:4], msg.Type) @@ -77,26 +81,27 @@ func (msg *IfInfomsg) ToWireFormat() []byte { return b } +func (msg *IfInfomsg) Len() int { + return syscall.SizeofIfInfomsg +} + type IfAddrmsg struct { syscall.IfAddrmsg } func newIfAddrmsg(family int) *IfAddrmsg { - msg := &IfAddrmsg{} - msg.Family = uint8(family) - msg.Prefixlen = uint8(0) - msg.Flags = uint8(0) - msg.Scope = uint8(0) - msg.Index = uint32(0) - - return msg + return &IfAddrmsg{ + IfAddrmsg: syscall.IfAddrmsg{ + Family: uint8(family), + }, + } } func (msg *IfAddrmsg) ToWireFormat() []byte { native := nativeEndian() - len := syscall.SizeofIfAddrmsg - b := make([]byte, len) + length := syscall.SizeofIfAddrmsg + b := make([]byte, length) b[0] = msg.Family b[1] = msg.Prefixlen b[2] = msg.Flags @@ -105,26 +110,31 @@ func (msg *IfAddrmsg) ToWireFormat() []byte { return b } +func (msg *IfAddrmsg) Len() int { + return syscall.SizeofIfAddrmsg +} + type RtMsg struct { syscall.RtMsg } func newRtMsg(family int) *RtMsg { - msg := &RtMsg{} - msg.Family = uint8(family) - msg.Table = syscall.RT_TABLE_MAIN - msg.Scope = syscall.RT_SCOPE_UNIVERSE - msg.Protocol = syscall.RTPROT_BOOT - msg.Type = syscall.RTN_UNICAST - - return msg + return &RtMsg{ + RtMsg: syscall.RtMsg{ + Family: uint8(family), + Table: syscall.RT_TABLE_MAIN, + Scope: syscall.RT_SCOPE_UNIVERSE, + Protocol: syscall.RTPROT_BOOT, + Type: syscall.RTN_UNICAST, + }, + } } func (msg *RtMsg) ToWireFormat() []byte { native := nativeEndian() - len := syscall.SizeofRtMsg - b := make([]byte, len) + length := syscall.SizeofRtMsg + b := make([]byte, length) b[0] = msg.Family b[1] = msg.Dst_len b[2] = msg.Src_len @@ -137,6 +147,10 @@ func (msg *RtMsg) ToWireFormat() []byte { return b } +func (msg *RtMsg) Len() int { + return syscall.SizeofRtMsg +} + func rtaAlignOf(attrlen int) int { return (attrlen + syscall.RTA_ALIGNTO - 1) & ^(syscall.RTA_ALIGNTO - 1) } @@ -144,18 +158,17 @@ func rtaAlignOf(attrlen int) int { type RtAttr struct { syscall.RtAttr Data []byte - children []*RtAttr - prefix int + children []NetlinkRequestData } func newRtAttr(attrType int, data []byte) *RtAttr { - attr := &RtAttr{ - children: []*RtAttr{}, + return &RtAttr{ + RtAttr: syscall.RtAttr{ + Type: uint16(attrType), + }, + children: []NetlinkRequestData{}, + Data: data, } - attr.Type = uint16(attrType) - attr.Data = data - - return attr } func newRtAttrChild(parent *RtAttr, attrType int, data []byte) *RtAttr { @@ -164,10 +177,10 @@ func newRtAttrChild(parent *RtAttr, attrType int, data []byte) *RtAttr { return attr } -func (a *RtAttr) length() int { +func (a *RtAttr) Len() int { l := 0 for _, child := range a.children { - l += child.length() + syscall.SizeofRtAttr + child.prefix + l += child.Len() + syscall.SizeofRtAttr } if l == 0 { l++ @@ -178,7 +191,7 @@ func (a *RtAttr) length() int { func (a *RtAttr) ToWireFormat() []byte { native := nativeEndian() - length := a.length() + length := a.Len() buf := make([]byte, rtaAlignOf(length+syscall.SizeofRtAttr)) if a.Data != nil { @@ -187,7 +200,7 @@ func (a *RtAttr) ToWireFormat() []byte { next := 4 for _, child := range a.children { childBuf := child.ToWireFormat() - copy(buf[next+child.prefix:], childBuf) + copy(buf[next:], childBuf) next += rtaAlignOf(len(childBuf)) } } @@ -212,7 +225,7 @@ func (rr *NetlinkRequest) ToWireFormat() []byte { dataBytes := make([][]byte, len(rr.Data)) for i, data := range rr.Data { dataBytes[i] = data.ToWireFormat() - length = length + uint32(len(dataBytes[i])) + length += uint32(len(dataBytes[i])) } b := make([]byte, length) native.PutUint32(b[0:4], length) @@ -221,12 +234,10 @@ func (rr *NetlinkRequest) ToWireFormat() []byte { native.PutUint32(b[8:12], rr.Seq) native.PutUint32(b[12:16], rr.Pid) - i := 16 + next := 16 for _, data := range dataBytes { - for _, dataByte := range data { - b[i] = dataByte - i = i + 1 - } + copy(b[next:], data) + next += len(data) } return b } @@ -238,12 +249,14 @@ func (rr *NetlinkRequest) AddData(data NetlinkRequestData) { } func newNetlinkRequest(proto, flags int) *NetlinkRequest { - rr := &NetlinkRequest{} - rr.Len = uint32(syscall.NLMSG_HDRLEN) - rr.Type = uint16(proto) - rr.Flags = syscall.NLM_F_REQUEST | uint16(flags) - rr.Seq = uint32(getSeq()) - return rr + return &NetlinkRequest{ + NlMsghdr: syscall.NlMsghdr{ + Len: uint32(syscall.NLMSG_HDRLEN), + Type: uint16(proto), + Flags: syscall.NLM_F_REQUEST | uint16(flags), + Seq: uint32(getSeq()), + }, + } } type NetlinkSocket struct { @@ -286,7 +299,7 @@ func (s *NetlinkSocket) Receive() ([]syscall.NetlinkMessage, error) { return nil, err } if nr < syscall.NLMSG_HDRLEN { - return nil, fmt.Errorf("Got short response from netlink") + return nil, ErrShortResponse } rb = rb[:nr] return syscall.ParseNetlinkMessage(rb) @@ -301,7 +314,7 @@ func (s *NetlinkSocket) GetPid() (uint32, error) { case *syscall.SockaddrNetlink: return v.Pid, nil } - return 0, fmt.Errorf("Wrong socket type") + return 0, ErrWrongSockType } func (s *NetlinkSocket) HandleAck(seq uint32) error { @@ -592,11 +605,7 @@ func zeroTerminated(s string) []byte { } func nonZeroTerminated(s string) []byte { - bytes := make([]byte, len(s)) - for i := 0; i < len(s); i++ { - bytes[i] = s[i] - } - return bytes + return []byte(s) } // Add a new network link of a specified type. This is identical to @@ -789,8 +798,8 @@ func NetworkCreateVethPair(name1, name2 string) error { nest2 := newRtAttrChild(nest1, IFLA_INFO_DATA, nil) nest3 := newRtAttrChild(nest2, VETH_INFO_PEER, nil) - last := newRtAttrChild(nest3, syscall.IFLA_IFNAME, zeroTerminated(name2)) - last.prefix = syscall.SizeofIfInfomsg + newIfInfomsgChild(nest3, syscall.AF_UNSPEC) + newRtAttrChild(nest3, syscall.IFLA_IFNAME, zeroTerminated(name2)) wb.AddData(nest1) From b2a134f727cd50d3d8b06214bed297c7af27c3e9 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 5 Feb 2014 14:35:44 -0600 Subject: [PATCH 072/403] Added sysvinit/sysconfig files for redhat family of distros (RHEL/CentOS/SL/etc.) Docker-DCO-1.1-Signed-off-by: Adam Miller (github: maxamillion) Upstream-commit: 2222cba5acd3fe663d492d7bc3fd4787d4c9869f Component: engine --- .../contrib/init/sysvinit-redhat/docker | 128 ++++++++++++++++++ .../init/sysvinit-redhat/docker.sysconfig | 7 + 2 files changed, 135 insertions(+) create mode 100755 components/engine/contrib/init/sysvinit-redhat/docker create mode 100644 components/engine/contrib/init/sysvinit-redhat/docker.sysconfig diff --git a/components/engine/contrib/init/sysvinit-redhat/docker b/components/engine/contrib/init/sysvinit-redhat/docker new file mode 100755 index 0000000000..4ee46bb02b --- /dev/null +++ b/components/engine/contrib/init/sysvinit-redhat/docker @@ -0,0 +1,128 @@ +#!/bin/sh +# +# /etc/rc.d/init.d/docker +# +# Daemon for docker.io +# +# chkconfig: 2345 95 95 +# description: Daemon for docker.io + +### BEGIN INIT INFO +# Provides: docker +# Required-Start: $network cgconfig +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: start and stop docker +# Description: Daemon for docker.io +### END INIT INFO + +# Source function library. +. /etc/rc.d/init.d/functions + +prog="docker" +exec="/usr/bin/$prog" +pidfile="/var/run/$prog.pid" +lockfile="/var/lock/subsys/$prog" +logfile="/var/log/$prog" + +[ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog + +prestart() { + service cgconfig status > /dev/null + + if [[ $? != 0 ]]; then + service cgconfig start + fi + + preexec="/sbin/sysctl" + [ -x $preexec ] || exit 6 + $preexec -w net.ipv4.ip_forward=1 > /dev/null 2>&1 + $preexec -w net.ipv6.conf.all.forwarding=1 > /dev/null 2>&1 + +} + +start() { + [ -x $exec ] || exit 5 + + if ! [ -f $pidfile ]; then + prestart + printf "Starting $prog:\t" + echo "\n$(date)\n" >> $logfile + $exec -d $other_args &>> $logfile & + pid=$! + touch $lockfile + success + echo + else + failure + echo + printf "$pidfile still exists...\n" + exit 7 + fi +} + +stop() { + echo -n $"Stopping $prog: " + killproc -p $pidfile $prog + retval=$? + echo + [ $retval -eq 0 ] && rm -f $lockfile + return $retval +} + +restart() { + stop + start +} + +reload() { + restart +} + +force_reload() { + restart +} + +rh_status() { + status -p $pidfile $prog +} + +rh_status_q() { + rh_status >/dev/null 2>&1 +} + +case "$1" in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 2 +esac + +exit $? diff --git a/components/engine/contrib/init/sysvinit-redhat/docker.sysconfig b/components/engine/contrib/init/sysvinit-redhat/docker.sysconfig new file mode 100644 index 0000000000..9c99dd1966 --- /dev/null +++ b/components/engine/contrib/init/sysvinit-redhat/docker.sysconfig @@ -0,0 +1,7 @@ +# /etc/sysconfig/docker +# +# Other arguments to pass to the docker daemon process +# These will be parsed by the sysv initscript and appended +# to the arguments list passed to docker -d + +other_args="" From 1e41d2ac045c1cec79250334d4df33df8d66847b Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Wed, 12 Feb 2014 23:23:42 +0000 Subject: [PATCH 073/403] add error to docekr build --rm Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) Upstream-commit: a895c7238d3f32b22989c409a5a3b09aa3463054 Component: engine --- components/engine/buildfile.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index a121276c21..b4024bd8c8 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -65,8 +65,11 @@ type buildFile struct { func (b *buildFile) clearTmp(containers map[string]struct{}) { for c := range containers { tmp := b.runtime.Get(c) - b.runtime.Destroy(tmp) - fmt.Fprintf(b.outStream, "Removing intermediate container %s\n", utils.TruncateID(c)) + if err := b.runtime.Destroy(tmp); err != nil { + fmt.Fprintf(b.outStream, "Error removing intermediate container %s: %s\n", utils.TruncateID(c), err.Error()) + } else { + fmt.Fprintf(b.outStream, "Removing intermediate container %s\n", utils.TruncateID(c)) + } } } From 8934e64a02f2593d1ce49220e3c7d5923295d2aa Mon Sep 17 00:00:00 2001 From: Charles Merriam Date: Wed, 12 Feb 2014 18:11:01 -0800 Subject: [PATCH 074/403] (resubmit) Add notes about OS/X and remote daemons Docker-DCO-1.1-Signed-off-by: Charles Merriam (github: merriam) Upstream-commit: 2a17bdce9e6e25e9be225fede7aac0b12729d82f Component: engine --- .../engine/docs/sources/use/working_with_volumes.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/components/engine/docs/sources/use/working_with_volumes.rst b/components/engine/docs/sources/use/working_with_volumes.rst index 34728cbd3d..c639efff4f 100644 --- a/components/engine/docs/sources/use/working_with_volumes.rst +++ b/components/engine/docs/sources/use/working_with_volumes.rst @@ -108,6 +108,16 @@ container with read only permissions as ``/var/host_logs``. .. versionadded:: v0.5.0 + +Note for OS/X users and remote daemon users: +-------------------------------------------- + +OS/X users run ``boot2docker`` to create a minimalist virtual machine running the docker daemon. That +virtual machine then launches docker commands on behalf of the OS/X command line. The means that ``host +directories`` refer to directories in the ``boot2docker`` virtual machine, not the OS/X filesystem. + +Similarly, anytime when the docker daemon is on a remote machine, the ``host directories`` always refer to directories on the daemon's machine. + Known Issues ............ From 60f7a02d072884f03780c4c6becfa452a79a5220 Mon Sep 17 00:00:00 2001 From: Charles Merriam Date: Wed, 12 Feb 2014 18:13:46 -0800 Subject: [PATCH 075/403] Minor error in example code (resubmit #4109) Docker-DCO-1.1-Signed-off-by: Charles Merriam (github: merriam) Upstream-commit: ba650e05177bcc2597606b998a104a29698d5064 Component: engine --- components/engine/docs/sources/use/working_with_volumes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/use/working_with_volumes.rst b/components/engine/docs/sources/use/working_with_volumes.rst index 34728cbd3d..d0fc5efef8 100644 --- a/components/engine/docs/sources/use/working_with_volumes.rst +++ b/components/engine/docs/sources/use/working_with_volumes.rst @@ -101,7 +101,7 @@ might not work on any other machine. For example:: - sudo docker run -v /var/logs:/var/host_logs:ro ubuntu bash + sudo docker run -t -i -v /var/logs:/var/host_logs:ro ubuntu bash The command above mounts the host directory ``/var/logs`` into the container with read only permissions as ``/var/host_logs``. From 171ba8d692f686f7600736091fcbb83156d97f3b Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 13 Feb 2014 03:26:35 +0200 Subject: [PATCH 076/403] disallow tcp:// from defaulting to 127.0.0.1:4243 This stops docker from accepting tcp:// as a valid bind address. Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: 31dde3ea05e1cf4c1f45083c1b6129e8c2a362df Component: engine --- components/engine/api/api.go | 3 +-- .../engine/docs/sources/installation/mac.rst | 2 +- .../sources/reference/commandline/cli.rst | 2 +- components/engine/utils/utils.go | 9 ++++--- components/engine/utils/utils_test.go | 24 ++++++++++--------- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index e0077a94e3..5ec9b778e0 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -31,12 +31,11 @@ import ( const ( APIVERSION = 1.9 DEFAULTHTTPHOST = "127.0.0.1" - DEFAULTHTTPPORT = 4243 DEFAULTUNIXSOCKET = "/var/run/docker.sock" ) func ValidateHost(val string) (string, error) { - host, err := utils.ParseHost(DEFAULTHTTPHOST, DEFAULTHTTPPORT, DEFAULTUNIXSOCKET, val) + host, err := utils.ParseHost(DEFAULTHTTPHOST, DEFAULTUNIXSOCKET, val) if err != nil { return val, err } diff --git a/components/engine/docs/sources/installation/mac.rst b/components/engine/docs/sources/installation/mac.rst index fce9b00f7e..efb999ee1f 100644 --- a/components/engine/docs/sources/installation/mac.rst +++ b/components/engine/docs/sources/installation/mac.rst @@ -72,7 +72,7 @@ Run the following commands to get it downloaded and set up: chmod +x docker # Set the environment variable for the docker daemon - export DOCKER_HOST=tcp:// + export DOCKER_HOST=tcp://127.0.0.1:4243 # Copy the executable file sudo cp docker /usr/local/bin/ diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index 18cef685aa..032076b941 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -12,7 +12,7 @@ To list available commands, either run ``docker`` with no parameters or execute $ sudo docker Usage: docker [OPTIONS] COMMAND [arg...] - -H=[unix:///var/run/docker.sock]: tcp://[host[:port]] to bind/connect to or unix://[/path/to/socket] to use. When host=[0.0.0.0], port=[4243] or path=[/var/run/docker.sock] is omitted, default values are used. + -H=[unix:///var/run/docker.sock]: tcp://[host]:port to bind/connect to or unix://[/path/to/socket] to use. When host=[127.0.0.1] is omitted for tcp or path=[/var/run/docker.sock] is omitted for unix sockets, default values are used. A self-sufficient runtime for linux containers. diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 60952606d5..2f7e408621 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -745,7 +745,7 @@ func GetNameserversAsCIDR(resolvConf []byte) []string { } // FIXME: Change this not to receive default value as parameter -func ParseHost(defaultHost string, defaultPort int, defaultUnix, addr string) (string, error) { +func ParseHost(defaultHost string, defaultUnix, addr string) (string, error) { var ( proto string host string @@ -753,6 +753,8 @@ func ParseHost(defaultHost string, defaultPort int, defaultUnix, addr string) (s ) addr = strings.TrimSpace(addr) switch { + case addr == "tcp://": + return "", fmt.Errorf("Invalid bind address format: %s", addr) case strings.HasPrefix(addr, "unix://"): proto = "unix" addr = strings.TrimPrefix(addr, "unix://") @@ -788,12 +790,13 @@ func ParseHost(defaultHost string, defaultPort int, defaultUnix, addr string) (s if p, err := strconv.Atoi(hostParts[1]); err == nil && p != 0 { port = p } else { - port = defaultPort + return "", fmt.Errorf("Invalid bind address format: %s", addr) } + } else if proto == "tcp" && !strings.Contains(addr, ":") { + return "", fmt.Errorf("Invalid bind address format: %s", addr) } else { host = addr - port = defaultPort } if proto == "unix" { return fmt.Sprintf("%s://%s", proto, host), nil diff --git a/components/engine/utils/utils_test.go b/components/engine/utils/utils_test.go index b0a5acb170..7e63a45cf7 100644 --- a/components/engine/utils/utils_test.go +++ b/components/engine/utils/utils_test.go @@ -301,34 +301,36 @@ func assertRAMInBytes(t *testing.T, size string, expectError bool, expectedBytes func TestParseHost(t *testing.T) { var ( defaultHttpHost = "127.0.0.1" - defaultHttpPort = 4243 defaultUnix = "/var/run/docker.sock" ) - if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "0.0.0.0"); err != nil || addr != "tcp://0.0.0.0:4243" { - t.Errorf("0.0.0.0 -> expected tcp://0.0.0.0:4243, got %s", addr) + if addr, err := ParseHost(defaultHttpHost, defaultUnix, "0.0.0.0"); err == nil { + t.Errorf("tcp 0.0.0.0 address expected error return, but err == nil, got %s", addr) } - if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "0.0.0.1:5555"); err != nil || addr != "tcp://0.0.0.1:5555" { + if addr, err := ParseHost(defaultHttpHost, defaultUnix, "tcp://"); err == nil { + t.Errorf("default tcp:// address expected error return, but err == nil, got %s", addr) + } + if addr, err := ParseHost(defaultHttpHost, defaultUnix, "0.0.0.1:5555"); err != nil || addr != "tcp://0.0.0.1:5555" { t.Errorf("0.0.0.1:5555 -> expected tcp://0.0.0.1:5555, got %s", addr) } - if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, ":6666"); err != nil || addr != "tcp://127.0.0.1:6666" { + if addr, err := ParseHost(defaultHttpHost, defaultUnix, ":6666"); err != nil || addr != "tcp://127.0.0.1:6666" { t.Errorf(":6666 -> expected tcp://127.0.0.1:6666, got %s", addr) } - if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "tcp://:7777"); err != nil || addr != "tcp://127.0.0.1:7777" { + if addr, err := ParseHost(defaultHttpHost, defaultUnix, "tcp://:7777"); err != nil || addr != "tcp://127.0.0.1:7777" { t.Errorf("tcp://:7777 -> expected tcp://127.0.0.1:7777, got %s", addr) } - if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, ""); err != nil || addr != "unix:///var/run/docker.sock" { + if addr, err := ParseHost(defaultHttpHost, defaultUnix, ""); err != nil || addr != "unix:///var/run/docker.sock" { t.Errorf("empty argument -> expected unix:///var/run/docker.sock, got %s", addr) } - if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "unix:///var/run/docker.sock"); err != nil || addr != "unix:///var/run/docker.sock" { + if addr, err := ParseHost(defaultHttpHost, defaultUnix, "unix:///var/run/docker.sock"); err != nil || addr != "unix:///var/run/docker.sock" { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) } - if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "unix://"); err != nil || addr != "unix:///var/run/docker.sock" { + if addr, err := ParseHost(defaultHttpHost, defaultUnix, "unix://"); err != nil || addr != "unix:///var/run/docker.sock" { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) } - if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "udp://127.0.0.1"); err == nil { + if addr, err := ParseHost(defaultHttpHost, defaultUnix, "udp://127.0.0.1"); err == nil { t.Errorf("udp protocol address expected error return, but err == nil. Got %s", addr) } - if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "udp://127.0.0.1:4243"); err == nil { + if addr, err := ParseHost(defaultHttpHost, defaultUnix, "udp://127.0.0.1:4243"); err == nil { t.Errorf("udp protocol address expected error return, but err == nil. Got %s", addr) } } From 173cbcacdb43b73ac7371acbced6ab91fd094d12 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 13 Feb 2014 10:07:53 -0700 Subject: [PATCH 077/403] Add shasum fallback to hack/make/dynbinary for Darwin (where sha1sum is not available) Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: c7840f522c37d2c964e442c5acac745f702b8801 Component: engine --- components/engine/hack/make/dynbinary | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/components/engine/hack/make/dynbinary b/components/engine/hack/make/dynbinary index 7de3a6cb59..d5ea6ebe54 100644 --- a/components/engine/hack/make/dynbinary +++ b/components/engine/hack/make/dynbinary @@ -7,8 +7,19 @@ CGO_ENABLED=0 go build -o $DEST/dockerinit-$VERSION -ldflags "$LDFLAGS -d" $BUIL echo "Created binary: $DEST/dockerinit-$VERSION" ln -sf dockerinit-$VERSION $DEST/dockerinit +sha1sum= +if command -v sha1sum &> /dev/null; then + sha1sum=sha1sum +elif command -v shasum &> /dev/null; then + # Mac OS X - why couldn't they just use the same command name and be happy? + sha1sum=shasum +else + echo >&2 'error: cannot find sha1sum command or equivalent' + exit 1 +fi + # sha1 our new dockerinit to ensure separate docker and dockerinit always run in a perfect pair compiled for one another -export DOCKER_INITSHA1="$(sha1sum $DEST/dockerinit-$VERSION | cut -d' ' -f1)" +export DOCKER_INITSHA1="$($sha1sum $DEST/dockerinit-$VERSION | cut -d' ' -f1)" # exported so that "dyntest" can easily access it later without recalculating it ( From 9d6155e45c3da9e90251a6453e2c90eda626eb52 Mon Sep 17 00:00:00 2001 From: apocas Date: Thu, 13 Feb 2014 13:34:04 +0000 Subject: [PATCH 078/403] Image tag endpoint doc typo. Docker-DCO-1.1-Signed-off-by: Pedro Dias (github: apocas) Upstream-commit: 3417dd4b3bef88d4da0deac520bbf150ba9c183b Component: engine --- .../docs/sources/reference/api/docker_remote_api_v1.0.rst | 4 ++-- .../docs/sources/reference/api/docker_remote_api_v1.1.rst | 4 ++-- .../docs/sources/reference/api/docker_remote_api_v1.2.rst | 4 ++-- .../docs/sources/reference/api/docker_remote_api_v1.3.rst | 4 ++-- .../docs/sources/reference/api/docker_remote_api_v1.4.rst | 4 ++-- .../docs/sources/reference/api/docker_remote_api_v1.5.rst | 4 ++-- .../docs/sources/reference/api/docker_remote_api_v1.6.rst | 4 ++-- .../docs/sources/reference/api/docker_remote_api_v1.7.rst | 4 ++-- .../docs/sources/reference/api/docker_remote_api_v1.8.rst | 4 ++-- .../docs/sources/reference/api/docker_remote_api_v1.9.rst | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.0.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.0.rst index dc06a27fc0..fa4b969758 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.0.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.0.rst @@ -732,11 +732,11 @@ Tag an image into a repository .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 201 OK :query repo: The repository to tag in :query force: 1/True/true or 0/False/false, default false - :statuscode 200: no error + :statuscode 201: no error :statuscode 400: bad parameter :statuscode 404: no such image :statuscode 500: server error diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.1.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.1.rst index 31b34caf5a..92b5039aa6 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.1.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.1.rst @@ -742,11 +742,11 @@ Tag an image into a repository .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 201 OK :query repo: The repository to tag in :query force: 1/True/true or 0/False/false, default false - :statuscode 200: no error + :statuscode 201: no error :statuscode 400: bad parameter :statuscode 404: no such image :statuscode 409: conflict diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.2.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.2.rst index 555ec14b75..1ae2db696f 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.2.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.2.rst @@ -761,11 +761,11 @@ Tag an image into a repository .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 201 OK :query repo: The repository to tag in :query force: 1/True/true or 0/False/false, default false - :statuscode 200: no error + :statuscode 201: no error :statuscode 400: bad parameter :statuscode 404: no such image :statuscode 409: conflict diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.3.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.3.rst index ab452798b9..cb4c54642d 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.3.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.3.rst @@ -808,11 +808,11 @@ Tag an image into a repository .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 201 OK :query repo: The repository to tag in :query force: 1/True/true or 0/False/false, default false - :statuscode 200: no error + :statuscode 201: no error :statuscode 400: bad parameter :statuscode 404: no such image :statuscode 409: conflict diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.4.rst index 5c8884b16f..39c8839653 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.4.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.4.rst @@ -852,11 +852,11 @@ Tag an image into a repository .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 201 OK :query repo: The repository to tag in :query force: 1/True/true or 0/False/false, default false - :statuscode 200: no error + :statuscode 201: no error :statuscode 400: bad parameter :statuscode 404: no such image :statuscode 409: conflict diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.5.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.5.rst index 609fc6b056..0cdbaf747a 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.5.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.5.rst @@ -831,11 +831,11 @@ Tag an image into a repository .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 201 OK :query repo: The repository to tag in :query force: 1/True/true or 0/False/false, default false - :statuscode 200: no error + :statuscode 201: no error :statuscode 400: bad parameter :statuscode 404: no such image :statuscode 409: conflict diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.6.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.6.rst index df53275a4f..a9ddfb2c13 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.6.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.6.rst @@ -958,11 +958,11 @@ Tag an image into a repository .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 201 OK :query repo: The repository to tag in :query force: 1/True/true or 0/False/false, default false - :statuscode 200: no error + :statuscode 201: no error :statuscode 400: bad parameter :statuscode 404: no such image :statuscode 409: conflict diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.7.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.7.rst index 28c5ba30f2..cacd7ab6f7 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.7.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.7.rst @@ -877,11 +877,11 @@ Tag an image into a repository .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 201 OK :query repo: The repository to tag in :query force: 1/True/true or 0/False/false, default false - :statuscode 200: no error + :statuscode 201: no error :statuscode 400: bad parameter :statuscode 404: no such image :statuscode 409: conflict diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.8.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.8.rst index 6ccc6eca94..5033f34210 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.8.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.8.rst @@ -892,11 +892,11 @@ Tag an image into a repository .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 201 OK :query repo: The repository to tag in :query force: 1/True/true or 0/False/false, default false - :statuscode 200: no error + :statuscode 201: no error :statuscode 400: bad parameter :statuscode 404: no such image :statuscode 409: conflict diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst index cb406da82b..47cdb46b28 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst @@ -892,11 +892,11 @@ Tag an image into a repository .. sourcecode:: http - HTTP/1.1 200 OK + HTTP/1.1 201 OK :query repo: The repository to tag in :query force: 1/True/true or 0/False/false, default false - :statuscode 200: no error + :statuscode 201: no error :statuscode 400: bad parameter :statuscode 404: no such image :statuscode 409: conflict From f734c535334b35787803b7c24b3405eacbe81c9b Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Thu, 13 Feb 2014 12:11:38 -0600 Subject: [PATCH 079/403] remove unneeded sysctl changes in sysvinit-redhat init script Docker-DCO-1.1-Signed-off-by: Adam Miller (github: maxamillion) Upstream-commit: 4317011e21b4e4025a9c92c7dc255fbf62af573a Component: engine --- components/engine/contrib/init/sysvinit-redhat/docker | 5 ----- 1 file changed, 5 deletions(-) diff --git a/components/engine/contrib/init/sysvinit-redhat/docker b/components/engine/contrib/init/sysvinit-redhat/docker index 4ee46bb02b..2b75c6903f 100755 --- a/components/engine/contrib/init/sysvinit-redhat/docker +++ b/components/engine/contrib/init/sysvinit-redhat/docker @@ -37,11 +37,6 @@ prestart() { service cgconfig start fi - preexec="/sbin/sysctl" - [ -x $preexec ] || exit 6 - $preexec -w net.ipv4.ip_forward=1 > /dev/null 2>&1 - $preexec -w net.ipv6.conf.all.forwarding=1 > /dev/null 2>&1 - } start() { From 1395c9829d11863ed6e64e225d7df9bc2dc767c6 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 13 Feb 2014 19:21:27 +0000 Subject: [PATCH 080/403] fix content-type for job.Stdout.Add Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) Upstream-commit: 4611a6bdd3270e4a404cae2d23c54dd94521c4ae Component: engine --- components/engine/api/api.go | 63 +++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index e0077a94e3..efab08ab3d 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -108,6 +108,15 @@ func writeJSON(w http.ResponseWriter, code int, v engine.Env) error { return v.Encode(w) } +func streamJSON(job *engine.Job, w http.ResponseWriter, flush bool) { + w.Header().Set("Content-Type", "application/json") + if flush { + job.Stdout.Add(utils.NewWriteFlusher(w)) + } else { + job.Stdout.Add(w) + } +} + func getBoolParam(value string) (bool, error) { if value == "" { return false, nil @@ -214,7 +223,7 @@ func getImagesJSON(eng *engine.Engine, version float64, w http.ResponseWriter, r job.Setenv("all", r.Form.Get("all")) if version >= 1.7 { - job.Stdout.Add(w) + streamJSON(job, w, false) } else if outs, err = job.Stdout.AddListTable(); err != nil { return err } @@ -265,9 +274,8 @@ func getEvents(eng *engine.Engine, version float64, w http.ResponseWriter, r *ht return err } - w.Header().Set("Content-Type", "application/json") var job = eng.Job("events", r.RemoteAddr) - job.Stdout.Add(utils.NewWriteFlusher(w)) + streamJSON(job, w, true) job.Setenv("since", r.Form.Get("since")) return job.Run() } @@ -278,7 +286,7 @@ func getImagesHistory(eng *engine.Engine, version float64, w http.ResponseWriter } var job = eng.Job("history", vars["name"]) - job.Stdout.Add(w) + streamJSON(job, w, false) if err := job.Run(); err != nil { return err @@ -291,7 +299,7 @@ func getContainersChanges(eng *engine.Engine, version float64, w http.ResponseWr return fmt.Errorf("Missing parameter") } var job = eng.Job("changes", vars["name"]) - job.Stdout.Add(w) + streamJSON(job, w, false) return job.Run() } @@ -308,7 +316,7 @@ func getContainersTop(eng *engine.Engine, version float64, w http.ResponseWriter } job := eng.Job("top", vars["name"], r.Form.Get("ps_args")) - job.Stdout.Add(w) + streamJSON(job, w, false) return job.Run() } @@ -329,8 +337,7 @@ func getContainersJSON(eng *engine.Engine, version float64, w http.ResponseWrite job.Setenv("limit", r.Form.Get("limit")) if version >= 1.5 { - w.Header().Set("Content-Type", "application/json") - job.Stdout.Add(w) + streamJSON(job, w, false) } else if outs, err = job.Stdout.AddTable(); err != nil { return err } @@ -435,8 +442,12 @@ func postImagesCreate(eng *engine.Engine, version float64, w http.ResponseWriter job.Stdin.Add(r.Body) } - job.SetenvBool("json", version > 1.0) - job.Stdout.Add(utils.NewWriteFlusher(w)) + if version > 1.0 { + job.SetenvBool("json", true) + streamJSON(job, w, true) + } else { + job.Stdout.Add(utils.NewWriteFlusher(w)) + } if err := job.Run(); err != nil { if !job.Stdout.Used() { return err @@ -475,7 +486,7 @@ func getImagesSearch(eng *engine.Engine, version float64, w http.ResponseWriter, var job = eng.Job("search", r.Form.Get("term")) job.SetenvJson("metaHeaders", metaHeaders) job.SetenvJson("authConfig", authConfig) - job.Stdout.Add(w) + streamJSON(job, w, false) return job.Run() } @@ -492,8 +503,12 @@ func postImagesInsert(eng *engine.Engine, version float64, w http.ResponseWriter } job := eng.Job("insert", vars["name"], r.Form.Get("url"), r.Form.Get("path")) - job.SetenvBool("json", version > 1.0) - job.Stdout.Add(w) + if version > 1.0 { + job.SetenvBool("json", true) + streamJSON(job, w, false) + } else { + job.Stdout.Add(w) + } if err := job.Run(); err != nil { if !job.Stdout.Used() { return err @@ -542,8 +557,12 @@ func postImagesPush(eng *engine.Engine, version float64, w http.ResponseWriter, job := eng.Job("push", vars["name"]) job.SetenvJson("metaHeaders", metaHeaders) job.SetenvJson("authConfig", authConfig) - job.SetenvBool("json", version > 1.0) - job.Stdout.Add(utils.NewWriteFlusher(w)) + if version > 1.0 { + job.SetenvBool("json", true) + streamJSON(job, w, true) + } else { + job.Stdout.Add(utils.NewWriteFlusher(w)) + } if err := job.Run(); err != nil { if !job.Stdout.Used() { @@ -645,7 +664,7 @@ func deleteImages(eng *engine.Engine, version float64, w http.ResponseWriter, r return fmt.Errorf("Missing parameter") } var job = eng.Job("image_delete", vars["name"]) - job.Stdout.Add(w) + streamJSON(job, w, false) job.SetenvBool("autoPrune", version > 1.1) return job.Run() @@ -825,7 +844,7 @@ func getContainersByName(eng *engine.Engine, version float64, w http.ResponseWri return fmt.Errorf("Missing parameter") } var job = eng.Job("inspect", vars["name"], "container") - job.Stdout.Add(w) + streamJSON(job, w, false) job.SetenvBool("conflict", true) //conflict=true to detect conflict between containers and images in the job return job.Run() } @@ -835,7 +854,7 @@ func getImagesByName(eng *engine.Engine, version float64, w http.ResponseWriter, return fmt.Errorf("Missing parameter") } var job = eng.Job("inspect", vars["name"], "image") - job.Stdout.Add(w) + streamJSON(job, w, false) job.SetenvBool("conflict", true) //conflict=true to detect conflict between containers and images in the job return job.Run() } @@ -875,11 +894,11 @@ func postBuild(eng *engine.Engine, version float64, w http.ResponseWriter, r *ht } if version >= 1.8 { - w.Header().Set("Content-Type", "application/json") job.SetenvBool("json", true) + streamJSON(job, w, true) + } else { + job.Stdout.Add(utils.NewWriteFlusher(w)) } - - job.Stdout.Add(utils.NewWriteFlusher(w)) job.Stdin.Add(r.Body) job.Setenv("remote", r.FormValue("remote")) job.Setenv("t", r.FormValue("t")) @@ -920,7 +939,7 @@ func postContainersCopy(eng *engine.Engine, version float64, w http.ResponseWrit } job := eng.Job("container_copy", vars["name"], copyData.Get("Resource")) - job.Stdout.Add(w) + streamJSON(job, w, false) if err := job.Run(); err != nil { utils.Errorf("%s", err.Error()) } From 6ca6193a8d91bb4fc8cfe6d3fefac73857bdf37f Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 13 Feb 2014 19:24:40 +0000 Subject: [PATCH 081/403] fix content-type for legacy Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) Upstream-commit: 0b403b35318a68d848bd9d7cddcf850d2fa7bfa7 Component: engine --- components/engine/api/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index efab08ab3d..67f1d909cf 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -247,6 +247,7 @@ func getImagesJSON(eng *engine.Engine, version float64, w http.ResponseWriter, r outsLegacy.Add(outLegacy) } } + w.Header().Set("Content-Type", "application/json") if _, err := outsLegacy.WriteListTo(w); err != nil { return err } @@ -350,6 +351,7 @@ func getContainersJSON(eng *engine.Engine, version float64, w http.ResponseWrite ports.ReadListFrom([]byte(out.Get("Ports"))) out.Set("Ports", displayablePorts(ports)) } + w.Header().Set("Content-Type", "application/json") if _, err = outs.WriteListTo(w); err != nil { return err } From 4415a052720dd36f6c79af5f50acf9f56cfc29f7 Mon Sep 17 00:00:00 2001 From: Johan Euphrosine Date: Thu, 13 Feb 2014 11:42:14 -0800 Subject: [PATCH 082/403] hack/RELEASE: add step for updating doc branch Docker-DCO-1.1-Signed-off-by: Johan Euphrosine (github: google) Upstream-commit: da3a52746b3ec43049dc3a8ca58b44d0188f043e Component: engine --- components/engine/hack/RELEASE-CHECKLIST.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/components/engine/hack/RELEASE-CHECKLIST.md b/components/engine/hack/RELEASE-CHECKLIST.md index a7ae45f2ff..84a0ff70e1 100644 --- a/components/engine/hack/RELEASE-CHECKLIST.md +++ b/components/engine/hack/RELEASE-CHECKLIST.md @@ -173,9 +173,13 @@ git push origin $VERSION It's very important that we don't make the tag until after the official release is uploaded to get.docker.io! -### 10. Go to github to merge the `bump_$VERSION` into release +### 10. Go to github to merge the `bump_$VERSION` branch into release -Merging the pull request to the release branch will automatically +Don't delete the leftover branch just yet, as we will need it for the next step. + +### 11. Go to github to merge the `bump_$VERSION` branch into docs + +Merging the pull request to the docs branch will automatically update the documentation on the "latest" revision of the docs. You should see the updated docs 5-10 minutes after the merge. The docs will appear on http://docs.docker.io/. For more information about @@ -184,7 +188,7 @@ documentation releases, see `docs/README.md`. Don't forget to push that pretty blue button to delete the leftover branch afterwards! -### 11. Create a new pull request to merge release back into master +### 12. Create a new pull request to merge release back into master ```bash git checkout master @@ -202,7 +206,7 @@ echo "https://github.com/dotcloud/docker/compare/master...merge_release_$VERSION Again, get two maintainers to validate, then merge, then push that pretty blue button to delete your branch. -### 12. Rejoice and Evangelize! +### 13. Rejoice and Evangelize! Congratulations! You're done. From 20b4cea3236dce641d7599e469b13c460d09e19b Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 13 Feb 2014 17:18:38 -0800 Subject: [PATCH 083/403] Fix regression with ADD of tar files Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: cad7f7ee5002206511daee4a29f399e6665f5fab Component: engine --- components/engine/buildfile.go | 36 ++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index a121276c21..995e995490 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -359,20 +359,36 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { } return err } + if fi.IsDir() { if err := archive.CopyWithTar(origPath, destPath); err != nil { return err } - // First try to unpack the source as an archive - } else if err := archive.UntarPath(origPath, destPath); err != nil { - utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err) - // If that fails, just copy it as a regular file - if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil { - return err - } - if err := archive.CopyWithTar(origPath, destPath); err != nil { - return err - } + return nil + } + + // First try to unpack the source as an archive + // to support the untar feature we need to clean up the path a little bit + // because tar is very forgiving. First we need to strip off the archive's + // filename from the path but this is only added if it does not end in / . + tarDest := destPath + if strings.HasSuffix(tarDest, "/") { + tarDest = filepath.Dir(destPath) + } + + // try to successfully untar the orig + if err := archive.UntarPath(origPath, tarDest); err == nil { + return nil + } + utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err) + + // If that fails, just copy it as a regular file + // but do not use all the magic path handling for the tar path + if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil { + return err + } + if err := archive.CopyWithTar(origPath, destPath); err != nil { + return err } return nil } From 3dbd63bc33062958bd766716952fb3b630e655b0 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 13 Feb 2014 17:23:09 -0800 Subject: [PATCH 084/403] Mount-bind the PTY as container console - allow for tmux/screen to run Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 408ea0771a9cb3046b8e484dcfefe5233fcbb27a Component: engine --- components/engine/container.go | 1 + components/engine/execdriver/driver.go | 2 ++ components/engine/execdriver/lxc/lxc_template.go | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/components/engine/container.go b/components/engine/container.go index b3c139269b..af28689810 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -231,6 +231,7 @@ func (container *Container) setupPty() error { container.ptyMaster = ptyMaster container.command.Stdout = ptySlave container.command.Stderr = ptySlave + container.command.Console = ptySlave.Name() // Copy the PTYs to our broadcasters go func() { diff --git a/components/engine/execdriver/driver.go b/components/engine/execdriver/driver.go index 1ea086075d..32b39771b6 100644 --- a/components/engine/execdriver/driver.go +++ b/components/engine/execdriver/driver.go @@ -99,6 +99,8 @@ type Command struct { Network *Network `json:"network"` // if network is nil then networking is disabled Config []string `json:"config"` // generic values that specific drivers can consume Resources *Resources `json:"resources"` + + Console string `json:"-"` } // Return the pid of the process diff --git a/components/engine/execdriver/lxc/lxc_template.go b/components/engine/execdriver/lxc/lxc_template.go index a89365f989..639780f5d8 100644 --- a/components/engine/execdriver/lxc/lxc_template.go +++ b/components/engine/execdriver/lxc/lxc_template.go @@ -80,6 +80,10 @@ lxc.mount.entry = proc {{escapeFstabSpaces $ROOTFS}}/proc proc nosuid,nodev,noex # if your userspace allows it. eg. see http://bit.ly/T9CkqJ lxc.mount.entry = sysfs {{escapeFstabSpaces $ROOTFS}}/sys sysfs nosuid,nodev,noexec 0 0 +{{if .Tty}} +lxc.mount.entry = {{.Console}} {{escapeFstabSpaces $ROOTFS}}/dev/console none bind,rw 0 0 +{{end}} + lxc.mount.entry = devpts {{escapeFstabSpaces $ROOTFS}}/dev/pts devpts newinstance,ptmxmode=0666,nosuid,noexec 0 0 lxc.mount.entry = shm {{escapeFstabSpaces $ROOTFS}}/dev/shm tmpfs size=65536k,nosuid,nodev,noexec 0 0 From ef1966871e0e7cabec76dcae8a809c1a9e0ed28f Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 13 Feb 2014 16:12:21 +1000 Subject: [PATCH 085/403] update the sshd example to use just a Dockerfile Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: a6182dc62ea335c40961c7e528b38b85d2569d07 Component: engine --- .../examples/running_ssh_service.Dockerfile | 17 +++ .../sources/examples/running_ssh_service.rst | 112 ++++-------------- 2 files changed, 43 insertions(+), 86 deletions(-) create mode 100644 components/engine/docs/sources/examples/running_ssh_service.Dockerfile diff --git a/components/engine/docs/sources/examples/running_ssh_service.Dockerfile b/components/engine/docs/sources/examples/running_ssh_service.Dockerfile new file mode 100644 index 0000000000..dd2acb7a4b --- /dev/null +++ b/components/engine/docs/sources/examples/running_ssh_service.Dockerfile @@ -0,0 +1,17 @@ +# sshd +# +# VERSION 0.0.1 + +FROM ubuntu +MAINTAINER Thatcher R. Peskens "thatcher@dotcloud.com" + +# make sure the package repository is up to date +RUN echo "deb http://archive.ubuntu.com/ubuntu precise main universe" > /etc/apt/sources.list +RUN apt-get update + +RUN apt-get install -y openssh-server +RUN mkdir /var/run/sshd +RUN echo 'root:screencast' |chpasswd + +EXPOSE 22 +CMD /usr/sbin/sshd -D diff --git a/components/engine/docs/sources/examples/running_ssh_service.rst b/components/engine/docs/sources/examples/running_ssh_service.rst index 52fe1f5914..d27799bee7 100644 --- a/components/engine/docs/sources/examples/running_ssh_service.rst +++ b/components/engine/docs/sources/examples/running_ssh_service.rst @@ -1,5 +1,5 @@ :title: Running an SSH service -:description: A screencast of installing and running an sshd service +:description: Installing and running an sshd service :keywords: docker, example, package installation, networking .. _running_ssh_service: @@ -9,101 +9,41 @@ SSH Daemon Service .. include:: example_header.inc +The following Dockerfile sets up an sshd service in a container that you can use +to connect to and inspect other container's volumes, or to get quick access to a +test container. -**Video:** +.. literalinclude:: running_ssh_service.Dockerfile -I've created a little screencast to show how to create an SSHd service -and connect to it. It is something like 11 minutes and not entirely -smooth, but it gives you a good idea. - -.. note:: - This screencast was created before Docker version 0.5.2, so the - daemon is unprotected and available via a TCP port. When you run - through the same steps in a newer version of Docker, you will - need to add ``sudo`` in front of each ``docker`` command in order - to reach the daemon over its protected Unix socket. - -.. raw:: html - - - -You can also get this sshd container by using: +Build the image using: .. code-block:: bash - sudo docker pull dhrp/sshd + $ sudo docker build -rm -t eg_sshd . - -The password is ``screencast``. - -**Video's Transcription:** +Then run it. You can then use ``docker port`` to find out what host port the container's +port 22 is mapped to: .. code-block:: bash - # Hello! We are going to try and install openssh on a container and run it as a service - # let's pull ubuntu to get a base ubuntu image. - $ docker pull ubuntu - # I had it so it was quick - # now let's connect using -i for interactive and with -t for terminal - # we execute /bin/bash to get a prompt. - $ docker run -i -t ubuntu /bin/bash - # yes! we are in! - # now lets install openssh - $ apt-get update - $ apt-get install openssh-server - # ok. lets see if we can run it. - $ which sshd - # we need to create privilege separation directory - $ mkdir /var/run/sshd - $ /usr/sbin/sshd - $ exit - # now let's commit it - # which container was it? - $ docker ps -a |more - $ docker commit a30a3a2f2b130749995f5902f079dc6ad31ea0621fac595128ec59c6da07feea dhrp/sshd - # I gave the name dhrp/sshd for the container - # now we can run it again - $ docker run -d dhrp/sshd /usr/sbin/sshd -D # D for daemon mode - # is it running? - $ docker ps - # yes! - # let's stop it - $ docker stop 0ebf7cec294755399d063f4b1627980d4cbff7d999f0bc82b59c300f8536a562 - $ docker ps - # and reconnect, but now open a port to it - $ docker run -d -p 22 dhrp/sshd /usr/sbin/sshd -D - $ docker port b2b407cf22cf8e7fa3736fa8852713571074536b1d31def3fdfcd9fa4fd8c8c5 22 - # it has now given us a port to connect to - # we have to connect using a public ip of our host - $ hostname - # *ifconfig* is deprecated, better use *ip addr show* now - $ ifconfig - $ ssh root@192.168.33.10 -p 49153 - # Ah! forgot to set root passwd - $ docker commit b2b407cf22cf8e7fa3736fa8852713571074536b1d31def3fdfcd9fa4fd8c8c5 dhrp/sshd - $ docker ps -a - $ docker run -i -t dhrp/sshd /bin/bash - $ passwd - $ exit - $ docker commit 9e863f0ca0af31c8b951048ba87641d67c382d08d655c2e4879c51410e0fedc1 dhrp/sshd - $ docker run -d -p 22 dhrp/sshd /usr/sbin/sshd -D - $ docker port a0aaa9558c90cf5c7782648df904a82365ebacce523e4acc085ac1213bfe2206 22 - # *ifconfig* is deprecated, better use *ip addr show* now - $ ifconfig - $ ssh root@192.168.33.10 -p 49154 - # Thanks for watching, Thatcher thatcher@dotcloud.com - -Update: -------- + $ sudo docker run -d -P -name test_sshd eg_sshd + $ sudo docker port test_sshd 22 + 0.0.0.0:49154 -For Ubuntu 13.10 using stackbrew/ubuntu, you may need do these additional steps: +And now you can ssh to port ``49154`` on the Docker daemon's host IP address +(``ip address`` or ``ifconfig`` can tell you that): -1. change /etc/pam.d/sshd, pam_loginuid line 'required' to 'optional' -2. echo LANG=\"en_US.UTF-8\" > /etc/default/locale +.. code-block:: bash + $ ssh root@192.168.1.2 -p 49154 + # The password is ``screencast``. + $$ +Finally, clean up after your test by stopping and removing the container, and +then removing the image. + +.. code-block:: bash + + $ sudo docker stop test_sshd + $ sudo docker rm test_sshd + $ sudo docker rmi eg_sshd From 77b539f03ccfe22a04bbc9c11bd1a82a82e2149b Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 14 Feb 2014 11:12:49 +0100 Subject: [PATCH 086/403] archive: Close decompressed streams Various kinds of decompressed streams are really ReadClosers. For instance gzip.NewReader() is, and the one returned from CmdStream is changed to be because it returns a PipeReader which is a ReadCloser. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 0d9213f859e00459e9efee2c5164417c94d66d2f Component: engine --- components/engine/archive/archive.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index 3a1c111ea2..a2b395d4b3 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -65,13 +65,13 @@ func DetectCompression(source []byte) Compression { return Uncompressed } -func xzDecompress(archive io.Reader) (io.Reader, error) { +func xzDecompress(archive io.Reader) (io.ReadCloser, error) { args := []string{"xz", "-d", "-c", "-q"} return CmdStream(exec.Command(args[0], args[1:]...), archive) } -func DecompressStream(archive io.Reader) (io.Reader, error) { +func DecompressStream(archive io.Reader) (io.ReadCloser, error) { buf := make([]byte, 10) totalN := 0 for totalN < 10 { @@ -90,11 +90,11 @@ func DecompressStream(archive io.Reader) (io.Reader, error) { switch compression { case Uncompressed: - return wrap, nil + return ioutil.NopCloser(wrap), nil case Gzip: return gzip.NewReader(wrap) case Bzip2: - return bzip2.NewReader(wrap), nil + return ioutil.NopCloser(bzip2.NewReader(wrap)), nil case Xz: return xzDecompress(wrap) default: @@ -352,12 +352,13 @@ func Untar(archive io.Reader, dest string, options *TarOptions) error { return fmt.Errorf("Empty archive") } - archive, err := DecompressStream(archive) + decompressedArchive, err := DecompressStream(archive) if err != nil { return err } + defer decompressedArchive.Close() - tr := tar.NewReader(archive) + tr := tar.NewReader(decompressedArchive) var dirs []*tar.Header @@ -528,7 +529,7 @@ func CopyFileWithTar(src, dst string) (err error) { // CmdStream executes a command, and returns its stdout as a stream. // If the command fails to run or doesn't complete successfully, an error // will be returned, including anything written on stderr. -func CmdStream(cmd *exec.Cmd, input io.Reader) (io.Reader, error) { +func CmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, error) { if input != nil { stdin, err := cmd.StdinPipe() if err != nil { From 12b909af8857fe5f6cc17672e463b7c249acff7f Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 14 Feb 2014 11:33:40 +0100 Subject: [PATCH 087/403] archive.TarFilter() - properly close readers CompressStream() now always returns a stream that is closable, and it never closes the underlying writer. TarFilter() makes sure the decompressed stream is closed at the and, as well as the PipeWriter. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 804690bd07dcc1da6d0707d081e40ec9a189a3b6 Component: engine --- components/engine/archive/archive.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index a2b395d4b3..dd7dd70b13 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -106,7 +106,7 @@ func CompressStream(dest io.WriteCloser, compression Compression) (io.WriteClose switch compression { case Uncompressed: - return dest, nil + return utils.NopWriteCloser(dest), nil case Gzip: return gzip.NewWriter(dest), nil case Bzip2, Xz: @@ -337,6 +337,9 @@ func TarFilter(srcPath string, options *TarOptions) (io.Reader, error) { if err := compressWriter.Close(); err != nil { utils.Debugf("Can't close compress writer: %s\n", err) } + if err := pipeWriter.Close(); err != nil { + utils.Debugf("Can't close pipe writer: %s\n", err) + } }() return pipeReader, nil From e12f62e6dff83331e2bbbf8d2e3218e67ac1e5d6 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 14 Feb 2014 12:41:46 +0100 Subject: [PATCH 088/403] Properly close archives All archive that are created from somewhere generally have to be closed, because at some point there is a file or a pipe or something that backs them. So, we make archive.Archive a ReadCloser. However, code consuming archives does not typically close them so we add an archive.ArchiveReader and use that when we're only reading. We then change all the Tar/Archive places to create ReadClosers, and to properly close them everywhere. As an added bonus we can use ReadCloserWrapper rather than EofReader in several places, which is good as EofReader doesn't always work right. For instance, many compression schemes like gzip knows it is EOF before having read the EOF from the stream, so the EofCloser never sees an EOF. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: f198ee525ad6862dce3940e08c72e0a092380a7b Component: engine --- components/engine/archive/archive.go | 19 ++++++++++++------- components/engine/archive/archive_test.go | 5 +++-- components/engine/archive/diff.go | 2 +- components/engine/buildfile.go | 1 + components/engine/commands.go | 4 ++-- components/engine/container.go | 18 +++++++++++++++--- components/engine/graph.go | 8 +++++--- components/engine/graphdriver/aufs/aufs.go | 2 +- components/engine/graphdriver/driver.go | 2 +- components/engine/image.go | 14 +++++++++++--- components/engine/integration/utils_test.go | 4 ++-- components/engine/runtime.go | 8 +++++++- components/engine/server.go | 14 ++++++++++---- 13 files changed, 71 insertions(+), 30 deletions(-) diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index dd7dd70b13..3bd3af2761 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -19,9 +19,10 @@ import ( ) type ( - Archive io.Reader - Compression int - TarOptions struct { + Archive io.ReadCloser + ArchiveReader io.Reader + Compression int + TarOptions struct { Includes []string Compression Compression } @@ -269,7 +270,7 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader *tar.Reader) // Tar creates an archive from the directory at `path`, and returns it as a // stream of bytes. -func Tar(path string, compression Compression) (io.Reader, error) { +func Tar(path string, compression Compression) (io.ReadCloser, error) { return TarFilter(path, &TarOptions{Compression: compression}) } @@ -291,7 +292,7 @@ func escapeName(name string) string { // Tar creates an archive from the directory at `path`, only including files whose relative // paths are included in `filter`. If `filter` is nil, then all files are included. -func TarFilter(srcPath string, options *TarOptions) (io.Reader, error) { +func TarFilter(srcPath string, options *TarOptions) (io.ReadCloser, error) { pipeReader, pipeWriter := io.Pipe() compressWriter, err := CompressStream(pipeWriter, options.Compression) @@ -436,15 +437,19 @@ func TarUntar(src string, dst string) error { if err != nil { return err } + defer archive.Close() return Untar(archive, dst, nil) } // UntarPath is a convenience function which looks for an archive // at filesystem path `src`, and unpacks it at `dst`. func UntarPath(src, dst string) error { - if archive, err := os.Open(src); err != nil { + archive, err := os.Open(src) + if err != nil { return err - } else if err := Untar(archive, dst, nil); err != nil { + } + defer archive.Close() + if err := Untar(archive, dst, nil); err != nil { return err } return nil diff --git a/components/engine/archive/archive_test.go b/components/engine/archive/archive_test.go index 891f977dcf..164fc8fe27 100644 --- a/components/engine/archive/archive_test.go +++ b/components/engine/archive/archive_test.go @@ -67,12 +67,13 @@ func tarUntar(t *testing.T, origin string, compression Compression) error { if err != nil { t.Fatal(err) } + defer archive.Close() buf := make([]byte, 10) if _, err := archive.Read(buf); err != nil { return err } - archive = io.MultiReader(bytes.NewReader(buf), archive) + wrap := io.MultiReader(bytes.NewReader(buf), archive) detectedCompression := DetectCompression(buf) if detectedCompression.Extension() != compression.Extension() { @@ -84,7 +85,7 @@ func tarUntar(t *testing.T, origin string, compression Compression) error { return err } defer os.RemoveAll(tmp) - if err := Untar(archive, tmp, nil); err != nil { + if err := Untar(wrap, tmp, nil); err != nil { return err } if _, err := os.Stat(tmp); err != nil { diff --git a/components/engine/archive/diff.go b/components/engine/archive/diff.go index cdf06dd055..de1efacf34 100644 --- a/components/engine/archive/diff.go +++ b/components/engine/archive/diff.go @@ -28,7 +28,7 @@ func timeToTimespec(time time.Time) (ts syscall.Timespec) { // ApplyLayer parses a diff in the standard layer format from `layer`, and // applies it to the directory `dest`. -func ApplyLayer(dest string, layer Archive) error { +func ApplyLayer(dest string, layer ArchiveReader) error { // We need to be able to set any perms oldmask := syscall.Umask(0) defer syscall.Umask(oldmask) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 445c4ae8ac..3ee63ea3fb 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -464,6 +464,7 @@ func (b *buildFile) CmdAdd(args string) error { } tarSum := utils.TarSum{Reader: r, DisableCompression: true} remoteHash = tarSum.Sum(nil) + r.Close() // If the destination is a directory, figure out the filename. if strings.HasSuffix(dest, "/") { diff --git a/components/engine/commands.go b/components/engine/commands.go index 7efbcc7085..2d897385c7 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -158,7 +158,7 @@ func MkBuildContext(dockerfile string, files [][2]string) (archive.Archive, erro if err := tw.Close(); err != nil { return nil, err } - return buf, nil + return ioutil.NopCloser(buf), nil } func (cli *DockerCli) CmdBuild(args ...string) error { @@ -206,7 +206,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { // FIXME: ProgressReader shouldn't be this annoying to use if context != nil { sf := utils.NewStreamFormatter(false) - body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf, true, "", "Uploading context") + body = utils.ProgressReader(context, 0, cli.err, sf, true, "", "Uploading context") } // Upload the build context v := &url.Values{} diff --git a/components/engine/container.go b/components/engine/container.go index af28689810..3740a7fb73 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -1288,7 +1288,11 @@ func (container *Container) ExportRw() (archive.Archive, error) { container.Unmount() return nil, err } - return EofReader(archive, func() { container.Unmount() }), nil + return utils.NewReadCloserWrapper(archive, func() error { + err := archive.Close() + container.Unmount() + return err + }), nil } func (container *Container) Export() (archive.Archive, error) { @@ -1301,7 +1305,11 @@ func (container *Container) Export() (archive.Archive, error) { container.Unmount() return nil, err } - return EofReader(archive, func() { container.Unmount() }), nil + return utils.NewReadCloserWrapper(archive, func() error { + err := archive.Close() + container.Unmount() + return err + }), nil } func (container *Container) WaitTimeout(timeout time.Duration) error { @@ -1455,7 +1463,11 @@ func (container *Container) Copy(resource string) (io.ReadCloser, error) { if err != nil { return nil, err } - return utils.NewReadCloserWrapper(archive, container.Unmount), nil + return utils.NewReadCloserWrapper(archive, func() error { + err := archive.Close() + container.Unmount() + return err + }), nil } // Returns true if the container exposes a certain port diff --git a/components/engine/graph.go b/components/engine/graph.go index 01cd50f4f0..a16a6ab7f7 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -127,7 +127,7 @@ func (graph *Graph) Get(name string) (*Image, error) { } // Create creates a new image and registers it in the graph. -func (graph *Graph) Create(layerData archive.Archive, container *Container, comment, author string, config *runconfig.Config) (*Image, error) { +func (graph *Graph) Create(layerData archive.ArchiveReader, container *Container, comment, author string, config *runconfig.Config) (*Image, error) { img := &Image{ ID: GenerateID(), Comment: comment, @@ -151,7 +151,7 @@ func (graph *Graph) Create(layerData archive.Archive, container *Container, comm // Register imports a pre-existing image into the graph. // FIXME: pass img as first argument -func (graph *Graph) Register(jsonData []byte, layerData archive.Archive, img *Image) (err error) { +func (graph *Graph) Register(jsonData []byte, layerData archive.ArchiveReader, img *Image) (err error) { defer func() { // If any error occurs, remove the new dir from the driver. // Don't check for errors since the dir might not have been created. @@ -226,7 +226,9 @@ func (graph *Graph) TempLayerArchive(id string, compression archive.Compression, if err != nil { return nil, err } - return archive.NewTempArchive(utils.ProgressReader(ioutil.NopCloser(a), 0, output, sf, false, utils.TruncateID(id), "Buffering to disk"), tmp) + progress := utils.ProgressReader(a, 0, output, sf, false, utils.TruncateID(id), "Buffering to disk") + defer progress.Close() + return archive.NewTempArchive(progress, tmp) } // Mktemp creates a temporary sub-directory inside the graph's filesystem. diff --git a/components/engine/graphdriver/aufs/aufs.go b/components/engine/graphdriver/aufs/aufs.go index d1cf87d1a0..f2a88a7ace 100644 --- a/components/engine/graphdriver/aufs/aufs.go +++ b/components/engine/graphdriver/aufs/aufs.go @@ -271,7 +271,7 @@ func (a *Driver) Diff(id string) (archive.Archive, error) { }) } -func (a *Driver) ApplyDiff(id string, diff archive.Archive) error { +func (a *Driver) ApplyDiff(id string, diff archive.ArchiveReader) error { return archive.Untar(diff, path.Join(a.rootPath(), "diff", id), nil) } diff --git a/components/engine/graphdriver/driver.go b/components/engine/graphdriver/driver.go index c0ed00b0ad..89fd03a624 100644 --- a/components/engine/graphdriver/driver.go +++ b/components/engine/graphdriver/driver.go @@ -28,7 +28,7 @@ type Driver interface { type Differ interface { Diff(id string) (archive.Archive, error) Changes(id string) ([]archive.Change, error) - ApplyDiff(id string, diff archive.Archive) error + ApplyDiff(id string, diff archive.ArchiveReader) error DiffSize(id string) (bytes int64, err error) } diff --git a/components/engine/image.go b/components/engine/image.go index 593dc14f00..fa5b65787c 100644 --- a/components/engine/image.go +++ b/components/engine/image.go @@ -67,7 +67,7 @@ func LoadImage(root string) (*Image, error) { return img, nil } -func StoreImage(img *Image, jsonData []byte, layerData archive.Archive, root, layer string) error { +func StoreImage(img *Image, jsonData []byte, layerData archive.ArchiveReader, root, layer string) error { // Store the layer var ( size int64 @@ -174,7 +174,11 @@ func (img *Image) TarLayer() (arch archive.Archive, err error) { if err != nil { return nil, err } - return EofReader(archive, func() { driver.Put(img.ID) }), nil + return utils.NewReadCloserWrapper(archive, func() error { + err := archive.Close() + driver.Put(img.ID) + return err + }), nil } parentFs, err := driver.Get(img.Parent) @@ -190,7 +194,11 @@ func (img *Image) TarLayer() (arch archive.Archive, err error) { if err != nil { return nil, err } - return EofReader(archive, func() { driver.Put(img.ID) }), nil + return utils.NewReadCloserWrapper(archive, func() error { + err := archive.Close() + driver.Put(img.ID) + return err + }), nil } func ValidateID(id string) error { diff --git a/components/engine/integration/utils_test.go b/components/engine/integration/utils_test.go index 6b0f458564..77328b2511 100644 --- a/components/engine/integration/utils_test.go +++ b/components/engine/integration/utils_test.go @@ -319,7 +319,7 @@ func runContainer(eng *engine.Engine, r *docker.Runtime, args []string, t *testi } // FIXME: this is duplicated from graph_test.go in the docker package. -func fakeTar() (io.Reader, error) { +func fakeTar() (io.ReadCloser, error) { content := []byte("Hello world!\n") buf := new(bytes.Buffer) tw := tar.NewWriter(buf) @@ -333,7 +333,7 @@ func fakeTar() (io.Reader, error) { tw.Write([]byte(content)) } tw.Close() - return buf, nil + return ioutil.NopCloser(buf), nil } func getAllImages(eng *engine.Engine, t *testing.T) *engine.Table { diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 828d3f0e66..872808d941 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -531,6 +531,8 @@ func (runtime *Runtime) Commit(container *Container, repository, tag, comment, a if err != nil { return nil, err } + defer rwTar.Close() + // Create a new image from the container's base layers + a new layer from container changes img, err := runtime.graph.Create(rwTar, container, comment, author, config) if err != nil { @@ -817,7 +819,11 @@ func (runtime *Runtime) Diff(container *Container) (archive.Archive, error) { if err != nil { return nil, err } - return EofReader(archive, func() { runtime.driver.Put(container.ID) }), nil + return utils.NewReadCloserWrapper(archive, func() error { + err := archive.Close() + runtime.driver.Put(container.ID) + return err + }), nil } func (runtime *Runtime) Run(c *Container, startCallback execdriver.StartCallback) (int, error) { diff --git a/components/engine/server.go b/components/engine/server.go index 440bab2bbd..daf05ec67a 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -292,6 +292,7 @@ func (srv *Server) ContainerExport(job *engine.Job) engine.Status { if err != nil { return job.Errorf("%s: %s", name, err) } + defer data.Close() // Stream the entire contents of the container (basically a volatile snapshot) if _, err := io.Copy(job.Stdout, data); err != nil { @@ -361,6 +362,7 @@ func (srv *Server) ImageExport(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } + defer fs.Close() if _, err := io.Copy(job.Stdout, fs); err != nil { return job.Error(err) @@ -400,6 +402,7 @@ func (srv *Server) exportImage(image *Image, tempdir string) error { if err != nil { return err } + defer fs.Close() fsTar, err := os.Create(path.Join(tmpImageDir, "layer.tar")) if err != nil { @@ -436,14 +439,14 @@ func (srv *Server) Build(job *engine.Job) engine.Status { authConfig = &auth.AuthConfig{} configFile = &auth.ConfigFile{} tag string - context io.Reader + context io.ReadCloser ) job.GetenvJson("authConfig", authConfig) job.GetenvJson("configFile", configFile) repoName, tag = utils.ParseRepositoryTag(repoName) if remoteURL == "" { - context = job.Stdin + context = ioutil.NopCloser(job.Stdin) } else if utils.IsGIT(remoteURL) { if !strings.HasPrefix(remoteURL, "git://") { remoteURL = "https://" + remoteURL @@ -479,6 +482,7 @@ func (srv *Server) Build(job *engine.Job) engine.Status { } context = c } + defer context.Close() sf := utils.NewStreamFormatter(job.GetenvBool("json")) b := NewBuildFile(srv, @@ -1575,7 +1579,7 @@ func (srv *Server) ImageImport(job *engine.Job) engine.Status { repo = job.Args[1] tag string sf = utils.NewStreamFormatter(job.GetenvBool("json")) - archive io.Reader + archive archive.ArchiveReader resp *http.Response ) if len(job.Args) > 2 { @@ -1601,7 +1605,9 @@ func (srv *Server) ImageImport(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), job.Stdout, sf, true, "", "Importing") + progressReader := utils.ProgressReader(resp.Body, int(resp.ContentLength), job.Stdout, sf, true, "", "Importing") + defer progressReader.Close() + archive = progressReader } img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil) if err != nil { From 13aaf53921c4020482f0c7869dd75ed97fd1d28f Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 14 Feb 2014 13:00:18 +0100 Subject: [PATCH 089/403] Drop EofReader This is not used any more Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: d8c888b3f85a99e9f94348bb71a852b59162464a Component: engine --- components/engine/utils.go | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/components/engine/utils.go b/components/engine/utils.go index 68cd2f24e5..ef666b0de1 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -6,8 +6,6 @@ import ( "github.com/dotcloud/docker/pkg/namesgenerator" "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" - "io" - "sync/atomic" ) type Change struct { @@ -56,28 +54,3 @@ func (c *checker) Exists(name string) bool { func generateRandomName(runtime *Runtime) (string, error) { return namesgenerator.GenerateRandomName(&checker{runtime}) } - -// Read an io.Reader and call a function when it returns EOF -func EofReader(r io.Reader, callback func()) *eofReader { - return &eofReader{ - Reader: r, - callback: callback, - } -} - -type eofReader struct { - io.Reader - gotEOF int32 - callback func() -} - -func (r *eofReader) Read(p []byte) (n int, err error) { - n, err = r.Reader.Read(p) - if err == io.EOF { - // Use atomics to make the gotEOF check threadsafe - if atomic.CompareAndSwapInt32(&r.gotEOF, 0, 1) { - r.callback() - } - } - return -} From c4982cc0521a53ff159d5bd79fbf0c4e8787e3ed Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 14 Feb 2014 11:42:47 -0800 Subject: [PATCH 090/403] Improve error message for 404 returned by daemon Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 190504472b4598eff7d7f6f53df4f813975ff5bb Component: engine --- components/engine/commands.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/engine/commands.go b/components/engine/commands.go index 7efbcc7085..bd9bdfd876 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -2118,7 +2118,9 @@ func (cli *DockerCli) call(method, path string, data interface{}, passAuthInfo b return nil, -1, err } - if resp.StatusCode < 200 || resp.StatusCode >= 400 { + if resp.StatusCode == 404 { + return nil, resp.StatusCode, fmt.Errorf("Error: request for %s returned 404 Not Found for the api version", req.URL) + } else if resp.StatusCode < 200 || resp.StatusCode >= 400 { body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, -1, err From 07f1d747db96ced7924cb20a4775290197d3965f Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 14 Feb 2014 12:12:35 -0800 Subject: [PATCH 091/403] Add new functions to unsupported file Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 17719cab91e175a7bd11f9852e27638df1202b8b Component: engine --- .../engine/pkg/netlink/netlink_unsupported.go | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/components/engine/pkg/netlink/netlink_unsupported.go b/components/engine/pkg/netlink/netlink_unsupported.go index cd796b373f..bd9e962d35 100644 --- a/components/engine/pkg/netlink/netlink_unsupported.go +++ b/components/engine/pkg/netlink/netlink_unsupported.go @@ -3,31 +3,59 @@ package netlink import ( - "fmt" + "errors" "net" ) +var ( + ErrNotImplemented = errors.New("not implemented") +) + func NetworkGetRoutes() ([]Route, error) { - return nil, fmt.Errorf("Not implemented") + return nil, ErrNotImplemented } func NetworkLinkAdd(name string, linkType string) error { - return fmt.Errorf("Not implemented") + return ErrNotImplemented } func NetworkLinkUp(iface *net.Interface) error { - return fmt.Errorf("Not implemented") + return ErrNotImplemented } func NetworkLinkAddIp(iface *net.Interface, ip net.IP, ipNet *net.IPNet) error { - return fmt.Errorf("Not implemented") + return ErrNotImplemented } func AddDefaultGw(ip net.IP) error { - return fmt.Errorf("Not implemented") + return ErrNotImplemented } func NetworkSetMTU(iface *net.Interface, mtu int) error { - return fmt.Errorf("Not implemented") + return ErrNotImplemented +} + +func NetworkCreateVethPair(name1, name2 string) error { + return ErrNotImplemented +} + +func NetworkChangeName(iface *net.Interface, newName string) error { + return ErrNotImplemented +} + +func NetworkSetNsFd(iface *net.Interface, fd int) error { + return ErrNotImplemented +} + +func NetworkSetNsPid(iface *net.Interface, nspid int) error { + return ErrNotImplemented +} + +func NetworkSetMaster(iface, master *net.Interface) error { + return ErrNotImplemented +} + +func NetworkLinkDown(iface *net.Interface) error { + return ErrNotImplemented } From afbbdb7fc1a7d9fc78ed96e8aa9f17407c50743e Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 14 Feb 2014 15:17:12 -0800 Subject: [PATCH 092/403] Update fixme Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 106079daca36e8c5595c3f05eb183c9919016e4e Component: engine --- components/engine/FIXME | 6 ------ 1 file changed, 6 deletions(-) diff --git a/components/engine/FIXME b/components/engine/FIXME index 91c8d92835..4f27d36856 100644 --- a/components/engine/FIXME +++ b/components/engine/FIXME @@ -11,20 +11,14 @@ They are just like FIXME comments in the source code, except we're not sure wher to put them - so we put them here :) -* Merge Runtime, Server and Builder into Runtime * Run linter on codebase * Unify build commands and regular commands * Move source code into src/ subdir for clarity * docker build: on non-existent local path for ADD, don't show full absolute path on the host -* docker tag foo REPO:TAG * use size header for progress bar in pull * Clean up context upload in build!!! * Parallel pull -* Always generate a resolv.conf per container, to avoid changing resolv.conf under thne container's feet -* Save metadata with import/export (#1974) * Upgrade dockerd without stopping containers * Simple command to remove all untagged images (`docker rmi $(docker images | awk '/^/ { print $3 }')`) * Simple command to clean up containers for disk space -* Caching after an ADD (#880) * Clean up the ProgressReader api, it's a PITA to use -* Use netlink instead of iproute2/iptables (#925) From 48bceec5a64412ac1b38acab904046808822b9aa Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 14 Feb 2014 15:32:39 -0800 Subject: [PATCH 093/403] Remove unused reflink files Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 055bfb699bb232c7e490c38d3a71c81870781c74 Component: engine --- components/engine/reflink_copy_linux.go | 55 ------------------- components/engine/reflink_copy_unsupported.go | 16 ------ 2 files changed, 71 deletions(-) delete mode 100644 components/engine/reflink_copy_linux.go delete mode 100644 components/engine/reflink_copy_unsupported.go diff --git a/components/engine/reflink_copy_linux.go b/components/engine/reflink_copy_linux.go deleted file mode 100644 index 74a0cb98f7..0000000000 --- a/components/engine/reflink_copy_linux.go +++ /dev/null @@ -1,55 +0,0 @@ -// +build amd64 - -package docker - -// FIXME: This could be easily rewritten in pure Go - -/* -#include -#include -#include - -// See linux.git/fs/btrfs/ioctl.h -#define BTRFS_IOCTL_MAGIC 0x94 -#define BTRFS_IOC_CLONE _IOW(BTRFS_IOCTL_MAGIC, 9, int) - -int -btrfs_reflink(int fd_out, int fd_in) -{ - int res; - res = ioctl(fd_out, BTRFS_IOC_CLONE, fd_in); - if (res < 0) - return errno; - return 0; -} - -*/ -import "C" - -import ( - "io" - "os" - "syscall" -) - -// FIXME: Move this to btrfs package? - -func BtrfsReflink(fd_out, fd_in uintptr) error { - res := C.btrfs_reflink(C.int(fd_out), C.int(fd_in)) - if res != 0 { - return syscall.Errno(res) - } - return nil -} - -func CopyFile(dstFile, srcFile *os.File) error { - err := BtrfsReflink(dstFile.Fd(), srcFile.Fd()) - if err == nil { - return nil - } - - // Fall back to normal copy - // FIXME: Check the return of Copy and compare with dstFile.Stat().Size - _, err = io.Copy(dstFile, srcFile) - return err -} diff --git a/components/engine/reflink_copy_unsupported.go b/components/engine/reflink_copy_unsupported.go deleted file mode 100644 index 271ed0178f..0000000000 --- a/components/engine/reflink_copy_unsupported.go +++ /dev/null @@ -1,16 +0,0 @@ -// +build !linux !amd64 - -package docker - -import ( - "io" - "os" -) - -func CopyFile(dstFile, srcFile *os.File) error { - // No BTRFS reflink suppport, Fall back to normal copy - - // FIXME: Check the return of Copy and compare with dstFile.Stat().Size - _, err := io.Copy(dstFile, srcFile) - return err -} From 690b93143ab879f8b0b347ae1e1be521cceaa873 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 14 Feb 2014 15:25:25 -0800 Subject: [PATCH 094/403] Move proxy into pkg Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 8c39db8f96ac2a7ec479a679bc060410cee1a567 Component: engine --- .../engine/networkdriver/portmapper/mapper.go | 2 +- .../networkdriver/portmapper/mapper_test.go | 2 +- components/engine/{ => pkg}/proxy/MAINTAINERS | 0 .../{ => pkg}/proxy/network_proxy_test.go | 0 components/engine/{ => pkg}/proxy/proxy.go | 0 .../engine/{ => pkg}/proxy/stub_proxy.go | 0 .../engine/{ => pkg}/proxy/tcp_proxy.go | 10 +++---- .../engine/{ => pkg}/proxy/udp_proxy.go | 26 +++++++++++++------ components/engine/utils/utils.go | 10 ------- 9 files changed, 25 insertions(+), 25 deletions(-) rename components/engine/{ => pkg}/proxy/MAINTAINERS (100%) rename components/engine/{ => pkg}/proxy/network_proxy_test.go (100%) rename components/engine/{ => pkg}/proxy/proxy.go (100%) rename components/engine/{ => pkg}/proxy/stub_proxy.go (100%) rename components/engine/{ => pkg}/proxy/tcp_proxy.go (81%) rename components/engine/{ => pkg}/proxy/udp_proxy.go (77%) diff --git a/components/engine/networkdriver/portmapper/mapper.go b/components/engine/networkdriver/portmapper/mapper.go index f052c48143..e29959a245 100644 --- a/components/engine/networkdriver/portmapper/mapper.go +++ b/components/engine/networkdriver/portmapper/mapper.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" "github.com/dotcloud/docker/pkg/iptables" - "github.com/dotcloud/docker/proxy" + "github.com/dotcloud/docker/pkg/proxy" "net" "sync" ) diff --git a/components/engine/networkdriver/portmapper/mapper_test.go b/components/engine/networkdriver/portmapper/mapper_test.go index 05718063e3..4c09f3c651 100644 --- a/components/engine/networkdriver/portmapper/mapper_test.go +++ b/components/engine/networkdriver/portmapper/mapper_test.go @@ -2,7 +2,7 @@ package portmapper import ( "github.com/dotcloud/docker/pkg/iptables" - "github.com/dotcloud/docker/proxy" + "github.com/dotcloud/docker/pkg/proxy" "net" "testing" ) diff --git a/components/engine/proxy/MAINTAINERS b/components/engine/pkg/proxy/MAINTAINERS similarity index 100% rename from components/engine/proxy/MAINTAINERS rename to components/engine/pkg/proxy/MAINTAINERS diff --git a/components/engine/proxy/network_proxy_test.go b/components/engine/pkg/proxy/network_proxy_test.go similarity index 100% rename from components/engine/proxy/network_proxy_test.go rename to components/engine/pkg/proxy/network_proxy_test.go diff --git a/components/engine/proxy/proxy.go b/components/engine/pkg/proxy/proxy.go similarity index 100% rename from components/engine/proxy/proxy.go rename to components/engine/pkg/proxy/proxy.go diff --git a/components/engine/proxy/stub_proxy.go b/components/engine/pkg/proxy/stub_proxy.go similarity index 100% rename from components/engine/proxy/stub_proxy.go rename to components/engine/pkg/proxy/stub_proxy.go diff --git a/components/engine/proxy/tcp_proxy.go b/components/engine/pkg/proxy/tcp_proxy.go similarity index 81% rename from components/engine/proxy/tcp_proxy.go rename to components/engine/pkg/proxy/tcp_proxy.go index e7c460f61d..b84483eff9 100644 --- a/components/engine/proxy/tcp_proxy.go +++ b/components/engine/pkg/proxy/tcp_proxy.go @@ -1,7 +1,6 @@ package proxy import ( - "github.com/dotcloud/docker/utils" "io" "log" "net" @@ -49,7 +48,8 @@ func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) { to.CloseRead() event <- written } - utils.Debugf("Forwarding traffic between tcp/%v and tcp/%v", client.RemoteAddr(), backend.RemoteAddr()) + + log.Printf("Forwarding traffic between tcp/%v and tcp/%v", client.RemoteAddr(), backend.RemoteAddr()) go broker(client, backend) go broker(backend, client) @@ -71,17 +71,17 @@ func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) { client.Close() backend.Close() done: - utils.Debugf("%v bytes transferred between tcp/%v and tcp/%v", transferred, client.RemoteAddr(), backend.RemoteAddr()) + log.Printf("%v bytes transferred between tcp/%v and tcp/%v", transferred, client.RemoteAddr(), backend.RemoteAddr()) } func (proxy *TCPProxy) Run() { quit := make(chan bool) defer close(quit) - utils.Debugf("Starting proxy on tcp/%v for tcp/%v", proxy.frontendAddr, proxy.backendAddr) + log.Printf("Starting proxy on tcp/%v for tcp/%v", proxy.frontendAddr, proxy.backendAddr) for { client, err := proxy.listener.Accept() if err != nil { - utils.Debugf("Stopping proxy on tcp/%v for tcp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) + log.Printf("Stopping proxy on tcp/%v for tcp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) return } go proxy.clientLoop(client.(*net.TCPConn), quit) diff --git a/components/engine/proxy/udp_proxy.go b/components/engine/pkg/proxy/udp_proxy.go similarity index 77% rename from components/engine/proxy/udp_proxy.go rename to components/engine/pkg/proxy/udp_proxy.go index 7d34988f70..9395516464 100644 --- a/components/engine/proxy/udp_proxy.go +++ b/components/engine/pkg/proxy/udp_proxy.go @@ -2,9 +2,9 @@ package proxy import ( "encoding/binary" - "github.com/dotcloud/docker/utils" "log" "net" + "strings" "sync" "syscall" "time" @@ -66,7 +66,7 @@ func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr proxy.connTrackLock.Lock() delete(proxy.connTrackTable, *clientKey) proxy.connTrackLock.Unlock() - utils.Debugf("Done proxying between udp/%v and udp/%v", clientAddr.String(), proxy.backendAddr.String()) + log.Printf("Done proxying between udp/%v and udp/%v", clientAddr.String(), proxy.backendAddr.String()) proxyConn.Close() }() @@ -92,24 +92,24 @@ func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr return } i += written - utils.Debugf("Forwarded %v/%v bytes to udp/%v", i, read, clientAddr.String()) + log.Printf("Forwarded %v/%v bytes to udp/%v", i, read, clientAddr.String()) } } } func (proxy *UDPProxy) Run() { readBuf := make([]byte, UDPBufSize) - utils.Debugf("Starting proxy on udp/%v for udp/%v", proxy.frontendAddr, proxy.backendAddr) + log.Printf("Starting proxy on udp/%v for udp/%v", proxy.frontendAddr, proxy.backendAddr) for { read, from, err := proxy.listener.ReadFromUDP(readBuf) if err != nil { // NOTE: Apparently ReadFrom doesn't return // ECONNREFUSED like Read do (see comment in // UDPProxy.replyLoop) - if utils.IsClosedError(err) { - utils.Debugf("Stopping proxy on udp/%v for udp/%v (socket was closed)", proxy.frontendAddr, proxy.backendAddr) + if isClosedError(err) { + log.Printf("Stopping proxy on udp/%v for udp/%v (socket was closed)", proxy.frontendAddr, proxy.backendAddr) } else { - utils.Errorf("Stopping proxy on udp/%v for udp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) + log.Printf("Stopping proxy on udp/%v for udp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) } break } @@ -134,7 +134,7 @@ func (proxy *UDPProxy) Run() { break } i += written - utils.Debugf("Forwarded %v/%v bytes to udp/%v", i, read, proxy.backendAddr.String()) + log.Printf("Forwarded %v/%v bytes to udp/%v", i, read, proxy.backendAddr.String()) } } } @@ -150,3 +150,13 @@ func (proxy *UDPProxy) Close() { func (proxy *UDPProxy) FrontendAddr() net.Addr { return proxy.frontendAddr } func (proxy *UDPProxy) BackendAddr() net.Addr { return proxy.backendAddr } + +func isClosedError(err error) bool { + /* This comparison is ugly, but unfortunately, net.go doesn't export errClosing. + * See: + * http://golang.org/src/pkg/net/net.go + * https://code.google.com/p/go/issues/detail?id=4337 + * https://groups.google.com/forum/#!msg/golang-nuts/0_aaCvBmOcM/SptmDyX1XJMJ + */ + return strings.HasSuffix(err.Error(), "use of closed network connection") +} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 2f7e408621..1aba80ff41 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -879,16 +879,6 @@ func ShellQuoteArguments(args []string) string { return buf.String() } -func IsClosedError(err error) bool { - /* This comparison is ugly, but unfortunately, net.go doesn't export errClosing. - * See: - * http://golang.org/src/pkg/net/net.go - * https://code.google.com/p/go/issues/detail?id=4337 - * https://groups.google.com/forum/#!msg/golang-nuts/0_aaCvBmOcM/SptmDyX1XJMJ - */ - return strings.HasSuffix(err.Error(), "use of closed network connection") -} - func PartParser(template, data string) (map[string]string, error) { // ip:public:private var ( From 39bda86fa5f3d6bb9331e109fcf26d7f062a2767 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 14 Feb 2014 17:15:40 -0800 Subject: [PATCH 095/403] Move volumes out of container.go and into volumes.go This helps clean up the container file and move volumes into one location. We currently cannot move volumes to a sub pkg because they depend on Container and also modify fields on the container. Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: bd54d40f68f78093d6a252e96f8d17f7021af7b5 Component: engine --- components/engine/container.go | 308 +----------------------------- components/engine/volumes.go | 332 +++++++++++++++++++++++++++++++++ 2 files changed, 339 insertions(+), 301 deletions(-) create mode 100644 components/engine/volumes.go diff --git a/components/engine/container.go b/components/engine/container.go index 3740a7fb73..95b48e07f7 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -9,7 +9,6 @@ import ( "github.com/dotcloud/docker/execdriver" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/nat" - "github.com/dotcloud/docker/pkg/mount" "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" @@ -19,7 +18,6 @@ import ( "log" "os" "path" - "path/filepath" "strings" "sync" "syscall" @@ -27,8 +25,10 @@ import ( ) var ( - ErrNotATTY = errors.New("The PTY is not a file") - ErrNoTTY = errors.New("No PTY found") + ErrNotATTY = errors.New("The PTY is not a file") + ErrNoTTY = errors.New("No PTY found") + ErrContainerStart = errors.New("The container failed to start. Unknown error") + ErrContainerStartTimeout = errors.New("The container failed to start due to timed out.") ) type Container struct { @@ -74,17 +74,6 @@ type Container struct { activeLinks map[string]*Link } -type BindMap struct { - SrcPath string - DstPath string - Mode string -} - -var ( - ErrContainerStart = errors.New("The container failed to start. Unknown error") - ErrContainerStartTimeout = errors.New("The container failed to start due to timed out.") -) - // FIXME: move deprecated port stuff to nat to clean up the core. type PortMapping map[string]string // Deprecated @@ -503,17 +492,7 @@ func (container *Container) Start() (err error) { log.Printf("WARNING: IPv4 forwarding is disabled. Networking will not work") } - if container.Volumes == nil || len(container.Volumes) == 0 { - container.Volumes = make(map[string]string) - container.VolumesRW = make(map[string]bool) - } - - // Apply volumes from another container if requested - if err := container.applyExternalVolumes(); err != nil { - return err - } - - if err := container.createVolumes(); err != nil { + if err := prepareVolumesForContainer(container); err != nil { return err } @@ -587,62 +566,10 @@ func (container *Container) Start() (err error) { return err } - // Setup the root fs as a bind mount of the base fs - root := container.RootfsPath() - if err := os.MkdirAll(root, 0755); err != nil && !os.IsExist(err) { - return nil - } - - // Create a bind mount of the base fs as a place where we can add mounts - // without affecting the ability to access the base fs - if err := mount.Mount(container.basefs, root, "none", "bind,rw"); err != nil { + if err := mountVolumesForContainer(container, envPath); err != nil { return err } - // Make sure the root fs is private so the mounts here don't propagate to basefs - if err := mount.ForceMount(root, root, "none", "private"); err != nil { - return err - } - - // Mount docker specific files into the containers root fs - if err := mount.Mount(runtime.sysInitPath, path.Join(root, "/.dockerinit"), "none", "bind,ro"); err != nil { - return err - } - if err := mount.Mount(envPath, path.Join(root, "/.dockerenv"), "none", "bind,ro"); err != nil { - return err - } - if err := mount.Mount(container.ResolvConfPath, path.Join(root, "/etc/resolv.conf"), "none", "bind,ro"); err != nil { - return err - } - - if container.HostnamePath != "" && container.HostsPath != "" { - if err := mount.Mount(container.HostnamePath, path.Join(root, "/etc/hostname"), "none", "bind,ro"); err != nil { - return err - } - if err := mount.Mount(container.HostsPath, path.Join(root, "/etc/hosts"), "none", "bind,ro"); err != nil { - return err - } - } - - // Mount user specified volumes - for r, v := range container.Volumes { - mountAs := "ro" - if container.VolumesRW[r] { - mountAs = "rw" - } - - r = path.Join(root, r) - if p, err := utils.FollowSymlinkInScope(r, root); err != nil { - return err - } else { - r = p - } - - if err := mount.Mount(v, r, "none", fmt.Sprintf("bind,%s", mountAs)); err != nil { - return err - } - } - populateCommand(container) // Setup logging of stdout and stderr to disk @@ -695,205 +622,6 @@ func (container *Container) Start() (err error) { return nil } -func (container *Container) getBindMap() (map[string]BindMap, error) { - // Create the requested bind mounts - binds := make(map[string]BindMap) - // Define illegal container destinations - illegalDsts := []string{"/", "."} - - for _, bind := range container.hostConfig.Binds { - // FIXME: factorize bind parsing in parseBind - var src, dst, mode string - arr := strings.Split(bind, ":") - if len(arr) == 2 { - src = arr[0] - dst = arr[1] - mode = "rw" - } else if len(arr) == 3 { - src = arr[0] - dst = arr[1] - mode = arr[2] - } else { - return nil, fmt.Errorf("Invalid bind specification: %s", bind) - } - - // Bail if trying to mount to an illegal destination - for _, illegal := range illegalDsts { - if dst == illegal { - return nil, fmt.Errorf("Illegal bind destination: %s", dst) - } - } - - bindMap := BindMap{ - SrcPath: src, - DstPath: dst, - Mode: mode, - } - binds[path.Clean(dst)] = bindMap - } - return binds, nil -} - -func (container *Container) createVolumes() error { - binds, err := container.getBindMap() - if err != nil { - return err - } - volumesDriver := container.runtime.volumes.driver - // Create the requested volumes if they don't exist - for volPath := range container.Config.Volumes { - volPath = path.Clean(volPath) - volIsDir := true - // Skip existing volumes - if _, exists := container.Volumes[volPath]; exists { - continue - } - var srcPath string - var isBindMount bool - srcRW := false - // If an external bind is defined for this volume, use that as a source - if bindMap, exists := binds[volPath]; exists { - isBindMount = true - srcPath = bindMap.SrcPath - if strings.ToLower(bindMap.Mode) == "rw" { - srcRW = true - } - if stat, err := os.Stat(bindMap.SrcPath); err != nil { - return err - } else { - volIsDir = stat.IsDir() - } - // Otherwise create an directory in $ROOT/volumes/ and use that - } else { - - // Do not pass a container as the parameter for the volume creation. - // The graph driver using the container's information ( Image ) to - // create the parent. - c, err := container.runtime.volumes.Create(nil, nil, "", "", nil) - if err != nil { - return err - } - srcPath, err = volumesDriver.Get(c.ID) - if err != nil { - return fmt.Errorf("Driver %s failed to get volume rootfs %s: %s", volumesDriver, c.ID, err) - } - srcRW = true // RW by default - } - - if p, err := filepath.EvalSymlinks(srcPath); err != nil { - return err - } else { - srcPath = p - } - - container.Volumes[volPath] = srcPath - container.VolumesRW[volPath] = srcRW - - // Create the mountpoint - volPath = path.Join(container.basefs, volPath) - rootVolPath, err := utils.FollowSymlinkInScope(volPath, container.basefs) - if err != nil { - return err - } - - if _, err := os.Stat(rootVolPath); err != nil { - if os.IsNotExist(err) { - if volIsDir { - if err := os.MkdirAll(rootVolPath, 0755); err != nil { - return err - } - } else { - if err := os.MkdirAll(path.Dir(rootVolPath), 0755); err != nil { - return err - } - if f, err := os.OpenFile(rootVolPath, os.O_CREATE, 0755); err != nil { - return err - } else { - f.Close() - } - } - } - } - - // Do not copy or change permissions if we are mounting from the host - if srcRW && !isBindMount { - volList, err := ioutil.ReadDir(rootVolPath) - if err != nil { - return err - } - if len(volList) > 0 { - srcList, err := ioutil.ReadDir(srcPath) - if err != nil { - return err - } - if len(srcList) == 0 { - // If the source volume is empty copy files from the root into the volume - if err := archive.CopyWithTar(rootVolPath, srcPath); err != nil { - return err - } - - var stat syscall.Stat_t - if err := syscall.Stat(rootVolPath, &stat); err != nil { - return err - } - var srcStat syscall.Stat_t - if err := syscall.Stat(srcPath, &srcStat); err != nil { - return err - } - // Change the source volume's ownership if it differs from the root - // files that were just copied - if stat.Uid != srcStat.Uid || stat.Gid != srcStat.Gid { - if err := os.Chown(srcPath, int(stat.Uid), int(stat.Gid)); err != nil { - return err - } - } - } - } - } - } - return nil -} - -func (container *Container) applyExternalVolumes() error { - if container.Config.VolumesFrom != "" { - containerSpecs := strings.Split(container.Config.VolumesFrom, ",") - for _, containerSpec := range containerSpecs { - mountRW := true - specParts := strings.SplitN(containerSpec, ":", 2) - switch len(specParts) { - case 0: - return fmt.Errorf("Malformed volumes-from specification: %s", container.Config.VolumesFrom) - case 2: - switch specParts[1] { - case "ro": - mountRW = false - case "rw": // mountRW is already true - default: - return fmt.Errorf("Malformed volumes-from specification: %s", containerSpec) - } - } - c := container.runtime.Get(specParts[0]) - if c == nil { - return fmt.Errorf("Container %s not found. Impossible to mount its volumes", container.ID) - } - for volPath, id := range c.Volumes { - if _, exists := container.Volumes[volPath]; exists { - continue - } - if err := os.MkdirAll(path.Join(container.basefs, volPath), 0755); err != nil { - return err - } - container.Volumes[volPath] = id - if isRW, exists := c.VolumesRW[volPath]; exists { - container.VolumesRW[volPath] = isRW && mountRW - } - } - - } - } - return nil -} - func (container *Container) Run() error { if err := container.Start(); err != nil { return err @@ -1166,29 +894,7 @@ func (container *Container) cleanup() { } } - var ( - root = container.RootfsPath() - mounts = []string{ - root, - path.Join(root, "/.dockerinit"), - path.Join(root, "/.dockerenv"), - path.Join(root, "/etc/resolv.conf"), - } - ) - - if container.HostnamePath != "" && container.HostsPath != "" { - mounts = append(mounts, path.Join(root, "/etc/hostname"), path.Join(root, "/etc/hosts")) - } - - for r := range container.Volumes { - mounts = append(mounts, path.Join(root, r)) - } - - for i := len(mounts) - 1; i >= 0; i-- { - if lastError := mount.Unmount(mounts[i]); lastError != nil { - log.Printf("Failed to umount %v: %v", mounts[i], lastError) - } - } + unmountVolumesForContainer(container) if err := container.Unmount(); err != nil { log.Printf("%v: Failed to umount filesystem: %v", container.ID, err) diff --git a/components/engine/volumes.go b/components/engine/volumes.go new file mode 100644 index 0000000000..a53ef722e7 --- /dev/null +++ b/components/engine/volumes.go @@ -0,0 +1,332 @@ +package docker + +import ( + "fmt" + "github.com/dotcloud/docker/archive" + "github.com/dotcloud/docker/pkg/mount" + "github.com/dotcloud/docker/utils" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + "syscall" +) + +type BindMap struct { + SrcPath string + DstPath string + Mode string +} + +func prepareVolumesForContainer(container *Container) error { + if container.Volumes == nil || len(container.Volumes) == 0 { + container.Volumes = make(map[string]string) + container.VolumesRW = make(map[string]bool) + } + + if err := applyVolumesFrom(container); err != nil { + return err + } + if err := createVolumes(container); err != nil { + return err + } + return nil +} + +func mountVolumesForContainer(container *Container, envPath string) error { + // Setup the root fs as a bind mount of the base fs + var ( + root = container.RootfsPath() + runtime = container.runtime + ) + if err := os.MkdirAll(root, 0755); err != nil && !os.IsExist(err) { + return nil + } + + // Create a bind mount of the base fs as a place where we can add mounts + // without affecting the ability to access the base fs + if err := mount.Mount(container.basefs, root, "none", "bind,rw"); err != nil { + return err + } + + // Make sure the root fs is private so the mounts here don't propagate to basefs + if err := mount.ForceMount(root, root, "none", "private"); err != nil { + return err + } + + // Mount docker specific files into the containers root fs + if err := mount.Mount(runtime.sysInitPath, filepath.Join(root, "/.dockerinit"), "none", "bind,ro"); err != nil { + return err + } + if err := mount.Mount(envPath, filepath.Join(root, "/.dockerenv"), "none", "bind,ro"); err != nil { + return err + } + if err := mount.Mount(container.ResolvConfPath, filepath.Join(root, "/etc/resolv.conf"), "none", "bind,ro"); err != nil { + return err + } + + if container.HostnamePath != "" && container.HostsPath != "" { + if err := mount.Mount(container.HostnamePath, filepath.Join(root, "/etc/hostname"), "none", "bind,ro"); err != nil { + return err + } + if err := mount.Mount(container.HostsPath, filepath.Join(root, "/etc/hosts"), "none", "bind,ro"); err != nil { + return err + } + } + + // Mount user specified volumes + for r, v := range container.Volumes { + mountAs := "ro" + if container.VolumesRW[r] { + mountAs = "rw" + } + + r = filepath.Join(root, r) + if p, err := utils.FollowSymlinkInScope(r, root); err != nil { + return err + } else { + r = p + } + + if err := mount.Mount(v, r, "none", fmt.Sprintf("bind,%s", mountAs)); err != nil { + return err + } + } + return nil +} + +func unmountVolumesForContainer(container *Container) { + var ( + root = container.RootfsPath() + mounts = []string{ + root, + filepath.Join(root, "/.dockerinit"), + filepath.Join(root, "/.dockerenv"), + filepath.Join(root, "/etc/resolv.conf"), + } + ) + + if container.HostnamePath != "" && container.HostsPath != "" { + mounts = append(mounts, filepath.Join(root, "/etc/hostname"), filepath.Join(root, "/etc/hosts")) + } + + for r := range container.Volumes { + mounts = append(mounts, filepath.Join(root, r)) + } + + for i := len(mounts) - 1; i >= 0; i-- { + if lastError := mount.Unmount(mounts[i]); lastError != nil { + log.Printf("Failed to umount %v: %v", mounts[i], lastError) + } + } +} + +func applyVolumesFrom(container *Container) error { + if container.Config.VolumesFrom != "" { + for _, containerSpec := range strings.Split(container.Config.VolumesFrom, ",") { + var ( + mountRW = true + specParts = strings.SplitN(containerSpec, ":", 2) + ) + + switch len(specParts) { + case 0: + return fmt.Errorf("Malformed volumes-from specification: %s", container.Config.VolumesFrom) + case 2: + switch specParts[1] { + case "ro": + mountRW = false + case "rw": // mountRW is already true + default: + return fmt.Errorf("Malformed volumes-from specification: %s", containerSpec) + } + } + + c := container.runtime.Get(specParts[0]) + if c == nil { + return fmt.Errorf("Container %s not found. Impossible to mount its volumes", container.ID) + } + + for volPath, id := range c.Volumes { + if _, exists := container.Volumes[volPath]; exists { + continue + } + if err := os.MkdirAll(filepath.Join(container.basefs, volPath), 0755); err != nil { + return err + } + container.Volumes[volPath] = id + if isRW, exists := c.VolumesRW[volPath]; exists { + container.VolumesRW[volPath] = isRW && mountRW + } + } + + } + } + return nil +} + +func getBindMap(container *Container) (map[string]BindMap, error) { + var ( + // Create the requested bind mounts + binds = make(map[string]BindMap) + // Define illegal container destinations + illegalDsts = []string{"/", "."} + ) + + for _, bind := range container.hostConfig.Binds { + // FIXME: factorize bind parsing in parseBind + var ( + src, dst, mode string + arr = strings.Split(bind, ":") + ) + + if len(arr) == 2 { + src = arr[0] + dst = arr[1] + mode = "rw" + } else if len(arr) == 3 { + src = arr[0] + dst = arr[1] + mode = arr[2] + } else { + return nil, fmt.Errorf("Invalid bind specification: %s", bind) + } + + // Bail if trying to mount to an illegal destination + for _, illegal := range illegalDsts { + if dst == illegal { + return nil, fmt.Errorf("Illegal bind destination: %s", dst) + } + } + + bindMap := BindMap{ + SrcPath: src, + DstPath: dst, + Mode: mode, + } + binds[filepath.Clean(dst)] = bindMap + } + return binds, nil +} + +func createVolumes(container *Container) error { + binds, err := getBindMap(container) + if err != nil { + return err + } + + volumesDriver := container.runtime.volumes.driver + // Create the requested volumes if they don't exist + for volPath := range container.Config.Volumes { + volPath = filepath.Clean(volPath) + volIsDir := true + // Skip existing volumes + if _, exists := container.Volumes[volPath]; exists { + continue + } + var srcPath string + var isBindMount bool + srcRW := false + // If an external bind is defined for this volume, use that as a source + if bindMap, exists := binds[volPath]; exists { + isBindMount = true + srcPath = bindMap.SrcPath + if strings.ToLower(bindMap.Mode) == "rw" { + srcRW = true + } + if stat, err := os.Stat(bindMap.SrcPath); err != nil { + return err + } else { + volIsDir = stat.IsDir() + } + // Otherwise create an directory in $ROOT/volumes/ and use that + } else { + + // Do not pass a container as the parameter for the volume creation. + // The graph driver using the container's information ( Image ) to + // create the parent. + c, err := container.runtime.volumes.Create(nil, nil, "", "", nil) + if err != nil { + return err + } + srcPath, err = volumesDriver.Get(c.ID) + if err != nil { + return fmt.Errorf("Driver %s failed to get volume rootfs %s: %s", volumesDriver, c.ID, err) + } + srcRW = true // RW by default + } + + if p, err := filepath.EvalSymlinks(srcPath); err != nil { + return err + } else { + srcPath = p + } + + container.Volumes[volPath] = srcPath + container.VolumesRW[volPath] = srcRW + + // Create the mountpoint + volPath = filepath.Join(container.basefs, volPath) + rootVolPath, err := utils.FollowSymlinkInScope(volPath, container.basefs) + if err != nil { + return err + } + + if _, err := os.Stat(rootVolPath); err != nil { + if os.IsNotExist(err) { + if volIsDir { + if err := os.MkdirAll(rootVolPath, 0755); err != nil { + return err + } + } else { + if err := os.MkdirAll(filepath.Dir(rootVolPath), 0755); err != nil { + return err + } + if f, err := os.OpenFile(rootVolPath, os.O_CREATE, 0755); err != nil { + return err + } else { + f.Close() + } + } + } + } + + // Do not copy or change permissions if we are mounting from the host + if srcRW && !isBindMount { + volList, err := ioutil.ReadDir(rootVolPath) + if err != nil { + return err + } + if len(volList) > 0 { + srcList, err := ioutil.ReadDir(srcPath) + if err != nil { + return err + } + if len(srcList) == 0 { + // If the source volume is empty copy files from the root into the volume + if err := archive.CopyWithTar(rootVolPath, srcPath); err != nil { + return err + } + + var stat syscall.Stat_t + if err := syscall.Stat(rootVolPath, &stat); err != nil { + return err + } + var srcStat syscall.Stat_t + if err := syscall.Stat(srcPath, &srcStat); err != nil { + return err + } + // Change the source volume's ownership if it differs from the root + // files that were just copied + if stat.Uid != srcStat.Uid || stat.Gid != srcStat.Gid { + if err := os.Chown(srcPath, int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + } + } + } + } + } + return nil +} From a7d9996138921a5ef49b9b7e3d807994177fd9b0 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 14 Feb 2014 17:28:50 -0800 Subject: [PATCH 096/403] Cleanup some statements from exec driver work Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 9e3da87a3a6fea21194ceb9dbd30a39d043a48a4 Component: engine --- components/engine/execdriver/lxc/init.go | 1 - components/engine/runtime.go | 14 -------------- components/engine/sysinit/sysinit.go | 8 +------- 3 files changed, 1 insertion(+), 22 deletions(-) diff --git a/components/engine/execdriver/lxc/init.go b/components/engine/execdriver/lxc/init.go index 1ed0606631..e138915212 100644 --- a/components/engine/execdriver/lxc/init.go +++ b/components/engine/execdriver/lxc/init.go @@ -100,7 +100,6 @@ func changeUser(args *execdriver.InitArgs) error { } func setupCapabilities(args *execdriver.InitArgs) error { - if args.Privileged { return nil } diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 872808d941..eed28f92ab 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -7,7 +7,6 @@ import ( "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/execdriver" - "github.com/dotcloud/docker/execdriver/chroot" "github.com/dotcloud/docker/execdriver/lxc" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/graphdriver/aufs" @@ -704,19 +703,6 @@ func NewRuntimeFromDirectory(config *DaemonConfig, eng *engine.Engine) (*Runtime sysInfo := sysinfo.New(false) - /* - temporarilly disabled. - */ - if false { - var ed execdriver.Driver - if driver := os.Getenv("EXEC_DRIVER"); driver == "lxc" { - ed, err = lxc.NewDriver(config.Root, sysInfo.AppArmor) - } else { - ed, err = chroot.NewDriver() - } - if ed != nil { - } - } ed, err := lxc.NewDriver(config.Root, sysInfo.AppArmor) if err != nil { return nil, err diff --git a/components/engine/sysinit/sysinit.go b/components/engine/sysinit/sysinit.go index dcf0eddf56..b02cf027aa 100644 --- a/components/engine/sysinit/sysinit.go +++ b/components/engine/sysinit/sysinit.go @@ -27,18 +27,12 @@ func setupEnv(args *execdriver.InitArgs) { func executeProgram(args *execdriver.InitArgs) error { setupEnv(args) + dockerInitFct, err := execdriver.GetInitFunc(args.Driver) if err != nil { panic(err) } return dockerInitFct(args) - - if args.Driver == "lxc" { - // Will never reach - } else if args.Driver == "chroot" { - } - - return nil } // Sys Init code From 8a2b18fccdba04aac5325181f392e6258ecbf3ad Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 14 Feb 2014 18:07:33 -0800 Subject: [PATCH 097/403] Remove container dependency for links Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: fc952e0de94aa4283c677ce6c153b1a766db2fc0 Component: engine --- components/engine/container.go | 15 +++++++++++-- components/engine/links.go | 23 +++++++++----------- components/engine/links_test.go | 37 ++------------------------------- 3 files changed, 25 insertions(+), 50 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index 3740a7fb73..d054bcf50d 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -548,8 +548,19 @@ func (container *Container) Start() (err error) { container.activeLinks = nil } - for p, child := range children { - link, err := NewLink(container, child, p, runtime.eng) + for linkAlias, child := range children { + if !child.State.IsRunning() { + return fmt.Errorf("Cannot link to a non running container: %s AS %s", child.Name, linkAlias) + } + + link, err := NewLink( + container.NetworkSettings.IPAddress, + child.NetworkSettings.IPAddress, + linkAlias, + child.Config.Env, + child.Config.ExposedPorts, + runtime.eng) + if err != nil { rollback() return err diff --git a/components/engine/links.go b/components/engine/links.go index ff39947a0d..1a198b0814 100644 --- a/components/engine/links.go +++ b/components/engine/links.go @@ -18,26 +18,23 @@ type Link struct { eng *engine.Engine } -func NewLink(parent, child *Container, name string, eng *engine.Engine) (*Link, error) { - if parent.ID == child.ID { - return nil, fmt.Errorf("Cannot link to self: %s == %s", parent.ID, child.ID) - } - if !child.State.IsRunning() { - return nil, fmt.Errorf("Cannot link to a non running container: %s AS %s", child.Name, name) - } +func NewLink(parentIP, childIP, name string, env []string, exposedPorts map[nat.Port]struct{}, eng *engine.Engine) (*Link, error) { - ports := make([]nat.Port, len(child.Config.ExposedPorts)) - var i int - for p := range child.Config.ExposedPorts { + var ( + i int + ports = make([]nat.Port, len(exposedPorts)) + ) + + for p := range exposedPorts { ports[i] = p i++ } l := &Link{ Name: name, - ChildIP: child.NetworkSettings.IPAddress, - ParentIP: parent.NetworkSettings.IPAddress, - ChildEnvironment: child.Config.Env, + ChildIP: childIP, + ParentIP: parentIP, + ChildEnvironment: env, Ports: ports, eng: eng, } diff --git a/components/engine/links_test.go b/components/engine/links_test.go index 7b85a8e86d..d84fe04061 100644 --- a/components/engine/links_test.go +++ b/components/engine/links_test.go @@ -2,37 +2,15 @@ package docker import ( "github.com/dotcloud/docker/nat" - "github.com/dotcloud/docker/runconfig" "strings" "testing" ) -func newMockLinkContainer(id string, ip string) *Container { - return &Container{ - Config: &runconfig.Config{}, - ID: id, - NetworkSettings: &NetworkSettings{ - IPAddress: ip, - }, - } -} - func TestLinkNew(t *testing.T) { - toID := GenerateID() - fromID := GenerateID() - - from := newMockLinkContainer(fromID, "172.0.17.2") - from.Config.Env = []string{} - from.State = State{Running: true} ports := make(nat.PortSet) - ports[nat.Port("6379/tcp")] = struct{}{} - from.Config.ExposedPorts = ports - - to := newMockLinkContainer(toID, "172.0.17.3") - - link, err := NewLink(to, from, "/db/docker", nil) + link, err := NewLink("172.0.17.3", "172.0.17.2", "/db/docker", nil, ports, nil) if err != nil { t.Fatal(err) } @@ -60,21 +38,10 @@ func TestLinkNew(t *testing.T) { } func TestLinkEnv(t *testing.T) { - toID := GenerateID() - fromID := GenerateID() - - from := newMockLinkContainer(fromID, "172.0.17.2") - from.Config.Env = []string{"PASSWORD=gordon"} - from.State = State{Running: true} ports := make(nat.PortSet) - ports[nat.Port("6379/tcp")] = struct{}{} - from.Config.ExposedPorts = ports - - to := newMockLinkContainer(toID, "172.0.17.3") - - link, err := NewLink(to, from, "/db/docker", nil) + link, err := NewLink("172.0.17.3", "172.0.17.2", "/db/docker", []string{"PASSWORD=gordon"}, ports, nil) if err != nil { t.Fatal(err) } From a10cdc2a0cba6a135fe542e072bc912145e082a3 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 14 Feb 2014 18:18:16 -0800 Subject: [PATCH 098/403] Move links to sub pkg Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 147b09fae9527e760d74b5ef2f9558ee109f4009 Component: engine --- components/engine/container.go | 7 ++++--- components/engine/{ => links}/links.go | 2 +- components/engine/{ => links}/links_test.go | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) rename components/engine/{ => links}/links.go (99%) rename components/engine/{ => links}/links_test.go (99%) diff --git a/components/engine/container.go b/components/engine/container.go index d054bcf50d..361dfeaf67 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -8,6 +8,7 @@ import ( "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/execdriver" "github.com/dotcloud/docker/graphdriver" + "github.com/dotcloud/docker/links" "github.com/dotcloud/docker/nat" "github.com/dotcloud/docker/pkg/mount" "github.com/dotcloud/docker/pkg/term" @@ -71,7 +72,7 @@ type Container struct { VolumesRW map[string]bool hostConfig *runconfig.HostConfig - activeLinks map[string]*Link + activeLinks map[string]*links.Link } type BindMap struct { @@ -537,7 +538,7 @@ func (container *Container) Start() (err error) { } if len(children) > 0 { - container.activeLinks = make(map[string]*Link, len(children)) + container.activeLinks = make(map[string]*links.Link, len(children)) // If we encounter an error make sure that we rollback any network // config and ip table changes @@ -553,7 +554,7 @@ func (container *Container) Start() (err error) { return fmt.Errorf("Cannot link to a non running container: %s AS %s", child.Name, linkAlias) } - link, err := NewLink( + link, err := links.NewLink( container.NetworkSettings.IPAddress, child.NetworkSettings.IPAddress, linkAlias, diff --git a/components/engine/links.go b/components/engine/links/links.go similarity index 99% rename from components/engine/links.go rename to components/engine/links/links.go index 1a198b0814..68ac98ee07 100644 --- a/components/engine/links.go +++ b/components/engine/links/links.go @@ -1,4 +1,4 @@ -package docker +package links import ( "fmt" diff --git a/components/engine/links_test.go b/components/engine/links/links_test.go similarity index 99% rename from components/engine/links_test.go rename to components/engine/links/links_test.go index d84fe04061..e66f9bfb78 100644 --- a/components/engine/links_test.go +++ b/components/engine/links/links_test.go @@ -1,4 +1,4 @@ -package docker +package links import ( "github.com/dotcloud/docker/nat" From d029fb179f659f24205b57f6517ffcb84323b78a Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 15 Feb 2014 14:27:01 -0800 Subject: [PATCH 099/403] Fix engine tests on systems where temp directories are symlinked. Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: 353cc8c2a5d9ae4c02dc94e14cd2ad6d3393992d Component: engine --- components/engine/engine/engine_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/components/engine/engine/engine_test.go b/components/engine/engine/engine_test.go index 065a19f492..da59610727 100644 --- a/components/engine/engine/engine_test.go +++ b/components/engine/engine/engine_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "testing" ) @@ -64,6 +65,18 @@ func TestEngineRoot(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmp) + // We expect Root to resolve to an absolute path. + // FIXME: this should not be necessary. + // Until the above FIXME is implemented, let's check for the + // current behavior. + tmp, err = filepath.EvalSymlinks(tmp) + if err != nil { + t.Fatal(err) + } + tmp, err = filepath.Abs(tmp) + if err != nil { + t.Fatal(err) + } dir := path.Join(tmp, "dir") eng, err := New(dir) if err != nil { From 0efb658af55bd59a2862577bf862d75b1c3ce1cc Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Sat, 15 Feb 2014 14:37:51 -0800 Subject: [PATCH 100/403] Document Darwin test failure in a FIXME Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: 7286842a12e3620224e4de65730a3b40d17f1f3a Component: engine --- components/engine/tags_unit_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/tags_unit_test.go b/components/engine/tags_unit_test.go index 1341b989fe..b6236280a8 100644 --- a/components/engine/tags_unit_test.go +++ b/components/engine/tags_unit_test.go @@ -31,6 +31,8 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { t.Fatal(err) } img := &Image{ID: testImageID} + // FIXME: this fails on Darwin with: + // tags_unit_test.go:36: mkdir /var/folders/7g/b3ydb5gx4t94ndr_cljffbt80000gq/T/docker-test569b-tRunner-075013689/vfs/dir/foo/etc/postgres: permission denied if err := graph.Register(nil, archive, img); err != nil { t.Fatal(err) } From 0f43c3eb059a9986cde87372daad805d344a1871 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Wed, 12 Feb 2014 15:10:41 -0800 Subject: [PATCH 101/403] Refactor utility `MkBuildContext` to the more generic `archive.Generate` This facilitates refactoring commands.go. Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: 9b56da78e0b1e4e1da9b2bacc44a22143745a2b0 Component: engine --- components/engine/archive/wrap.go | 59 +++++++++++++++++++ components/engine/commands.go | 28 +-------- .../engine/integration/buildfile_test.go | 28 +++++---- components/engine/server.go | 2 +- 4 files changed, 76 insertions(+), 41 deletions(-) create mode 100644 components/engine/archive/wrap.go diff --git a/components/engine/archive/wrap.go b/components/engine/archive/wrap.go new file mode 100644 index 0000000000..dfb335c0b6 --- /dev/null +++ b/components/engine/archive/wrap.go @@ -0,0 +1,59 @@ +package archive + +import ( + "archive/tar" + "bytes" + "io/ioutil" +) + +// Generate generates a new archive from the content provided +// as input. +// +// `files` is a sequence of path/content pairs. A new file is +// added to the archive for each pair. +// If the last pair is incomplete, the file is created with an +// empty content. For example: +// +// Generate("foo.txt", "hello world", "emptyfile") +// +// The above call will return an archive with 2 files: +// * ./foo.txt with content "hello world" +// * ./empty with empty content +// +// FIXME: stream content instead of buffering +// FIXME: specify permissions and other archive metadata +func Generate(input ...string) (Archive, error) { + files := parseStringPairs(input...) + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + for _, file := range files { + name, content := file[0], file[1] + hdr := &tar.Header{ + Name: name, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + if _, err := tw.Write([]byte(content)); err != nil { + return nil, err + } + } + if err := tw.Close(); err != nil { + return nil, err + } + return ioutil.NopCloser(buf), nil +} + +func parseStringPairs(input ...string) (output [][2]string) { + output = make([][2]string, 0, len(input)/2+1) + for i := 0; i < len(input); i += 2 { + var pair [2]string + pair[0] = input[i] + if i+1 < len(input) { + pair[1] = input[i+1] + } + output = append(output, pair) + } + return +} diff --git a/components/engine/commands.go b/components/engine/commands.go index e272d9b94a..96608aa674 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -1,7 +1,6 @@ package docker import ( - "archive/tar" "bufio" "bytes" "encoding/base64" @@ -136,31 +135,6 @@ func (cli *DockerCli) CmdInsert(args ...string) error { return cli.stream("POST", "/images/"+cmd.Arg(0)+"/insert?"+v.Encode(), nil, cli.out, nil) } -// mkBuildContext returns an archive of an empty context with the contents -// of `dockerfile` at the path ./Dockerfile -func MkBuildContext(dockerfile string, files [][2]string) (archive.Archive, error) { - buf := new(bytes.Buffer) - tw := tar.NewWriter(buf) - files = append(files, [2]string{"Dockerfile", dockerfile}) - for _, file := range files { - name, content := file[0], file[1] - hdr := &tar.Header{ - Name: name, - Size: int64(len(content)), - } - if err := tw.WriteHeader(hdr); err != nil { - return nil, err - } - if _, err := tw.Write([]byte(content)); err != nil { - return nil, err - } - } - if err := tw.Close(); err != nil { - return nil, err - } - return ioutil.NopCloser(buf), nil -} - func (cli *DockerCli) CmdBuild(args ...string) error { cmd := cli.Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH") tag := cmd.String([]string{"t", "-tag"}, "", "Repository name (and optionally a tag) to be applied to the resulting image in case of success") @@ -188,7 +162,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if err != nil { return err } - context, err = MkBuildContext(string(dockerfile), nil) + context, err = archive.Generate("Dockerfile", string(dockerfile)) } else if utils.IsURL(cmd.Arg(0)) || utils.IsGIT(cmd.Arg(0)) { isRemote = true } else { diff --git a/components/engine/integration/buildfile_test.go b/components/engine/integration/buildfile_test.go index 199637789e..805932b57a 100644 --- a/components/engine/integration/buildfile_test.go +++ b/components/engine/integration/buildfile_test.go @@ -14,16 +14,6 @@ import ( "testing" ) -// mkTestContext generates a build context from the contents of the provided dockerfile. -// This context is suitable for use as an argument to BuildFile.Build() -func mkTestContext(dockerfile string, files [][2]string, t *testing.T) archive.Archive { - context, err := docker.MkBuildContext(dockerfile, files) - if err != nil { - t.Fatal(err) - } - return context -} - // A testContextTemplate describes a build context and how to test it type testContextTemplate struct { // Contents of the Dockerfile @@ -34,6 +24,18 @@ type testContextTemplate struct { remoteFiles [][2]string } +func (context testContextTemplate) Archive(dockerfile string, t *testing.T) archive.Archive { + input := []string{"Dockerfile", dockerfile} + for _, pair := range context.files { + input = append(input, pair[0], pair[1]) + } + a, err := archive.Generate(input...) + if err != nil { + t.Fatal(err) + } + return a +} + // A table of all the contexts to build and test. // A new docker runtime will be created and torn down for each context. var testContexts = []testContextTemplate{ @@ -381,7 +383,7 @@ func buildImage(context testContextTemplate, t *testing.T, eng *engine.Engine, u dockerfile := constructDockerfile(context.dockerfile, ip, port) buildfile := docker.NewBuildFile(srv, ioutil.Discard, ioutil.Discard, false, useCache, false, ioutil.Discard, utils.NewStreamFormatter(false), nil, nil) - id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) + id, err := buildfile.Build(context.Archive(dockerfile, t)) if err != nil { return nil, err } @@ -785,7 +787,7 @@ func TestForbiddenContextPath(t *testing.T) { dockerfile := constructDockerfile(context.dockerfile, ip, port) buildfile := docker.NewBuildFile(srv, ioutil.Discard, ioutil.Discard, false, true, false, ioutil.Discard, utils.NewStreamFormatter(false), nil, nil) - _, err = buildfile.Build(mkTestContext(dockerfile, context.files, t)) + _, err = buildfile.Build(context.Archive(dockerfile, t)) if err == nil { t.Log("Error should not be nil") @@ -831,7 +833,7 @@ func TestBuildADDFileNotFound(t *testing.T) { dockerfile := constructDockerfile(context.dockerfile, ip, port) buildfile := docker.NewBuildFile(mkServerFromEngine(eng, t), ioutil.Discard, ioutil.Discard, false, true, false, ioutil.Discard, utils.NewStreamFormatter(false), nil, nil) - _, err = buildfile.Build(mkTestContext(dockerfile, context.files, t)) + _, err = buildfile.Build(context.Archive(dockerfile, t)) if err == nil { t.Log("Error should not be nil") diff --git a/components/engine/server.go b/components/engine/server.go index daf05ec67a..bad8d8bfb5 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -476,7 +476,7 @@ func (srv *Server) Build(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } - c, err := MkBuildContext(string(dockerFile), nil) + c, err := archive.Generate("Dockerfile", string(dockerFile)) if err != nil { return job.Error(err) } From 024e6f564e1dbbda68901d57f38bd89fd1415de1 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 11 Feb 2014 17:59:00 -0800 Subject: [PATCH 102/403] Move remote api client to api/ This moves `commands.go` out of the core and into `api/client.go` Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: ee59ba969fdd1837fe0151638cc01637e18139c3 Component: engine --- .../engine/{commands.go => api/client.go} | 13 +++--- components/engine/docker/docker.go | 3 +- .../engine/integration/commands_test.go | 45 ++++++++++--------- 3 files changed, 30 insertions(+), 31 deletions(-) rename components/engine/{commands.go => api/client.go} (99%) diff --git a/components/engine/commands.go b/components/engine/api/client.go similarity index 99% rename from components/engine/commands.go rename to components/engine/api/client.go index 96608aa674..a7098c1bea 100644 --- a/components/engine/commands.go +++ b/components/engine/api/client.go @@ -1,4 +1,4 @@ -package docker +package api import ( "bufio" @@ -7,7 +7,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/dockerversion" @@ -76,7 +75,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error { return nil } } - help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=[unix://%s]: tcp://host:port to bind/connect to or unix://path/to/socket to use\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", api.DEFAULTUNIXSOCKET) + help := fmt.Sprintf("Usage: docker [OPTIONS] COMMAND [arg...]\n -H=[unix://%s]: tcp://host:port to bind/connect to or unix://path/to/socket to use\n\nA self-sufficient runtime for linux containers.\n\nCommands:\n", DEFAULTUNIXSOCKET) for _, command := range [][]string{ {"attach", "Attach to a running container"}, {"build", "Build a container from a Dockerfile"}, @@ -2044,7 +2043,7 @@ func (cli *DockerCli) call(method, path string, data interface{}, passAuthInfo b re := regexp.MustCompile("/+") path = re.ReplaceAllString(path, "/") - req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", api.APIVERSION, path), params) + req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), params) if err != nil { return nil, -1, err } @@ -2123,7 +2122,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, h re := regexp.MustCompile("/+") path = re.ReplaceAllString(path, "/") - req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", api.APIVERSION, path), in) + req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), in) if err != nil { return err } @@ -2168,7 +2167,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer, h return fmt.Errorf("Error: %s", bytes.TrimSpace(body)) } - if api.MatchesContentType(resp.Header.Get("Content-Type"), "application/json") { + if MatchesContentType(resp.Header.Get("Content-Type"), "application/json") { return utils.DisplayJSONMessagesStream(resp.Body, out, cli.terminalFd, cli.isTerminal) } if _, err := io.Copy(out, resp.Body); err != nil { @@ -2187,7 +2186,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea re := regexp.MustCompile("/+") path = re.ReplaceAllString(path, "/") - req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", api.APIVERSION, path), nil) + req, err := http.NewRequest(method, fmt.Sprintf("/v%g%s", APIVERSION, path), nil) if err != nil { return err } diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 5ab8f65d65..5b0d279a5b 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -6,7 +6,6 @@ import ( "os" "strings" - "github.com/dotcloud/docker" "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" @@ -108,7 +107,7 @@ func main() { log.Fatal("Please specify only one -H") } protoAddrParts := strings.SplitN(flHosts.GetAll()[0], "://", 2) - if err := docker.ParseCommands(protoAddrParts[0], protoAddrParts[1], flag.Args()...); err != nil { + if err := api.ParseCommands(protoAddrParts[0], protoAddrParts[1], flag.Args()...); err != nil { if sterr, ok := err.(*utils.StatusError); ok { if sterr.Status != "" { log.Println(sterr.Status) diff --git a/components/engine/integration/commands_test.go b/components/engine/integration/commands_test.go index a0fc4b9523..a3359ec631 100644 --- a/components/engine/integration/commands_test.go +++ b/components/engine/integration/commands_test.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "github.com/dotcloud/docker" + "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/utils" @@ -119,7 +120,7 @@ func assertPipe(input, output string, r io.Reader, w io.Writer, count int) error func TestRunHostname(t *testing.T) { stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) c := make(chan struct{}) @@ -164,7 +165,7 @@ func TestRunHostname(t *testing.T) { func TestRunWorkdir(t *testing.T) { stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) c := make(chan struct{}) @@ -209,7 +210,7 @@ func TestRunWorkdir(t *testing.T) { func TestRunWorkdirExists(t *testing.T) { stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) c := make(chan struct{}) @@ -254,7 +255,7 @@ func TestRunExit(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) c1 := make(chan struct{}) @@ -307,7 +308,7 @@ func TestRunDisconnect(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) c1 := make(chan struct{}) @@ -353,7 +354,7 @@ func TestRunDisconnectTty(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) c1 := make(chan struct{}) @@ -405,7 +406,7 @@ func TestRunAttachStdin(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) ch := make(chan struct{}) @@ -469,7 +470,7 @@ func TestRunDetach(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) ch := make(chan struct{}) @@ -516,7 +517,7 @@ func TestAttachDetach(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) ch := make(chan struct{}) @@ -549,7 +550,7 @@ func TestAttachDetach(t *testing.T) { stdin, stdinPipe = io.Pipe() stdout, stdoutPipe = io.Pipe() - cli = docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli = api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) ch = make(chan struct{}) go func() { @@ -597,7 +598,7 @@ func TestAttachDetachTruncatedID(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) // Discard the CmdRun output @@ -615,7 +616,7 @@ func TestAttachDetachTruncatedID(t *testing.T) { stdin, stdinPipe = io.Pipe() stdout, stdoutPipe = io.Pipe() - cli = docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli = api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) ch := make(chan struct{}) go func() { @@ -662,7 +663,7 @@ func TestAttachDisconnect(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) go func() { @@ -731,7 +732,7 @@ func TestAttachDisconnect(t *testing.T) { func TestRunAutoRemove(t *testing.T) { t.Skip("Fixme. Skipping test for now, race condition") stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) c := make(chan struct{}) @@ -767,7 +768,7 @@ func TestRunAutoRemove(t *testing.T) { func TestCmdLogs(t *testing.T) { t.Skip("Test not impemented") - cli := docker.NewDockerCli(nil, ioutil.Discard, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(nil, ioutil.Discard, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) if err := cli.CmdRun(unitTestImageID, "sh", "-c", "ls -l"); err != nil { @@ -785,7 +786,7 @@ func TestCmdLogs(t *testing.T) { // Expected behaviour: error out when attempting to bind mount non-existing source paths func TestRunErrorBindNonExistingSource(t *testing.T) { - cli := docker.NewDockerCli(nil, nil, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(nil, nil, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) c := make(chan struct{}) @@ -805,7 +806,7 @@ func TestRunErrorBindNonExistingSource(t *testing.T) { func TestImagesViz(t *testing.T) { stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) image := buildTestImages(t, globalEngine) @@ -855,7 +856,7 @@ func TestImagesViz(t *testing.T) { func TestImagesTree(t *testing.T) { stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) image := buildTestImages(t, globalEngine) @@ -938,7 +939,7 @@ func TestRunCidFile(t *testing.T) { } tmpCidFile := path.Join(tmpDir, "cid") - cli := docker.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) c := make(chan struct{}) @@ -988,7 +989,7 @@ func TestContainerOrphaning(t *testing.T) { defer os.RemoveAll(tmpDir) // setup a CLI and server - cli := docker.NewDockerCli(nil, ioutil.Discard, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(nil, ioutil.Discard, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) srv := mkServerFromEngine(globalEngine, t) @@ -1045,8 +1046,8 @@ func TestCmdKill(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() - cli := docker.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) - cli2 := docker.NewDockerCli(nil, ioutil.Discard, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli := api.NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + cli2 := api.NewDockerCli(nil, ioutil.Discard, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalEngine, t) ch := make(chan struct{}) From 377e1023122e4f7442578e587063da77ea0978a2 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 11 Feb 2014 18:03:50 -0800 Subject: [PATCH 103/403] Remove duplicate function displayablePorts Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: 567a422a65f71a58c3000aa66a42853c7e005bb4 Component: engine --- components/engine/api/client.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/components/engine/api/client.go b/components/engine/api/client.go index a7098c1bea..c724d08269 100644 --- a/components/engine/api/client.go +++ b/components/engine/api/client.go @@ -29,7 +29,6 @@ import ( "reflect" "regexp" "runtime" - "sort" "strconv" "strings" "syscall" @@ -1279,19 +1278,6 @@ func (cli *DockerCli) printTreeNode(noTrunc bool, image *engine.Env, prefix stri } } -func displayablePorts(ports *engine.Table) string { - result := []string{} - for _, port := range ports.Data { - if port.Get("IP") == "" { - result = append(result, fmt.Sprintf("%d/%s", port.GetInt("PublicPort"), port.Get("Type"))) - } else { - result = append(result, fmt.Sprintf("%s:%d->%d/%s", port.Get("IP"), port.GetInt("PublicPort"), port.GetInt("PrivatePort"), port.Get("Type"))) - } - } - sort.Strings(result) - return strings.Join(result, ", ") -} - func (cli *DockerCli) CmdPs(args ...string) error { cmd := cli.Subcmd("ps", "[OPTIONS]", "List containers") quiet := cmd.Bool([]string{"q", "-quiet"}, false, "Only display numeric IDs") From 9c1c314fd97094bf805a25d7aac4e5dae3f05d51 Mon Sep 17 00:00:00 2001 From: Solomon Hykes Date: Tue, 11 Feb 2014 17:59:45 -0800 Subject: [PATCH 104/403] api/container.go: an API-specific representation of a container This breaks the dependency from the remote API implementation to the internal representation of a container. Instead it uses its own partial representation of a container, with only required fields. * This preserves reverse-compatibility with all past implementations of the remote API. * This clarifies which fields are guaranteed to be present in a response A docker remote api server *may* return more fields in a Container object, but their presence and semantics are not guaranteed and should not be relied upon by client implementations. Docker-DCO-1.1-Signed-off-by: Solomon Hykes (github: shykes) Upstream-commit: 44e10433c7a3811d4d0897192418cb6b62996584 Component: engine --- components/engine/api/client.go | 4 ++-- components/engine/api/container.go | 18 ++++++++++++++++++ components/engine/docker/docker.go | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 components/engine/api/container.go diff --git a/components/engine/api/client.go b/components/engine/api/client.go index c724d08269..f1f336d5f4 100644 --- a/components/engine/api/client.go +++ b/components/engine/api/client.go @@ -1572,7 +1572,7 @@ func (cli *DockerCli) CmdAttach(args ...string) error { return err } - if !container.State.IsRunning() { + if !container.State.Running { return fmt.Errorf("Impossible to attach to a stopped container, start it first") } @@ -2355,7 +2355,7 @@ func getExitCode(cli *DockerCli, containerId string) (bool, int, error) { if err := json.Unmarshal(body, c); err != nil { return false, -1, err } - return c.State.IsRunning(), c.State.GetExitCode(), nil + return c.State.Running, c.State.ExitCode, nil } func readBody(stream io.ReadCloser, statusCode int, err error) ([]byte, int, error) { diff --git a/components/engine/api/container.go b/components/engine/api/container.go new file mode 100644 index 0000000000..4cc73b2252 --- /dev/null +++ b/components/engine/api/container.go @@ -0,0 +1,18 @@ +package api + +import ( + "github.com/dotcloud/docker/nat" + "github.com/dotcloud/docker/runconfig" +) + +type Container struct { + Config runconfig.Config + HostConfig runconfig.HostConfig + State struct { + Running bool + ExitCode int + } + NetworkSettings struct { + Ports nat.PortMap + } +} diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 5b0d279a5b..b4d7879397 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -6,6 +6,7 @@ import ( "os" "strings" + _ "github.com/dotcloud/docker" "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" From 7473a46910eda135316d20e58c9564d47ca133a7 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sat, 15 Feb 2014 20:02:54 -0800 Subject: [PATCH 105/403] Add socket activation for go apps Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: cfb7711a74dc4b54d879f79bc2f6435ed71163b9 Component: engine --- .../engine/pkg/socketactivation/activation.go | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 components/engine/pkg/socketactivation/activation.go diff --git a/components/engine/pkg/socketactivation/activation.go b/components/engine/pkg/socketactivation/activation.go new file mode 100644 index 0000000000..0edbcaac23 --- /dev/null +++ b/components/engine/pkg/socketactivation/activation.go @@ -0,0 +1,61 @@ +/* + Package to allow go applications to immediately start + listening on a socket, unix, tcp, udp but hold connections + until the application has booted and is ready to accept them +*/ +package socketactivation + +import ( + "fmt" + "net" + "time" +) + +// NewActivationListener returns a listener listening on addr with the protocol. It sets the +// timeout to wait on first connection before an error is returned +func NewActivationListener(proto, addr string, activate chan struct{}, timeout time.Duration) (net.Listener, error) { + wrapped, err := net.Listen(proto, addr) + if err != nil { + return nil, err + } + + return &defaultListener{ + wrapped: wrapped, + activate: activate, + timeout: timeout, + }, nil +} + +type defaultListener struct { + wrapped net.Listener // the real listener to wrap + ready bool // is the listner ready to start accpeting connections + activate chan struct{} + timeout time.Duration // how long to wait before we consider this an error +} + +func (l *defaultListener) Close() error { + return l.wrapped.Close() +} + +func (l *defaultListener) Addr() net.Addr { + return l.wrapped.Addr() +} + +func (l *defaultListener) Accept() (net.Conn, error) { + // if the listen has been told it is ready then we can go ahead and + // start returning connections + if l.ready { + return l.wrapped.Accept() + } + + select { + case <-time.After(l.timeout): + // close the connection so any clients are disconnected + l.Close() + return nil, fmt.Errorf("timeout (%s) reached waiting for listener to become ready", l.timeout.String()) + case <-l.activate: + l.ready = true + return l.Accept() + } + panic("unreachable") +} From 09d6a6c5dd981178401db046a36787d6c191c5ec Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sat, 15 Feb 2014 20:24:55 -0800 Subject: [PATCH 106/403] Use socket activation pkg for listeners Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 64716a21e464ac63dd9b287c832ae3431e1418d0 Component: engine --- components/engine/api/api.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index 2cfc75631e..ed8c8bb795 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -10,6 +10,7 @@ import ( "fmt" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/pkg/socketactivation" "github.com/dotcloud/docker/pkg/systemd" "github.com/dotcloud/docker/utils" "github.com/gorilla/mux" @@ -25,6 +26,7 @@ import ( "strconv" "strings" "syscall" + "time" ) // FIXME: move code common to client and server to common.go @@ -34,6 +36,10 @@ const ( DEFAULTUNIXSOCKET = "/var/run/docker.sock" ) +var ( + activationLock chan struct{} +) + func ValidateHost(val string) (string, error) { host, err := utils.ParseHost(DEFAULTHTTPHOST, DEFAULTUNIXSOCKET, val) if err != nil { @@ -46,6 +52,7 @@ type HttpApiFunc func(eng *engine.Engine, version float64, w http.ResponseWriter func init() { engine.Register("serveapi", ServeApi) + engine.Register("acceptconnections", AcceptConnections) } func hijackServer(w http.ResponseWriter) (io.ReadCloser, io.Writer, error) { @@ -1156,7 +1163,7 @@ func ListenAndServe(proto, addr string, eng *engine.Engine, logging, enableCors } } - l, err := net.Listen(proto, addr) + l, err := socketactivation.NewActivationListener(proto, addr, activationLock, 15*time.Minute) if err != nil { return err } @@ -1198,8 +1205,11 @@ func ListenAndServe(proto, addr string, eng *engine.Engine, logging, enableCors // ServeApi loops through all of the protocols sent in to docker and spawns // off a go routine to setup a serving http.Server for each. func ServeApi(job *engine.Job) engine.Status { - protoAddrs := job.Args - chErrors := make(chan error, len(protoAddrs)) + var ( + protoAddrs = job.Args + chErrors = make(chan error, len(protoAddrs)) + ) + activationLock = make(chan struct{}) for _, protoAddr := range protoAddrs { protoAddrParts := strings.SplitN(protoAddr, "://", 2) @@ -1209,6 +1219,8 @@ func ServeApi(job *engine.Job) engine.Status { }() } + AcceptConnections(nil) + for i := 0; i < len(protoAddrs); i += 1 { err := <-chErrors if err != nil { @@ -1216,8 +1228,15 @@ func ServeApi(job *engine.Job) engine.Status { } } + return engine.StatusOK +} + +func AcceptConnections(job *engine.Job) engine.Status { // Tell the init daemon we are accepting requests go systemd.SdNotify("READY=1") + // close the lock so the listeners start accepting connections + close(activationLock) + return engine.StatusOK } From 9fff3868b1e8e480bc4e3aac87b8ef708054e3f1 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sat, 15 Feb 2014 20:49:50 -0800 Subject: [PATCH 107/403] Integration generic socket wait for docker api Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 778f1bf6639e52004608ca2289e04b8f7bddb14b Component: engine --- components/engine/api/api.go | 2 - components/engine/docker/docker.go | 47 ++++++++++++------- components/engine/integration/runtime_test.go | 5 ++ 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index ed8c8bb795..8f6729ab61 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -1219,8 +1219,6 @@ func ServeApi(job *engine.Job) engine.Status { }() } - AcceptConnections(nil) - for i := 0; i < len(protoAddrs); i += 1 { err := <-chErrors if err != nil { diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index b4d7879397..fe214811cf 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -78,25 +78,36 @@ func main() { if err != nil { log.Fatal(err) } - // Load plugin: httpapi - job := eng.Job("initserver") - job.Setenv("Pidfile", *pidfile) - job.Setenv("Root", *flRoot) - job.SetenvBool("AutoRestart", *flAutoRestart) - job.SetenvList("Dns", flDns.GetAll()) - job.SetenvBool("EnableIptables", *flEnableIptables) - job.SetenvBool("EnableIpForward", *flEnableIpForward) - job.Setenv("BridgeIface", *bridgeName) - job.Setenv("BridgeIP", *bridgeIp) - job.Setenv("DefaultIp", *flDefaultIp) - job.SetenvBool("InterContainerCommunication", *flInterContainerComm) - job.Setenv("GraphDriver", *flGraphDriver) - job.SetenvInt("Mtu", *flMtu) - if err := job.Run(); err != nil { - log.Fatal(err) - } + // load the daemon in the background so we can immediately start + // the http api so that connections don't fail while the daemon + // is booting + go func() { + // Load plugin: httpapi + job := eng.Job("initserver") + job.Setenv("Pidfile", *pidfile) + job.Setenv("Root", *flRoot) + job.SetenvBool("AutoRestart", *flAutoRestart) + job.SetenvList("Dns", flDns.GetAll()) + job.SetenvBool("EnableIptables", *flEnableIptables) + job.SetenvBool("EnableIpForward", *flEnableIpForward) + job.Setenv("BridgeIface", *bridgeName) + job.Setenv("BridgeIP", *bridgeIp) + job.Setenv("DefaultIp", *flDefaultIp) + job.SetenvBool("InterContainerCommunication", *flInterContainerComm) + job.Setenv("GraphDriver", *flGraphDriver) + job.SetenvInt("Mtu", *flMtu) + if err := job.Run(); err != nil { + log.Fatal(err) + } + // after the daemon is done setting up we can tell the api to start + // accepting connections + if err := eng.Job("acceptconnections").Run(); err != nil { + log.Fatal(err) + } + }() + // Serve api - job = eng.Job("serveapi", flHosts.GetAll()...) + job := eng.Job("serveapi", flHosts.GetAll()...) job.SetenvBool("Logging", true) job.SetenvBool("EnableCors", *flEnableCors) job.Setenv("Version", dockerversion.VERSION) diff --git a/components/engine/integration/runtime_test.go b/components/engine/integration/runtime_test.go index 170f4c9638..ca2119ce1f 100644 --- a/components/engine/integration/runtime_test.go +++ b/components/engine/integration/runtime_test.go @@ -171,9 +171,14 @@ func spawnGlobalDaemon() { log.Fatalf("Unable to spawn the test daemon: %s", err) } }() + // Give some time to ListenAndServer to actually start // FIXME: use inmem transports instead of tcp time.Sleep(time.Second) + + if err := eng.Job("acceptconnections").Run(); err != nil { + log.Fatalf("Unable to accept connections for test api: %s", err) + } } // FIXME: test that ImagePull(json=true) send correct json output From 9fdd3cd967beaa871158b52edb78d2e899af0a00 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sat, 15 Feb 2014 21:10:37 -0800 Subject: [PATCH 108/403] Change name to listenbuffer Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: d5e41c1cb67b7ccb86fbbd80079642afd5055c2a Component: engine --- components/engine/api/api.go | 4 ++-- .../activation.go => listenbuffer/buffer.go} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename components/engine/pkg/{socketactivation/activation.go => listenbuffer/buffer.go} (85%) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index 8f6729ab61..8d7b1de685 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -10,7 +10,7 @@ import ( "fmt" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/engine" - "github.com/dotcloud/docker/pkg/socketactivation" + "github.com/dotcloud/docker/pkg/listenbuffer" "github.com/dotcloud/docker/pkg/systemd" "github.com/dotcloud/docker/utils" "github.com/gorilla/mux" @@ -1163,7 +1163,7 @@ func ListenAndServe(proto, addr string, eng *engine.Engine, logging, enableCors } } - l, err := socketactivation.NewActivationListener(proto, addr, activationLock, 15*time.Minute) + l, err := listenbuffer.NewListenBuffer(proto, addr, activationLock, 15*time.Minute) if err != nil { return err } diff --git a/components/engine/pkg/socketactivation/activation.go b/components/engine/pkg/listenbuffer/buffer.go similarity index 85% rename from components/engine/pkg/socketactivation/activation.go rename to components/engine/pkg/listenbuffer/buffer.go index 0edbcaac23..c350805a7d 100644 --- a/components/engine/pkg/socketactivation/activation.go +++ b/components/engine/pkg/listenbuffer/buffer.go @@ -3,7 +3,7 @@ listening on a socket, unix, tcp, udp but hold connections until the application has booted and is ready to accept them */ -package socketactivation +package listenbuffer import ( "fmt" @@ -11,9 +11,9 @@ import ( "time" ) -// NewActivationListener returns a listener listening on addr with the protocol. It sets the +// NewListenBuffer returns a listener listening on addr with the protocol. It sets the // timeout to wait on first connection before an error is returned -func NewActivationListener(proto, addr string, activate chan struct{}, timeout time.Duration) (net.Listener, error) { +func NewListenBuffer(proto, addr string, activate chan struct{}, timeout time.Duration) (net.Listener, error) { wrapped, err := net.Listen(proto, addr) if err != nil { return nil, err From 1bea948b6b7e8fc634fdee53c3c214dc767f7226 Mon Sep 17 00:00:00 2001 From: Fabio Falci Date: Sat, 15 Feb 2014 00:43:55 +0000 Subject: [PATCH 109/403] `docker cp` error when container doesn't exist Fix cp api to return a 404 notfound if container doesn't exist. Fixes #4119. Docker-DCO-1.1-Signed-off-by: Fabio Falci (github: fabiofalci) Upstream-commit: a51441278afbfde76ac1ec1e01c619fe5733b895 Component: engine --- components/engine/api/api.go | 3 +++ components/engine/api/client.go | 3 +++ components/engine/integration/api_test.go | 28 +++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index 2cfc75631e..2ba27288c0 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -943,6 +943,9 @@ func postContainersCopy(eng *engine.Engine, version float64, w http.ResponseWrit streamJSON(job, w, false) if err := job.Run(); err != nil { utils.Errorf("%s", err.Error()) + if strings.Contains(err.Error(), "No such container") { + w.WriteHeader(http.StatusNotFound) + } } return nil } diff --git a/components/engine/api/client.go b/components/engine/api/client.go index f1f336d5f4..61cabc8798 100644 --- a/components/engine/api/client.go +++ b/components/engine/api/client.go @@ -1961,6 +1961,9 @@ func (cli *DockerCli) CmdCp(args ...string) error { if stream != nil { defer stream.Close() } + if statusCode == 404 { + return fmt.Errorf("No such container: %v", info[0]) + } if err != nil { return err } diff --git a/components/engine/integration/api_test.go b/components/engine/integration/api_test.go index c587f111a2..9756ea1983 100644 --- a/components/engine/integration/api_test.go +++ b/components/engine/integration/api_test.go @@ -1217,6 +1217,34 @@ func TestPostContainersCopy(t *testing.T) { } } +func TestPostContainersCopyWhenContainerNotFound(t *testing.T) { + eng := NewTestEngine(t) + defer mkRuntimeFromEngine(eng, t).Nuke() + + r := httptest.NewRecorder() + + var copyData engine.Env + copyData.Set("Resource", "/test.txt") + copyData.Set("HostPath", ".") + + jsonData := bytes.NewBuffer(nil) + if err := copyData.Encode(jsonData); err != nil { + t.Fatal(err) + } + + req, err := http.NewRequest("POST", "/containers/id_not_found/copy", jsonData) + if err != nil { + t.Fatal(err) + } + req.Header.Add("Content-Type", "application/json") + if err := api.ServeRequest(eng, api.APIVERSION, r, req); err != nil { + t.Fatal(err) + } + if r.Code != http.StatusNotFound { + t.Fatalf("404 expected for id_not_found Container, received %v", r.Code) + } +} + // Mocked types for tests type NopConn struct { io.ReadCloser From ec297483f8bc7e7231cf7e0e23350158bb81f21d Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Mon, 17 Feb 2014 10:02:23 +1000 Subject: [PATCH 110/403] change the -H TCP / docker group warning to not reference a resolved issue Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 8871fd3a317240dbd6d5a9ae3309726281572bc1 Component: engine --- components/engine/docs/sources/use/basics.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/engine/docs/sources/use/basics.rst b/components/engine/docs/sources/use/basics.rst index 6bd1f0b7a0..d62f778b9d 100644 --- a/components/engine/docs/sources/use/basics.rst +++ b/components/engine/docs/sources/use/basics.rst @@ -59,10 +59,10 @@ Bind Docker to another host/port or a Unix socket .. warning:: Changing the default ``docker`` daemon binding to a TCP port or Unix *docker* user group will increase your security risks - by allowing non-root users to potentially gain *root* access on the - host (`e.g. #1369 - `_). Make sure you - control access to ``docker``. + by allowing non-root users to gain *root* access on the + host. Make sure you control access to ``docker``. If you are binding + to a TCP port, anyone with access to that port has full Docker access; + so it is not advisable on an open network. With ``-H`` it is possible to make the Docker daemon to listen on a specific IP and port. By default, it will listen on From 66f105510e64d81ba52a7e1a7b274c17d9c696ba Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 13 Feb 2014 14:48:55 +1000 Subject: [PATCH 111/403] add a little reenforcement to the help for docker build -q Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 797b7d3446304be01c220c29f3dc6c4dcc2711be Component: engine --- components/engine/api/client.go | 2 +- components/engine/docs/sources/reference/commandline/cli.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/api/client.go b/components/engine/api/client.go index f1f336d5f4..f3cdb456e3 100644 --- a/components/engine/api/client.go +++ b/components/engine/api/client.go @@ -136,7 +136,7 @@ func (cli *DockerCli) CmdInsert(args ...string) error { func (cli *DockerCli) CmdBuild(args ...string) error { cmd := cli.Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH") tag := cmd.String([]string{"t", "-tag"}, "", "Repository name (and optionally a tag) to be applied to the resulting image in case of success") - suppressOutput := cmd.Bool([]string{"q", "-quiet"}, false, "Suppress verbose build output") + suppressOutput := cmd.Bool([]string{"q", "-quiet"}, false, "Suppress the verbose output generated by the containers") noCache := cmd.Bool([]string{"#no-cache", "-no-cache"}, false, "Do not use cache when building the image") rm := cmd.Bool([]string{"#rm", "-rm"}, false, "Remove intermediate containers after a successful build") if err := cmd.Parse(args); err != nil { diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index 032076b941..191577eeec 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -186,7 +186,7 @@ Examples: Build a new container image from the source code at PATH -t, --time="": Repository name (and optionally a tag) to be applied to the resulting image in case of success. - -q, --quiet=false: Suppress verbose build output. + -q, --quiet=false: Suppress the verbose output generated by the containers. --no-cache: Do not use the cache when building the image. --rm: Remove intermediate containers after a successful build From e94461f78a72a69464ce62a968788e32d715c71e Mon Sep 17 00:00:00 2001 From: Fabio Falci Date: Mon, 17 Feb 2014 01:44:25 +0000 Subject: [PATCH 112/403] Test for save and load commands Improve test for save to tar and load from tar commands. Docker-DCO-1.1-Signed-off-by: Fabio Falci (github: fabiofalci) Upstream-commit: 4ab3c31b155997821d5f75689617235fa7386efa Component: engine --- components/engine/integration/api_test.go | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/components/engine/integration/api_test.go b/components/engine/integration/api_test.go index c587f111a2..73d6c4e78c 100644 --- a/components/engine/integration/api_test.go +++ b/components/engine/integration/api_test.go @@ -389,6 +389,77 @@ func TestGetContainersExport(t *testing.T) { } } +func TestSaveImageAndThenLoad(t *testing.T) { + eng := NewTestEngine(t) + defer mkRuntimeFromEngine(eng, t).Nuke() + + // save image + r := httptest.NewRecorder() + req, err := http.NewRequest("GET", "/images/"+unitTestImageID+"/get", nil) + if err != nil { + t.Fatal(err) + } + if err := api.ServeRequest(eng, api.APIVERSION, r, req); err != nil { + t.Fatal(err) + } + if r.Code != http.StatusOK { + t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code) + } + tarball := r.Body + + // delete the image + r = httptest.NewRecorder() + req, err = http.NewRequest("DELETE", "/images/"+unitTestImageID, nil) + if err != nil { + t.Fatal(err) + } + if err := api.ServeRequest(eng, api.APIVERSION, r, req); err != nil { + t.Fatal(err) + } + if r.Code != http.StatusOK { + t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code) + } + + // make sure there is no image + r = httptest.NewRecorder() + req, err = http.NewRequest("GET", "/images/"+unitTestImageID+"/get", nil) + if err != nil { + t.Fatal(err) + } + if err := api.ServeRequest(eng, api.APIVERSION, r, req); err != nil { + t.Fatal(err) + } + if r.Code != http.StatusNotFound { + t.Fatalf("%d NotFound expected, received %d\n", http.StatusNotFound, r.Code) + } + + // load the image + r = httptest.NewRecorder() + req, err = http.NewRequest("POST", "/images/load", tarball) + if err != nil { + t.Fatal(err) + } + if err := api.ServeRequest(eng, api.APIVERSION, r, req); err != nil { + t.Fatal(err) + } + if r.Code != http.StatusOK { + t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code) + } + + // finally make sure the image is there + r = httptest.NewRecorder() + req, err = http.NewRequest("GET", "/images/"+unitTestImageID+"/get", nil) + if err != nil { + t.Fatal(err) + } + if err := api.ServeRequest(eng, api.APIVERSION, r, req); err != nil { + t.Fatal(err) + } + if r.Code != http.StatusOK { + t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code) + } +} + func TestGetContainersChanges(t *testing.T) { eng := NewTestEngine(t) defer mkRuntimeFromEngine(eng, t).Nuke() From 4dc8b80613ae18f1d3c7dd67e56ce5163e2d0fa1 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Mon, 17 Feb 2014 12:24:19 +1000 Subject: [PATCH 113/403] add a little space above h2->h6 - it gets a bit dense in there when there are lots of steps Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 6a5a1d44f5d6c1281a66d562b7a4310dd6c3b997 Component: engine --- components/engine/docs/theme/docker/static/css/main.css | 3 +++ components/engine/docs/theme/docker/static/css/main.less | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/components/engine/docs/theme/docker/static/css/main.css b/components/engine/docs/theme/docker/static/css/main.css index 1195801542..ce4ba7b869 100755 --- a/components/engine/docs/theme/docker/static/css/main.css +++ b/components/engine/docs/theme/docker/static/css/main.css @@ -428,6 +428,9 @@ dt:hover > a.headerlink { float: right; visibility: hidden; } +h2, h3, h4, h5, h6 { + margin-top: 0.7em; +} /* ===================================== Miscellaneous information ====================================== */ diff --git a/components/engine/docs/theme/docker/static/css/main.less b/components/engine/docs/theme/docker/static/css/main.less index 8c9296d979..e248e21c08 100644 --- a/components/engine/docs/theme/docker/static/css/main.less +++ b/components/engine/docs/theme/docker/static/css/main.less @@ -631,6 +631,10 @@ dt:hover > a.headerlink { visibility: hidden; } +h2, h3, h4, h5, h6 { + margin-top: 0.7em; +} + /* ===================================== Miscellaneous information ====================================== */ From 6180b3b39051d971665acb2ecd8466a4162576d3 Mon Sep 17 00:00:00 2001 From: Mike MacCana Date: Tue, 11 Feb 2014 11:23:31 +0000 Subject: [PATCH 114/403] Add dockerode NodeJS implementation Also correctly case 'JavaScript' Docker-DCO-1.1-Signed-off-by: Mike MacCana (github: mikemaccana) Upstream-commit: 02d94dcf6ec2541c66b081d4dc251725f994747c Component: engine --- .../reference/api/remote_api_client_libraries.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/components/engine/docs/sources/reference/api/remote_api_client_libraries.rst b/components/engine/docs/sources/reference/api/remote_api_client_libraries.rst index c7ced0055e..362fa6fe3d 100644 --- a/components/engine/docs/sources/reference/api/remote_api_client_libraries.rst +++ b/components/engine/docs/sources/reference/api/remote_api_client_libraries.rst @@ -1,6 +1,6 @@ :title: Remote API Client Libraries :description: Various client libraries available to use with the Docker remote API -:keywords: API, Docker, index, registry, REST, documentation, clients, Python, Ruby, Javascript, Erlang, Go +:keywords: API, Docker, index, registry, REST, documentation, clients, Python, Ruby, JavaScript, Erlang, Go ================================== @@ -21,15 +21,18 @@ and we will add the libraries here. +----------------------+----------------+--------------------------------------------+----------+ | Ruby | docker-api | https://github.com/swipely/docker-api | Active | +----------------------+----------------+--------------------------------------------+----------+ -| Javascript (NodeJS) | docker.io | https://github.com/appersonlabs/docker.io | Active | +| JavaScript (NodeJS) | dockerode | https://github.com/apocas/dockerode | Active | +| | | Install via NPM: `npm install dockerode` | | ++----------------------+----------------+--------------------------------------------+----------+ +| JavaScript (NodeJS) | docker.io | https://github.com/appersonlabs/docker.io | Active | | | | Install via NPM: `npm install docker.io` | | +----------------------+----------------+--------------------------------------------+----------+ -| Javascript | docker-js | https://github.com/dgoujard/docker-js | Active | +| JavaScript | docker-js | https://github.com/dgoujard/docker-js | Active | +----------------------+----------------+--------------------------------------------+----------+ -| Javascript (Angular) | docker-cp | https://github.com/13W/docker-cp | Active | +| JavaScript (Angular) | docker-cp | https://github.com/13W/docker-cp | Active | | **WebUI** | | | | +----------------------+----------------+--------------------------------------------+----------+ -| Javascript (Angular) | dockerui | https://github.com/crosbymichael/dockerui | Active | +| JavaScript (Angular) | dockerui | https://github.com/crosbymichael/dockerui | Active | | **WebUI** | | | | +----------------------+----------------+--------------------------------------------+----------+ | Java | docker-java | https://github.com/kpelykh/docker-java | Active | From de5db2558539e45d452e5c5be13f88acba3951f6 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 17 Feb 2014 16:10:54 +0100 Subject: [PATCH 115/403] Avoid temporarily unmounting the container when restarting it Stopping the container will typicall cause it to unmount, to keep it mounted over the stop/start cycle we aquire a temporary reference to it during this time. This helps with https://github.com/dotcloud/docker/issues/4036 Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: ab0f3f86c8d05aa69415e6909a88b621d5b37e81 Component: engine --- components/engine/container.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/engine/container.go b/components/engine/container.go index 3740a7fb73..f55b8c3c7b 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -1256,6 +1256,13 @@ func (container *Container) Stop(seconds int) error { } func (container *Container) Restart(seconds int) error { + // Avoid unnecessarily unmounting and then directly mounting + // the container when the container stops and then starts + // again + if err := container.Mount(); err == nil { + defer container.Unmount() + } + if err := container.Stop(seconds); err != nil { return err } From fc1f23f527dc7d12df58fc8726c7c329909ace98 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 13 Feb 2014 15:11:39 -0700 Subject: [PATCH 116/403] Add vendored archive/tar that includes xattrs patch Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: b762d3a7391dd526826a0a1f1c983ea5e54852f5 Component: engine --- components/engine/archive/archive.go | 2 +- components/engine/archive/archive_test.go | 2 +- components/engine/archive/changes.go | 2 +- components/engine/archive/diff.go | 2 +- components/engine/archive/wrap.go | 2 +- components/engine/hack/vendor.sh | 8 + components/engine/integration/api_test.go | 2 +- components/engine/integration/utils_test.go | 2 +- components/engine/utils/tarsum.go | 2 +- components/engine/utils_test.go | 2 +- .../p/go/src/pkg/archive/tar/common.go | 304 ++++++++++++ .../p/go/src/pkg/archive/tar/example_test.go | 79 ++++ .../p/go/src/pkg/archive/tar/reader.go | 402 ++++++++++++++++ .../p/go/src/pkg/archive/tar/reader_test.go | 425 +++++++++++++++++ .../p/go/src/pkg/archive/tar/stat_atim.go | 20 + .../go/src/pkg/archive/tar/stat_atimespec.go | 20 + .../p/go/src/pkg/archive/tar/stat_unix.go | 32 ++ .../p/go/src/pkg/archive/tar/tar_test.go | 284 ++++++++++++ .../p/go/src/pkg/archive/tar/testdata/gnu.tar | Bin 0 -> 3072 bytes .../src/pkg/archive/tar/testdata/nil-uid.tar | Bin 0 -> 1024 bytes .../p/go/src/pkg/archive/tar/testdata/pax.tar | Bin 0 -> 10240 bytes .../go/src/pkg/archive/tar/testdata/small.txt | 1 + .../src/pkg/archive/tar/testdata/small2.txt | 1 + .../go/src/pkg/archive/tar/testdata/star.tar | Bin 0 -> 3072 bytes .../go/src/pkg/archive/tar/testdata/ustar.tar | Bin 0 -> 2048 bytes .../p/go/src/pkg/archive/tar/testdata/v7.tar | Bin 0 -> 3584 bytes .../pkg/archive/tar/testdata/writer-big.tar | Bin 0 -> 4096 bytes .../src/pkg/archive/tar/testdata/writer.tar | Bin 0 -> 3584 bytes .../src/pkg/archive/tar/testdata/xattrs.tar | Bin 0 -> 5120 bytes .../p/go/src/pkg/archive/tar/writer.go | 383 ++++++++++++++++ .../p/go/src/pkg/archive/tar/writer_test.go | 433 ++++++++++++++++++ 31 files changed, 2401 insertions(+), 9 deletions(-) create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/common.go create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/example_test.go create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader_test.go create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_atim.go create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_atimespec.go create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_unix.go create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/tar_test.go create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/gnu.tar create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/nil-uid.tar create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/pax.tar create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/small.txt create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/small2.txt create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/star.tar create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/ustar.tar create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/v7.tar create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/writer-big.tar create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/writer.tar create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/xattrs.tar create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go create mode 100644 components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer_test.go diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index 3bd3af2761..8212621bb9 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -1,8 +1,8 @@ package archive import ( - "archive/tar" "bytes" + "code.google.com/p/go/src/pkg/archive/tar" "compress/bzip2" "compress/gzip" "errors" diff --git a/components/engine/archive/archive_test.go b/components/engine/archive/archive_test.go index 164fc8fe27..8badd58bd7 100644 --- a/components/engine/archive/archive_test.go +++ b/components/engine/archive/archive_test.go @@ -1,8 +1,8 @@ package archive import ( - "archive/tar" "bytes" + "code.google.com/p/go/src/pkg/archive/tar" "fmt" "io" "io/ioutil" diff --git a/components/engine/archive/changes.go b/components/engine/archive/changes.go index 25406f5cec..b46b13bbe7 100644 --- a/components/engine/archive/changes.go +++ b/components/engine/archive/changes.go @@ -1,7 +1,7 @@ package archive import ( - "archive/tar" + "code.google.com/p/go/src/pkg/archive/tar" "fmt" "github.com/dotcloud/docker/utils" "io" diff --git a/components/engine/archive/diff.go b/components/engine/archive/diff.go index de1efacf34..de93fc69b2 100644 --- a/components/engine/archive/diff.go +++ b/components/engine/archive/diff.go @@ -1,7 +1,7 @@ package archive import ( - "archive/tar" + "code.google.com/p/go/src/pkg/archive/tar" "io" "os" "path/filepath" diff --git a/components/engine/archive/wrap.go b/components/engine/archive/wrap.go index dfb335c0b6..981420b3fe 100644 --- a/components/engine/archive/wrap.go +++ b/components/engine/archive/wrap.go @@ -1,8 +1,8 @@ package archive import ( - "archive/tar" "bytes" + "code.google.com/p/go/src/pkg/archive/tar" "io/ioutil" ) diff --git a/components/engine/hack/vendor.sh b/components/engine/hack/vendor.sh index d3e7ea9f43..184cb750a5 100755 --- a/components/engine/hack/vendor.sh +++ b/components/engine/hack/vendor.sh @@ -50,3 +50,11 @@ clone git github.com/syndtr/gocapability 3454319be2 clone hg code.google.com/p/go.net 84a4013f96e0 clone hg code.google.com/p/gosqlite 74691fb6f837 + +# get Go tip's archive/tar, for xattr support +# TODO after Go 1.3 drops, bump our minimum supported version and drop this vendored dep +clone hg code.google.com/p/go a15f344a9efa +mv src/code.google.com/p/go/src/pkg/archive/tar tmp-tar +rm -rf src/code.google.com/p/go +mkdir -p src/code.google.com/p/go/src/pkg/archive +mv tmp-tar src/code.google.com/p/go/src/pkg/archive/tar diff --git a/components/engine/integration/api_test.go b/components/engine/integration/api_test.go index c587f111a2..f0d0a7d097 100644 --- a/components/engine/integration/api_test.go +++ b/components/engine/integration/api_test.go @@ -1,9 +1,9 @@ package docker import ( - "archive/tar" "bufio" "bytes" + "code.google.com/p/go/src/pkg/archive/tar" "encoding/json" "fmt" "github.com/dotcloud/docker" diff --git a/components/engine/integration/utils_test.go b/components/engine/integration/utils_test.go index 77328b2511..947ace11d9 100644 --- a/components/engine/integration/utils_test.go +++ b/components/engine/integration/utils_test.go @@ -1,8 +1,8 @@ package docker import ( - "archive/tar" "bytes" + "code.google.com/p/go/src/pkg/archive/tar" "fmt" "io" "io/ioutil" diff --git a/components/engine/utils/tarsum.go b/components/engine/utils/tarsum.go index 786196b6b4..ddeecfb450 100644 --- a/components/engine/utils/tarsum.go +++ b/components/engine/utils/tarsum.go @@ -1,8 +1,8 @@ package utils import ( - "archive/tar" "bytes" + "code.google.com/p/go/src/pkg/archive/tar" "compress/gzip" "crypto/sha256" "encoding/hex" diff --git a/components/engine/utils_test.go b/components/engine/utils_test.go index 4b8cfba39f..6917007575 100644 --- a/components/engine/utils_test.go +++ b/components/engine/utils_test.go @@ -1,8 +1,8 @@ package docker import ( - "archive/tar" "bytes" + "code.google.com/p/go/src/pkg/archive/tar" "io" ) diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/common.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/common.go new file mode 100644 index 0000000000..e8b973c1fa --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/common.go @@ -0,0 +1,304 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tar implements access to tar archives. +// It aims to cover most of the variations, including those produced +// by GNU and BSD tars. +// +// References: +// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5 +// http://www.gnu.org/software/tar/manual/html_node/Standard.html +// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html +package tar + +import ( + "bytes" + "errors" + "fmt" + "os" + "path" + "time" +) + +const ( + blockSize = 512 + + // Types + TypeReg = '0' // regular file + TypeRegA = '\x00' // regular file + TypeLink = '1' // hard link + TypeSymlink = '2' // symbolic link + TypeChar = '3' // character device node + TypeBlock = '4' // block device node + TypeDir = '5' // directory + TypeFifo = '6' // fifo node + TypeCont = '7' // reserved + TypeXHeader = 'x' // extended header + TypeXGlobalHeader = 'g' // global extended header + TypeGNULongName = 'L' // Next file has a long name + TypeGNULongLink = 'K' // Next file symlinks to a file w/ a long name +) + +// A Header represents a single header in a tar archive. +// Some fields may not be populated. +type Header struct { + Name string // name of header file entry + Mode int64 // permission and mode bits + Uid int // user id of owner + Gid int // group id of owner + Size int64 // length in bytes + ModTime time.Time // modified time + Typeflag byte // type of header entry + Linkname string // target name of link + Uname string // user name of owner + Gname string // group name of owner + Devmajor int64 // major number of character or block device + Devminor int64 // minor number of character or block device + AccessTime time.Time // access time + ChangeTime time.Time // status change time + Xattrs map[string]string +} + +// File name constants from the tar spec. +const ( + fileNameSize = 100 // Maximum number of bytes in a standard tar name. + fileNamePrefixSize = 155 // Maximum number of ustar extension bytes. +) + +// FileInfo returns an os.FileInfo for the Header. +func (h *Header) FileInfo() os.FileInfo { + return headerFileInfo{h} +} + +// headerFileInfo implements os.FileInfo. +type headerFileInfo struct { + h *Header +} + +func (fi headerFileInfo) Size() int64 { return fi.h.Size } +func (fi headerFileInfo) IsDir() bool { return fi.Mode().IsDir() } +func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime } +func (fi headerFileInfo) Sys() interface{} { return fi.h } + +// Name returns the base name of the file. +func (fi headerFileInfo) Name() string { + if fi.IsDir() { + return path.Base(path.Clean(fi.h.Name)) + } + return path.Base(fi.h.Name) +} + +// Mode returns the permission and mode bits for the headerFileInfo. +func (fi headerFileInfo) Mode() (mode os.FileMode) { + // Set file permission bits. + mode = os.FileMode(fi.h.Mode).Perm() + + // Set setuid, setgid and sticky bits. + if fi.h.Mode&c_ISUID != 0 { + // setuid + mode |= os.ModeSetuid + } + if fi.h.Mode&c_ISGID != 0 { + // setgid + mode |= os.ModeSetgid + } + if fi.h.Mode&c_ISVTX != 0 { + // sticky + mode |= os.ModeSticky + } + + // Set file mode bits. + // clear perm, setuid, setgid and sticky bits. + m := os.FileMode(fi.h.Mode) &^ 07777 + if m == c_ISDIR { + // directory + mode |= os.ModeDir + } + if m == c_ISFIFO { + // named pipe (FIFO) + mode |= os.ModeNamedPipe + } + if m == c_ISLNK { + // symbolic link + mode |= os.ModeSymlink + } + if m == c_ISBLK { + // device file + mode |= os.ModeDevice + } + if m == c_ISCHR { + // Unix character device + mode |= os.ModeDevice + mode |= os.ModeCharDevice + } + if m == c_ISSOCK { + // Unix domain socket + mode |= os.ModeSocket + } + + switch fi.h.Typeflag { + case TypeLink, TypeSymlink: + // hard link, symbolic link + mode |= os.ModeSymlink + case TypeChar: + // character device node + mode |= os.ModeDevice + mode |= os.ModeCharDevice + case TypeBlock: + // block device node + mode |= os.ModeDevice + case TypeDir: + // directory + mode |= os.ModeDir + case TypeFifo: + // fifo node + mode |= os.ModeNamedPipe + } + + return mode +} + +// sysStat, if non-nil, populates h from system-dependent fields of fi. +var sysStat func(fi os.FileInfo, h *Header) error + +// Mode constants from the tar spec. +const ( + c_ISUID = 04000 // Set uid + c_ISGID = 02000 // Set gid + c_ISVTX = 01000 // Save text (sticky bit) + c_ISDIR = 040000 // Directory + c_ISFIFO = 010000 // FIFO + c_ISREG = 0100000 // Regular file + c_ISLNK = 0120000 // Symbolic link + c_ISBLK = 060000 // Block special file + c_ISCHR = 020000 // Character special file + c_ISSOCK = 0140000 // Socket +) + +// Keywords for the PAX Extended Header +const ( + paxAtime = "atime" + paxCharset = "charset" + paxComment = "comment" + paxCtime = "ctime" // please note that ctime is not a valid pax header. + paxGid = "gid" + paxGname = "gname" + paxLinkpath = "linkpath" + paxMtime = "mtime" + paxPath = "path" + paxSize = "size" + paxUid = "uid" + paxUname = "uname" + paxXattr = "SCHILY.xattr." + paxNone = "" +) + +// FileInfoHeader creates a partially-populated Header from fi. +// If fi describes a symlink, FileInfoHeader records link as the link target. +// If fi describes a directory, a slash is appended to the name. +// Because os.FileInfo's Name method returns only the base name of +// the file it describes, it may be necessary to modify the Name field +// of the returned header to provide the full path name of the file. +func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { + if fi == nil { + return nil, errors.New("tar: FileInfo is nil") + } + fm := fi.Mode() + h := &Header{ + Name: fi.Name(), + ModTime: fi.ModTime(), + Mode: int64(fm.Perm()), // or'd with c_IS* constants later + } + switch { + case fm.IsRegular(): + h.Mode |= c_ISREG + h.Typeflag = TypeReg + h.Size = fi.Size() + case fi.IsDir(): + h.Typeflag = TypeDir + h.Mode |= c_ISDIR + h.Name += "/" + case fm&os.ModeSymlink != 0: + h.Typeflag = TypeSymlink + h.Mode |= c_ISLNK + h.Linkname = link + case fm&os.ModeDevice != 0: + if fm&os.ModeCharDevice != 0 { + h.Mode |= c_ISCHR + h.Typeflag = TypeChar + } else { + h.Mode |= c_ISBLK + h.Typeflag = TypeBlock + } + case fm&os.ModeNamedPipe != 0: + h.Typeflag = TypeFifo + h.Mode |= c_ISFIFO + case fm&os.ModeSocket != 0: + h.Mode |= c_ISSOCK + default: + return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm) + } + if fm&os.ModeSetuid != 0 { + h.Mode |= c_ISUID + } + if fm&os.ModeSetgid != 0 { + h.Mode |= c_ISGID + } + if fm&os.ModeSticky != 0 { + h.Mode |= c_ISVTX + } + if sysStat != nil { + return h, sysStat(fi, h) + } + return h, nil +} + +var zeroBlock = make([]byte, blockSize) + +// POSIX specifies a sum of the unsigned byte values, but the Sun tar uses signed byte values. +// We compute and return both. +func checksum(header []byte) (unsigned int64, signed int64) { + for i := 0; i < len(header); i++ { + if i == 148 { + // The chksum field (header[148:156]) is special: it should be treated as space bytes. + unsigned += ' ' * 8 + signed += ' ' * 8 + i += 7 + continue + } + unsigned += int64(header[i]) + signed += int64(int8(header[i])) + } + return +} + +type slicer []byte + +func (sp *slicer) next(n int) (b []byte) { + s := *sp + b, *sp = s[0:n], s[n:] + return +} + +func isASCII(s string) bool { + for _, c := range s { + if c >= 0x80 { + return false + } + } + return true +} + +func toASCII(s string) string { + if isASCII(s) { + return s + } + var buf bytes.Buffer + for _, c := range s { + if c < 0x80 { + buf.WriteByte(byte(c)) + } + } + return buf.String() +} diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/example_test.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/example_test.go new file mode 100644 index 0000000000..351eaa0e6c --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/example_test.go @@ -0,0 +1,79 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar_test + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "log" + "os" +) + +func Example() { + // Create a buffer to write our archive to. + buf := new(bytes.Buffer) + + // Create a new tar archive. + tw := tar.NewWriter(buf) + + // Add some files to the archive. + var files = []struct { + Name, Body string + }{ + {"readme.txt", "This archive contains some text files."}, + {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"}, + {"todo.txt", "Get animal handling licence."}, + } + for _, file := range files { + hdr := &tar.Header{ + Name: file.Name, + Size: int64(len(file.Body)), + } + if err := tw.WriteHeader(hdr); err != nil { + log.Fatalln(err) + } + if _, err := tw.Write([]byte(file.Body)); err != nil { + log.Fatalln(err) + } + } + // Make sure to check the error on Close. + if err := tw.Close(); err != nil { + log.Fatalln(err) + } + + // Open the tar archive for reading. + r := bytes.NewReader(buf.Bytes()) + tr := tar.NewReader(r) + + // Iterate through the files in the archive. + for { + hdr, err := tr.Next() + if err == io.EOF { + // end of tar archive + break + } + if err != nil { + log.Fatalln(err) + } + fmt.Printf("Contents of %s:\n", hdr.Name) + if _, err := io.Copy(os.Stdout, tr); err != nil { + log.Fatalln(err) + } + fmt.Println() + } + + // Output: + // Contents of readme.txt: + // This archive contains some text files. + // Contents of gopher.txt: + // Gopher names: + // George + // Geoffrey + // Gonzo + // Contents of todo.txt: + // Get animal handling licence. +} diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go new file mode 100644 index 0000000000..7cb6e649c7 --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go @@ -0,0 +1,402 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +// TODO(dsymonds): +// - pax extensions + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "os" + "strconv" + "strings" + "time" +) + +var ( + ErrHeader = errors.New("archive/tar: invalid tar header") +) + +const maxNanoSecondIntSize = 9 + +// A Reader provides sequential access to the contents of a tar archive. +// A tar archive consists of a sequence of files. +// The Next method advances to the next file in the archive (including the first), +// and then it can be treated as an io.Reader to access the file's data. +type Reader struct { + r io.Reader + err error + nb int64 // number of unread bytes for current file entry + pad int64 // amount of padding (ignored) after current file entry +} + +// NewReader creates a new Reader reading from r. +func NewReader(r io.Reader) *Reader { return &Reader{r: r} } + +// Next advances to the next entry in the tar archive. +func (tr *Reader) Next() (*Header, error) { + var hdr *Header + if tr.err == nil { + tr.skipUnread() + } + if tr.err != nil { + return hdr, tr.err + } + hdr = tr.readHeader() + if hdr == nil { + return hdr, tr.err + } + // Check for PAX/GNU header. + switch hdr.Typeflag { + case TypeXHeader: + // PAX extended header + headers, err := parsePAX(tr) + if err != nil { + return nil, err + } + // We actually read the whole file, + // but this skips alignment padding + tr.skipUnread() + hdr = tr.readHeader() + mergePAX(hdr, headers) + return hdr, nil + case TypeGNULongName: + // We have a GNU long name header. Its contents are the real file name. + realname, err := ioutil.ReadAll(tr) + if err != nil { + return nil, err + } + hdr, err := tr.Next() + hdr.Name = cString(realname) + return hdr, err + case TypeGNULongLink: + // We have a GNU long link header. + realname, err := ioutil.ReadAll(tr) + if err != nil { + return nil, err + } + hdr, err := tr.Next() + hdr.Linkname = cString(realname) + return hdr, err + } + return hdr, tr.err +} + +// mergePAX merges well known headers according to PAX standard. +// In general headers with the same name as those found +// in the header struct overwrite those found in the header +// struct with higher precision or longer values. Esp. useful +// for name and linkname fields. +func mergePAX(hdr *Header, headers map[string]string) error { + for k, v := range headers { + switch k { + case paxPath: + hdr.Name = v + case paxLinkpath: + hdr.Linkname = v + case paxGname: + hdr.Gname = v + case paxUname: + hdr.Uname = v + case paxUid: + uid, err := strconv.ParseInt(v, 10, 0) + if err != nil { + return err + } + hdr.Uid = int(uid) + case paxGid: + gid, err := strconv.ParseInt(v, 10, 0) + if err != nil { + return err + } + hdr.Gid = int(gid) + case paxAtime: + t, err := parsePAXTime(v) + if err != nil { + return err + } + hdr.AccessTime = t + case paxMtime: + t, err := parsePAXTime(v) + if err != nil { + return err + } + hdr.ModTime = t + case paxCtime: + t, err := parsePAXTime(v) + if err != nil { + return err + } + hdr.ChangeTime = t + case paxSize: + size, err := strconv.ParseInt(v, 10, 0) + if err != nil { + return err + } + hdr.Size = int64(size) + default: + if strings.HasPrefix(k, paxXattr) { + if hdr.Xattrs == nil { + hdr.Xattrs = make(map[string]string) + } + hdr.Xattrs[k[len(paxXattr):]] = v + } + } + } + return nil +} + +// parsePAXTime takes a string of the form %d.%d as described in +// the PAX specification. +func parsePAXTime(t string) (time.Time, error) { + buf := []byte(t) + pos := bytes.IndexByte(buf, '.') + var seconds, nanoseconds int64 + var err error + if pos == -1 { + seconds, err = strconv.ParseInt(t, 10, 0) + if err != nil { + return time.Time{}, err + } + } else { + seconds, err = strconv.ParseInt(string(buf[:pos]), 10, 0) + if err != nil { + return time.Time{}, err + } + nano_buf := string(buf[pos+1:]) + // Pad as needed before converting to a decimal. + // For example .030 -> .030000000 -> 30000000 nanoseconds + if len(nano_buf) < maxNanoSecondIntSize { + // Right pad + nano_buf += strings.Repeat("0", maxNanoSecondIntSize-len(nano_buf)) + } else if len(nano_buf) > maxNanoSecondIntSize { + // Right truncate + nano_buf = nano_buf[:maxNanoSecondIntSize] + } + nanoseconds, err = strconv.ParseInt(string(nano_buf), 10, 0) + if err != nil { + return time.Time{}, err + } + } + ts := time.Unix(seconds, nanoseconds) + return ts, nil +} + +// parsePAX parses PAX headers. +// If an extended header (type 'x') is invalid, ErrHeader is returned +func parsePAX(r io.Reader) (map[string]string, error) { + buf, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + headers := make(map[string]string) + // Each record is constructed as + // "%d %s=%s\n", length, keyword, value + for len(buf) > 0 { + // or the header was empty to start with. + var sp int + // The size field ends at the first space. + sp = bytes.IndexByte(buf, ' ') + if sp == -1 { + return nil, ErrHeader + } + // Parse the first token as a decimal integer. + n, err := strconv.ParseInt(string(buf[:sp]), 10, 0) + if err != nil { + return nil, ErrHeader + } + // Extract everything between the decimal and the n -1 on the + // beginning to to eat the ' ', -1 on the end to skip the newline. + var record []byte + record, buf = buf[sp+1:n-1], buf[n:] + // The first equals is guaranteed to mark the end of the key. + // Everything else is value. + eq := bytes.IndexByte(record, '=') + if eq == -1 { + return nil, ErrHeader + } + key, value := record[:eq], record[eq+1:] + headers[string(key)] = string(value) + } + return headers, nil +} + +// cString parses bytes as a NUL-terminated C-style string. +// If a NUL byte is not found then the whole slice is returned as a string. +func cString(b []byte) string { + n := 0 + for n < len(b) && b[n] != 0 { + n++ + } + return string(b[0:n]) +} + +func (tr *Reader) octal(b []byte) int64 { + // Check for binary format first. + if len(b) > 0 && b[0]&0x80 != 0 { + var x int64 + for i, c := range b { + if i == 0 { + c &= 0x7f // ignore signal bit in first byte + } + x = x<<8 | int64(c) + } + return x + } + + // Because unused fields are filled with NULs, we need + // to skip leading NULs. Fields may also be padded with + // spaces or NULs. + // So we remove leading and trailing NULs and spaces to + // be sure. + b = bytes.Trim(b, " \x00") + + if len(b) == 0 { + return 0 + } + x, err := strconv.ParseUint(cString(b), 8, 64) + if err != nil { + tr.err = err + } + return int64(x) +} + +// skipUnread skips any unread bytes in the existing file entry, as well as any alignment padding. +func (tr *Reader) skipUnread() { + nr := tr.nb + tr.pad // number of bytes to skip + tr.nb, tr.pad = 0, 0 + if sr, ok := tr.r.(io.Seeker); ok { + if _, err := sr.Seek(nr, os.SEEK_CUR); err == nil { + return + } + } + _, tr.err = io.CopyN(ioutil.Discard, tr.r, nr) +} + +func (tr *Reader) verifyChecksum(header []byte) bool { + if tr.err != nil { + return false + } + + given := tr.octal(header[148:156]) + unsigned, signed := checksum(header) + return given == unsigned || given == signed +} + +func (tr *Reader) readHeader() *Header { + header := make([]byte, blockSize) + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { + return nil + } + + // Two blocks of zero bytes marks the end of the archive. + if bytes.Equal(header, zeroBlock[0:blockSize]) { + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { + return nil + } + if bytes.Equal(header, zeroBlock[0:blockSize]) { + tr.err = io.EOF + } else { + tr.err = ErrHeader // zero block and then non-zero block + } + return nil + } + + if !tr.verifyChecksum(header) { + tr.err = ErrHeader + return nil + } + + // Unpack + hdr := new(Header) + s := slicer(header) + + hdr.Name = cString(s.next(100)) + hdr.Mode = tr.octal(s.next(8)) + hdr.Uid = int(tr.octal(s.next(8))) + hdr.Gid = int(tr.octal(s.next(8))) + hdr.Size = tr.octal(s.next(12)) + hdr.ModTime = time.Unix(tr.octal(s.next(12)), 0) + s.next(8) // chksum + hdr.Typeflag = s.next(1)[0] + hdr.Linkname = cString(s.next(100)) + + // The remainder of the header depends on the value of magic. + // The original (v7) version of tar had no explicit magic field, + // so its magic bytes, like the rest of the block, are NULs. + magic := string(s.next(8)) // contains version field as well. + var format string + switch magic { + case "ustar\x0000": // POSIX tar (1003.1-1988) + if string(header[508:512]) == "tar\x00" { + format = "star" + } else { + format = "posix" + } + case "ustar \x00": // old GNU tar + format = "gnu" + } + + switch format { + case "posix", "gnu", "star": + hdr.Uname = cString(s.next(32)) + hdr.Gname = cString(s.next(32)) + devmajor := s.next(8) + devminor := s.next(8) + if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock { + hdr.Devmajor = tr.octal(devmajor) + hdr.Devminor = tr.octal(devminor) + } + var prefix string + switch format { + case "posix", "gnu": + prefix = cString(s.next(155)) + case "star": + prefix = cString(s.next(131)) + hdr.AccessTime = time.Unix(tr.octal(s.next(12)), 0) + hdr.ChangeTime = time.Unix(tr.octal(s.next(12)), 0) + } + if len(prefix) > 0 { + hdr.Name = prefix + "/" + hdr.Name + } + } + + if tr.err != nil { + tr.err = ErrHeader + return nil + } + + // Maximum value of hdr.Size is 64 GB (12 octal digits), + // so there's no risk of int64 overflowing. + tr.nb = int64(hdr.Size) + tr.pad = -tr.nb & (blockSize - 1) // blockSize is a power of two + + return hdr +} + +// Read reads from the current entry in the tar archive. +// It returns 0, io.EOF when it reaches the end of that entry, +// until Next is called to advance to the next entry. +func (tr *Reader) Read(b []byte) (n int, err error) { + if tr.nb == 0 { + // file consumed + return 0, io.EOF + } + + if int64(len(b)) > tr.nb { + b = b[0:tr.nb] + } + n, err = tr.r.Read(b) + tr.nb -= int64(n) + + if err == io.EOF && tr.nb > 0 { + err = io.ErrUnexpectedEOF + } + tr.err = err + return +} diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader_test.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader_test.go new file mode 100644 index 0000000000..f84dbebe98 --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader_test.go @@ -0,0 +1,425 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + "os" + "reflect" + "strings" + "testing" + "time" +) + +type untarTest struct { + file string + headers []*Header + cksums []string +} + +var gnuTarTest = &untarTest{ + file: "testdata/gnu.tar", + headers: []*Header{ + { + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + ModTime: time.Unix(1244428340, 0), + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + { + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + ModTime: time.Unix(1244436044, 0), + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + }, + cksums: []string{ + "e38b27eaccb4391bdec553a7f3ae6b2f", + "c65bd2e50a56a2138bf1716f2fd56fe9", + }, +} + +var untarTests = []*untarTest{ + gnuTarTest, + { + file: "testdata/star.tar", + headers: []*Header{ + { + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + ModTime: time.Unix(1244592783, 0), + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + AccessTime: time.Unix(1244592783, 0), + ChangeTime: time.Unix(1244592783, 0), + }, + { + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + ModTime: time.Unix(1244592783, 0), + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + AccessTime: time.Unix(1244592783, 0), + ChangeTime: time.Unix(1244592783, 0), + }, + }, + }, + { + file: "testdata/v7.tar", + headers: []*Header{ + { + Name: "small.txt", + Mode: 0444, + Uid: 73025, + Gid: 5000, + Size: 5, + ModTime: time.Unix(1244593104, 0), + Typeflag: '\x00', + }, + { + Name: "small2.txt", + Mode: 0444, + Uid: 73025, + Gid: 5000, + Size: 11, + ModTime: time.Unix(1244593104, 0), + Typeflag: '\x00', + }, + }, + }, + { + file: "testdata/pax.tar", + headers: []*Header{ + { + Name: "a/123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", + Mode: 0664, + Uid: 1000, + Gid: 1000, + Uname: "shane", + Gname: "shane", + Size: 7, + ModTime: time.Unix(1350244992, 23960108), + ChangeTime: time.Unix(1350244992, 23960108), + AccessTime: time.Unix(1350244992, 23960108), + Typeflag: TypeReg, + }, + { + Name: "a/b", + Mode: 0777, + Uid: 1000, + Gid: 1000, + Uname: "shane", + Gname: "shane", + Size: 0, + ModTime: time.Unix(1350266320, 910238425), + ChangeTime: time.Unix(1350266320, 910238425), + AccessTime: time.Unix(1350266320, 910238425), + Typeflag: TypeSymlink, + Linkname: "123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100", + }, + }, + }, + { + file: "testdata/nil-uid.tar", // golang.org/issue/5290 + headers: []*Header{ + { + Name: "P1050238.JPG.log", + Mode: 0664, + Uid: 0, + Gid: 0, + Size: 14, + ModTime: time.Unix(1365454838, 0), + Typeflag: TypeReg, + Linkname: "", + Uname: "eyefi", + Gname: "eyefi", + Devmajor: 0, + Devminor: 0, + }, + }, + }, + { + file: "testdata/xattrs.tar", + headers: []*Header{ + { + Name: "small.txt", + Mode: 0644, + Uid: 1000, + Gid: 10, + Size: 5, + ModTime: time.Unix(1386065770, 448252320), + Typeflag: '0', + Uname: "alex", + Gname: "wheel", + AccessTime: time.Unix(1389782991, 419875220), + ChangeTime: time.Unix(1389782956, 794414986), + Xattrs: map[string]string{ + "user.key": "value", + "user.key2": "value2", + // Interestingly, selinux encodes the terminating null inside the xattr + "security.selinux": "unconfined_u:object_r:default_t:s0\x00", + }, + }, + { + Name: "small2.txt", + Mode: 0644, + Uid: 1000, + Gid: 10, + Size: 11, + ModTime: time.Unix(1386065770, 449252304), + Typeflag: '0', + Uname: "alex", + Gname: "wheel", + AccessTime: time.Unix(1389782991, 419875220), + ChangeTime: time.Unix(1386065770, 449252304), + Xattrs: map[string]string{ + "security.selinux": "unconfined_u:object_r:default_t:s0\x00", + }, + }, + }, + }, +} + +func TestReader(t *testing.T) { +testLoop: + for i, test := range untarTests { + f, err := os.Open(test.file) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + continue + } + defer f.Close() + tr := NewReader(f) + for j, header := range test.headers { + hdr, err := tr.Next() + if err != nil || hdr == nil { + t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err) + f.Close() + continue testLoop + } + if !reflect.DeepEqual(*hdr, *header) { + t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v", + i, j, *hdr, *header) + } + } + hdr, err := tr.Next() + if err == io.EOF { + continue testLoop + } + if hdr != nil || err != nil { + t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, hdr, err) + } + } +} + +func TestPartialRead(t *testing.T) { + f, err := os.Open("testdata/gnu.tar") + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + tr := NewReader(f) + + // Read the first four bytes; Next() should skip the last byte. + hdr, err := tr.Next() + if err != nil || hdr == nil { + t.Fatalf("Didn't get first file: %v", err) + } + buf := make([]byte, 4) + if _, err := io.ReadFull(tr, buf); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if expected := []byte("Kilt"); !bytes.Equal(buf, expected) { + t.Errorf("Contents = %v, want %v", buf, expected) + } + + // Second file + hdr, err = tr.Next() + if err != nil || hdr == nil { + t.Fatalf("Didn't get second file: %v", err) + } + buf = make([]byte, 6) + if _, err := io.ReadFull(tr, buf); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if expected := []byte("Google"); !bytes.Equal(buf, expected) { + t.Errorf("Contents = %v, want %v", buf, expected) + } +} + +func TestIncrementalRead(t *testing.T) { + test := gnuTarTest + f, err := os.Open(test.file) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + tr := NewReader(f) + + headers := test.headers + cksums := test.cksums + nread := 0 + + // loop over all files + for ; ; nread++ { + hdr, err := tr.Next() + if hdr == nil || err == io.EOF { + break + } + + // check the header + if !reflect.DeepEqual(*hdr, *headers[nread]) { + t.Errorf("Incorrect header:\nhave %+v\nwant %+v", + *hdr, headers[nread]) + } + + // read file contents in little chunks EOF, + // checksumming all the way + h := md5.New() + rdbuf := make([]uint8, 8) + for { + nr, err := tr.Read(rdbuf) + if err == io.EOF { + break + } + if err != nil { + t.Errorf("Read: unexpected error %v\n", err) + break + } + h.Write(rdbuf[0:nr]) + } + // verify checksum + have := fmt.Sprintf("%x", h.Sum(nil)) + want := cksums[nread] + if want != have { + t.Errorf("Bad checksum on file %s:\nhave %+v\nwant %+v", hdr.Name, have, want) + } + } + if nread != len(headers) { + t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(headers), nread) + } +} + +func TestNonSeekable(t *testing.T) { + test := gnuTarTest + f, err := os.Open(test.file) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer f.Close() + + type readerOnly struct { + io.Reader + } + tr := NewReader(readerOnly{f}) + nread := 0 + + for ; ; nread++ { + _, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + } + + if nread != len(test.headers) { + t.Errorf("Didn't process all files\nexpected: %d\nprocessed %d\n", len(test.headers), nread) + } +} + +func TestParsePAXHeader(t *testing.T) { + paxTests := [][3]string{ + {"a", "a=name", "10 a=name\n"}, // Test case involving multiple acceptable lengths + {"a", "a=name", "9 a=name\n"}, // Test case involving multiple acceptable length + {"mtime", "mtime=1350244992.023960108", "30 mtime=1350244992.023960108\n"}} + for _, test := range paxTests { + key, expected, raw := test[0], test[1], test[2] + reader := bytes.NewReader([]byte(raw)) + headers, err := parsePAX(reader) + if err != nil { + t.Errorf("Couldn't parse correctly formatted headers: %v", err) + continue + } + if strings.EqualFold(headers[key], expected) { + t.Errorf("mtime header incorrectly parsed: got %s, wanted %s", headers[key], expected) + continue + } + trailer := make([]byte, 100) + n, err := reader.Read(trailer) + if err != io.EOF || n != 0 { + t.Error("Buffer wasn't consumed") + } + } + badHeader := bytes.NewReader([]byte("3 somelongkey=")) + if _, err := parsePAX(badHeader); err != ErrHeader { + t.Fatal("Unexpected success when parsing bad header") + } +} + +func TestParsePAXTime(t *testing.T) { + // Some valid PAX time values + timestamps := map[string]time.Time{ + "1350244992.023960108": time.Unix(1350244992, 23960108), // The commoon case + "1350244992.02396010": time.Unix(1350244992, 23960100), // Lower precision value + "1350244992.0239601089": time.Unix(1350244992, 23960108), // Higher precision value + "1350244992": time.Unix(1350244992, 0), // Low precision value + } + for input, expected := range timestamps { + ts, err := parsePAXTime(input) + if err != nil { + t.Fatal(err) + } + if !ts.Equal(expected) { + t.Fatalf("Time parsing failure %s %s", ts, expected) + } + } +} + +func TestMergePAX(t *testing.T) { + hdr := new(Header) + // Test a string, integer, and time based value. + headers := map[string]string{ + "path": "a/b/c", + "uid": "1000", + "mtime": "1350244992.023960108", + } + err := mergePAX(hdr, headers) + if err != nil { + t.Fatal(err) + } + want := &Header{ + Name: "a/b/c", + Uid: 1000, + ModTime: time.Unix(1350244992, 23960108), + } + if !reflect.DeepEqual(hdr, want) { + t.Errorf("incorrect merge: got %+v, want %+v", hdr, want) + } +} diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_atim.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_atim.go new file mode 100644 index 0000000000..cf9cc79c59 --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_atim.go @@ -0,0 +1,20 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux dragonfly openbsd solaris + +package tar + +import ( + "syscall" + "time" +) + +func statAtime(st *syscall.Stat_t) time.Time { + return time.Unix(st.Atim.Unix()) +} + +func statCtime(st *syscall.Stat_t) time.Time { + return time.Unix(st.Ctim.Unix()) +} diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_atimespec.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_atimespec.go new file mode 100644 index 0000000000..6f17dbe307 --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_atimespec.go @@ -0,0 +1,20 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin freebsd netbsd + +package tar + +import ( + "syscall" + "time" +) + +func statAtime(st *syscall.Stat_t) time.Time { + return time.Unix(st.Atimespec.Unix()) +} + +func statCtime(st *syscall.Stat_t) time.Time { + return time.Unix(st.Ctimespec.Unix()) +} diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_unix.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_unix.go new file mode 100644 index 0000000000..cb843db4cf --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/stat_unix.go @@ -0,0 +1,32 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux darwin dragonfly freebsd openbsd netbsd solaris + +package tar + +import ( + "os" + "syscall" +) + +func init() { + sysStat = statUnix +} + +func statUnix(fi os.FileInfo, h *Header) error { + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return nil + } + h.Uid = int(sys.Uid) + h.Gid = int(sys.Gid) + // TODO(bradfitz): populate username & group. os/user + // doesn't cache LookupId lookups, and lacks group + // lookup functions. + h.AccessTime = statAtime(sys) + h.ChangeTime = statCtime(sys) + // TODO(bradfitz): major/minor device numbers? + return nil +} diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/tar_test.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/tar_test.go new file mode 100644 index 0000000000..ed333f3ea4 --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/tar_test.go @@ -0,0 +1,284 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +import ( + "bytes" + "io/ioutil" + "os" + "path" + "reflect" + "strings" + "testing" + "time" +) + +func TestFileInfoHeader(t *testing.T) { + fi, err := os.Stat("testdata/small.txt") + if err != nil { + t.Fatal(err) + } + h, err := FileInfoHeader(fi, "") + if err != nil { + t.Fatalf("FileInfoHeader: %v", err) + } + if g, e := h.Name, "small.txt"; g != e { + t.Errorf("Name = %q; want %q", g, e) + } + if g, e := h.Mode, int64(fi.Mode().Perm())|c_ISREG; g != e { + t.Errorf("Mode = %#o; want %#o", g, e) + } + if g, e := h.Size, int64(5); g != e { + t.Errorf("Size = %v; want %v", g, e) + } + if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) { + t.Errorf("ModTime = %v; want %v", g, e) + } + // FileInfoHeader should error when passing nil FileInfo + if _, err := FileInfoHeader(nil, ""); err == nil { + t.Fatalf("Expected error when passing nil to FileInfoHeader") + } +} + +func TestFileInfoHeaderDir(t *testing.T) { + fi, err := os.Stat("testdata") + if err != nil { + t.Fatal(err) + } + h, err := FileInfoHeader(fi, "") + if err != nil { + t.Fatalf("FileInfoHeader: %v", err) + } + if g, e := h.Name, "testdata/"; g != e { + t.Errorf("Name = %q; want %q", g, e) + } + // Ignoring c_ISGID for golang.org/issue/4867 + if g, e := h.Mode&^c_ISGID, int64(fi.Mode().Perm())|c_ISDIR; g != e { + t.Errorf("Mode = %#o; want %#o", g, e) + } + if g, e := h.Size, int64(0); g != e { + t.Errorf("Size = %v; want %v", g, e) + } + if g, e := h.ModTime, fi.ModTime(); !g.Equal(e) { + t.Errorf("ModTime = %v; want %v", g, e) + } +} + +func TestFileInfoHeaderSymlink(t *testing.T) { + h, err := FileInfoHeader(symlink{}, "some-target") + if err != nil { + t.Fatal(err) + } + if g, e := h.Name, "some-symlink"; g != e { + t.Errorf("Name = %q; want %q", g, e) + } + if g, e := h.Linkname, "some-target"; g != e { + t.Errorf("Linkname = %q; want %q", g, e) + } +} + +type symlink struct{} + +func (symlink) Name() string { return "some-symlink" } +func (symlink) Size() int64 { return 0 } +func (symlink) Mode() os.FileMode { return os.ModeSymlink } +func (symlink) ModTime() time.Time { return time.Time{} } +func (symlink) IsDir() bool { return false } +func (symlink) Sys() interface{} { return nil } + +func TestRoundTrip(t *testing.T) { + data := []byte("some file contents") + + var b bytes.Buffer + tw := NewWriter(&b) + hdr := &Header{ + Name: "file.txt", + Uid: 1 << 21, // too big for 8 octal digits + Size: int64(len(data)), + ModTime: time.Now(), + } + // tar only supports second precision. + hdr.ModTime = hdr.ModTime.Add(-time.Duration(hdr.ModTime.Nanosecond()) * time.Nanosecond) + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("tw.WriteHeader: %v", err) + } + if _, err := tw.Write(data); err != nil { + t.Fatalf("tw.Write: %v", err) + } + if err := tw.Close(); err != nil { + t.Fatalf("tw.Close: %v", err) + } + + // Read it back. + tr := NewReader(&b) + rHdr, err := tr.Next() + if err != nil { + t.Fatalf("tr.Next: %v", err) + } + if !reflect.DeepEqual(rHdr, hdr) { + t.Errorf("Header mismatch.\n got %+v\nwant %+v", rHdr, hdr) + } + rData, err := ioutil.ReadAll(tr) + if err != nil { + t.Fatalf("Read: %v", err) + } + if !bytes.Equal(rData, data) { + t.Errorf("Data mismatch.\n got %q\nwant %q", rData, data) + } +} + +type headerRoundTripTest struct { + h *Header + fm os.FileMode +} + +func TestHeaderRoundTrip(t *testing.T) { + golden := []headerRoundTripTest{ + // regular file. + { + h: &Header{ + Name: "test.txt", + Mode: 0644 | c_ISREG, + Size: 12, + ModTime: time.Unix(1360600916, 0), + Typeflag: TypeReg, + }, + fm: 0644, + }, + // hard link. + { + h: &Header{ + Name: "hard.txt", + Mode: 0644 | c_ISLNK, + Size: 0, + ModTime: time.Unix(1360600916, 0), + Typeflag: TypeLink, + }, + fm: 0644 | os.ModeSymlink, + }, + // symbolic link. + { + h: &Header{ + Name: "link.txt", + Mode: 0777 | c_ISLNK, + Size: 0, + ModTime: time.Unix(1360600852, 0), + Typeflag: TypeSymlink, + }, + fm: 0777 | os.ModeSymlink, + }, + // character device node. + { + h: &Header{ + Name: "dev/null", + Mode: 0666 | c_ISCHR, + Size: 0, + ModTime: time.Unix(1360578951, 0), + Typeflag: TypeChar, + }, + fm: 0666 | os.ModeDevice | os.ModeCharDevice, + }, + // block device node. + { + h: &Header{ + Name: "dev/sda", + Mode: 0660 | c_ISBLK, + Size: 0, + ModTime: time.Unix(1360578954, 0), + Typeflag: TypeBlock, + }, + fm: 0660 | os.ModeDevice, + }, + // directory. + { + h: &Header{ + Name: "dir/", + Mode: 0755 | c_ISDIR, + Size: 0, + ModTime: time.Unix(1360601116, 0), + Typeflag: TypeDir, + }, + fm: 0755 | os.ModeDir, + }, + // fifo node. + { + h: &Header{ + Name: "dev/initctl", + Mode: 0600 | c_ISFIFO, + Size: 0, + ModTime: time.Unix(1360578949, 0), + Typeflag: TypeFifo, + }, + fm: 0600 | os.ModeNamedPipe, + }, + // setuid. + { + h: &Header{ + Name: "bin/su", + Mode: 0755 | c_ISREG | c_ISUID, + Size: 23232, + ModTime: time.Unix(1355405093, 0), + Typeflag: TypeReg, + }, + fm: 0755 | os.ModeSetuid, + }, + // setguid. + { + h: &Header{ + Name: "group.txt", + Mode: 0750 | c_ISREG | c_ISGID, + Size: 0, + ModTime: time.Unix(1360602346, 0), + Typeflag: TypeReg, + }, + fm: 0750 | os.ModeSetgid, + }, + // sticky. + { + h: &Header{ + Name: "sticky.txt", + Mode: 0600 | c_ISREG | c_ISVTX, + Size: 7, + ModTime: time.Unix(1360602540, 0), + Typeflag: TypeReg, + }, + fm: 0600 | os.ModeSticky, + }, + } + + for i, g := range golden { + fi := g.h.FileInfo() + h2, err := FileInfoHeader(fi, "") + if err != nil { + t.Error(err) + continue + } + if strings.Contains(fi.Name(), "/") { + t.Errorf("FileInfo of %q contains slash: %q", g.h.Name, fi.Name()) + } + name := path.Base(g.h.Name) + if fi.IsDir() { + name += "/" + } + if got, want := h2.Name, name; got != want { + t.Errorf("i=%d: Name: got %v, want %v", i, got, want) + } + if got, want := h2.Size, g.h.Size; got != want { + t.Errorf("i=%d: Size: got %v, want %v", i, got, want) + } + if got, want := h2.Mode, g.h.Mode; got != want { + t.Errorf("i=%d: Mode: got %o, want %o", i, got, want) + } + if got, want := fi.Mode(), g.fm; got != want { + t.Errorf("i=%d: fi.Mode: got %o, want %o", i, got, want) + } + if got, want := h2.ModTime, g.h.ModTime; got != want { + t.Errorf("i=%d: ModTime: got %v, want %v", i, got, want) + } + if sysh, ok := fi.Sys().(*Header); !ok || sysh != g.h { + t.Errorf("i=%d: Sys didn't return original *Header", i) + } + } +} diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/gnu.tar b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/gnu.tar new file mode 100644 index 0000000000000000000000000000000000000000..fc899dc8dc2ad9952f5c5f67a0c76ca2d87249e9 GIT binary patch literal 3072 zcmeHH%L>9U5Ztq0(Jv@FdDKtv;8zq|ijXv5BIw^6DOh^2UK)_HbK1>@VQ0c5`qsHR zJrb1zXEcV16&lMRW}rdtKd=NSXg->JkvP|Esp4`g&CK_h+FMmo7oR?iU7RP&svn2t z!9Ke4)upeR_aRYKtT+(g`B!B>fS>t?p7IZ9V9LKWlK+)w+iY|SVQ_tY3I4Ddrx1w) M;($0H4*b6ZFWOBnBLDyZ literal 0 HcmV?d00001 diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/nil-uid.tar b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/nil-uid.tar new file mode 100644 index 0000000000000000000000000000000000000000..cc9cfaa33cc5de0a28b4183c1705d801f788c96a GIT binary patch literal 1024 zcmWGAG%z(VGPcn33UJrU$xmmX0WbgpGcywmlR@HOU}(l*Xk=(?U}j`)Zf3?{U78BOoLqCLtvwr=Z5b$i&RT%Er#Y zO+ZjcSVUAHn~8MUp(~vBV+cg7LjpEKCCE6D7E@EwTcMv_>l+&bbg`j1Cv0A776ym5t@+ zSt9MDBFtXbKY&m4pMN0f`l~hhD>#q(-`x$5n+q@eEPmAevA;0XTM8XMYkTvSmQ-t5 zkihVw{(qQ#_JjT})&KMa&-FhG0c8or{CPvw|Jf69WL!B2Wa1KoKYcMW6^2fg(@@ia-%40!5$*6oDd81d2cr_`3;w E2V3|JA^-pY literal 0 HcmV?d00001 diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/small.txt b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/small.txt new file mode 100644 index 0000000000..b249bfc518 --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/small.txt @@ -0,0 +1 @@ +Kilts \ No newline at end of file diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/small2.txt b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/small2.txt new file mode 100644 index 0000000000..394ee3ecd0 --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/small2.txt @@ -0,0 +1 @@ +Google.com diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/star.tar b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/star.tar new file mode 100644 index 0000000000000000000000000000000000000000..59e2d4e604611eeac3e2a0f3d6f71d2623c50449 GIT binary patch literal 3072 zcmeHHT?)e>4DRzz;R#BjQ;)ERouag*479>@u-$$NbG3!-WyoMlUh?xvNIv}HZD&jy zuA!-C5KZlY0Y@bP833Zfm_JQ2M2yBQ2dTK6K{>VDLBV=D{z>IvVF` zUD#xgUGh?F1AikeIW6NnOIrMRGU4UU`62nAWxyx>^STEhN#m{lQEc_E}GZPaA5N&Q|3Z@N=AbgM*P?o{a$k4#V#K6eR)QrKv#MIE( zgh9c8hHiozU0Pg{SOj!ZaYkZZDqIwk0aTWjhA9jefq29K;yD8YhMfGo^t{B}RQ&;E gz@3MSk&&8{lh1`qc2s;c1V%$(Gz3ONV7P_=0FTf|dgec!sXT^AK{$jKc@-kWdUe(&uh-&e0L+xARj zwLzm>LI~3|1sT#R&XkBIzWbfCPrYEK7fr^Q@7vXO;&pw$QCTT3-?&yO+jq(<{6qS`FS_vP zIBhMBjnmsnS~{|C9LMN8#r!W{zj5l&zcE?^U_t*||1zJ{zqInH{-Zy}2$O|c?WSFx zxn8RtM3-UpAJiW`Z@Zar#$ojz)NjtWBfnULUzD=jj5!>iG>O2k{o(=ZAg=$-urC7q zVm{n!{kK`S@p|Vk`q%aFg#nw)bMB-40yAj*%7=F37m@ziFINBH7pTSD@Cfil^^9T6 zxL-iu+Aq)#ev#CF(l2&S@A^eC<`;^e4{ZQ#s9$Y4r}$iP3;;e3V;a&MNN*s$f%FFc H(;N5+1FUK9 literal 0 HcmV?d00001 diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/writer-big.tar b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/writer-big.tar new file mode 100644 index 0000000000000000000000000000000000000000..753e883cebf52ac1291f1b7bf1b7a37ae517b2d9 GIT binary patch literal 4096 zcmeIuF%E+;3tu-|!r}ytVByrmfae ipO37m$1T~NWs?FFpa2CZKmiI+fC3bt00k&;vcMnFf)<_t literal 0 HcmV?d00001 diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/writer.tar b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/writer.tar new file mode 100644 index 0000000000000000000000000000000000000000..e6d816ad0775d56d09242d6f5d1dbe56af310a32 GIT binary patch literal 3584 zcmeHIK@P$o5ajGDd_l9j6nKIMUt!cVjT92WM1L@81h#LhDgML6Bon)c?rO_kPgyt^3D0fH9$GJM`O*&4VCw= zv#H)UKC-TtzNwGuV$*%C{bm zsdIMLR{C5VZL^vBE!S4cfUeCYt@>GOiAt%sq7tp|_iN{x5cDreh9ME=K+wOCQm`$x j!znSk-v6Dy)}|V_!f*AilYjI7l|Jj-R%ReG@B;%+QQ}au literal 0 HcmV?d00001 diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/xattrs.tar b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/testdata/xattrs.tar new file mode 100644 index 0000000000000000000000000000000000000000..9701950edd1f0dc82858b7117136b37391be0b08 GIT binary patch literal 5120 zcmeHJv2KGf5M|~o_yWg1+khiw>d;i}P^nX=$R$ooYd`|ilD{uBAv6g^kxC>6-(uu< zHg^v_-l5r}td>fyRbC(vfcdOQq}Iq(#u+Ja9X?}Dv(|CCVoJF~09ZgF;2a!G7^%~| zYNYoMUQ-rE=5KzzBJ^EKyr-Mx-NQ4gq%k=v3zee}wOxElT`HH-ei(K*xV|_} zC{$GDvDuoW?o>&odUrVuVHkt_w?IH zW3PV_@V!Jxt@A^i>Yrj(>;K=H?5X8!tJS~MYVd#a^`?|QJKb&Uduf~MfN4M7$J!Lr zF40zZMF!9x{tqJ#0F5+;{2!=)=Knre|G(mAKU`hAc#r>!#{V(9d;sW1hxVv7@B_zF ze)#eKF~#1~>@WTI`#+&4`lkel_5U6!N8h^5vRAE8lqGgr9-Ul!p=H1_U>TS&1K)l2 B)fNB% literal 0 HcmV?d00001 diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go new file mode 100644 index 0000000000..9ee9499297 --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go @@ -0,0 +1,383 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +// TODO(dsymonds): +// - catch more errors (no first header, etc.) + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path" + "strconv" + "strings" + "time" +) + +var ( + ErrWriteTooLong = errors.New("archive/tar: write too long") + ErrFieldTooLong = errors.New("archive/tar: header field too long") + ErrWriteAfterClose = errors.New("archive/tar: write after close") + errNameTooLong = errors.New("archive/tar: name too long") + errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values") +) + +// A Writer provides sequential writing of a tar archive in POSIX.1 format. +// A tar archive consists of a sequence of files. +// Call WriteHeader to begin a new file, and then call Write to supply that file's data, +// writing at most hdr.Size bytes in total. +type Writer struct { + w io.Writer + err error + nb int64 // number of unwritten bytes for current file entry + pad int64 // amount of padding to write after current file entry + closed bool + usedBinary bool // whether the binary numeric field extension was used + preferPax bool // use pax header instead of binary numeric header +} + +// NewWriter creates a new Writer writing to w. +func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } + +// Flush finishes writing the current file (optional). +func (tw *Writer) Flush() error { + if tw.nb > 0 { + tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb) + return tw.err + } + + n := tw.nb + tw.pad + for n > 0 && tw.err == nil { + nr := n + if nr > blockSize { + nr = blockSize + } + var nw int + nw, tw.err = tw.w.Write(zeroBlock[0:nr]) + n -= int64(nw) + } + tw.nb = 0 + tw.pad = 0 + return tw.err +} + +// Write s into b, terminating it with a NUL if there is room. +// If the value is too long for the field and allowPax is true add a paxheader record instead +func (tw *Writer) cString(b []byte, s string, allowPax bool, paxKeyword string, paxHeaders map[string]string) { + needsPaxHeader := allowPax && len(s) > len(b) || !isASCII(s) + if needsPaxHeader { + paxHeaders[paxKeyword] = s + return + } + if len(s) > len(b) { + if tw.err == nil { + tw.err = ErrFieldTooLong + } + return + } + ascii := toASCII(s) + copy(b, ascii) + if len(ascii) < len(b) { + b[len(ascii)] = 0 + } +} + +// Encode x as an octal ASCII string and write it into b with leading zeros. +func (tw *Writer) octal(b []byte, x int64) { + s := strconv.FormatInt(x, 8) + // leading zeros, but leave room for a NUL. + for len(s)+1 < len(b) { + s = "0" + s + } + tw.cString(b, s, false, paxNone, nil) +} + +// Write x into b, either as octal or as binary (GNUtar/star extension). +// If the value is too long for the field and writingPax is enabled both for the field and the add a paxheader record instead +func (tw *Writer) numeric(b []byte, x int64, allowPax bool, paxKeyword string, paxHeaders map[string]string) { + // Try octal first. + s := strconv.FormatInt(x, 8) + if len(s) < len(b) { + tw.octal(b, x) + return + } + + // If it is too long for octal, and pax is preferred, use a pax header + if allowPax && tw.preferPax { + tw.octal(b, 0) + s := strconv.FormatInt(x, 10) + paxHeaders[paxKeyword] = s + return + } + + // Too big: use binary (big-endian). + tw.usedBinary = true + for i := len(b) - 1; x > 0 && i >= 0; i-- { + b[i] = byte(x) + x >>= 8 + } + b[0] |= 0x80 // highest bit indicates binary format +} + +var ( + minTime = time.Unix(0, 0) + // There is room for 11 octal digits (33 bits) of mtime. + maxTime = minTime.Add((1<<33 - 1) * time.Second) +) + +// WriteHeader writes hdr and prepares to accept the file's contents. +// WriteHeader calls Flush if it is not the first header. +// Calling after a Close will return ErrWriteAfterClose. +func (tw *Writer) WriteHeader(hdr *Header) error { + return tw.writeHeader(hdr, true) +} + +// WriteHeader writes hdr and prepares to accept the file's contents. +// WriteHeader calls Flush if it is not the first header. +// Calling after a Close will return ErrWriteAfterClose. +// As this method is called internally by writePax header to allow it to +// suppress writing the pax header. +func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { + if tw.closed { + return ErrWriteAfterClose + } + if tw.err == nil { + tw.Flush() + } + if tw.err != nil { + return tw.err + } + + // a map to hold pax header records, if any are needed + paxHeaders := make(map[string]string) + + // TODO(shanemhansen): we might want to use PAX headers for + // subsecond time resolution, but for now let's just capture + // too long fields or non ascii characters + + header := make([]byte, blockSize) + s := slicer(header) + + // keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax + pathHeaderBytes := s.next(fileNameSize) + + tw.cString(pathHeaderBytes, hdr.Name, true, paxPath, paxHeaders) + + // Handle out of range ModTime carefully. + var modTime int64 + if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) { + modTime = hdr.ModTime.Unix() + } + + tw.octal(s.next(8), hdr.Mode) // 100:108 + tw.numeric(s.next(8), int64(hdr.Uid), true, paxUid, paxHeaders) // 108:116 + tw.numeric(s.next(8), int64(hdr.Gid), true, paxGid, paxHeaders) // 116:124 + tw.numeric(s.next(12), hdr.Size, true, paxSize, paxHeaders) // 124:136 + tw.numeric(s.next(12), modTime, false, paxNone, nil) // 136:148 --- consider using pax for finer granularity + s.next(8) // chksum (148:156) + s.next(1)[0] = hdr.Typeflag // 156:157 + + tw.cString(s.next(100), hdr.Linkname, true, paxLinkpath, paxHeaders) + + copy(s.next(8), []byte("ustar\x0000")) // 257:265 + tw.cString(s.next(32), hdr.Uname, true, paxUname, paxHeaders) // 265:297 + tw.cString(s.next(32), hdr.Gname, true, paxGname, paxHeaders) // 297:329 + tw.numeric(s.next(8), hdr.Devmajor, false, paxNone, nil) // 329:337 + tw.numeric(s.next(8), hdr.Devminor, false, paxNone, nil) // 337:345 + + // keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax + prefixHeaderBytes := s.next(155) + tw.cString(prefixHeaderBytes, "", false, paxNone, nil) // 345:500 prefix + + // Use the GNU magic instead of POSIX magic if we used any GNU extensions. + if tw.usedBinary { + copy(header[257:265], []byte("ustar \x00")) + } + + _, paxPathUsed := paxHeaders[paxPath] + // try to use a ustar header when only the name is too long + if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed { + suffix := hdr.Name + prefix := "" + if len(hdr.Name) > fileNameSize && isASCII(hdr.Name) { + var err error + prefix, suffix, err = tw.splitUSTARLongName(hdr.Name) + if err == nil { + // ok we can use a ustar long name instead of pax, now correct the fields + + // remove the path field from the pax header. this will suppress the pax header + delete(paxHeaders, paxPath) + + // update the path fields + tw.cString(pathHeaderBytes, suffix, false, paxNone, nil) + tw.cString(prefixHeaderBytes, prefix, false, paxNone, nil) + + // Use the ustar magic if we used ustar long names. + if len(prefix) > 0 { + copy(header[257:265], []byte("ustar\000")) + } + } + } + } + + // The chksum field is terminated by a NUL and a space. + // This is different from the other octal fields. + chksum, _ := checksum(header) + tw.octal(header[148:155], chksum) + header[155] = ' ' + + if tw.err != nil { + // problem with header; probably integer too big for a field. + return tw.err + } + + if allowPax { + for k, v := range hdr.Xattrs { + paxHeaders[paxXattr+k] = v + } + } + + if len(paxHeaders) > 0 { + if !allowPax { + return errInvalidHeader + } + if err := tw.writePAXHeader(hdr, paxHeaders); err != nil { + return err + } + } + tw.nb = int64(hdr.Size) + tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize + + _, tw.err = tw.w.Write(header) + return tw.err +} + +// writeUSTARLongName splits a USTAR long name hdr.Name. +// name must be < 256 characters. errNameTooLong is returned +// if hdr.Name can't be split. The splitting heuristic +// is compatible with gnu tar. +func (tw *Writer) splitUSTARLongName(name string) (prefix, suffix string, err error) { + length := len(name) + if length > fileNamePrefixSize+1 { + length = fileNamePrefixSize + 1 + } else if name[length-1] == '/' { + length-- + } + i := strings.LastIndex(name[:length], "/") + // nlen contains the resulting length in the name field. + // plen contains the resulting length in the prefix field. + nlen := len(name) - i - 1 + plen := i + if i <= 0 || nlen > fileNameSize || nlen == 0 || plen > fileNamePrefixSize { + err = errNameTooLong + return + } + prefix, suffix = name[:i], name[i+1:] + return +} + +// writePaxHeader writes an extended pax header to the +// archive. +func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error { + // Prepare extended header + ext := new(Header) + ext.Typeflag = TypeXHeader + // Setting ModTime is required for reader parsing to + // succeed, and seems harmless enough. + ext.ModTime = hdr.ModTime + // The spec asks that we namespace our pseudo files + // with the current pid. + pid := os.Getpid() + dir, file := path.Split(hdr.Name) + fullName := path.Join(dir, + fmt.Sprintf("PaxHeaders.%d", pid), file) + + ascii := toASCII(fullName) + if len(ascii) > 100 { + ascii = ascii[:100] + } + ext.Name = ascii + // Construct the body + var buf bytes.Buffer + + for k, v := range paxHeaders { + fmt.Fprint(&buf, paxHeader(k+"="+v)) + } + + ext.Size = int64(len(buf.Bytes())) + if err := tw.writeHeader(ext, false); err != nil { + return err + } + if _, err := tw.Write(buf.Bytes()); err != nil { + return err + } + if err := tw.Flush(); err != nil { + return err + } + return nil +} + +// paxHeader formats a single pax record, prefixing it with the appropriate length +func paxHeader(msg string) string { + const padding = 2 // Extra padding for space and newline + size := len(msg) + padding + size += len(strconv.Itoa(size)) + record := fmt.Sprintf("%d %s\n", size, msg) + if len(record) != size { + // Final adjustment if adding size increased + // the number of digits in size + size = len(record) + record = fmt.Sprintf("%d %s\n", size, msg) + } + return record +} + +// Write writes to the current entry in the tar archive. +// Write returns the error ErrWriteTooLong if more than +// hdr.Size bytes are written after WriteHeader. +func (tw *Writer) Write(b []byte) (n int, err error) { + if tw.closed { + err = ErrWriteTooLong + return + } + overwrite := false + if int64(len(b)) > tw.nb { + b = b[0:tw.nb] + overwrite = true + } + n, err = tw.w.Write(b) + tw.nb -= int64(n) + if err == nil && overwrite { + err = ErrWriteTooLong + return + } + tw.err = err + return +} + +// Close closes the tar archive, flushing any unwritten +// data to the underlying writer. +func (tw *Writer) Close() error { + if tw.err != nil || tw.closed { + return tw.err + } + tw.Flush() + tw.closed = true + if tw.err != nil { + return tw.err + } + + // trailer: two zero blocks + for i := 0; i < 2; i++ { + _, tw.err = tw.w.Write(zeroBlock) + if tw.err != nil { + break + } + } + return tw.err +} diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer_test.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer_test.go new file mode 100644 index 0000000000..2b9ea658db --- /dev/null +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer_test.go @@ -0,0 +1,433 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tar + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "reflect" + "strings" + "testing" + "testing/iotest" + "time" +) + +type writerTestEntry struct { + header *Header + contents string +} + +type writerTest struct { + file string // filename of expected output + entries []*writerTestEntry +} + +var writerTests = []*writerTest{ + // The writer test file was produced with this command: + // tar (GNU tar) 1.26 + // ln -s small.txt link.txt + // tar -b 1 --format=ustar -c -f writer.tar small.txt small2.txt link.txt + { + file: "testdata/writer.tar", + entries: []*writerTestEntry{ + { + header: &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + ModTime: time.Unix(1246508266, 0), + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + contents: "Kilts", + }, + { + header: &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + ModTime: time.Unix(1245217492, 0), + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + contents: "Google.com\n", + }, + { + header: &Header{ + Name: "link.txt", + Mode: 0777, + Uid: 1000, + Gid: 1000, + Size: 0, + ModTime: time.Unix(1314603082, 0), + Typeflag: '2', + Linkname: "small.txt", + Uname: "strings", + Gname: "strings", + }, + // no contents + }, + }, + }, + // The truncated test file was produced using these commands: + // dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt + // tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar + { + file: "testdata/writer-big.tar", + entries: []*writerTestEntry{ + { + header: &Header{ + Name: "tmp/16gig.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 16 << 30, + ModTime: time.Unix(1254699560, 0), + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + // fake contents + contents: strings.Repeat("\x00", 4<<10), + }, + }, + }, + // This file was produced using gnu tar 1.17 + // gnutar -b 4 --format=ustar (longname/)*15 + file.txt + { + file: "testdata/ustar.tar", + entries: []*writerTestEntry{ + { + header: &Header{ + Name: strings.Repeat("longname/", 15) + "file.txt", + Mode: 0644, + Uid: 0765, + Gid: 024, + Size: 06, + ModTime: time.Unix(1360135598, 0), + Typeflag: '0', + Uname: "shane", + Gname: "staff", + }, + contents: "hello\n", + }, + }, + }, +} + +// Render byte array in a two-character hexadecimal string, spaced for easy visual inspection. +func bytestr(offset int, b []byte) string { + const rowLen = 32 + s := fmt.Sprintf("%04x ", offset) + for _, ch := range b { + switch { + case '0' <= ch && ch <= '9', 'A' <= ch && ch <= 'Z', 'a' <= ch && ch <= 'z': + s += fmt.Sprintf(" %c", ch) + default: + s += fmt.Sprintf(" %02x", ch) + } + } + return s +} + +// Render a pseudo-diff between two blocks of bytes. +func bytediff(a []byte, b []byte) string { + const rowLen = 32 + s := fmt.Sprintf("(%d bytes vs. %d bytes)\n", len(a), len(b)) + for offset := 0; len(a)+len(b) > 0; offset += rowLen { + na, nb := rowLen, rowLen + if na > len(a) { + na = len(a) + } + if nb > len(b) { + nb = len(b) + } + sa := bytestr(offset, a[0:na]) + sb := bytestr(offset, b[0:nb]) + if sa != sb { + s += fmt.Sprintf("-%v\n+%v\n", sa, sb) + } + a = a[na:] + b = b[nb:] + } + return s +} + +func TestWriter(t *testing.T) { +testLoop: + for i, test := range writerTests { + expected, err := ioutil.ReadFile(test.file) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + continue + } + + buf := new(bytes.Buffer) + tw := NewWriter(iotest.TruncateWriter(buf, 4<<10)) // only catch the first 4 KB + big := false + for j, entry := range test.entries { + big = big || entry.header.Size > 1<<10 + if err := tw.WriteHeader(entry.header); err != nil { + t.Errorf("test %d, entry %d: Failed writing header: %v", i, j, err) + continue testLoop + } + if _, err := io.WriteString(tw, entry.contents); err != nil { + t.Errorf("test %d, entry %d: Failed writing contents: %v", i, j, err) + continue testLoop + } + } + // Only interested in Close failures for the small tests. + if err := tw.Close(); err != nil && !big { + t.Errorf("test %d: Failed closing archive: %v", i, err) + continue testLoop + } + + actual := buf.Bytes() + if !bytes.Equal(expected, actual) { + t.Errorf("test %d: Incorrect result: (-=expected, +=actual)\n%v", + i, bytediff(expected, actual)) + } + if testing.Short() { // The second test is expensive. + break + } + } +} + +func TestPax(t *testing.T) { + // Create an archive with a large name + fileinfo, err := os.Stat("testdata/small.txt") + if err != nil { + t.Fatal(err) + } + hdr, err := FileInfoHeader(fileinfo, "") + if err != nil { + t.Fatalf("os.Stat: %v", err) + } + // Force a PAX long name to be written + longName := strings.Repeat("ab", 100) + contents := strings.Repeat(" ", int(hdr.Size)) + hdr.Name = longName + var buf bytes.Buffer + writer := NewWriter(&buf) + if err := writer.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err = writer.Write([]byte(contents)); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + // Simple test to make sure PAX extensions are in effect + if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { + t.Fatal("Expected at least one PAX header to be written.") + } + // Test that we can get a long name back out of the archive. + reader := NewReader(&buf) + hdr, err = reader.Next() + if err != nil { + t.Fatal(err) + } + if hdr.Name != longName { + t.Fatal("Couldn't recover long file name") + } +} + +func TestPaxSymlink(t *testing.T) { + // Create an archive with a large linkname + fileinfo, err := os.Stat("testdata/small.txt") + if err != nil { + t.Fatal(err) + } + hdr, err := FileInfoHeader(fileinfo, "") + hdr.Typeflag = TypeSymlink + if err != nil { + t.Fatalf("os.Stat:1 %v", err) + } + // Force a PAX long linkname to be written + longLinkname := strings.Repeat("1234567890/1234567890", 10) + hdr.Linkname = longLinkname + + hdr.Size = 0 + var buf bytes.Buffer + writer := NewWriter(&buf) + if err := writer.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + // Simple test to make sure PAX extensions are in effect + if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { + t.Fatal("Expected at least one PAX header to be written.") + } + // Test that we can get a long name back out of the archive. + reader := NewReader(&buf) + hdr, err = reader.Next() + if err != nil { + t.Fatal(err) + } + if hdr.Linkname != longLinkname { + t.Fatal("Couldn't recover long link name") + } +} + +func TestPaxNonAscii(t *testing.T) { + // Create an archive with non ascii. These should trigger a pax header + // because pax headers have a defined utf-8 encoding. + fileinfo, err := os.Stat("testdata/small.txt") + if err != nil { + t.Fatal(err) + } + + hdr, err := FileInfoHeader(fileinfo, "") + if err != nil { + t.Fatalf("os.Stat:1 %v", err) + } + + // some sample data + chineseFilename := "文件名" + chineseGroupname := "組" + chineseUsername := "用戶名" + + hdr.Name = chineseFilename + hdr.Gname = chineseGroupname + hdr.Uname = chineseUsername + + contents := strings.Repeat(" ", int(hdr.Size)) + + var buf bytes.Buffer + writer := NewWriter(&buf) + if err := writer.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err = writer.Write([]byte(contents)); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + // Simple test to make sure PAX extensions are in effect + if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) { + t.Fatal("Expected at least one PAX header to be written.") + } + // Test that we can get a long name back out of the archive. + reader := NewReader(&buf) + hdr, err = reader.Next() + if err != nil { + t.Fatal(err) + } + if hdr.Name != chineseFilename { + t.Fatal("Couldn't recover unicode name") + } + if hdr.Gname != chineseGroupname { + t.Fatal("Couldn't recover unicode group") + } + if hdr.Uname != chineseUsername { + t.Fatal("Couldn't recover unicode user") + } +} + +func TestPaxXattrs(t *testing.T) { + xattrs := map[string]string{ + "user.key": "value", + } + + // Create an archive with an xattr + fileinfo, err := os.Stat("testdata/small.txt") + if err != nil { + t.Fatal(err) + } + hdr, err := FileInfoHeader(fileinfo, "") + if err != nil { + t.Fatalf("os.Stat: %v", err) + } + contents := "Kilts" + hdr.Xattrs = xattrs + var buf bytes.Buffer + writer := NewWriter(&buf) + if err := writer.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if _, err = writer.Write([]byte(contents)); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + // Test that we can get the xattrs back out of the archive. + reader := NewReader(&buf) + hdr, err = reader.Next() + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(hdr.Xattrs, xattrs) { + t.Fatalf("xattrs did not survive round trip: got %+v, want %+v", + hdr.Xattrs, xattrs) + } +} + +func TestPAXHeader(t *testing.T) { + medName := strings.Repeat("CD", 50) + longName := strings.Repeat("AB", 100) + paxTests := [][2]string{ + {paxPath + "=/etc/hosts", "19 path=/etc/hosts\n"}, + {"a=b", "6 a=b\n"}, // Single digit length + {"a=names", "11 a=names\n"}, // Test case involving carries + {paxPath + "=" + longName, fmt.Sprintf("210 path=%s\n", longName)}, + {paxPath + "=" + medName, fmt.Sprintf("110 path=%s\n", medName)}} + + for _, test := range paxTests { + key, expected := test[0], test[1] + if result := paxHeader(key); result != expected { + t.Fatalf("paxHeader: got %s, expected %s", result, expected) + } + } +} + +func TestUSTARLongName(t *testing.T) { + // Create an archive with a path that failed to split with USTAR extension in previous versions. + fileinfo, err := os.Stat("testdata/small.txt") + if err != nil { + t.Fatal(err) + } + hdr, err := FileInfoHeader(fileinfo, "") + hdr.Typeflag = TypeDir + if err != nil { + t.Fatalf("os.Stat:1 %v", err) + } + // Force a PAX long name to be written. The name was taken from a practical example + // that fails and replaced ever char through numbers to anonymize the sample. + longName := "/0000_0000000/00000-000000000/0000_0000000/00000-0000000000000/0000_0000000/00000-0000000-00000000/0000_0000000/00000000/0000_0000000/000/0000_0000000/00000000v00/0000_0000000/000000/0000_0000000/0000000/0000_0000000/00000y-00/0000/0000/00000000/0x000000/" + hdr.Name = longName + + hdr.Size = 0 + var buf bytes.Buffer + writer := NewWriter(&buf) + if err := writer.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + // Test that we can get a long name back out of the archive. + reader := NewReader(&buf) + hdr, err = reader.Next() + if err != nil { + t.Fatal(err) + } + if hdr.Name != longName { + t.Fatal("Couldn't recover long name") + } +} From e184f368050d01398e3fcd5a0ed353ad816bf982 Mon Sep 17 00:00:00 2001 From: Song Gao Date: Mon, 6 Jan 2014 12:06:05 -0600 Subject: [PATCH 117/403] Support submodules when building from a gh repo Docker-DCO-1.1-Signed-off-by: Song Gao (github: songgao) Upstream-commit: c9ae66ffe375156ddf39cb41664224d6e1a6f096 Component: engine --- components/engine/docs/sources/reference/commandline/cli.rst | 3 ++- components/engine/server.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index 032076b941..87222484c5 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -194,7 +194,8 @@ The files at ``PATH`` or ``URL`` are called the "context" of the build. The build process may refer to any of the files in the context, for example when using an :ref:`ADD ` instruction. When a single ``Dockerfile`` is given as ``URL``, then no context is set. When a Git repository is set as -``URL``, then the repository is used as the context +``URL``, then the repository is used as the context. Git repositories are +cloned with their submodules (`git clone --recursive`). .. _cli_build_examples: diff --git a/components/engine/server.go b/components/engine/server.go index bad8d8bfb5..46ab33b467 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -457,7 +457,7 @@ func (srv *Server) Build(job *engine.Job) engine.Status { } defer os.RemoveAll(root) - if output, err := exec.Command("git", "clone", remoteURL, root).CombinedOutput(); err != nil { + if output, err := exec.Command("git", "clone", "--recursive", remoteURL, root).CombinedOutput(); err != nil { return job.Errorf("Error trying to use git: %s (%s)", err, output) } From 077c2ff0d64d7d0004f4b4b3034dfee713e4d85e Mon Sep 17 00:00:00 2001 From: Travis Cline Date: Mon, 17 Feb 2014 10:23:07 -0800 Subject: [PATCH 118/403] Quote volume path to allow spaces Docker-DCO-1.1-Signed-off-by: Travis Cline (github: tmc) Upstream-commit: 5006509131986093a9529a748e80e023e19548f3 Component: engine --- components/engine/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index 168707a80f..e124d1d7e6 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -3,7 +3,7 @@ GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) DOCKER_IMAGE := docker:$(GIT_BRANCH) DOCKER_DOCS_IMAGE := docker-docs:$(GIT_BRANCH) -DOCKER_RUN_DOCKER := docker run -rm -i -t -privileged -e TESTFLAGS -v $(CURDIR)/bundles:/go/src/github.com/dotcloud/docker/bundles "$(DOCKER_IMAGE)" +DOCKER_RUN_DOCKER := docker run -rm -i -t -privileged -e TESTFLAGS -v "$(CURDIR)/bundles:/go/src/github.com/dotcloud/docker/bundles" "$(DOCKER_IMAGE)" default: binary From d96c32338087d2d4f61af1fa34985c2cfed62307 Mon Sep 17 00:00:00 2001 From: Daniel Mizyrycki Date: Fri, 14 Feb 2014 20:50:16 -0800 Subject: [PATCH 119/403] docker-ci 0.5.6: Fully dockerize docker-ci. Add build test coverage. Add backup builder. Docker-DCO-1.1-Signed-off-by: Daniel Mizyrycki (github: mzdaniel) Upstream-commit: b7db2d5f8022e24faff80d312977c49a4c0408f3 Component: engine --- .../engine/hack/infrastructure/docker-ci.rst | 56 ------ .../hack/infrastructure/docker-ci/Dockerfile | 66 +++---- .../hack/infrastructure/docker-ci/README.rst | 75 ++++++-- .../hack/infrastructure/docker-ci/VERSION | 2 +- .../docker-ci/buildbot/README.rst | 1 - .../docker-ci/buildbot/buildbot.conf | 18 -- .../docker-ci/buildbot/github.py | 9 +- .../docker-ci/buildbot/master.cfg | 179 ++++++++++-------- .../docker-ci/buildbot/requirements.txt | 9 - .../docker-ci/buildbot/setup.sh | 59 ------ .../docker-ci/dcr/prod/docker-ci.yml | 22 +++ .../docker-ci/dcr/prod/settings.yml | 5 + .../docker-ci/dcr/stage/docker-ci.yml | 22 +++ .../docker-ci/dcr/stage/settings.yml | 5 + .../infrastructure/docker-ci/deployment.py | 171 ----------------- .../docker-coverage/coverage-docker.sh | 32 ---- .../docker-ci/docker-test/Dockerfile | 25 --- .../docker-ci/docker-test/test_docker.sh | 33 ---- .../docker-ci/dockertest/docker | 1 + .../docker-ci/dockertest/docker-registry | 1 + .../docker-ci/dockertest/nightlyrelease | 13 ++ .../docker-ci/dockertest/project | 8 + .../infrastructure/docker-ci/nginx/nginx.conf | 12 ++ .../docker-ci/nightlyrelease/Dockerfile | 30 --- .../docker-ci/nightlyrelease/dockerbuild.sh | 40 ---- .../docker-ci/registry-coverage/Dockerfile | 18 -- .../registry-coverage/registry_coverage.sh | 18 -- .../hack/infrastructure/docker-ci/setup.sh | 54 ++++++ .../docker-ci/testbuilder/Dockerfile | 12 ++ .../docker-ci/testbuilder/docker-registry.sh | 12 ++ .../docker-ci/testbuilder/docker.sh | 18 ++ .../docker-ci/testbuilder/testbuilder.sh | 40 ++++ .../infrastructure/docker-ci/tool/backup.py | 47 +++++ 33 files changed, 463 insertions(+), 650 deletions(-) delete mode 100644 components/engine/hack/infrastructure/docker-ci.rst delete mode 100644 components/engine/hack/infrastructure/docker-ci/buildbot/README.rst delete mode 100644 components/engine/hack/infrastructure/docker-ci/buildbot/buildbot.conf delete mode 100644 components/engine/hack/infrastructure/docker-ci/buildbot/requirements.txt delete mode 100755 components/engine/hack/infrastructure/docker-ci/buildbot/setup.sh create mode 100644 components/engine/hack/infrastructure/docker-ci/dcr/prod/docker-ci.yml create mode 100644 components/engine/hack/infrastructure/docker-ci/dcr/prod/settings.yml create mode 100644 components/engine/hack/infrastructure/docker-ci/dcr/stage/docker-ci.yml create mode 100644 components/engine/hack/infrastructure/docker-ci/dcr/stage/settings.yml delete mode 100755 components/engine/hack/infrastructure/docker-ci/deployment.py delete mode 100755 components/engine/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh delete mode 100644 components/engine/hack/infrastructure/docker-ci/docker-test/Dockerfile delete mode 100755 components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh create mode 120000 components/engine/hack/infrastructure/docker-ci/dockertest/docker create mode 120000 components/engine/hack/infrastructure/docker-ci/dockertest/docker-registry create mode 100755 components/engine/hack/infrastructure/docker-ci/dockertest/nightlyrelease create mode 100755 components/engine/hack/infrastructure/docker-ci/dockertest/project create mode 100644 components/engine/hack/infrastructure/docker-ci/nginx/nginx.conf delete mode 100644 components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile delete mode 100644 components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh delete mode 100644 components/engine/hack/infrastructure/docker-ci/registry-coverage/Dockerfile delete mode 100755 components/engine/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh create mode 100755 components/engine/hack/infrastructure/docker-ci/setup.sh create mode 100644 components/engine/hack/infrastructure/docker-ci/testbuilder/Dockerfile create mode 100755 components/engine/hack/infrastructure/docker-ci/testbuilder/docker-registry.sh create mode 100755 components/engine/hack/infrastructure/docker-ci/testbuilder/docker.sh create mode 100755 components/engine/hack/infrastructure/docker-ci/testbuilder/testbuilder.sh create mode 100755 components/engine/hack/infrastructure/docker-ci/tool/backup.py diff --git a/components/engine/hack/infrastructure/docker-ci.rst b/components/engine/hack/infrastructure/docker-ci.rst deleted file mode 100644 index 0be530d302..0000000000 --- a/components/engine/hack/infrastructure/docker-ci.rst +++ /dev/null @@ -1,56 +0,0 @@ -docker-ci -========= - -docker-ci is our buildbot continuous integration server, -building and testing docker, hosted on EC2 and reachable at -http://docker-ci.dotcloud.com - - -Deployment -========== - -# Load AWS credentials -export AWS_ACCESS_KEY_ID='' -export AWS_SECRET_ACCESS_KEY='' -export AWS_KEYPAIR_NAME='' -export AWS_SSH_PRIVKEY='' - -# Load buildbot credentials and config -export BUILDBOT_PWD='' -export IRC_PWD='' -export IRC_CHANNEL='docker-dev' -export SMTP_USER='' -export SMTP_PWD='' -export EMAIL_RCP='' - -# Load registry test credentials -export REGISTRY_USER='' -export REGISTRY_PWD='' - -cd docker/testing -vagrant up --provider=aws - - -github pull request -=================== - -The entire docker pull request test workflow is event driven by github. Its -usage is fully automatic and the results are logged in docker-ci.dotcloud.com - -Each time there is a pull request on docker's github project, github connects -to docker-ci using github's rest API documented in http://developer.github.com/v3/repos/hooks -The issued command to program github's notification PR event was: -curl -u GITHUB_USER:GITHUB_PASSWORD -d '{"name":"web","active":true,"events":["pull_request"],"config":{"url":"http://docker-ci.dotcloud.com:8011/change_hook/github?project=docker"}}' https://api.github.com/repos/dotcloud/docker/hooks - -buildbot (0.8.7p1) was patched using ./testing/buildbot/github.py, so it -can understand the PR data github sends to it. Originally PR #1603 (ee64e099e0) -implemented this capability. Also we added a new scheduler to exclusively filter -PRs. and the 'pullrequest' builder to rebase the PR on top of master and test it. - - -nighthly release -================ - -The nightly release process is done by buildbot, running a DinD container that downloads -the docker repository and builds the release container. The resulting docker -binary is then tested, and if everything is fine, the release is done. diff --git a/components/engine/hack/infrastructure/docker-ci/Dockerfile b/components/engine/hack/infrastructure/docker-ci/Dockerfile index d894330ffa..fd795f4d45 100644 --- a/components/engine/hack/infrastructure/docker-ci/Dockerfile +++ b/components/engine/hack/infrastructure/docker-ci/Dockerfile @@ -1,47 +1,29 @@ -# VERSION: 0.25 -# DOCKER-VERSION 0.6.6 -# AUTHOR: Daniel Mizyrycki -# DESCRIPTION: Deploy docker-ci on Digital Ocean -# COMMENTS: -# CONFIG_JSON is an environment variable json string loaded as: -# -# export CONFIG_JSON=' -# { "DROPLET_NAME": "docker-ci", -# "DO_CLIENT_ID": "Digital_Ocean_client_id", -# "DO_API_KEY": "Digital_Ocean_api_key", -# "DOCKER_KEY_ID": "Digital_Ocean_ssh_key_id", -# "DOCKER_CI_KEY_PATH": "docker-ci_private_key_path", -# "DOCKER_CI_PUB": "$(cat docker-ci_ssh_public_key.pub)", -# "DOCKER_CI_KEY": "$(cat docker-ci_ssh_private_key.key)", -# "BUILDBOT_PWD": "Buildbot_server_password", -# "IRC_PWD": "Buildbot_IRC_password", -# "SMTP_USER": "SMTP_server_user", -# "SMTP_PWD": "SMTP_server_password", -# "PKG_ACCESS_KEY": "Docker_release_S3_bucket_access_key", -# "PKG_SECRET_KEY": "Docker_release_S3_bucket_secret_key", -# "PKG_GPG_PASSPHRASE": "Docker_release_gpg_passphrase", -# "INDEX_AUTH": "Index_encripted_user_password", -# "REGISTRY_USER": "Registry_test_user", -# "REGISTRY_PWD": "Registry_test_password", -# "REGISTRY_BUCKET": "Registry_S3_bucket_name", -# "REGISTRY_ACCESS_KEY": "Registry_S3_bucket_access_key", -# "REGISTRY_SECRET_KEY": "Registry_S3_bucket_secret_key", -# "IRC_CHANNEL": "Buildbot_IRC_channel", -# "EMAIL_RCP": "Buildbot_mailing_receipient" }' -# -# -# TO_BUILD: docker build -t docker-ci . -# TO_DEPLOY: docker run -e CONFIG_JSON="${CONFIG_JSON}" docker-ci +# DOCKER-VERSION: 0.7.6 +# AUTHOR: Daniel Mizyrycki +# DESCRIPTION: docker-ci continuous integration service +# TO_BUILD: docker build -rm -t docker-ci/docker-ci . +# TO_RUN: docker run -rm -i -t -p 8000:80 -p 2222:22 -v /run:/var/socket \ +# -v /data/docker-ci:/data/docker-ci docker-ci/docker-ci from ubuntu:12.04 +maintainer Daniel Mizyrycki -run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' \ - > /etc/apt/sources.list -run apt-get update; apt-get install -y git python2.7 python-dev libevent-dev \ - python-pip ssh rsync less vim -run pip install requests fabric +ENV DEBIAN_FRONTEND noninteractive +RUN echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > \ + /etc/apt/sources.list; apt-get update +RUN apt-get install -y --no-install-recommends python2.7 python-dev \ + libevent-dev git supervisor ssh rsync less vim sudo gcc wget nginx +RUN cd /tmp; wget http://python-distribute.org/distribute_setup.py +RUN cd /tmp; python distribute_setup.py; easy_install pip; rm distribute_setup.py -# Add deployment code and set default container command -add . /docker-ci -cmd "/docker-ci/deployment.py" +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 +RUN echo 'deb http://get.docker.io/ubuntu docker main' > \ + /etc/apt/sources.list.d/docker.list; apt-get update +RUN apt-get install -y lxc-docker-0.8.0 +RUN pip install SQLAlchemy==0.7.10 buildbot buildbot-slave pyopenssl boto +RUN ln -s /var/socket/docker.sock /run/docker.sock +ADD . /docker-ci +RUN /docker-ci/setup.sh + +ENTRYPOINT ["supervisord", "-n"] diff --git a/components/engine/hack/infrastructure/docker-ci/README.rst b/components/engine/hack/infrastructure/docker-ci/README.rst index 33a14359bf..3e429ffdd5 100644 --- a/components/engine/hack/infrastructure/docker-ci/README.rst +++ b/components/engine/hack/infrastructure/docker-ci/README.rst @@ -1,26 +1,65 @@ -======= -testing -======= +========= +docker-ci +========= -This directory contains docker-ci testing related files. +This directory contains docker-ci continuous integration system. +As expected, it is a fully dockerized and deployed using +docker-container-runner. +docker-ci is based on Buildbot, a continuous integration system designed +to automate the build/test cycle. By automatically rebuilding and testing +the tree each time something has changed, build problems are pinpointed +quickly, before other developers are inconvenienced by the failure. +We are running buildbot at Rackspace to verify docker and docker-registry +pass tests, and check for coverage code details. + +docker-ci instance is at https://docker-ci.docker.io/waterfall + +Inside docker-ci container we have the following directory structure: + +/docker-ci source code of docker-ci +/data/backup/docker-ci/ daily backup (replicated over S3) +/data/docker-ci/coverage/{docker,docker-registry}/ mapped to host volumes +/data/buildbot/{master,slave}/ main docker-ci buildbot config and database +/var/socket/{docker.sock} host volume access to docker socket -Buildbot -======== +Production deployment +===================== -Buildbot is a continuous integration system designed to automate the -build/test cycle. By automatically rebuilding and testing the tree each time -something has changed, build problems are pinpointed quickly, before other -developers are inconvenienced by the failure. +:: -We are running buildbot in Amazon's EC2 to verify docker passes all -tests when commits get pushed to the master branch and building -nightly releases using Docker in Docker awesome implementation made -by Jerome Petazzoni. + # Clone docker-ci repository + git clone https://github.com/dotcloud/docker + cd docker/hack/infrastructure/docker-ci -https://github.com/jpetazzo/dind + export DOCKER_PROD=[PRODUCTION_SERVER_IP] -Docker's buildbot instance is at http://docker-ci.dotcloud.com/waterfall + # Create data host volume. (only once) + docker -H $DOCKER_PROD run -v /home:/data ubuntu:12.04 \ + mkdir -p /data/docker-ci/coverage/docker + docker -H $DOCKER_PROD run -v /home:/data ubuntu:12.04 \ + mkdir -p /data/docker-ci/coverage/docker-registry + docker -H $DOCKER_PROD run -v /home:/data ubuntu:12.04 \ + chown -R 1000.1000 /data/docker-ci -For deployment instructions, please take a look at -hack/infrastructure/docker-ci/Dockerfile + # dcr deployment. Define credentials and special environment dcr variables + # ( retrieved at /hack/infrastructure/docker-ci/dcr/prod/docker-ci.yml ) + export WEB_USER=[DOCKER-CI-WEBSITE-USERNAME] + export WEB_IRC_PWD=[DOCKER-CI-WEBSITE-PASSWORD] + export BUILDBOT_PWD=[BUILDSLAVE_PASSWORD] + export AWS_ACCESS_KEY=[DOCKER_RELEASE_S3_ACCESS] + export AWS_SECRET_KEY=[DOCKER_RELEASE_S3_SECRET] + export GPG_PASSPHRASE=[DOCKER_RELEASE_PASSPHRASE] + export BACKUP_AWS_ID=[S3_BUCKET_CREDENTIAL_ACCESS] + export BACKUP_AWS_SECRET=[S3_BUCKET_CREDENTIAL_SECRET] + export SMTP_USER=[MAILGUN_SMTP_USERNAME] + export SMTP_PWD=[MAILGUN_SMTP_PASSWORD] + export EMAIL_RCP=[EMAIL_FOR_BUILD_ERRORS] + + # Build docker-ci and testbuilder docker images + docker -H $DOCKER_PROD build -rm -t docker-ci/docker-ci . + (cd testbuilder; docker -H $DOCKER_PROD build -rm -t docker-ci/testbuilder .) + + # Run docker-ci container ( assuming no previous container running ) + (cd dcr/prod; dcr docker-ci.yml start) + (cd dcr/prod; dcr docker-ci.yml register docker-ci.docker.io) diff --git a/components/engine/hack/infrastructure/docker-ci/VERSION b/components/engine/hack/infrastructure/docker-ci/VERSION index 0bfccb0804..b49b25336d 100644 --- a/components/engine/hack/infrastructure/docker-ci/VERSION +++ b/components/engine/hack/infrastructure/docker-ci/VERSION @@ -1 +1 @@ -0.4.5 +0.5.6 diff --git a/components/engine/hack/infrastructure/docker-ci/buildbot/README.rst b/components/engine/hack/infrastructure/docker-ci/buildbot/README.rst deleted file mode 100644 index 6cbcb8d93a..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/buildbot/README.rst +++ /dev/null @@ -1 +0,0 @@ -Buildbot configuration and setup files diff --git a/components/engine/hack/infrastructure/docker-ci/buildbot/buildbot.conf b/components/engine/hack/infrastructure/docker-ci/buildbot/buildbot.conf deleted file mode 100644 index e07b2e3c8c..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/buildbot/buildbot.conf +++ /dev/null @@ -1,18 +0,0 @@ -[program:buildmaster] -command=twistd --nodaemon --no_save -y buildbot.tac -directory=/data/buildbot/master -chown= root:root -redirect_stderr=true -stdout_logfile=/var/log/supervisor/buildbot-master.log -stderr_logfile=/var/log/supervisor/buildbot-master.log - -[program:buildworker] -command=twistd --nodaemon --no_save -y buildbot.tac -directory=/data/buildbot/slave -chown= root:root -redirect_stderr=true -stdout_logfile=/var/log/supervisor/buildbot-slave.log -stderr_logfile=/var/log/supervisor/buildbot-slave.log - -[group:buildbot] -programs=buildmaster,buildworker diff --git a/components/engine/hack/infrastructure/docker-ci/buildbot/github.py b/components/engine/hack/infrastructure/docker-ci/buildbot/github.py index ff6b6c62dd..5316e13282 100644 --- a/components/engine/hack/infrastructure/docker-ci/buildbot/github.py +++ b/components/engine/hack/infrastructure/docker-ci/buildbot/github.py @@ -17,7 +17,7 @@ """ github_buildbot.py is based on git_buildbot.py -github_buildbot.py will determine the repository information from the JSON +github_buildbot.py will determine the repository information from the JSON HTTP POST it receives from github.com and build the appropriate repository. If your github repository is private, you must add a ssh key to the github repository for the user who initiated the build on the buildslave. @@ -88,7 +88,8 @@ def getChanges(request, options = None): payload = json.loads(request.args['payload'][0]) import urllib,datetime fname = str(datetime.datetime.now()).replace(' ','_').replace(':','-')[:19] - open('github_{0}.json'.format(fname),'w').write(json.dumps(json.loads(urllib.unquote(request.args['payload'][0])), sort_keys = True, indent = 2)) + # Github event debug + # open('github_{0}.json'.format(fname),'w').write(json.dumps(json.loads(urllib.unquote(request.args['payload'][0])), sort_keys = True, indent = 2)) if 'pull_request' in payload: user = payload['pull_request']['user']['login'] @@ -142,13 +143,13 @@ def process_change(payload, user, repo, repo_url, project): 'category' : 'github_pullrequest', 'who' : '{0} - PR#{1}'.format(user,payload['number']), 'files' : [], - 'comments' : payload['pull_request']['title'], + 'comments' : payload['pull_request']['title'], 'revision' : newrev, 'when' : convertTime(payload['pull_request']['updated_at']), 'branch' : branch, 'revlink' : '{0}/commit/{1}'.format(repo_url,newrev), 'repository' : repo_url, - 'project' : project }] + 'project' : project }] return changes for commit in payload['commits']: files = [] diff --git a/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg b/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg index 9ca5fc035a..75605da8ab 100644 --- a/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg +++ b/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg @@ -1,4 +1,4 @@ -import os +import os, re from buildbot.buildslave import BuildSlave from buildbot.schedulers.forcesched import ForceScheduler from buildbot.schedulers.basic import SingleBranchScheduler @@ -6,127 +6,156 @@ from buildbot.schedulers.timed import Nightly from buildbot.changes import filter from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory -from buildbot.process.properties import Interpolate +from buildbot.process.properties import Property from buildbot.steps.shell import ShellCommand from buildbot.status import html, words from buildbot.status.web import authz, auth from buildbot.status.mail import MailNotifier -PORT_WEB = 80 # Buildbot webserver port -PORT_GITHUB = 8011 # Buildbot github hook port -PORT_MASTER = 9989 # Port where buildbot master listen buildworkers -TEST_USER = 'buildbot' # Credential to authenticate build triggers -TEST_PWD = 'docker' # Credential to authenticate build triggers -GITHUB_DOCKER = 'github.com/dotcloud/docker' -BUILDBOT_PATH = '/data/buildbot' -DOCKER_PATH = '/go/src/github.com/dotcloud/docker' -DOCKER_CI_PATH = '/docker-ci' + +def ENV(x): + '''Promote an environment variable for global use returning its value''' + retval = os.environ.get(x, '') + globals()[x] = retval + return retval + + +class TestCommand(ShellCommand): + '''Extend ShellCommand with optional summary logs''' + def __init__(self, *args, **kwargs): + super(TestCommand, self).__init__(*args, **kwargs) + + def createSummary(self, log): + exit_status = re.sub(r'.+\n\+ exit (\d+).+', + r'\1', log.getText()[-100:], flags=re.DOTALL) + if exit_status != '0': + return + # Infer coverage path from log + if '+ COVERAGE_PATH' in log.getText(): + path = re.sub(r'.+\+ COVERAGE_PATH=((.+?)-\d+).+', + r'\2/\1', log.getText(), flags=re.DOTALL) + url = '{}coverage/{}/index.html'.format(c['buildbotURL'], path) + self.addURL('coverage', url) + elif 'COVERAGE_FILE' in log.getText(): + path = re.sub(r'.+\+ COVERAGE_FILE=((.+?)-\d+).+', + r'\2/\1', log.getText(), flags=re.DOTALL) + url = '{}coverage/{}/index.html'.format(c['buildbotURL'], path) + self.addURL('coverage', url) + + +PORT_WEB = 8000 # Buildbot webserver port +PORT_GITHUB = 8011 # Buildbot github hook port +PORT_MASTER = 9989 # Port where buildbot master listen buildworkers + +BUILDBOT_URL = '//localhost:{}/'.format(PORT_WEB) +DOCKER_REPO = 'https://github.com/docker-test/docker' +DOCKER_TEST_ARGV = 'HEAD {}'.format(DOCKER_REPO) +REGISTRY_REPO = 'https://github.com/docker-test/docker-registry' +REGISTRY_TEST_ARGV = 'HEAD {}'.format(REGISTRY_REPO) +if ENV('DEPLOYMENT') == 'staging': + BUILDBOT_URL = "//docker-ci-stage.docker.io/" +if ENV('DEPLOYMENT') == 'production': + BUILDBOT_URL = '//docker-ci.docker.io/' + DOCKER_REPO = 'https://github.com/dotcloud/docker' + DOCKER_TEST_ARGV = '' + REGISTRY_REPO = 'https://github.com/dotcloud/docker-registry' + REGISTRY_TEST_ARGV = '' # Credentials set by setup.sh from deployment.py -BUILDBOT_PWD = '' -IRC_PWD = '' -IRC_CHANNEL = '' -SMTP_USER = '' -SMTP_PWD = '' -EMAIL_RCP = '' +ENV('WEB_USER') +ENV('WEB_IRC_PWD') +ENV('BUILDBOT_PWD') +ENV('SMTP_USER') +ENV('SMTP_PWD') +ENV('EMAIL_RCP') +ENV('IRC_CHANNEL') c = BuildmasterConfig = {} -c['title'] = "Docker" +c['title'] = "docker-ci" c['titleURL'] = "waterfall" -c['buildbotURL'] = "http://docker-ci.dotcloud.com/" +c['buildbotURL'] = BUILDBOT_URL c['db'] = {'db_url':"sqlite:///state.sqlite"} c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)] c['slavePortnum'] = PORT_MASTER # Schedulers -c['schedulers'] = [ForceScheduler(name='trigger', builderNames=['docker', - 'index','registry','docker-coverage','registry-coverage','nightlyrelease'])] -c['schedulers'] += [SingleBranchScheduler(name="all", treeStableTimer=None, +c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[ + 'docker', 'docker-registry', 'nightlyrelease', 'backup'])] +c['schedulers'] += [SingleBranchScheduler(name="docker", treeStableTimer=None, change_filter=filter.ChangeFilter(branch='master', - repository='https://github.com/dotcloud/docker'), builderNames=['docker'])] -c['schedulers'] += [SingleBranchScheduler(name='pullrequest', - change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None, - builderNames=['pullrequest'])] -c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease', - 'docker-coverage','registry-coverage'], hour=7, minute=00)] -c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'], - hour=range(0,24,4), minute=15)] + repository=DOCKER_REPO), builderNames=['docker'])] +c['schedulers'] += [SingleBranchScheduler(name="registry", treeStableTimer=None, + change_filter=filter.ChangeFilter(branch='master', + repository=REGISTRY_REPO), builderNames=['docker-registry'])] +c['schedulers'] += [SingleBranchScheduler(name='docker-pr', treeStableTimer=None, + change_filter=filter.ChangeFilter(category='github_pullrequest', + project='docker'), builderNames=['docker-pr'])] +c['schedulers'] += [SingleBranchScheduler(name='docker-registry-pr', treeStableTimer=None, + change_filter=filter.ChangeFilter(category='github_pullrequest', + project='docker-registry'), builderNames=['docker-registry-pr'])] +c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=[ + 'nightlyrelease', 'backup'], hour=7, minute=00)] + # Builders -# Docker commit test -test_cmd = ('docker run -privileged mzdaniel/test_docker hack/dind' - ' test_docker.sh %(src::revision)s') + +# Backup factory = BuildFactory() -factory.addStep(ShellCommand(description='Docker', logEnviron=False, - usePTY=True, command=["sh", "-c", Interpolate(test_cmd)])) -c['builders'] = [BuilderConfig(name='docker',slavenames=['buildworker'], +factory.addStep(TestCommand(description='backup', logEnviron=False, + usePTY=True, command='/docker-ci/tool/backup.py')) +c['builders'] = [BuilderConfig(name='backup',slavenames=['buildworker'], + factory=factory)] + +# Docker test +factory = BuildFactory() +factory.addStep(TestCommand(description='docker', logEnviron=False, + usePTY=True, command='/docker-ci/dockertest/docker {}'.format(DOCKER_TEST_ARGV))) +c['builders'] += [BuilderConfig(name='docker',slavenames=['buildworker'], factory=factory)] # Docker pull request test -test_cmd = ('docker run -privileged mzdaniel/test_docker hack/dind' - ' test_docker.sh %(src::revision)s %(src::repository)s %(src::branch)s') factory = BuildFactory() -factory.addStep(ShellCommand(description='pull_request', logEnviron=False, - usePTY=True, command=["sh", "-c", Interpolate(test_cmd)])) -c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'], +factory.addStep(TestCommand(description='docker-pr', logEnviron=False, + usePTY=True, command=['/docker-ci/dockertest/docker', + Property('revision'), Property('repository'), Property('branch')])) +c['builders'] += [BuilderConfig(name='docker-pr',slavenames=['buildworker'], factory=factory)] -# Docker coverage test +# docker-registry test factory = BuildFactory() -factory.addStep(ShellCommand(description='docker-coverage', logEnviron=False, - usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format( - DOCKER_CI_PATH))) -c['builders'] += [BuilderConfig(name='docker-coverage',slavenames=['buildworker'], +factory.addStep(TestCommand(description='docker-registry', logEnviron=False, + usePTY=True, command='/docker-ci/dockertest/docker-registry {}'.format(REGISTRY_TEST_ARGV))) +c['builders'] += [BuilderConfig(name='docker-registry',slavenames=['buildworker'], factory=factory)] -# Docker registry coverage test +# Docker registry pull request test factory = BuildFactory() -factory.addStep(ShellCommand(description='registry-coverage', logEnviron=False, - usePTY=True, command='docker run registry_coverage'.format( - DOCKER_CI_PATH))) -c['builders'] += [BuilderConfig(name='registry-coverage',slavenames=['buildworker'], - factory=factory)] - -# Registry functional test -factory = BuildFactory() -factory.addStep(ShellCommand(description='registry', logEnviron=False, - command='. {0}/master/credentials.cfg; ' - '{1}/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, DOCKER_CI_PATH), - usePTY=True)) -c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], - factory=factory)] - -# Index functional test -factory = BuildFactory() -factory.addStep(ShellCommand(description='index', logEnviron=False, - command='. {0}/master/credentials.cfg; ' - '{1}/functionaltests/test_index.py'.format(BUILDBOT_PATH, DOCKER_CI_PATH), - usePTY=True)) -c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'], +factory.addStep(TestCommand(description='docker-registry-pr', logEnviron=False, + usePTY=True, command=['/docker-ci/dockertest/docker-registry', + Property('revision'), Property('repository'), Property('branch')])) +c['builders'] += [BuilderConfig(name='docker-registry-pr',slavenames=['buildworker'], factory=factory)] # Docker nightly release -nightlyrelease_cmd = ('docker version; docker run -i -t -privileged -e AWS_S3_BUCKET=' - 'test.docker.io dockerbuilder hack/dind dockerbuild.sh') factory = BuildFactory() factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False, - usePTY=True, command=nightlyrelease_cmd)) + usePTY=True, command=['/docker-ci/dockertest/nightlyrelease'])) c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'], factory=factory)] # Status -authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]), +authz_cfg = authz.Authz(auth=auth.BasicAuth([(WEB_USER, WEB_IRC_PWD)]), forceBuild='auth') c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)] c['status'].append(html.WebStatus(http_port=PORT_GITHUB, allowForce=True, change_hook_dialects={ 'github': True })) -c['status'].append(MailNotifier(fromaddr='buildbot@docker.io', +c['status'].append(MailNotifier(fromaddr='docker-test@docker.io', sendToInterestedUsers=False, extraRecipients=[EMAIL_RCP], mode='failing', relayhost='smtp.mailgun.org', smtpPort=587, useTls=True, smtpUser=SMTP_USER, smtpPassword=SMTP_PWD)) c['status'].append(words.IRC("irc.freenode.net", "dockerqabot", - channels=[IRC_CHANNEL], password=IRC_PWD, allowForce=True, + channels=[IRC_CHANNEL], password=WEB_IRC_PWD, allowForce=True, notify_events={'exception':1, 'successToFailure':1, 'failureToSuccess':1})) diff --git a/components/engine/hack/infrastructure/docker-ci/buildbot/requirements.txt b/components/engine/hack/infrastructure/docker-ci/buildbot/requirements.txt deleted file mode 100644 index d2dcf1d125..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/buildbot/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -sqlalchemy<=0.7.9 -sqlalchemy-migrate>=0.7.2 -buildbot==0.8.7p1 -buildbot_slave==0.8.7p1 -nose==1.2.1 -requests==1.1.0 -flask==0.10.1 -simplejson==2.3.2 -selenium==2.35.0 diff --git a/components/engine/hack/infrastructure/docker-ci/buildbot/setup.sh b/components/engine/hack/infrastructure/docker-ci/buildbot/setup.sh deleted file mode 100755 index c5d9cb988e..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/buildbot/setup.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash - -# Setup of buildbot configuration. Package installation is being done by -# Vagrantfile -# Dependencies: buildbot, buildbot-slave, supervisor - -USER=$1 -CFG_PATH=$2 -DOCKER_PATH=$3 -BUILDBOT_PWD=$4 -IRC_PWD=$5 -IRC_CHANNEL=$6 -SMTP_USER=$7 -SMTP_PWD=$8 -EMAIL_RCP=$9 -REGISTRY_USER=${10} -REGISTRY_PWD=${11} -REGISTRY_BUCKET=${12} -REGISTRY_ACCESS_KEY=${13} -REGISTRY_SECRET_KEY=${14} -BUILDBOT_PATH="/data/buildbot" -SLAVE_NAME="buildworker" -SLAVE_SOCKET="localhost:9989" - -export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" - -function run { su $USER -c "$1"; } - -# Exit if buildbot has already been installed -[ -d "$BUILDBOT_PATH" ] && exit 0 - -# Setup buildbot -run "mkdir -p $BUILDBOT_PATH" -cd $BUILDBOT_PATH -run "buildbot create-master master" -run "cp $CFG_PATH/master.cfg master" -run "sed -i -E 's#(BUILDBOT_PWD = ).+#\1\"$BUILDBOT_PWD\"#' master/master.cfg" -run "sed -i -E 's#(IRC_PWD = ).+#\1\"$IRC_PWD\"#' master/master.cfg" -run "sed -i -E 's#(IRC_CHANNEL = ).+#\1\"$IRC_CHANNEL\"#' master/master.cfg" -run "sed -i -E 's#(SMTP_USER = ).+#\1\"$SMTP_USER\"#' master/master.cfg" -run "sed -i -E 's#(SMTP_PWD = ).+#\1\"$SMTP_PWD\"#' master/master.cfg" -run "sed -i -E 's#(EMAIL_RCP = ).+#\1\"$EMAIL_RCP\"#' master/master.cfg" -run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD" -run "echo 'export DOCKER_CREDS=\"$REGISTRY_USER:$REGISTRY_PWD\"' > $BUILDBOT_PATH/master/credentials.cfg" -run "echo 'export S3_BUCKET=\"$REGISTRY_BUCKET\"' >> $BUILDBOT_PATH/master/credentials.cfg" -run "echo 'export S3_ACCESS_KEY=\"$REGISTRY_ACCESS_KEY\"' >> $BUILDBOT_PATH/master/credentials.cfg" -run "echo 'export S3_SECRET_KEY=\"$REGISTRY_SECRET_KEY\"' >> $BUILDBOT_PATH/master/credentials.cfg" - -# Patch github webstatus to capture pull requests -cp $CFG_PATH/github.py /usr/local/lib/python2.7/dist-packages/buildbot/status/web/hooks - -# Allow buildbot subprocesses (docker tests) to properly run in containers, -# in particular with docker -u -run "sed -i 's/^umask = None/umask = 000/' slave/buildbot.tac" - -# Setup supervisor -cp $CFG_PATH/buildbot.conf /etc/supervisor/conf.d/buildbot.conf -sed -i -E "s/^chmod=0700.+/chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf -kill -HUP $(pgrep -f "/usr/bin/python /usr/bin/supervisord") diff --git a/components/engine/hack/infrastructure/docker-ci/dcr/prod/docker-ci.yml b/components/engine/hack/infrastructure/docker-ci/dcr/prod/docker-ci.yml new file mode 100644 index 0000000000..523535446a --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/dcr/prod/docker-ci.yml @@ -0,0 +1,22 @@ +docker-ci: + image: "docker-ci/docker-ci" + release_name: "docker-ci-0.5.6" + ports: ["80","2222:22","8011:8011"] + register: "80" + volumes: ["/run:/var/socket","/home/docker-ci:/data/docker-ci"] + command: [] + env: + - "DEPLOYMENT=production" + - "IRC_CHANNEL=docker-testing" + - "BACKUP_BUCKET=backup-ci" + - "$WEB_USER" + - "$WEB_IRC_PWD" + - "$BUILDBOT_PWD" + - "$AWS_ACCESS_KEY" + - "$AWS_SECRET_KEY" + - "$GPG_PASSPHRASE" + - "$BACKUP_AWS_ID" + - "$BACKUP_AWS_SECRET" + - "$SMTP_USER" + - "$SMTP_PWD" + - "$EMAIL_RCP" diff --git a/components/engine/hack/infrastructure/docker-ci/dcr/prod/settings.yml b/components/engine/hack/infrastructure/docker-ci/dcr/prod/settings.yml new file mode 100644 index 0000000000..9831afa6dd --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/dcr/prod/settings.yml @@ -0,0 +1,5 @@ +default: + hipaches: ['192.168.100.67:6379'] + daemons: ['192.168.100.67:4243'] + use_ssh: False + diff --git a/components/engine/hack/infrastructure/docker-ci/dcr/stage/docker-ci.yml b/components/engine/hack/infrastructure/docker-ci/dcr/stage/docker-ci.yml new file mode 100644 index 0000000000..8eba84825c --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/dcr/stage/docker-ci.yml @@ -0,0 +1,22 @@ +docker-ci: + image: "docker-ci/docker-ci" + release_name: "docker-ci-stage" + ports: ["80","2222:22","8011:8011"] + register: "80" + volumes: ["/run:/var/socket","/home/docker-ci:/data/docker-ci"] + command: [] + env: + - "DEPLOYMENT=staging" + - "IRC_CHANNEL=docker-testing-staging" + - "BACKUP_BUCKET=ci-backup-stage" + - "$BACKUP_AWS_ID" + - "$BACKUP_AWS_SECRET" + - "$WEB_USER" + - "$WEB_IRC_PWD" + - "$BUILDBOT_PWD" + - "$AWS_ACCESS_KEY" + - "$AWS_SECRET_KEY" + - "$GPG_PASSPHRASE" + - "$SMTP_USER" + - "$SMTP_PWD" + - "$EMAIL_RCP" diff --git a/components/engine/hack/infrastructure/docker-ci/dcr/stage/settings.yml b/components/engine/hack/infrastructure/docker-ci/dcr/stage/settings.yml new file mode 100644 index 0000000000..a7d37acff3 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/dcr/stage/settings.yml @@ -0,0 +1,5 @@ +default: + hipaches: ['192.168.100.65:6379'] + daemons: ['192.168.100.65:4243'] + use_ssh: False + diff --git a/components/engine/hack/infrastructure/docker-ci/deployment.py b/components/engine/hack/infrastructure/docker-ci/deployment.py deleted file mode 100755 index fd0fdb0fe8..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/deployment.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python - -import os, sys, re, json, requests, base64 -from subprocess import call -from fabric import api -from fabric.api import cd, run, put, sudo -from os import environ as env -from datetime import datetime -from time import sleep - -# Remove SSH private key as it needs more processing -CONFIG = json.loads(re.sub(r'("DOCKER_CI_KEY".+?"(.+?)",)','', - env['CONFIG_JSON'], flags=re.DOTALL)) - -# Populate environment variables -for key in CONFIG: - env[key] = CONFIG[key] - -# Load SSH private key -env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1', - env['CONFIG_JSON'],flags=re.DOTALL) - -DROPLET_NAME = env.get('DROPLET_NAME','docker-ci') -TIMEOUT = 120 # Seconds before timeout droplet creation -IMAGE_ID = 1004145 # Docker on Ubuntu 13.04 -REGION_ID = 4 # New York 2 -SIZE_ID = 62 # memory 2GB -DO_IMAGE_USER = 'root' # Image user on Digital Ocean -API_URL = 'https://api.digitalocean.com/' -DOCKER_PATH = '/go/src/github.com/dotcloud/docker' -DOCKER_CI_PATH = '/docker-ci' -CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH) - - -class DigitalOcean(): - - def __init__(self, key, client): - '''Set default API parameters''' - self.key = key - self.client = client - self.api_url = API_URL - - def api(self, cmd_path, api_arg={}): - '''Make api call''' - api_arg.update({'api_key':self.key, 'client_id':self.client}) - resp = requests.get(self.api_url + cmd_path, params=api_arg).text - resp = json.loads(resp) - if resp['status'] != 'OK': - raise Exception(resp['error_message']) - return resp - - def droplet_data(self, name): - '''Get droplet data''' - data = self.api('droplets') - data = [droplet for droplet in data['droplets'] - if droplet['name'] == name] - return data[0] if data else {} - - -def json_fmt(data): - '''Format json output''' - return json.dumps(data, sort_keys = True, indent = 2) - - -do = DigitalOcean(env['DO_API_KEY'], env['DO_CLIENT_ID']) - -# Get DROPLET_NAME data -data = do.droplet_data(DROPLET_NAME) - -# Stop processing if DROPLET_NAME exists on Digital Ocean -if data: - print ('Droplet: {} already deployed. Not further processing.' - .format(DROPLET_NAME)) - exit(1) - -# Create droplet -do.api('droplets/new', {'name':DROPLET_NAME, 'region_id':REGION_ID, - 'image_id':IMAGE_ID, 'size_id':SIZE_ID, - 'ssh_key_ids':[env['DOCKER_KEY_ID']]}) - -# Wait for droplet to be created. -start_time = datetime.now() -while (data.get('status','') != 'active' and ( - datetime.now()-start_time).seconds < TIMEOUT): - data = do.droplet_data(DROPLET_NAME) - print data['status'] - sleep(3) - -# Wait for the machine to boot -sleep(15) - -# Get droplet IP -ip = str(data['ip_address']) -print 'droplet: {} ip: {}'.format(DROPLET_NAME, ip) - -# Create docker-ci ssh private key so docker-ci docker container can communicate -# with its EC2 instance -os.makedirs('/root/.ssh') -open('/root/.ssh/id_rsa','w').write(env['DOCKER_CI_KEY']) -os.chmod('/root/.ssh/id_rsa',0600) -open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n') - -api.env.host_string = ip -api.env.user = DO_IMAGE_USER -api.env.key_filename = '/root/.ssh/id_rsa' - -# Correct timezone -sudo('echo "America/Los_Angeles" >/etc/timezone') -sudo('dpkg-reconfigure --frontend noninteractive tzdata') - -# Load public docker-ci key -sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB'])) - -# Create docker nightly release credentials file -credentials = { - 'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'], - 'AWS_SECRET_KEY': env['PKG_SECRET_KEY'], - 'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE']} -open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write( - base64.b64encode(json.dumps(credentials))) - -# Transfer docker -sudo('mkdir -p ' + DOCKER_CI_PATH) -sudo('chown {}.{} {}'.format(DO_IMAGE_USER, DO_IMAGE_USER, DOCKER_CI_PATH)) -call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, DO_IMAGE_USER, ip, - os.path.dirname(DOCKER_CI_PATH)), shell=True) - -# Install Docker and Buildbot dependencies -sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker') -sudo('apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9') -sudo('echo deb https://get.docker.io/ubuntu docker main >' - ' /etc/apt/sources.list.d/docker.list') -sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n' - 'deb http://us.archive.ubuntu.com/ubuntu/ raring-security main universe\n"' - ' > /etc/apt/sources.list; apt-get update') -sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev' - ' python-pip supervisor git mercurial linux-image-extra-$(uname -r)' - ' aufs-tools make libfontconfig libevent-dev libsqlite3-dev libssl-dev') -sudo('wget -O - https://go.googlecode.com/files/go1.2.linux-amd64.tar.gz | ' - 'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go') -sudo('GOPATH=/go go get -d github.com/dotcloud/docker') -sudo('pip install -r {}/requirements.txt'.format(CFG_PATH)) - -# Install docker and testing dependencies -sudo('apt-get install -y -q lxc-docker') -sudo('curl -s https://phantomjs.googlecode.com/files/' - 'phantomjs-1.9.1-linux-x86_64.tar.bz2 | tar jx -C /usr/bin' - ' --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs') - -# Build docker-ci containers -sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH)) -sudo('cd {}; docker build -t docker-ci .'.format(DOCKER_CI_PATH)) -sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format( - DOCKER_CI_PATH)) -sudo('cd {}/registry-coverage; docker build -t registry_coverage .'.format( - DOCKER_CI_PATH)) - -# Download docker-ci testing container -sudo('docker pull mzdaniel/test_docker') - -# Setup buildbot -sudo('mkdir /data') -sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}' - ' {11} {12}'.format(CFG_PATH, DOCKER_PATH, env['BUILDBOT_PWD'], - env['IRC_PWD'], env['IRC_CHANNEL'], env['SMTP_USER'], - env['SMTP_PWD'], env['EMAIL_RCP'], env['REGISTRY_USER'], - env['REGISTRY_PWD'], env['REGISTRY_BUCKET'], env['REGISTRY_ACCESS_KEY'], - env['REGISTRY_SECRET_KEY'])) - -# Preventively reboot docker-ci daily -sudo('ln -s /sbin/reboot /etc/cron.daily') diff --git a/components/engine/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh b/components/engine/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh deleted file mode 100755 index c29ede5b81..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -x -# Generate a random string of $1 characters -function random { - cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1 -} - -# Compute test paths -BASE_PATH=`pwd`/test_docker_$(random 12) -DOCKER_PATH=$BASE_PATH/go/src/github.com/dotcloud/docker -export GOPATH=$BASE_PATH/go:$DOCKER_PATH/vendor - -# Fetch latest master -mkdir -p $DOCKER_PATH -cd $DOCKER_PATH -git init . -git fetch -q http://github.com/dotcloud/docker master -git reset --hard FETCH_HEAD - -# Fetch go coverage -cd $BASE_PATH/go -GOPATH=$BASE_PATH/go go get github.com/axw/gocov/gocov -sudo -E GOPATH=$GOPATH ./bin/gocov test -deps -exclude-goroot -v\ - -exclude github.com/gorilla/context,github.com/gorilla/mux,github.com/kr/pty,\ -code.google.com/p/go.net/websocket\ - github.com/dotcloud/docker | ./bin/gocov report; exit_status=$? - -# Cleanup testing directory -rm -rf $BASE_PATH - -exit $exit_status diff --git a/components/engine/hack/infrastructure/docker-ci/docker-test/Dockerfile b/components/engine/hack/infrastructure/docker-ci/docker-test/Dockerfile deleted file mode 100644 index 0f3a63f5f1..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/docker-test/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -# VERSION: 0.4 -# DOCKER-VERSION 0.6.6 -# AUTHOR: Daniel Mizyrycki -# DESCRIPTION: Testing docker PRs and commits on top of master using -# REFERENCES: This code reuses the excellent implementation of -# Docker in Docker made by Jerome Petazzoni. -# https://github.com/jpetazzo/dind -# COMMENTS: -# This Dockerfile adapts /Dockerfile to enable docker PRs and commits testing -# Optional arguments: -# [commit] (default: 'HEAD') -# [repo] (default: 'http://github.com/dotcloud/docker') -# [branch] (default: 'master') -# TO_BUILD: docker build -t test_docker . -# TO_RUN: docker run -privileged test_docker hack/dind test_docker.sh [commit] [repo] [branch] - -from docker -maintainer Daniel Mizyrycki - -# Setup go in PATH. Extracted from /Dockerfile -env PATH /usr/local/go/bin:$PATH - -# Add test_docker.sh -add test_docker.sh /usr/bin/test_docker.sh -run chmod +x /usr/bin/test_docker.sh diff --git a/components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh b/components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh deleted file mode 100755 index 14816706ed..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -x -COMMIT=${1-HEAD} -REPO=${2-http://github.com/dotcloud/docker} -BRANCH=${3-master} - -# Compute test paths -DOCKER_PATH=/go/src/github.com/dotcloud/docker - -# Timestamp -echo -date; echo - -# Fetch latest master -cd / -rm -rf /go -git clone -q -b master http://github.com/dotcloud/docker $DOCKER_PATH -cd $DOCKER_PATH - -# Merge commit -git fetch -q "$REPO" "$BRANCH" -git merge --no-edit $COMMIT || exit 255 - -# Test commit -./hack/make.sh test; exit_status=$? - -# Display load if test fails -if [ $exit_status -ne 0 ] ; then - uptime; echo; free -fi - -exit $exit_status diff --git a/components/engine/hack/infrastructure/docker-ci/dockertest/docker b/components/engine/hack/infrastructure/docker-ci/dockertest/docker new file mode 120000 index 0000000000..e3f094ee63 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/dockertest/docker @@ -0,0 +1 @@ +project \ No newline at end of file diff --git a/components/engine/hack/infrastructure/docker-ci/dockertest/docker-registry b/components/engine/hack/infrastructure/docker-ci/dockertest/docker-registry new file mode 120000 index 0000000000..e3f094ee63 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/dockertest/docker-registry @@ -0,0 +1 @@ +project \ No newline at end of file diff --git a/components/engine/hack/infrastructure/docker-ci/dockertest/nightlyrelease b/components/engine/hack/infrastructure/docker-ci/dockertest/nightlyrelease new file mode 100755 index 0000000000..475b088065 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/dockertest/nightlyrelease @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +if [ "$DEPLOYMENT" == "production" ]; then + AWS_S3_BUCKET='test.docker.io' +else + AWS_S3_BUCKET='get-staging.docker.io' +fi + +docker run -rm -privileged -v /run:/var/socket \ + -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY \ + -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE \ + -e DOCKER_RELEASE=1 -e DEPLOYMENT=$DEPLOYMENT docker-ci/testbuilder docker + diff --git a/components/engine/hack/infrastructure/docker-ci/dockertest/project b/components/engine/hack/infrastructure/docker-ci/dockertest/project new file mode 100755 index 0000000000..160f2d5d59 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/dockertest/project @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -x + +PROJECT_NAME=$(basename $0) + +docker run -rm -u sysadmin -e DEPLOYMENT=$DEPLOYMENT -v /run:/var/socket \ + -v /home/docker-ci/coverage/$PROJECT_NAME:/data docker-ci/testbuilder $PROJECT_NAME $1 $2 $3 + diff --git a/components/engine/hack/infrastructure/docker-ci/nginx/nginx.conf b/components/engine/hack/infrastructure/docker-ci/nginx/nginx.conf new file mode 100644 index 0000000000..6649741134 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/nginx/nginx.conf @@ -0,0 +1,12 @@ +server { + listen 80; + root /data/docker-ci; + + location / { + proxy_pass http://localhost:8000/; + } + + location /coverage { + root /data/docker-ci; + } +} diff --git a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile deleted file mode 100644 index 2100a9e8e9..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# VERSION: 1.6 -# DOCKER-VERSION 0.6.6 -# AUTHOR: Daniel Mizyrycki -# DESCRIPTION: Build docker nightly release using Docker in Docker. -# REFERENCES: This code reuses the excellent implementation of docker in docker -# made by Jerome Petazzoni. https://github.com/jpetazzo/dind -# COMMENTS: -# release_credentials.json is a base64 json encoded file containing: -# { "AWS_ACCESS_KEY": "Test_docker_AWS_S3_bucket_id", -# "AWS_SECRET_KEY": "Test_docker_AWS_S3_bucket_key", -# "GPG_PASSPHRASE": "Test_docker_GPG_passphrase_signature" } -# TO_BUILD: docker build -t dockerbuilder . -# TO_RELEASE: docker run -i -t -privileged -e AWS_S3_BUCKET="test.docker.io" dockerbuilder hack/dind dockerbuild.sh - -from docker -maintainer Daniel Mizyrycki - -# Add docker dependencies and downloading packages -run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list -run apt-get update; apt-get install -y -q wget python2.7 - -# Add production docker binary -run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker - -# Add proto docker builder -add ./dockerbuild.sh /usr/bin/dockerbuild.sh -run chmod +x /usr/bin/dockerbuild.sh - -# Add release credentials -add ./release_credentials.json /root/release_credentials.json diff --git a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh deleted file mode 100644 index d5e58da7e1..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY and PG_PASSPHRASE are decoded -# from /root/release_credentials.json -# Variable AWS_S3_BUCKET is passed to the environment from docker run -e - -# Turn debug off to load credentials from the environment -set +x -eval $(cat /root/release_credentials.json | python -c ' -import sys,json,base64; -d=json.loads(base64.b64decode(sys.stdin.read())); -exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")') - -# Fetch docker master branch -set -x -cd / -rm -rf /go -git clone -q -b master http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker -cd /go/src/github.com/dotcloud/docker - -# Launch docker daemon using dind inside the container -/usr/bin/docker version -/usr/bin/docker -d & -sleep 5 - -# Build Docker release container -docker build -t docker . - -# Test docker and if everything works well, release -echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX docker hack/release.sh -set +x -docker run -privileged -i -t -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE docker hack/release.sh -exit_status=$? - -# Display load if test fails -set -x -if [ $exit_status -ne 0 ] ; then - uptime; echo; free - exit 1 -fi diff --git a/components/engine/hack/infrastructure/docker-ci/registry-coverage/Dockerfile b/components/engine/hack/infrastructure/docker-ci/registry-coverage/Dockerfile deleted file mode 100644 index e544645b67..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/registry-coverage/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# VERSION: 0.1 -# DOCKER-VERSION 0.6.4 -# AUTHOR: Daniel Mizyrycki -# DESCRIPTION: Docker registry coverage -# COMMENTS: Add registry coverage into the docker-ci image -# TO_BUILD: docker build -t registry_coverage . -# TO_RUN: docker run registry_coverage - -from docker-ci -maintainer Daniel Mizyrycki - -# Add registry_coverager.sh and dependencies -run pip install coverage flask pyyaml requests simplejson python-glanceclient \ - blinker redis boto gevent rsa mock -add registry_coverage.sh /usr/bin/registry_coverage.sh -run chmod +x /usr/bin/registry_coverage.sh - -cmd "/usr/bin/registry_coverage.sh" diff --git a/components/engine/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh b/components/engine/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh deleted file mode 100755 index c67b17eba0..0000000000 --- a/components/engine/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -x - -# Setup the environment -REGISTRY_PATH=/data/docker-registry -export SETTINGS_FLAVOR=test -export DOCKER_REGISTRY_CONFIG=config_test.yml -export PYTHONPATH=$REGISTRY_PATH/test - -# Fetch latest docker-registry master -rm -rf $REGISTRY_PATH -git clone https://github.com/dotcloud/docker-registry -b master $REGISTRY_PATH -cd $REGISTRY_PATH - -# Generate coverage -coverage run -m unittest discover test || exit 1 -coverage report --include='./*' --omit='./test/*' diff --git a/components/engine/hack/infrastructure/docker-ci/setup.sh b/components/engine/hack/infrastructure/docker-ci/setup.sh new file mode 100755 index 0000000000..65a00f6dd0 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/setup.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# Set timezone +echo "GMT" >/etc/timezone +dpkg-reconfigure --frontend noninteractive tzdata + +# Set ssh superuser +mkdir -p /data/buildbot /var/run/sshd /run +useradd -m -d /home/sysadmin -s /bin/bash -G sudo,docker -p '*' sysadmin +sed -Ei 's/(\%sudo.*) ALL/\1 NOPASSWD:ALL/' /etc/sudoers +cd /home/sysadmin +mkdir .ssh +chmod 700 .ssh +cat > .ssh/authorized_keys << 'EOF' +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC7ALVhwQ68q1SjrKaAduOuOEAcWmb8kDZf5qA7T1fM8AP07EDC7nSKRJ8PXUBGTOQfxm89coJDuSJsTAZ+1PvglXhA0Mq6+knc6ZrZY+SuZlDIDAk4TOdVPoDZnmR1YW2McxHkhcGIOKeC8MMig5NeEjtgQwXzauUSPqeh8HMlLZRMooFYyyluIpn7NaCLzyWjwAQz2s3KyI7VE7hl+ncCrW86v+dciEdwqtzNoUMFb3iDpPxaiCl3rv+SB7co/5eUDTs1FZvUcYMXKQuf8R+2ZKzXOpwr0Zs8sKQXvXavCeWykwGgXLBjVkvrDcHuDD6UXCW63UKgmRECpLZaMBVIIRWLEEgTS5OSQTcxpMVe5zUW6sDvXHTcdPwWrcn1dE9F/0vLC0HJ4ADKelLX5zyTpmXGbuZuntIf1JO67D/K/P++uV1rmVIH+zgtOf23w5rX2zKb4BSTqP0sv61pmWV7MEVoEz6yXswcTjS92tb775v7XLU9vKAkt042ORFdE4/++hejhL/Lj52IRgjt1CJZHZsR9JywJZrz3kYuf8eU2J2FYh0Cpz5gmf0f+12Rt4HztnZxGPP4KuMa66e4+hpx1jynjMZ7D5QUnNYEmuvJByopn8HSluuY/kS5MMyZCZtJLEPGX4+yECX0Di/S0vCRl2NyqfCBqS+yXXT5SA1nFw== docker-test@docker.io +EOF +chmod 600 .ssh/authorized_keys +chown -R sysadmin .ssh + +# Fix docker group id for use of host dockerd by sysadmin +sed -Ei 's/(docker:x:)[^:]+/\1999/' /etc/group + +# Create buildbot configuration +cd /data/buildbot; buildbot create-master master +cp -a /data/buildbot/master/master.cfg.sample \ + /data/buildbot/master/master.cfg +cd /data/buildbot; \ + buildslave create-slave slave localhost:9989 buildworker pass +cp /docker-ci/buildbot/master.cfg /data/buildbot/master + +# Patch github webstatus to capture pull requests +cp /docker-ci/buildbot/github.py /usr/local/lib/python2.7/dist-packages/buildbot/status/web/hooks +chown -R sysadmin.sysadmin /data + +# Create nginx configuration +rm /etc/nginx/sites-enabled/default +cp /docker-ci/nginx/nginx.conf /etc/nginx/conf.d/buildbot.conf +/bin/echo -e '\ndaemon off;\n' >> /etc/nginx/nginx.conf + +# Set supervisord buildbot, nginx and sshd processes +/bin/echo -e "\ +[program:buildmaster]\n\ +command=twistd --nodaemon --no_save -y buildbot.tac\n\ +directory=/data/buildbot/master\n\ +user=sysadmin\n\n\ +[program:buildworker]\n\ +command=twistd --nodaemon --no_save -y buildbot.tac\n\ +directory=/data/buildbot/slave\n\ +user=sysadmin\n" > \ + /etc/supervisor/conf.d/buildbot.conf +/bin/echo -e "[program:nginx]\ncommand=/usr/sbin/nginx\n" > \ + /etc/supervisor/conf.d/nginx.conf +/bin/echo -e "[program:sshd]\ncommand=/usr/sbin/sshd -D\n" > \ + /etc/supervisor/conf.d/sshd.conf diff --git a/components/engine/hack/infrastructure/docker-ci/testbuilder/Dockerfile b/components/engine/hack/infrastructure/docker-ci/testbuilder/Dockerfile new file mode 100644 index 0000000000..a008da6843 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/testbuilder/Dockerfile @@ -0,0 +1,12 @@ +# TO_BUILD: docker build -rm -no-cache -t docker-ci/testbuilder . +# TO_RUN: docker run -rm -u sysadmin \ +# -v /run:/var/socket docker-ci/testbuilder docker-registry +# + +FROM docker-ci/docker-ci +ENV HOME /home/sysadmin + +RUN mkdir /testbuilder +ADD . /testbuilder + +ENTRYPOINT ["/testbuilder/testbuilder.sh"] diff --git a/components/engine/hack/infrastructure/docker-ci/testbuilder/docker-registry.sh b/components/engine/hack/infrastructure/docker-ci/testbuilder/docker-registry.sh new file mode 100755 index 0000000000..72087462ad --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/testbuilder/docker-registry.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -x +set -e +PROJECT_PATH=$1 + +# Build the docker project +cd /data/$PROJECT_PATH +sg docker -c "docker build -q -rm -t registry ." +cd test; sg docker -c "docker build -q -rm -t docker-registry-test ." + +# Run the tests +sg docker -c "docker run -rm -v /home/docker-ci/coverage/docker-registry:/data docker-registry-test" diff --git a/components/engine/hack/infrastructure/docker-ci/testbuilder/docker.sh b/components/engine/hack/infrastructure/docker-ci/testbuilder/docker.sh new file mode 100755 index 0000000000..b365dd7eaf --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/testbuilder/docker.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -x +set -e +PROJECT_PATH=$1 + +# Build the docker project +cd /data/$PROJECT_PATH +sg docker -c "docker build -q -rm -t docker ." + +if [ "$DOCKER_RELEASE" == "1" ]; then + # Do nightly release + echo sg docker -c "docker run -rm -privileged -v /run:/var/socket -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY= -e AWS_SECRET_KEY= -e GPG_PASSPHRASE= docker hack/release.sh" + set +x + sg docker -c "docker run -rm -privileged -v /run:/var/socket -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE docker hack/release.sh" +else + # Run the tests + sg docker -c "docker run -rm -privileged -v /home/docker-ci/coverage/docker:/data docker ./hack/infrastructure/docker-ci/docker-coverage/gocoverage.sh" +fi diff --git a/components/engine/hack/infrastructure/docker-ci/testbuilder/testbuilder.sh b/components/engine/hack/infrastructure/docker-ci/testbuilder/testbuilder.sh new file mode 100755 index 0000000000..70701343c2 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/testbuilder/testbuilder.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Download, build and run a docker project tests +# Environment variables: DEPLOYMENT + +cat $0 +set -e +set -x + +PROJECT=$1 +COMMIT=${2-HEAD} +REPO=${3-https://github.com/dotcloud/$PROJECT} +BRANCH=${4-master} +REPO_PROJ="https://github.com/docker-test/$PROJECT" +if [ "$DEPLOYMENT" == "production" ]; then + REPO_PROJ="https://github.com/dotcloud/$PROJECT" +fi +set +x + +# Generate a random string of $1 characters +function random { + cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1 +} + +PROJECT_PATH="$PROJECT-tmp-$(random 12)" + +# Set docker-test git user +set -x +git config --global user.email "docker-test@docker.io" +git config --global user.name "docker-test" + +# Fetch project +git clone -q $REPO_PROJ -b master /data/$PROJECT_PATH +cd /data/$PROJECT_PATH +echo "Git commit: $(git rev-parse HEAD)" +git fetch -q $REPO $BRANCH +git merge --no-edit $COMMIT + +# Build the project dockertest +/testbuilder/$PROJECT.sh $PROJECT_PATH +rm -rf /data/$PROJECT_PATH diff --git a/components/engine/hack/infrastructure/docker-ci/tool/backup.py b/components/engine/hack/infrastructure/docker-ci/tool/backup.py new file mode 100755 index 0000000000..2db633e526 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/tool/backup.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +import os,sys,json +from datetime import datetime +from filecmp import cmp +from subprocess import check_call +from boto.s3.key import Key +from boto.s3.connection import S3Connection + +def ENV(x): + '''Promote an environment variable for global use returning its value''' + retval = os.environ.get(x, '') + globals()[x] = retval + return retval + +ROOT_PATH = '/data/backup/docker-ci' +TODAY = str(datetime.today())[:10] +BACKUP_FILE = '{}/docker-ci_{}.tgz'.format(ROOT_PATH, TODAY) +BACKUP_LINK = '{}/docker-ci.tgz'.format(ROOT_PATH) +ENV('BACKUP_BUCKET') +ENV('BACKUP_AWS_ID') +ENV('BACKUP_AWS_SECRET') + +'''Create full master buildbot backup, avoiding duplicates''' +# Ensure backup path exist +if not os.path.exists(ROOT_PATH): + os.makedirs(ROOT_PATH) +# Make actual backups +check_call('/bin/tar czf {} -C /data --exclude=backup --exclude=buildbot/slave' + ' . 1>/dev/null 2>&1'.format(BACKUP_FILE),shell=True) +# remove previous dump if it is the same as the latest +if (os.path.exists(BACKUP_LINK) and cmp(BACKUP_FILE, BACKUP_LINK) and + os.path._resolve_link(BACKUP_LINK) != BACKUP_FILE): + os.unlink(os.path._resolve_link(BACKUP_LINK)) +# Recreate backup link pointing to latest backup +try: + os.unlink(BACKUP_LINK) +except: + pass +os.symlink(BACKUP_FILE, BACKUP_LINK) + +# Make backup on S3 +bucket = S3Connection(BACKUP_AWS_ID,BACKUP_AWS_SECRET).get_bucket(BACKUP_BUCKET) +k = Key(bucket) +k.key = BACKUP_FILE +k.set_contents_from_filename(BACKUP_FILE) +bucket.copy_key(os.path.basename(BACKUP_LINK),BACKUP_BUCKET,BACKUP_FILE[1:]) From d3c56b4c4af84743fd5ea42a7252b7f1aa6c1ef2 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 17 Feb 2014 11:17:05 -0800 Subject: [PATCH 120/403] Make crosbymichael and creack netlink maintainers Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: a75c6907b48f730fe79c6c9ebfe22eeea6389f4a Component: engine --- components/engine/pkg/netlink/MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 components/engine/pkg/netlink/MAINTAINERS diff --git a/components/engine/pkg/netlink/MAINTAINERS b/components/engine/pkg/netlink/MAINTAINERS new file mode 100644 index 0000000000..e53d933d47 --- /dev/null +++ b/components/engine/pkg/netlink/MAINTAINERS @@ -0,0 +1,2 @@ +Michael Crosby (@crosbymichael) +Guillaume Charmes (@creack) From 3192f2a68b973da91ee925697b015ddf4d326258 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 17 Feb 2014 11:35:26 -0800 Subject: [PATCH 121/403] Fix DOCKER_HOST=tcp:// panic Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 513d86488046bfd898f5f1fdab39bfb302607535 Component: engine --- components/engine/docker/docker.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index b4d7879397..a552b8318a 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -58,6 +58,9 @@ func main() { // If we do not have a host, default to unix socket defaultHost = fmt.Sprintf("unix://%s", api.DEFAULTUNIXSOCKET) } + if _, err := api.ValidateHost(defaultHost); err != nil { + log.Fatal(err) + } flHosts.Set(defaultHost) } From e821f31734dd13ef8a965d75fb276c28101e04ba Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 17 Feb 2014 13:41:52 +0100 Subject: [PATCH 122/403] archive: Handle aufs plink hardlinks in ApplyLayer Files in the .wh..wh.plnk directory are ignored, but other files inside the tarfile can be hardlinks to these files. This is not something that normally happens, as on aufs unmount such files are supposed to be dropped via the "auplink" too, yet images on the index (such as shipyard/shipyard, e.g. layer f73c835af6d58b6fc827b400569f79a8f28e54f5bb732be063e1aacefbc374d0) contains such files. We handle these by extracting these files to a temporary directory and resolve such hardlinks via the temporary files. This fixes https://github.com/dotcloud/docker/issues/3884 Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 6889cd9f9cfa3a6439b7702d73f27ab3fac3d3ef Component: engine --- components/engine/archive/archive.go | 2 +- components/engine/archive/diff.go | 42 +++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index 8212621bb9..c0551d9fa3 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -186,7 +186,7 @@ func addTarFile(path, name string, tw *tar.Writer) error { return nil } -func createTarFile(path, extractDir string, hdr *tar.Header, reader *tar.Reader) error { +func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader) error { switch hdr.Typeflag { case tar.TypeDir: // Create directory unless it exists as a directory already. diff --git a/components/engine/archive/diff.go b/components/engine/archive/diff.go index de93fc69b2..6a778390bb 100644 --- a/components/engine/archive/diff.go +++ b/components/engine/archive/diff.go @@ -2,7 +2,9 @@ package archive import ( "code.google.com/p/go/src/pkg/archive/tar" + "fmt" "io" + "io/ioutil" "os" "path/filepath" "strings" @@ -42,6 +44,9 @@ func ApplyLayer(dest string, layer ArchiveReader) error { var dirs []*tar.Header + aufsTempdir := "" + aufsHardlinks := make(map[string]*tar.Header) + // Iterate through the files in the archive. for { hdr, err := tr.Next() @@ -72,6 +77,22 @@ func ApplyLayer(dest string, layer ArchiveReader) error { // Skip AUFS metadata dirs if strings.HasPrefix(hdr.Name, ".wh..wh.") { + // Regular files inside /.wh..wh.plnk can be used as hardlink targets + // We don't want this directory, but we need the files in them so that + // such hardlinks can be resolved. + if strings.HasPrefix(hdr.Name, ".wh..wh.plnk") && hdr.Typeflag == tar.TypeReg { + basename := filepath.Base(hdr.Name) + aufsHardlinks[basename] = hdr + if aufsTempdir == "" { + if aufsTempdir, err = ioutil.TempDir("", "dockerplnk"); err != nil { + return err + } + defer os.RemoveAll(aufsTempdir) + } + if err := createTarFile(filepath.Join(aufsTempdir, basename), dest, hdr, tr); err != nil { + return err + } + } continue } @@ -96,7 +117,26 @@ func ApplyLayer(dest string, layer ArchiveReader) error { } } - if err := createTarFile(path, dest, hdr, tr); err != nil { + srcData := io.Reader(tr) + srcHdr := hdr + + // Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so + // we manually retarget these into the temporary files we extracted them into + if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), ".wh..wh.plnk") { + linkBasename := filepath.Base(hdr.Linkname) + srcHdr = aufsHardlinks[linkBasename] + if srcHdr == nil { + return fmt.Errorf("Invalid aufs hardlink") + } + tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename)) + if err != nil { + return err + } + defer tmpFile.Close() + srcData = tmpFile + } + + if err := createTarFile(path, dest, srcHdr, srcData); err != nil { return err } From 7cfa188a7564ba2faceb1a39546a7efb45161aa2 Mon Sep 17 00:00:00 2001 From: unclejack Date: Mon, 17 Feb 2014 23:09:09 +0200 Subject: [PATCH 123/403] update warning about local dns resolvers This changes the local DNS resolver warning to explain what the warning is about and why the default external DNS server is used. Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: 85b8025a35c29e0832fd13adc55177758ecbb309 Component: engine --- components/engine/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/server.go b/components/engine/server.go index 46ab33b467..0ed96fee31 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -1647,7 +1647,7 @@ func (srv *Server) ContainerCreate(job *engine.Job) engine.Status { return job.Error(err) } if !config.NetworkDisabled && len(config.Dns) == 0 && len(srv.runtime.config.Dns) == 0 && utils.CheckLocalDns(resolvConf) { - job.Errorf("WARNING: Docker detected local DNS server on resolv.conf. Using default external servers: %v\n", defaultDns) + job.Errorf("WARNING: Local (127.0.0.1) DNS resolver found in resolv.conf and containers can't use it. Using default external servers : %v\n", defaultDns) config.Dns = defaultDns } From 5f4d37b73db36fa8c970b73ff7f2558d2acfe87c Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 17 Feb 2014 13:31:13 -0800 Subject: [PATCH 124/403] Remove verbose logging for non errors Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 055f1a1f8181d51d3386b4270f810efd59d51d30 Component: engine --- components/engine/pkg/proxy/tcp_proxy.go | 10 +++------- components/engine/pkg/proxy/udp_proxy.go | 14 ++++---------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/components/engine/pkg/proxy/tcp_proxy.go b/components/engine/pkg/proxy/tcp_proxy.go index b84483eff9..1aa6d9fd70 100644 --- a/components/engine/pkg/proxy/tcp_proxy.go +++ b/components/engine/pkg/proxy/tcp_proxy.go @@ -30,7 +30,7 @@ func NewTCPProxy(frontendAddr, backendAddr *net.TCPAddr) (*TCPProxy, error) { func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) { backend, err := net.DialTCP("tcp", nil, proxy.backendAddr) if err != nil { - log.Printf("Can't forward traffic to backend tcp/%v: %v\n", proxy.backendAddr, err.Error()) + log.Printf("Can't forward traffic to backend tcp/%v: %s\n", proxy.backendAddr, err) client.Close() return } @@ -49,7 +49,6 @@ func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) { event <- written } - log.Printf("Forwarding traffic between tcp/%v and tcp/%v", client.RemoteAddr(), backend.RemoteAddr()) go broker(client, backend) go broker(backend, client) @@ -65,23 +64,20 @@ func (proxy *TCPProxy) clientLoop(client *net.TCPConn, quit chan bool) { for ; i < 2; i++ { transferred += <-event } - goto done + return } } client.Close() backend.Close() -done: - log.Printf("%v bytes transferred between tcp/%v and tcp/%v", transferred, client.RemoteAddr(), backend.RemoteAddr()) } func (proxy *TCPProxy) Run() { quit := make(chan bool) defer close(quit) - log.Printf("Starting proxy on tcp/%v for tcp/%v", proxy.frontendAddr, proxy.backendAddr) for { client, err := proxy.listener.Accept() if err != nil { - log.Printf("Stopping proxy on tcp/%v for tcp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) + log.Printf("Stopping proxy on tcp/%v for tcp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) return } go proxy.clientLoop(client.(*net.TCPConn), quit) diff --git a/components/engine/pkg/proxy/udp_proxy.go b/components/engine/pkg/proxy/udp_proxy.go index 9395516464..14f2306a5a 100644 --- a/components/engine/pkg/proxy/udp_proxy.go +++ b/components/engine/pkg/proxy/udp_proxy.go @@ -66,7 +66,6 @@ func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr proxy.connTrackLock.Lock() delete(proxy.connTrackTable, *clientKey) proxy.connTrackLock.Unlock() - log.Printf("Done proxying between udp/%v and udp/%v", clientAddr.String(), proxy.backendAddr.String()) proxyConn.Close() }() @@ -92,24 +91,20 @@ func (proxy *UDPProxy) replyLoop(proxyConn *net.UDPConn, clientAddr *net.UDPAddr return } i += written - log.Printf("Forwarded %v/%v bytes to udp/%v", i, read, clientAddr.String()) } } } func (proxy *UDPProxy) Run() { readBuf := make([]byte, UDPBufSize) - log.Printf("Starting proxy on udp/%v for udp/%v", proxy.frontendAddr, proxy.backendAddr) for { read, from, err := proxy.listener.ReadFromUDP(readBuf) if err != nil { // NOTE: Apparently ReadFrom doesn't return // ECONNREFUSED like Read do (see comment in // UDPProxy.replyLoop) - if isClosedError(err) { - log.Printf("Stopping proxy on udp/%v for udp/%v (socket was closed)", proxy.frontendAddr, proxy.backendAddr) - } else { - log.Printf("Stopping proxy on udp/%v for udp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) + if !isClosedError(err) { + log.Printf("Stopping proxy on udp/%v for udp/%v (%s)", proxy.frontendAddr, proxy.backendAddr, err) } break } @@ -120,7 +115,7 @@ func (proxy *UDPProxy) Run() { if !hit { proxyConn, err = net.DialUDP("udp", nil, proxy.backendAddr) if err != nil { - log.Printf("Can't proxy a datagram to udp/%s: %v\n", proxy.backendAddr.String(), err) + log.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) continue } proxy.connTrackTable[*fromKey] = proxyConn @@ -130,11 +125,10 @@ func (proxy *UDPProxy) Run() { for i := 0; i != read; { written, err := proxyConn.Write(readBuf[i:read]) if err != nil { - log.Printf("Can't proxy a datagram to udp/%s: %v\n", proxy.backendAddr.String(), err) + log.Printf("Can't proxy a datagram to udp/%s: %s\n", proxy.backendAddr, err) break } i += written - log.Printf("Forwarded %v/%v bytes to udp/%v", i, read, proxy.backendAddr.String()) } } } From 7dcf535f27fc2a7e2d316ce001c8ba9930ae5ffe Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 17 Feb 2014 15:10:51 -0800 Subject: [PATCH 125/403] Move job register into servapi Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: e743021193e589b1358d0eef0521e9d8878dd66d Component: engine --- components/engine/api/api.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index 8d7b1de685..bd4283e301 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -52,7 +52,6 @@ type HttpApiFunc func(eng *engine.Engine, version float64, w http.ResponseWriter func init() { engine.Register("serveapi", ServeApi) - engine.Register("acceptconnections", AcceptConnections) } func hijackServer(w http.ResponseWriter) (io.ReadCloser, io.Writer, error) { @@ -1211,6 +1210,10 @@ func ServeApi(job *engine.Job) engine.Status { ) activationLock = make(chan struct{}) + if err := job.Eng.Register("acceptconnections", AcceptConnections); err != nil { + return job.Error(err) + } + for _, protoAddr := range protoAddrs { protoAddrParts := strings.SplitN(protoAddr, "://", 2) go func() { From 78488bf1e626adbed1aa96ecca423f3c372fa0bd Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 17 Feb 2014 15:25:00 -0800 Subject: [PATCH 126/403] Revert "Merge pull request #4151 from crosbymichael/improve-404-client-message" This reverts commit 6dd1bb9eb08dc144bc172b0052df7c3904896e56, reversing changes made to 23aec9d7fcaa72db026c042046102b381e2e9d27. Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: d55cc64c4eb66c4ee536514e35e2e2094aa3d911 Component: engine --- components/engine/api/client.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/engine/api/client.go b/components/engine/api/client.go index 605970c0fd..3dd867f771 100644 --- a/components/engine/api/client.go +++ b/components/engine/api/client.go @@ -2080,9 +2080,7 @@ func (cli *DockerCli) call(method, path string, data interface{}, passAuthInfo b return nil, -1, err } - if resp.StatusCode == 404 { - return nil, resp.StatusCode, fmt.Errorf("Error: request for %s returned 404 Not Found for the api version", req.URL) - } else if resp.StatusCode < 200 || resp.StatusCode >= 400 { + if resp.StatusCode < 200 || resp.StatusCode >= 400 { body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, -1, err From cf7d93d809bb7e2bdba513a1e1d214417dcf2439 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 17 Feb 2014 15:28:30 -0800 Subject: [PATCH 127/403] Improve error message only if no body is returned Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: e23190b6b3ddd16b3a5f951a33e05fd75ebb8970 Component: engine --- components/engine/api/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/api/client.go b/components/engine/api/client.go index 3dd867f771..81e337b023 100644 --- a/components/engine/api/client.go +++ b/components/engine/api/client.go @@ -2086,7 +2086,7 @@ func (cli *DockerCli) call(method, path string, data interface{}, passAuthInfo b return nil, -1, err } if len(body) == 0 { - return nil, resp.StatusCode, fmt.Errorf("Error :%s", http.StatusText(resp.StatusCode)) + return nil, resp.StatusCode, fmt.Errorf("Error: request returned %s for api route and version %s, check if the server supports the requested api version", http.StatusText(resp.StatusCode), req.URL) } return nil, resp.StatusCode, fmt.Errorf("Error: %s", bytes.TrimSpace(body)) } From c39069efb079fc24fe29ecdabd17de2b75717d2a Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 17 Feb 2014 16:54:36 -0500 Subject: [PATCH 128/403] execdriver flag for docker daemon like the storage-driver flag, this implements a flag for choosing the execdriver to be used, defaulting to lxc. Docker-DCO-1.1-Signed-off-by: Vincent Batts (github: vbatts) Upstream-commit: 5f84d7f3146db1ac15974f1e28f77834de3e63ff Component: engine --- components/engine/config.go | 2 ++ components/engine/docker/docker.go | 2 ++ .../docs/sources/reference/commandline/cli.rst | 1 + components/engine/runtime.go | 12 +++++++++++- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/components/engine/config.go b/components/engine/config.go index dc6e8b554f..19aad9ed4a 100644 --- a/components/engine/config.go +++ b/components/engine/config.go @@ -25,6 +25,7 @@ type DaemonConfig struct { BridgeIP string InterContainerCommunication bool GraphDriver string + ExecDriver string Mtu int DisableNetwork bool } @@ -43,6 +44,7 @@ func DaemonConfigFromJob(job *engine.Job) *DaemonConfig { DefaultIp: net.ParseIP(job.Getenv("DefaultIp")), InterContainerCommunication: job.GetenvBool("InterContainerCommunication"), GraphDriver: job.Getenv("GraphDriver"), + ExecDriver: job.Getenv("ExecDriver"), } if dns := job.GetenvList("Dns"); dns != nil { config.Dns = dns diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 02c99b9316..51aaf334d1 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -39,6 +39,7 @@ func main() { flDefaultIp = flag.String([]string{"#ip", "-ip"}, "0.0.0.0", "Default IP address to use when binding container ports") flInterContainerComm = flag.Bool([]string{"#icc", "-icc"}, true, "Enable inter-container communication") flGraphDriver = flag.String([]string{"s", "-storage-driver"}, "", "Force the docker runtime to use a specific storage driver") + flExecDriver = flag.String([]string{"e", "-exec-driver"}, "", "Force the docker runtime to use a specific exec driver") flHosts = opts.NewListOpts(api.ValidateHost) flMtu = flag.Int([]string{"#mtu", "-mtu"}, 0, "Set the containers network MTU; if no value is provided: default to the default route MTU or 1500 if not default route is available") ) @@ -98,6 +99,7 @@ func main() { job.Setenv("DefaultIp", *flDefaultIp) job.SetenvBool("InterContainerCommunication", *flInterContainerComm) job.Setenv("GraphDriver", *flGraphDriver) + job.Setenv("ExecDriver", *flExecDriver) job.SetenvInt("Mtu", *flMtu) if err := job.Run(); err != nil { log.Fatal(err) diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index 7ba0123065..e0ec2b99cc 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -79,6 +79,7 @@ Commands -p, --pidfile="/var/run/docker.pid": Path to use for daemon PID file -r, --restart=true: Restart previously running containers -s, --storage-driver="": Force the docker runtime to use a specific storage driver + -e, --exec-driver="": Force the docker runtime to use a specific exec driver -v, --version=false: Print version information and quit -mtu, --mtu=0: Set the containers network MTU; if no value is provided: default to the default route MTU or 1500 if not default route is available diff --git a/components/engine/runtime.go b/components/engine/runtime.go index eed28f92ab..8e12a43402 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -7,6 +7,7 @@ import ( "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/execdriver" + "github.com/dotcloud/docker/execdriver/chroot" "github.com/dotcloud/docker/execdriver/lxc" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/graphdriver/aufs" @@ -703,7 +704,16 @@ func NewRuntimeFromDirectory(config *DaemonConfig, eng *engine.Engine) (*Runtime sysInfo := sysinfo.New(false) - ed, err := lxc.NewDriver(config.Root, sysInfo.AppArmor) + var ed execdriver.Driver + utils.Debugf("execDriver: provided %s", config.ExecDriver) + if config.ExecDriver == "chroot" && false { + // chroot is presently a noop driver https://github.com/dotcloud/docker/pull/4189#issuecomment-35330655 + ed, err = chroot.NewDriver() + utils.Debugf("execDriver: using chroot") + } else { + ed, err = lxc.NewDriver(config.Root, sysInfo.AppArmor) + utils.Debugf("execDriver: using lxc") + } if err != nil { return nil, err } From bf1c684667d674f839e920afad4428edcc63e947 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Mon, 17 Feb 2014 17:08:17 -0800 Subject: [PATCH 129/403] Fix remote tar ADD behavior Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 98ce0cdb4f80550462675ee3bbb927451bb76585 Component: engine --- components/engine/buildfile.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 3ee63ea3fb..c7181b9146 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -355,7 +355,7 @@ func (b *buildFile) checkPathForAddition(orig string) error { return nil } -func (b *buildFile) addContext(container *Container, orig, dest string) error { +func (b *buildFile) addContext(container *Container, orig, dest string, remote bool) error { var ( origPath = path.Join(b.contextPath, orig) destPath = path.Join(container.BasefsPath(), dest) @@ -388,11 +388,14 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { tarDest = filepath.Dir(destPath) } - // try to successfully untar the orig - if err := archive.UntarPath(origPath, tarDest); err == nil { - return nil + // If we are adding a remote file, do not try to untar it + if !remote { + // try to successfully untar the orig + if err := archive.UntarPath(origPath, tarDest); err == nil { + return nil + } + utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err) } - utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err) // If that fails, just copy it as a regular file // but do not use all the magic path handling for the tar path @@ -428,14 +431,15 @@ func (b *buildFile) CmdAdd(args string) error { b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)} b.config.Image = b.image - // FIXME: do we really need this? var ( origPath = orig destPath = dest remoteHash string + isRemote bool ) if utils.IsURL(orig) { + isRemote = true resp, err := utils.Download(orig) if err != nil { return err @@ -545,7 +549,7 @@ func (b *buildFile) CmdAdd(args string) error { } defer container.Unmount() - if err := b.addContext(container, origPath, destPath); err != nil { + if err := b.addContext(container, origPath, destPath, isRemote); err != nil { return err } From 5f957db92e780ba4af224a287ce863140bf7e5ab Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 14 Feb 2014 21:02:54 +1000 Subject: [PATCH 130/403] rejig the helloflask example as more advanced, and move it to the end Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 3036b5a196fdbb744f94299c82dc44078361bdae Component: engine --- .../engine/docs/sources/examples/index.rst | 2 +- .../docs/sources/examples/python_web_app.rst | 113 +++++++++--------- 2 files changed, 59 insertions(+), 56 deletions(-) diff --git a/components/engine/docs/sources/examples/index.rst b/components/engine/docs/sources/examples/index.rst index cd08107e7a..cf9ed9340a 100644 --- a/components/engine/docs/sources/examples/index.rst +++ b/components/engine/docs/sources/examples/index.rst @@ -16,7 +16,6 @@ to more substantial services like those which you might find in production. :maxdepth: 1 hello_world - python_web_app nodejs_web_app running_redis_service running_ssh_service @@ -26,3 +25,4 @@ to more substantial services like those which you might find in production. running_riak_service using_supervisord cfengine_process_management + python_web_app diff --git a/components/engine/docs/sources/examples/python_web_app.rst b/components/engine/docs/sources/examples/python_web_app.rst index f31b31b7d2..b44c0e1ee5 100644 --- a/components/engine/docs/sources/examples/python_web_app.rst +++ b/components/engine/docs/sources/examples/python_web_app.rst @@ -9,85 +9,86 @@ Python Web App .. include:: example_header.inc -The goal of this example is to show you how you can author your own -Docker images using a parent image, making changes to it, and then -saving the results as a new image. We will do that by making a simple -hello Flask web application image. +While using Dockerfiles is the prefered way to create maintainable +and repeatable images, its useful to know how you can try things out +and then commit your live changes to an image. + +The goal of this example is to show you how you can modify your own +Docker images by making changes to a running +container, and then saving the results as a new image. We will do +that by making a simple 'hello world' Flask web application image. **Steps:** .. code-block:: bash - sudo docker pull shykes/pybuilder + $ sudo docker pull shykes/pybuilder + +Download the ``shykes/pybuilder`` Docker image from the ``http://index.docker.io`` +registry. Note that this container was built with a very old version of docker +(May 2013), but can still be used now. -We are downloading the ``shykes/pybuilder`` Docker image .. code-block:: bash - URL=http://github.com/shykes/helloflask/archive/master.tar.gz + $ sudo docker run -i -t -name pybuilder_run shykes/pybuilder bash -We set a ``URL`` variable that points to a tarball of a simple helloflask web app + $$ URL=http://github.com/shykes/helloflask/archive/master.tar.gz + $$ /usr/local/bin/buildapp $URL + [lots of output later] + $$ exit + + +We then start a new container running interactively using the +image. +First, we set a ``URL`` variable that points to a tarball of a simple +helloflask web app, and then we run a command contained in the image called +``buildapp``, passing it the ``$URL`` variable. The container is +given a name ``pybuilder_run`` which we will use in the next steps. + +While this example is simple, you could run any number of interactive commands, +try things out, and then exit when you're done. .. code-block:: bash - BUILD_JOB=$(sudo docker run -d -t shykes/pybuilder:latest /usr/local/bin/buildapp $URL) - -Inside of the ``shykes/pybuilder`` image there is a command called -``buildapp``, we are running that command and passing the ``$URL`` variable -from step 2 to it, and running the whole thing inside of a new -container. The ``BUILD_JOB`` environment variable will be set with the new container ID. - -.. code-block:: bash - - sudo docker attach -sig-proxy=false $BUILD_JOB - [...] - -While this container is running, we can attach to the new container to -see what is going on. The flag ``--sig-proxy`` set as ``false`` allows you to connect and -disconnect (Ctrl-C) to it without stopping the container. - -.. code-block:: bash - - sudo docker ps -a - -List all Docker containers. If this container has already finished -running, it will still be listed here. - -.. code-block:: bash - - BUILD_IMG=$(sudo docker commit $BUILD_JOB _/builds/github.com/shykes/helloflask/master) + $ sudo docker commit pybuilder_run /builds/github.com/shykes/helloflask/master + c8b2e8228f11b8b3e492cbf9a49923ae66496230056d61e07880dc74c5f495f9 Save the changes we just made in the container to a new image called -``_/builds/github.com/hykes/helloflask/master`` and save the image ID in -the ``BUILD_IMG`` variable name. +``/builds/github.com/hykes/helloflask/master``. You now have 3 different +ways to refer to the container, name, short-id ``c8b2e8228f11``, or +long-id ``c8b2e8228f11b8b3e492cbf9a49923ae66496230056d61e07880dc74c5f495f9``. .. code-block:: bash - WEB_WORKER=$(sudo docker run -d -p 5000 $BUILD_IMG /usr/local/bin/runapp) + $ WEB_WORKER=$(sudo docker run -d -p 5000 /builds/github.com/hykes/helloflask/master /usr/local/bin/runapp) + +Use the new image to create a new container with +network port 5000, and return the container ID and store in the +``WEB_WORKER`` variable (rather than naming a container/image, you can use the ID's). - **"docker run -d "** run a command in a new container. We pass "-d" so it runs as a daemon. - **"-p 5000"** the web app is going to listen on this port, so it must be mapped from the container to the host system. -- **"$BUILD_IMG"** is the image we want to run the command inside of. - **/usr/local/bin/runapp** is the command which starts the web app. -Use the new image we just created and create a new container with -network port 5000, and return the container ID and store in the -``WEB_WORKER`` variable. .. code-block:: bash - sudo docker logs $WEB_WORKER + $ sudo docker logs -f $WEB_WORKER * Running on http://0.0.0.0:5000/ View the logs for the new container using the ``WEB_WORKER`` variable, and if everything worked as planned you should see the line ``Running on http://0.0.0.0:5000/`` in the log output. +To exit the view without stopping the container, hit Ctrl-C, or open another +terminal and continue with the example while watching the result in the logs. + .. code-block:: bash - WEB_PORT=$(sudo docker port $WEB_WORKER 5000 | awk -F: '{ print $2 }') + $ WEB_PORT=$(sudo docker port $WEB_WORKER 5000 | awk -F: '{ print $2 }') Look up the public-facing port which is NAT-ed. Find the private port used by the container and store it inside of the ``WEB_PORT`` variable. @@ -95,23 +96,25 @@ used by the container and store it inside of the ``WEB_PORT`` variable. .. code-block:: bash # install curl if necessary, then ... - curl http://127.0.0.1:$WEB_PORT - Hello world! + $ curl http://127.0.0.1:$WEB_PORT + Hello world! Access the web app using the ``curl`` binary. If everything worked as planned you should see the line ``Hello world!`` inside of your console. -**Video:** +.. code-block:: bash -See the example in action + $ sudo docker ps --all -.. raw:: html +List ``--all`` the Docker containers. If this container had already finished +running, it will still be listed here with a status of 'Exit 0'. - +.. code-block:: bash + + $ sudo docker stop $WEB_WORKER + $ sudo docker rm $WEB_WORKER pybuilder_run + $ sudo docker rmi /builds/github.com/shykes/helloflask/master shykes/pybuilder:latest + +And now stop the running web worker, and delete the containers, so that we can +then delete the images that we used. -Continue to :ref:`running_ssh_service`. From 244da2b52047f75e467631b38cf94d3296aa93c3 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 14 Feb 2014 21:27:57 +1000 Subject: [PATCH 131/403] swap busybox for ubuntu, its quicker.. Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: f535a58959809dbb613bf529019f26cc6ae0c6f9 Component: engine --- .../docs/sources/examples/hello_world.rst | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/components/engine/docs/sources/examples/hello_world.rst b/components/engine/docs/sources/examples/hello_world.rst index fd5d6421c3..c8ecb0f6c2 100644 --- a/components/engine/docs/sources/examples/hello_world.rst +++ b/components/engine/docs/sources/examples/hello_world.rst @@ -4,9 +4,6 @@ .. _examples: -Hello World ------------ - .. _running_examples: Check your Docker install @@ -18,7 +15,7 @@ your Docker install, run the following command: .. code-block:: bash # Check that you have a working install - docker info + $ sudo docker info If you get ``docker: command not found`` or something like ``/var/lib/docker/repositories: permission denied`` you may have an incomplete @@ -30,27 +27,28 @@ Please refer to :ref:`installation_list` for installation instructions. .. _hello_world: Hello World -=========== +----------- .. include:: example_header.inc This is the most basic example available for using Docker. -Download the base image which is named ``ubuntu``: +Download the small base image named ``busybox``: .. code-block:: bash - # Download an ubuntu image - sudo docker pull ubuntu + # Download an busybox image + $ sudo docker pull busybox -Alternatively to the ``ubuntu`` image, you can select ``busybox``, a bare -minimal Linux system. The images are retrieved from the Docker -repository. +The ``busybox`` image is a minimal Linux system. You can do the same +with any number of other images, such as ``debian``, ``ubuntu`` or ``centos``. +The images can be found and retrieved using the `Docker index`_. +.. _Docker index: http://index.docker.io .. code-block:: bash - sudo docker run ubuntu /bin/echo hello world + $ sudo docker run busybox /bin/echo hello world This command will run a simple ``echo`` command, that will echo ``hello world`` back to the console over standard out. @@ -58,7 +56,7 @@ This command will run a simple ``echo`` command, that will echo ``hello world`` - **"sudo"** execute the following commands as user *root* - **"docker run"** run a command in a new container -- **"ubuntu"** is the image we want to run the command inside of. +- **"busybox"** is the image we are running the command in. - **"/bin/echo"** is the command we want to run in the container - **"hello world"** is the input for the echo command @@ -82,7 +80,7 @@ See the example in action .. _hello_world_daemon: Hello World Daemon -================== +------------------ .. include:: example_header.inc @@ -172,14 +170,14 @@ See the example in action id="asciicast-2562" async>"> -The next example in the series is a :ref:`python_web_app` example, or +The next example in the series is a :ref:`nodejs_web_app` example, or you could skip to any of the other examples: -* :ref:`python_web_app` * :ref:`nodejs_web_app` * :ref:`running_redis_service` * :ref:`running_ssh_service` * :ref:`running_couchdb_service` * :ref:`postgresql_service` * :ref:`mongodb_image` +* :ref:`python_web_app` From 32cc7c320166636568232a77c71e162166cace98 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 14 Feb 2014 22:41:59 +1000 Subject: [PATCH 132/403] update a new movie: http://asciinema.org/a/7658 Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: b05602a0fe6d9b9731cee0008fbac27250f361bd Component: engine --- .../docs/sources/examples/hello_world.rst | 8 +- .../docs/sources/examples/python_web_app.rst | 80 ++++++++++++------- 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/components/engine/docs/sources/examples/hello_world.rst b/components/engine/docs/sources/examples/hello_world.rst index c8ecb0f6c2..63362e7d7b 100644 --- a/components/engine/docs/sources/examples/hello_world.rst +++ b/components/engine/docs/sources/examples/hello_world.rst @@ -2,8 +2,6 @@ :description: A simple hello world example with Docker :keywords: docker, example, hello world -.. _examples: - .. _running_examples: Check your Docker install @@ -37,7 +35,7 @@ Download the small base image named ``busybox``: .. code-block:: bash - # Download an busybox image + # Download a busybox image $ sudo docker pull busybox The ``busybox`` image is a minimal Linux system. You can do the same @@ -71,8 +69,8 @@ See the example in action ---- diff --git a/components/engine/docs/sources/examples/python_web_app.rst b/components/engine/docs/sources/examples/python_web_app.rst index b44c0e1ee5..6992d0f7f8 100644 --- a/components/engine/docs/sources/examples/python_web_app.rst +++ b/components/engine/docs/sources/examples/python_web_app.rst @@ -9,7 +9,7 @@ Python Web App .. include:: example_header.inc -While using Dockerfiles is the prefered way to create maintainable +While using Dockerfiles is the preffered way to create maintainable and repeatable images, its useful to know how you can try things out and then commit your live changes to an image. @@ -18,29 +18,29 @@ Docker images by making changes to a running container, and then saving the results as a new image. We will do that by making a simple 'hello world' Flask web application image. -**Steps:** +Download the initial image +-------------------------- + +Download the ``shykes/pybuilder`` Docker image from the ``http://index.docker.io`` +registry. + +This image contains a ``buildapp`` script to download the web app and then ``pip install`` +any required modules, and a ``runapp`` script that finds the ``app.py`` and runs it. + +.. _`shykes/pybuilder`: https://github.com/shykes/pybuilder .. code-block:: bash $ sudo docker pull shykes/pybuilder -Download the ``shykes/pybuilder`` Docker image from the ``http://index.docker.io`` -registry. Note that this container was built with a very old version of docker -(May 2013), but can still be used now. +.. note:: This container was built with a very old version of docker + (May 2013 - see `shykes/pybuilder`_ ), when the ``Dockerfile`` format was different, + but the image can still be used now. +Interactively make some modifications +------------------------------------- -.. code-block:: bash - - $ sudo docker run -i -t -name pybuilder_run shykes/pybuilder bash - - $$ URL=http://github.com/shykes/helloflask/archive/master.tar.gz - $$ /usr/local/bin/buildapp $URL - [lots of output later] - $$ exit - - -We then start a new container running interactively using the -image. +We then start a new container running interactively using the image. First, we set a ``URL`` variable that points to a tarball of a simple helloflask web app, and then we run a command contained in the image called ``buildapp``, passing it the ``$URL`` variable. The container is @@ -51,8 +51,15 @@ try things out, and then exit when you're done. .. code-block:: bash - $ sudo docker commit pybuilder_run /builds/github.com/shykes/helloflask/master - c8b2e8228f11b8b3e492cbf9a49923ae66496230056d61e07880dc74c5f495f9 + $ sudo docker run -i -t -name pybuilder_run shykes/pybuilder bash + + $$ URL=http://github.com/shykes/helloflask/archive/master.tar.gz + $$ /usr/local/bin/buildapp $URL + [...] + $$ exit + +Commit the container to create a new image +------------------------------------------ Save the changes we just made in the container to a new image called ``/builds/github.com/hykes/helloflask/master``. You now have 3 different @@ -61,12 +68,22 @@ long-id ``c8b2e8228f11b8b3e492cbf9a49923ae66496230056d61e07880dc74c5f495f9``. .. code-block:: bash - $ WEB_WORKER=$(sudo docker run -d -p 5000 /builds/github.com/hykes/helloflask/master /usr/local/bin/runapp) + $ sudo docker commit pybuilder_run /builds/github.com/shykes/helloflask/master + c8b2e8228f11b8b3e492cbf9a49923ae66496230056d61e07880dc74c5f495f9 + + +Run the new image to start the web worker +----------------------------------------- Use the new image to create a new container with network port 5000, and return the container ID and store in the ``WEB_WORKER`` variable (rather than naming a container/image, you can use the ID's). +.. code-block:: bash + + $ WEB_WORKER=$(sudo docker run -d -p 5000 /builds/github.com/hykes/helloflask/master /usr/local/bin/runapp) + + - **"docker run -d "** run a command in a new container. We pass "-d" so it runs as a daemon. - **"-p 5000"** the web app is going to listen on this port, so it @@ -74,10 +91,8 @@ network port 5000, and return the container ID and store in the - **/usr/local/bin/runapp** is the command which starts the web app. -.. code-block:: bash - - $ sudo docker logs -f $WEB_WORKER - * Running on http://0.0.0.0:5000/ +View the container logs +----------------------- View the logs for the new container using the ``WEB_WORKER`` variable, and if everything worked as planned you should see the line ``Running on @@ -88,19 +103,30 @@ terminal and continue with the example while watching the result in the logs. .. code-block:: bash - $ WEB_PORT=$(sudo docker port $WEB_WORKER 5000 | awk -F: '{ print $2 }') + $ sudo docker logs -f $WEB_WORKER + * Running on http://0.0.0.0:5000/ + + +See the webapp output +--------------------- Look up the public-facing port which is NAT-ed. Find the private port used by the container and store it inside of the ``WEB_PORT`` variable. +Access the web app using the ``curl`` binary. If everything worked as planned you +should see the line ``Hello world!`` inside of your console. + .. code-block:: bash + $ WEB_PORT=$(sudo docker port $WEB_WORKER 5000 | awk -F: '{ print $2 }') + # install curl if necessary, then ... $ curl http://127.0.0.1:$WEB_PORT Hello world! -Access the web app using the ``curl`` binary. If everything worked as planned you -should see the line ``Hello world!`` inside of your console. + +Clean up example containers and images +-------------------------------------- .. code-block:: bash From ca7b056023a7c43b57c178cb27cae4dafca389c1 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Tue, 18 Feb 2014 13:23:18 +1000 Subject: [PATCH 133/403] suggested improvements Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 29f7e1bfcbe6654d962a55eb5006e248284b9e0c Component: engine --- .../docs/sources/examples/python_web_app.rst | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/components/engine/docs/sources/examples/python_web_app.rst b/components/engine/docs/sources/examples/python_web_app.rst index 6992d0f7f8..5b8e3f6b4b 100644 --- a/components/engine/docs/sources/examples/python_web_app.rst +++ b/components/engine/docs/sources/examples/python_web_app.rst @@ -9,7 +9,7 @@ Python Web App .. include:: example_header.inc -While using Dockerfiles is the preffered way to create maintainable +While using Dockerfiles is the preferred way to create maintainable and repeatable images, its useful to know how you can try things out and then commit your live changes to an image. @@ -62,8 +62,8 @@ Commit the container to create a new image ------------------------------------------ Save the changes we just made in the container to a new image called -``/builds/github.com/hykes/helloflask/master``. You now have 3 different -ways to refer to the container, name, short-id ``c8b2e8228f11``, or +``/builds/github.com/shykes/helloflask/master``. You now have 3 different +ways to refer to the container: name ``pybuilder_run``, short-id ``c8b2e8228f11``, or long-id ``c8b2e8228f11b8b3e492cbf9a49923ae66496230056d61e07880dc74c5f495f9``. .. code-block:: bash @@ -76,12 +76,11 @@ Run the new image to start the web worker ----------------------------------------- Use the new image to create a new container with -network port 5000, and return the container ID and store in the -``WEB_WORKER`` variable (rather than naming a container/image, you can use the ID's). +network port 5000 mapped to a local port .. code-block:: bash - $ WEB_WORKER=$(sudo docker run -d -p 5000 /builds/github.com/hykes/helloflask/master /usr/local/bin/runapp) + $ sudo docker run -d -p 5000 --name web_worker /builds/github.com/shykes/helloflask/master /usr/local/bin/runapp - **"docker run -d "** run a command in a new container. We pass "-d" @@ -94,7 +93,7 @@ network port 5000, and return the container ID and store in the View the container logs ----------------------- -View the logs for the new container using the ``WEB_WORKER`` variable, and +View the logs for the new ``web_worker`` container and if everything worked as planned you should see the line ``Running on http://0.0.0.0:5000/`` in the log output. @@ -103,7 +102,7 @@ terminal and continue with the example while watching the result in the logs. .. code-block:: bash - $ sudo docker logs -f $WEB_WORKER + $ sudo docker logs -f web_worker * Running on http://0.0.0.0:5000/ @@ -118,7 +117,7 @@ should see the line ``Hello world!`` inside of your console. .. code-block:: bash - $ WEB_PORT=$(sudo docker port $WEB_WORKER 5000 | awk -F: '{ print $2 }') + $ WEB_PORT=$(sudo docker port web_worker 5000 | awk -F: '{ print $2 }') # install curl if necessary, then ... $ curl http://127.0.0.1:$WEB_PORT @@ -137,8 +136,8 @@ running, it will still be listed here with a status of 'Exit 0'. .. code-block:: bash - $ sudo docker stop $WEB_WORKER - $ sudo docker rm $WEB_WORKER pybuilder_run + $ sudo docker stop web_worker + $ sudo docker rm web_worker pybuilder_run $ sudo docker rmi /builds/github.com/shykes/helloflask/master shykes/pybuilder:latest And now stop the running web worker, and delete the containers, so that we can From 92abc1e6512fe6038ef3205a884a443b7a752d55 Mon Sep 17 00:00:00 2001 From: Yasunori Mahata Date: Mon, 17 Feb 2014 19:43:56 -0800 Subject: [PATCH 134/403] Fix typo in 'Using Supervisor with Docker' example Docker-DCO-1.1-Signed-off-by: Yasunori Mahata (github: mahata) Upstream-commit: 68de32888b6bad84cd37ccd165b0e7bb013574af Component: engine --- components/engine/docs/sources/examples/using_supervisord.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/examples/using_supervisord.rst b/components/engine/docs/sources/examples/using_supervisord.rst index eed063292d..750b6c2334 100644 --- a/components/engine/docs/sources/examples/using_supervisord.rst +++ b/components/engine/docs/sources/examples/using_supervisord.rst @@ -112,7 +112,7 @@ Once we've got a built image we can launch a container from it. .. code-block:: bash - sudo docker run -p 22 -p 80 -t -i /supervisor + sudo docker run -p 22 -p 80 -t -i /supervisord 2013-11-25 18:53:22,312 CRIT Supervisor running as root (no user in config file) 2013-11-25 18:53:22,312 WARN Included extra file "/etc/supervisor/conf.d/supervisord.conf" during parsing 2013-11-25 18:53:22,342 INFO supervisord started with pid 1 From f829291e78f9d587d834be1c91607ab8ef25dac5 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Tue, 18 Feb 2014 09:53:30 +0100 Subject: [PATCH 135/403] archive: Fix the storing of setuid bits, etc In commit 3dfc910d7774d57c533b067fbe59d6b24dd803cd we changed from syscall.Chmod() to os.Chmod(), but these take a different form of the Mode argument. The sycall one takes the raw linux form, wheras os.Chmod takes the os.FileMode form, and they differ for the higher bits (setuid, setgid, etc). The raw tar header uses a form which is compatible with the syscalls, but not the go calls. We fix this by using hdr.FileInfo() which properly converts the mode to what go expects. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 5ba24629610c27f08b54ff4c1f000ad6d787c156 Component: engine --- components/engine/archive/archive.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index c0551d9fa3..16c01993b7 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -187,19 +187,24 @@ func addTarFile(path, name string, tw *tar.Writer) error { } func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader) error { + // hdr.Mode is in linux format, which we can use for sycalls, + // but for os.Foo() calls we need the mode converted to os.FileMode, + // so use hdrInfo.Mode() (they differ for e.g. setuid bits) + hdrInfo := hdr.FileInfo() + switch hdr.Typeflag { case tar.TypeDir: // Create directory unless it exists as a directory already. // In that case we just want to merge the two if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { - if err := os.Mkdir(path, os.FileMode(hdr.Mode)); err != nil { + if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { return err } } case tar.TypeReg, tar.TypeRegA: // Source is regular file - file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode)) + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) if err != nil { return err } @@ -249,7 +254,7 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader) e // There is no LChmod, so ignore mode for symlink. Also, this // must happen after chown, as that can modify the file mode if hdr.Typeflag != tar.TypeSymlink { - if err := os.Chmod(path, os.FileMode(hdr.Mode&07777)); err != nil { + if err := os.Chmod(path, hdrInfo.Mode()); err != nil { return err } } From 3700ea5ec4b8a9e34816af9a64cb786d96eb736c Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 18 Dec 2013 21:12:49 +1000 Subject: [PATCH 136/403] my attempt to disentagle repository and registry (Issue #1439) Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 940c381dd5898df4dd863a2d76a122b6610f2940 Component: engine --- components/engine/api/client.go | 4 +-- .../sources/reference/commandline/cli.rst | 2 +- .../engine/docs/sources/terms/index.rst | 2 ++ .../engine/docs/sources/terms/registry.rst | 16 +++++++++ .../engine/docs/sources/terms/repository.rst | 27 ++++++++++++++ .../sources/use/workingwithrepository.rst | 35 +++++++++++-------- 6 files changed, 69 insertions(+), 17 deletions(-) create mode 100644 components/engine/docs/sources/terms/registry.rst create mode 100644 components/engine/docs/sources/terms/repository.rst diff --git a/components/engine/api/client.go b/components/engine/api/client.go index 81e337b023..eb345ae40b 100644 --- a/components/engine/api/client.go +++ b/components/engine/api/client.go @@ -1668,7 +1668,7 @@ func (cli *DockerCli) CmdSearch(args ...string) error { type ports []int func (cli *DockerCli) CmdTag(args ...string) error { - cmd := cli.Subcmd("tag", "[OPTIONS] IMAGE REPOSITORY[:TAG]", "Tag an image into a repository") + cmd := cli.Subcmd("tag", "[OPTIONS] IMAGE [REGISTRYHOST/][USERNAME/]NAME[:TAG]", "Tag an image into a repository") force := cmd.Bool([]string{"f", "#force", "-force"}, false, "Force") if err := cmd.Parse(args); err != nil { return nil @@ -1681,7 +1681,7 @@ func (cli *DockerCli) CmdTag(args ...string) error { var repository, tag string if cmd.NArg() == 3 { - fmt.Fprintf(cli.err, "[DEPRECATED] The format 'IMAGE [REPOSITORY [TAG]]' as been deprecated. Please use IMAGE [REPOSITORY[:TAG]]\n") + fmt.Fprintf(cli.err, "[DEPRECATED] The format 'IMAGE [REPOSITORY [TAG]]' as been deprecated. Please use IMAGE [REGISTRYHOST/][USERNAME/]NAME[:TAG]]\n") repository, tag = cmd.Arg(1), cmd.Arg(2) } else { repository, tag = utils.ParseRepositoryTag(cmd.Arg(1)) diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index 7ba0123065..927f6d3b2d 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -1301,7 +1301,7 @@ The main process inside the container will receive SIGTERM, and after a grace pe :: - Usage: docker tag [OPTIONS] IMAGE REPOSITORY[:TAG] + Usage: docker tag [OPTIONS] IMAGE [REGISTRYHOST/][USERNAME/]NAME[:TAG] Tag an image into a repository diff --git a/components/engine/docs/sources/terms/index.rst b/components/engine/docs/sources/terms/index.rst index 882d83f0d4..40851082b5 100644 --- a/components/engine/docs/sources/terms/index.rst +++ b/components/engine/docs/sources/terms/index.rst @@ -18,5 +18,7 @@ Contents: layer image container + registry + repository diff --git a/components/engine/docs/sources/terms/registry.rst b/components/engine/docs/sources/terms/registry.rst new file mode 100644 index 0000000000..90c3ee721c --- /dev/null +++ b/components/engine/docs/sources/terms/registry.rst @@ -0,0 +1,16 @@ +:title: Registry +:description: Definition of an Registry +:keywords: containers, lxc, concepts, explanation, image, repository, container + +.. _registry_def: + +Registry +========== + +A Registry is a hosted service containing :ref:`repositories` +of :ref:`images` which responds to the Registry API. + +The default registry can be accessed using a browser at http://images.docker.io +or using the ``sudo docker search`` command. + +For more information see :ref:`Working with Repositories` diff --git a/components/engine/docs/sources/terms/repository.rst b/components/engine/docs/sources/terms/repository.rst new file mode 100644 index 0000000000..cceaeec23a --- /dev/null +++ b/components/engine/docs/sources/terms/repository.rst @@ -0,0 +1,27 @@ +:title: Repository +:description: Definition of an Repository +:keywords: containers, lxc, concepts, explanation, image, repository, container + +.. _repository_def: + +Repository +========== + +A repository is a tagged set of images either on your local docker server, or +shared, by pushing it to a :ref:`Registry` server. + +Images can be labeld into a repository using ``sudo docker build -t LABEL``, +``sudo docker commit CONTAINERID LABEL`` or ``sudo docker tag IMAGEID LABEL``. + +The label can be made up of 3 parts: + +[registry_hostname[:port]/][user_name/]( repository_name[:version_tag] | image_id ) +[REGISTRYHOST/][USERNAME/]NAME[:TAG] + +TAG defaults to ``latest``, USERNAME and REGISTRYHOST default to an empty string. +When REGISTRYHOST is an empty string, then ``docker push`` will push to ``index.docker.io:80``. + +If you create a new repository which you want to share, you will need to set the +first part, as the 'default' blank REPOSITORY prefix is reserved for official Docker images. + +For more information see :ref:`Working with Repositories` diff --git a/components/engine/docs/sources/use/workingwithrepository.rst b/components/engine/docs/sources/use/workingwithrepository.rst index 38062556cb..36f9c590c0 100644 --- a/components/engine/docs/sources/use/workingwithrepository.rst +++ b/components/engine/docs/sources/use/workingwithrepository.rst @@ -7,9 +7,9 @@ Share Images via Repositories ============================= -A *repository* is a hosted collection of tagged :ref:`images -` that together create the file system for a container. The -repository's name is a tag that indicates the provenance of the +A *repository* is a shareable collection of tagged :ref:`images` +that together create the file systems for containers. The +repository's name is a label that indicates the provenance of the repository, i.e. who created it and where the original copy is located. @@ -19,7 +19,7 @@ tag. The implicit registry is located at ``index.docker.io``, the home of "top-level" repositories and the Central Index. This registry may also include public "user" repositories. -So Docker is not only a tool for creating and managing your own +Docker is not only a tool for creating and managing your own :ref:`containers ` -- **Docker is also a tool for sharing**. The Docker project provides a Central Registry to host public repositories, namespaced by user, and a Central Index which @@ -28,6 +28,12 @@ repositories. You can host your own Registry too! Docker acts as a client for these services via ``docker search, pull, login`` and ``push``. +Local Repositories +------------------ + +Docker images which have been created and labled on your local docker server +need to be pushed to a Public or Private registry to be shared. + .. _using_public_repositories: Public Repositories @@ -136,13 +142,13 @@ name for the image. .. _image_push: -Pushing an image to its repository ----------------------------------- +Pushing a repository to its registry +------------------------------------ -In order to push an image to its repository you need to have committed -your container to a named image (see above) +In order to push an repository to its registry you need to have named an image, +or committed your container to a named image (see above) -Now you can commit this image to the repository designated by its name +Now you can push this repository to the registry designated by its name or tag. .. code-block:: bash @@ -187,14 +193,15 @@ manage it by committing code to your GitHub repository. You can create multiple Trusted Builds per repository and configure them to point to specific ``Dockerfile``'s or Git branches. -Private Repositories --------------------- +Private Registry +---------------- -Right now (version 0.6), private repositories are only possible by -hosting `your own registry +Private registries and private shared repositories are +only possible by hosting `your own registry `_. To push or pull to a repository on your own registry, you must prefix the tag with the -address of the registry's host, like this: +address of the registry's host (a ``.`` or ``:`` is used to identify a host), +like this: .. code-block:: bash From 90a166ff8b1f5b7f6834430f0f30282789fb2eaf Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 18 Dec 2013 21:36:02 +1000 Subject: [PATCH 137/403] try out @metalivedev's suggestion, including FQIN Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: a049e93a4c164de25ce1fc1113f2929aa0aa0b5b Component: engine --- .../engine/docs/sources/terms/repository.rst | 23 +++++++++++-------- .../sources/use/workingwithrepository.rst | 10 ++++---- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/components/engine/docs/sources/terms/repository.rst b/components/engine/docs/sources/terms/repository.rst index cceaeec23a..e4fe4b8fd1 100644 --- a/components/engine/docs/sources/terms/repository.rst +++ b/components/engine/docs/sources/terms/repository.rst @@ -7,21 +7,24 @@ Repository ========== -A repository is a tagged set of images either on your local docker server, or +A repository is a set of images either on your local Docker server, or shared, by pushing it to a :ref:`Registry` server. -Images can be labeld into a repository using ``sudo docker build -t LABEL``, -``sudo docker commit CONTAINERID LABEL`` or ``sudo docker tag IMAGEID LABEL``. +Images can be associated with a repository (or multiple) by giving them an image name +using one of three different commands: -The label can be made up of 3 parts: +1. At build time (e.g. ``sudo docker build -t IMAGENAME``), +2. When committing a container (e.g. ``sudo docker commit CONTAINERID IMAGENAME``) or +3. When tagging an image id with an image name (e.g. ``sudo docker tag IMAGEID IMAGENAME``). -[registry_hostname[:port]/][user_name/]( repository_name[:version_tag] | image_id ) -[REGISTRYHOST/][USERNAME/]NAME[:TAG] +A `Fully Qualified Image Name` (FQIN) can be made up of 3 parts: -TAG defaults to ``latest``, USERNAME and REGISTRYHOST default to an empty string. -When REGISTRYHOST is an empty string, then ``docker push`` will push to ``index.docker.io:80``. +``[registry_hostname[:port]/][user_name/](repository_name[:version_tag])`` -If you create a new repository which you want to share, you will need to set the -first part, as the 'default' blank REPOSITORY prefix is reserved for official Docker images. +``version_tag`` defaults to ``latest``, ``username`` and ``registry_hostname`` default to an empty string. +When ``registry_hostname`` is an empty string, then ``docker push`` will push to ``index.docker.io:80``. + +If you create a new repository which you want to share, you will need to set at least the +``user_name``, as the 'default' blank ``user_name`` prefix is reserved for official Docker images. For more information see :ref:`Working with Repositories` diff --git a/components/engine/docs/sources/use/workingwithrepository.rst b/components/engine/docs/sources/use/workingwithrepository.rst index 36f9c590c0..cbde932cde 100644 --- a/components/engine/docs/sources/use/workingwithrepository.rst +++ b/components/engine/docs/sources/use/workingwithrepository.rst @@ -31,7 +31,7 @@ client for these services via ``docker search, pull, login`` and Local Repositories ------------------ -Docker images which have been created and labled on your local docker server +Docker images which have been created and labeled on your local Docker server need to be pushed to a Public or Private registry to be shared. .. _using_public_repositories: @@ -64,8 +64,8 @@ Find Public Images on the Central Index --------------------------------------- You can search the Central Index `online `_ -or by the CLI. Searching can find images by name, user name or -description: +or using the command line interface. Searching can find images by name, user +name or description: .. code-block:: bash @@ -162,7 +162,7 @@ Trusted Builds -------------- Trusted Builds automate the building and updating of images from GitHub, directly -on docker.io servers. It works by adding a commit hook to your selected repository, +on ``docker.io`` servers. It works by adding a commit hook to your selected repository, triggering a build and update when you push a commit. To setup a trusted build @@ -186,7 +186,7 @@ If you want to see the status of your Trusted Builds you can go to your `Trusted Builds page `_ on the Docker index, and it will show you the status of your builds, and the build history. -Once you've created a Trusted Build you can deactive or delete it. You cannot +Once you've created a Trusted Build you can deactivate or delete it. You cannot however push to a Trusted Build with the ``docker push`` command. You can only manage it by committing code to your GitHub repository. From e8870b2f18badaf5ed78296ca78513df7cc58528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20K=C3=BChnle?= Date: Fri, 14 Feb 2014 14:23:10 +0100 Subject: [PATCH 138/403] Env Clarification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clarified that the env variables here are an example output of env. The user should not set them to the example values. Docker-DCO-1.1-Signed-off-by: Matthias Kühnle (github: float64) Upstream-commit: a7f1b74dd812fbc86beefea6fcd3ba4c4abaa7e2 Component: engine --- .../engine/docs/sources/examples/running_redis_service.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/examples/running_redis_service.rst b/components/engine/docs/sources/examples/running_redis_service.rst index 9687f0cfa8..a07250c35a 100644 --- a/components/engine/docs/sources/examples/running_redis_service.rst +++ b/components/engine/docs/sources/examples/running_redis_service.rst @@ -68,13 +68,14 @@ Once inside our freshly created container we need to install Redis to get the service redis-server stop Now we can test the connection. Firstly, let's look at the available environmental -variables in our web application container. We can use these to get the IP and port +variables in our web application container that docker has setup for us. We can use these to get the IP and port of our ``redis`` container. .. code-block:: bash - env - . . . + env | grep DB_ + + # Should return something similar to this with your values DB_NAME=/violet_wolf/db DB_PORT_6379_TCP_PORT=6379 DB_PORT=tcp://172.17.0.33:6379 From efed40e022e5d427b01d2142dddd67f75f74507d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20K=C3=BChnle?= Date: Fri, 14 Feb 2014 14:29:09 +0100 Subject: [PATCH 139/403] Fixed misspelling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker-DCO-1.1-Signed-off-by: Matthias Kühnle (github: float64) Upstream-commit: 8a3cb0c65e7940e928e31d6e64cce5d81d9975a6 Component: engine --- .../engine/docs/sources/examples/running_redis_service.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/examples/running_redis_service.rst b/components/engine/docs/sources/examples/running_redis_service.rst index a07250c35a..c679547b7d 100644 --- a/components/engine/docs/sources/examples/running_redis_service.rst +++ b/components/engine/docs/sources/examples/running_redis_service.rst @@ -68,7 +68,7 @@ Once inside our freshly created container we need to install Redis to get the service redis-server stop Now we can test the connection. Firstly, let's look at the available environmental -variables in our web application container that docker has setup for us. We can use these to get the IP and port +variables in our web application container that Docker has setup for us. We can use these to get the IP and port of our ``redis`` container. .. code-block:: bash From e867f9248c7211bbc16e0c3235b7cd6a6db5a073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20K=C3=BChnle?= Date: Tue, 18 Feb 2014 12:36:14 +0100 Subject: [PATCH 140/403] Reworded description of the environment variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed the "test now..." b/c the code example is only about inspection of env-variables. Test follows in the next section. Docker-DCO-1.1-Signed-off-by: Matthias Kühnle (github: float64) Upstream-commit: d73d3b03e4216bf6cf44b393108dd520acac9670 Component: engine --- .../engine/docs/sources/examples/running_redis_service.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/examples/running_redis_service.rst b/components/engine/docs/sources/examples/running_redis_service.rst index c679547b7d..3b2f65c128 100644 --- a/components/engine/docs/sources/examples/running_redis_service.rst +++ b/components/engine/docs/sources/examples/running_redis_service.rst @@ -67,9 +67,8 @@ Once inside our freshly created container we need to install Redis to get the apt-get -y install redis-server service redis-server stop -Now we can test the connection. Firstly, let's look at the available environmental -variables in our web application container that Docker has setup for us. We can use these to get the IP and port -of our ``redis`` container. +Docker sets some environment variables in our web application container. +Let's inspect them with the ``env`` command. .. code-block:: bash From 983d3e7de300208ebb76b16dbe85cc9bdf6cd934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20K=C3=BChnle?= Date: Tue, 18 Feb 2014 12:47:44 +0100 Subject: [PATCH 141/403] Updated environment description MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .. to make it even more clear that this env variables are there because of the --link option. Docker-DCO-1.1-Signed-off-by: Matthias Kühnle (github: float64) Upstream-commit: c1dfc63845cdf08d7609e1890f45d01f10326bda Component: engine --- .../engine/docs/sources/examples/running_redis_service.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/engine/docs/sources/examples/running_redis_service.rst b/components/engine/docs/sources/examples/running_redis_service.rst index 3b2f65c128..7cbc58b58a 100644 --- a/components/engine/docs/sources/examples/running_redis_service.rst +++ b/components/engine/docs/sources/examples/running_redis_service.rst @@ -67,7 +67,8 @@ Once inside our freshly created container we need to install Redis to get the apt-get -y install redis-server service redis-server stop -Docker sets some environment variables in our web application container. +Since we used the ``--link redis:db`` option before. Docker has created some +environment variables in our web application container. Let's inspect them with the ``env`` command. .. code-block:: bash From d71035c5b7420916748c56c0d7a50bcbfb08fecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20K=C3=BChnle?= Date: Tue, 18 Feb 2014 16:00:21 +0100 Subject: [PATCH 142/403] Reword - one sentence :-) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker-DCO-1.1-Signed-off-by: Matthias Kühnle (github: float64) Upstream-commit: 311cb5ca45c15c32408d54194470536e73ae016d Component: engine --- .../engine/docs/sources/examples/running_redis_service.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/examples/running_redis_service.rst b/components/engine/docs/sources/examples/running_redis_service.rst index 7cbc58b58a..c9424867a4 100644 --- a/components/engine/docs/sources/examples/running_redis_service.rst +++ b/components/engine/docs/sources/examples/running_redis_service.rst @@ -67,9 +67,8 @@ Once inside our freshly created container we need to install Redis to get the apt-get -y install redis-server service redis-server stop -Since we used the ``--link redis:db`` option before. Docker has created some -environment variables in our web application container. -Let's inspect them with the ``env`` command. +As we've used the ``--link redis:db`` option, Docker has created some environment +variables in our web application container. .. code-block:: bash From b0d22755b4ad2d71e611f40537cd0f0b55aef1cf Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 18 Feb 2014 11:23:37 -0700 Subject: [PATCH 143/403] Fix dyntest problems finding dockerinit Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 6ccb30a0e5d1146dddaec340643d3950e6f992ef Component: engine --- components/engine/hack/make/dyntest | 2 +- components/engine/hack/make/dyntest-integration | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/hack/make/dyntest b/components/engine/hack/make/dyntest index eb5c2b73ed..555517fd05 100644 --- a/components/engine/hack/make/dyntest +++ b/components/engine/hack/make/dyntest @@ -12,6 +12,6 @@ fi ( export TEST_DOCKERINIT_PATH="$INIT" - export LDFLAGS_STATIC="-X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" + export LDFLAGS_STATIC="-X github.com/dotcloud/docker/dockerversion.INITSHA1 \"$DOCKER_INITSHA1\"" source "$(dirname "$BASH_SOURCE")/test" ) diff --git a/components/engine/hack/make/dyntest-integration b/components/engine/hack/make/dyntest-integration index 0887c45be0..b3a2d7be63 100644 --- a/components/engine/hack/make/dyntest-integration +++ b/components/engine/hack/make/dyntest-integration @@ -12,6 +12,6 @@ fi ( export TEST_DOCKERINIT_PATH="$INIT" - export LDFLAGS_STATIC="-X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" + export LDFLAGS_STATIC="-X github.com/dotcloud/docker/dockerversion.INITSHA1 \"$DOCKER_INITSHA1\"" source "$(dirname "$BASH_SOURCE")/test-integration" ) From 27672a2f55b8893768ca30c07894b1f3bf795438 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Tue, 18 Feb 2014 11:24:33 -0700 Subject: [PATCH 144/403] Add "Conflicts" to our lxc-docker package to make it more clear that we can't/shouldn't be installed alongside either "docker" or "docker.io" Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 6d1c0a91f89887bd0d73b161b1ffe8b9cc10365e Component: engine --- components/engine/hack/make/ubuntu | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/hack/make/ubuntu b/components/engine/hack/make/ubuntu index 1d309d2b5c..23af0ff7e0 100644 --- a/components/engine/hack/make/ubuntu +++ b/components/engine/hack/make/ubuntu @@ -119,6 +119,8 @@ EOF --deb-recommends xz-utils \ --description "$PACKAGE_DESCRIPTION" \ --maintainer "$PACKAGE_MAINTAINER" \ + --conflicts docker \ + --conflicts docker.io \ --conflicts lxc-docker-virtual-package \ --provides lxc-docker \ --provides lxc-docker-virtual-package \ From 24eab2e4178a18a68640278e5005bad272b335d3 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 18 Feb 2014 15:08:09 -0500 Subject: [PATCH 145/403] Add extra info regarding docker run funcationality Docker-DCO-1.1-Signed-off-by: Brian Goff (github: cpuguy83) Upstream-commit: 349adf4d4df63cd35ecfd74aa21de01629964ea5 Component: engine --- components/engine/docs/sources/reference/commandline/cli.rst | 3 +++ components/engine/docs/sources/use/basics.rst | 1 + 2 files changed, 4 insertions(+) diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index 927f6d3b2d..8ef0ab8b72 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -917,6 +917,8 @@ Running ``docker ps`` showing 2 linked containers. The last container is marked as a ``Ghost`` container. It is a container that was running when the docker daemon was restarted (upgraded, or ``-H`` settings changed). The container is still running, but as this docker daemon process is not able to manage it, you can't attach to it. To bring them out of ``Ghost`` Status, you need to use ``docker kill`` or ``docker restart``. +``docker ps`` will show only running containers by default. To see all containers: ``docker ps -a`` + .. _cli_pull: ``pull`` @@ -1085,6 +1087,7 @@ The ``docker run`` command first ``creates`` a writeable container layer over the specified image, and then ``starts`` it using the specified command. That is, ``docker run`` is equivalent to the API ``/containers/create`` then ``/containers/(id)/start``. +Once the container is stopped it still exists and can be started back up. See ``docker ps -a`` to view a list of all containers. The ``docker run`` command can be used in combination with ``docker commit`` to :ref:`change the command that a container runs `. diff --git a/components/engine/docs/sources/use/basics.rst b/components/engine/docs/sources/use/basics.rst index d62f778b9d..75907efa39 100644 --- a/components/engine/docs/sources/use/basics.rst +++ b/components/engine/docs/sources/use/basics.rst @@ -50,6 +50,7 @@ Running an interactive shell # allocate a tty, attach stdin and stdout # To detach the tty without exiting the shell, # use the escape sequence Ctrl-p + Ctrl-q + # note: This will continue to exist in a stopped state once exited (see "docker ps -a") sudo docker run -i -t ubuntu /bin/bash .. _bind_docker: From f51c1fe9adfafb8f55151713a9f3690a49cecfa2 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 18 Feb 2014 11:56:52 -0800 Subject: [PATCH 146/403] Bump to version 0.8.1 Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: a1598d1e1c157388f3b07415729de28f4d205e49 Component: engine --- components/engine/CHANGELOG.md | 45 ++++++++++++++++++++++++++++++++++ components/engine/VERSION | 2 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/components/engine/CHANGELOG.md b/components/engine/CHANGELOG.md index e016472406..a6a93dc97d 100644 --- a/components/engine/CHANGELOG.md +++ b/components/engine/CHANGELOG.md @@ -1,5 +1,50 @@ # Changelog +## 0.8.1 (2014-02-18) + +#### Builder + +- Avoid extra mount/unmount during build. This removes an unneeded mount/unmount operation which was causing problems with devicemapper +- Fix regression with ADD of tar files. This stops Docker from decompressing tarballs added via ADD from the local file system +- Add error to `docker build --rm`. This adds a missing error check to ensure failures to remove containers are detected and reported + +#### Documentation + +* Update issue filing instructions +* Warn against the use of symlinks for Docker's storage folder +* Replace the Firefox example with an IceWeasel example +* Rewrite the PostgresSQL example using a Dockerfile and add more details to it +* Improve the OS X documentation + +#### Remote API + +- Fix broken images API for version less than 1.7 +- Use the right encoding for all API endpoints which return JSON +- Move remote api client to api/ +- Queue calls to the API using generic socket wait + +#### Runtime + +- Fix the use of custom settings for bridges and custom bridges +- Refactor the devicemapper code to avoid many mount/unmount race conditions and failures +- Remove two panics which could make Docker crash in some situations +- Don't ping registry from the CLI client +- Enable skip_block_zeroing for devicemapper. This stops devicemapper from always zeroing entire blocks +- Fix --run in `docker commit`. This makes docker commit store `--run` in the image configuration +- Remove directory when removing devicemapper device. This cleans up leftover mount directories +- Drop NET_ADMIN capability for non-privileged containers. Unprivileged containers can't change their network configuration +- Ensure `docker cp` stream is closed properly +- Avoid extra mount/unmount during container registration. This removes an unneeded mount/unmount operation which was causing problems with devicemapper +- Stop allowing tcp:// as a default tcp bin address which binds to 127.0.0.1:4243 and remove the default port ++ Mount-bind the PTY as container console. This allows tmux and screen to run in a container +- Clean up archive closing. This fixes and improves archive handling +- Fix engine tests on systems where temp directories are symlinked +- Add test methods for save and load +- Avoid temporarily unmounting the container when restarting it. This fixes a race for devicemapper during restart +- Support submodules when building from a GitHub repository +- Quote volume path to allow spaces +- Fix remote tar ADD behavior. This fixes a regression which was causing Docker to extract tarballs + ## 0.8.0 (2014-02-04) #### Notable features since 0.7.0 diff --git a/components/engine/VERSION b/components/engine/VERSION index f8d71478f5..6f4eebdf6f 100644 --- a/components/engine/VERSION +++ b/components/engine/VERSION @@ -1 +1 @@ -0.8.0-dev +0.8.1 From c8dd504e3a38f79c8f09417d70fddeb2987e62bf Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 18 Feb 2014 16:01:04 -0800 Subject: [PATCH 147/403] Change version to v0.8.1 Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 97ed268a8281231cdab88b166e2fe3faf7ccb296 Component: engine --- components/engine/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/VERSION b/components/engine/VERSION index 6f4eebdf6f..d182dc9160 100644 --- a/components/engine/VERSION +++ b/components/engine/VERSION @@ -1 +1 @@ -0.8.1 +0.8.1-dev From ee616cd9b4a66c14ae7207f7bdbea35a08493615 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 19 Feb 2014 11:35:58 +1000 Subject: [PATCH 148/403] we moved the repo Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: e2da84ee63b759173d4919481eeb0c3ddf58cfb5 Component: engine --- components/engine/docs/sources/installation/mac.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/installation/mac.rst b/components/engine/docs/sources/installation/mac.rst index efb999ee1f..75e788b597 100644 --- a/components/engine/docs/sources/installation/mac.rst +++ b/components/engine/docs/sources/installation/mac.rst @@ -39,7 +39,7 @@ boot2docker ``docker`` daemon. It also takes care of the installation for the OS image that is used for the job. -.. _GitHub page: https://github.com/steeve/boot2docker +.. _GitHub page: https://github.com/boot2docker/boot2docker Open up a new terminal window, if you have not already. @@ -51,7 +51,7 @@ Run the following commands to get boot2docker: cd ~/bin # Get the file - curl https://raw.github.com/steeve/boot2docker/master/boot2docker > boot2docker + curl https://raw.github.com/boot2docker/boot2docker/master/boot2docker > boot2docker # Mark it executable chmod +x boot2docker @@ -153,7 +153,7 @@ boot2docker: See the GitHub page for `boot2docker`_. -.. _boot2docker: https://github.com/steeve/boot2docker +.. _boot2docker: https://github.com/boot2docker/boot2docker If SSH complains about keys: ---------------------------- From 2894395a0944902537125eea7d3d9871fb72682c Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 18 Feb 2014 21:45:48 -0500 Subject: [PATCH 149/403] Add basics for controlling a container's lifecycle Docker-DCO-1.1-Signed-off-by: Brian Goff (github: cpuguy83) Upstream-commit: 6dc2f5513648c42b2e71e55c4a4ea6cd3073ce55 Component: engine --- components/engine/docs/sources/use/basics.rst | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/use/basics.rst b/components/engine/docs/sources/use/basics.rst index 75907efa39..24c22bba39 100644 --- a/components/engine/docs/sources/use/basics.rst +++ b/components/engine/docs/sources/use/basics.rst @@ -122,12 +122,38 @@ Starting a long-running worker process sudo docker kill $JOB -Listing all running containers ------------------------------- +Listing containers +------------------ .. code-block:: bash - sudo docker ps + sudo docker ps # Lists only running containers + sudo docker ps -a # Lists all containers + + +Controlling containers +---------------------- +.. code-block:: bash + + # Start a new container + JOB=$(sudo docker run -d ubuntu /bin/sh -c "while true; do echo Hello world; sleep 1; done") + + # Stop the container + docker stop $JOB + + # Start the container + docker start $JOB + + # Restart the container + docker restart $JOB + + # SIGKILL a container + docker kill $JOB + + # Remove a container + docker stop $JOB # Container must be stopped to remove it + docker rm $JOB + Bind a service on a TCP port ------------------------------ From 83c5ea4b475e71aa053aa41a7b0d749d6f797e76 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Tue, 18 Feb 2014 19:21:12 -0800 Subject: [PATCH 150/403] Update the documentation docs with new branch information. Docker-DCO-1.1-Signed-off-by: Andy Rothfusz (github: metalivedev) Upstream-commit: 05c692d64ec9cabe6c16385300405406ebe63f40 Component: engine --- components/engine/docs/README.md | 63 +++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/components/engine/docs/README.md b/components/engine/docs/README.md index 3fdbad2ead..71b3a09995 100644 --- a/components/engine/docs/README.md +++ b/components/engine/docs/README.md @@ -19,10 +19,24 @@ post-commit hooks. The "release" branch maps to the "latest" documentation and the "master" branch maps to the "master" documentation. -**Warning**: The "master" documentation may include features not yet -part of any official docker release. "Master" docs should be used only -for understanding bleeding-edge development and "latest" should be -used for the latest official release. +## Branches + +**There are two branches related to editing docs**: ``master`` and a +``doc*`` branch (currently ``doc0.8.1``). You should normally edit +docs on the ``master`` branch. That way your fixes will automatically +get included in later releases, and docs maintainers can easily +cherry-pick your changes to bring over to the current docs branch. In +the rare case where your change is not forward-compatible, then you +could base your change on the appropriate ``doc*`` branch. + +Now that we have a ``doc*`` branch, we can keep the ``latest`` docs +up to date with any bugs found between ``docker`` code releases. + +**Warning**: When *reading* the docs, the ``master`` documentation may +include features not yet part of any official docker +release. ``Master`` docs should be used only for understanding +bleeding-edge development and ``latest`` (which points to the ``doc*`` +branch``) should be used for the latest official release. If you need to manually trigger a build of an existing branch, then you can do that through the [readthedocs @@ -39,7 +53,7 @@ Getting Started To edit and test the docs, you'll need to install the Sphinx tool and its dependencies. There are two main ways to install this tool: -###Native Installation +### Native Installation Install dependencies from `requirements.txt` file in your `docker/docs` directory: @@ -48,7 +62,7 @@ directory: * Mac OS X: `[sudo] pip-2.7 install -r docs/requirements.txt` -###Alternative Installation: Docker Container +### Alternative Installation: Docker Container If you're running ``docker`` on your development machine then you may find it easier and cleaner to use the docs Dockerfile. This installs Sphinx @@ -59,11 +73,16 @@ docs inside the container, even starting a simple HTTP server on port In the ``docker`` source directory, run: ```make docs``` -This is the equivalent to ``make clean server`` since each container starts clean. +This is the equivalent to ``make clean server`` since each container +starts clean. -Usage ------ -* Follow the contribution guidelines (``../CONTRIBUTING.md``) +# Contributing + +## Normal Case: + +* Follow the contribution guidelines ([see + ``../CONTRIBUTING.md``](../CONTRIBUTING)). +* Remember to sign your work! * Work in your own fork of the code, we accept pull requests. * Change the ``.rst`` files with your favorite editor -- try to keep the lines short and respect RST and Sphinx conventions. @@ -75,6 +94,20 @@ Usage ``make clean docs`` must complete without any warnings or errors. +## Special Case for RST Newbies: + +If you want to write a new doc or make substantial changes to an +existing doc, but **you don't know RST syntax**, we will accept pull +requests in Markdown and plain text formats. We really want to +encourage people to share their knowledge and don't want the markup +syntax to be the obstacle. So when you make the Pull Request, please +note in your comment that you need RST markup assistance, and we'll +make the changes for you, and then we will make a pull request to your +pull request so that you can get all the changes and learn about the +markup. You still need to follow the +[``CONTRIBUTING``](../CONTRIBUTING) guidelines, so please sign your +commits. + Working using GitHub's file editor ---------------------------------- @@ -93,8 +126,11 @@ exists. Notes ----- -* For the template the css is compiled from less. When changes are needed they can be compiled using -lessc ``lessc main.less`` or watched using watch-lessc ``watch-lessc -i main.less -o main.css`` + +* For the template the css is compiled from less. When changes are + needed they can be compiled using + + lessc ``lessc main.less`` or watched using watch-lessc ``watch-lessc -i main.less -o main.css`` Guides on using sphinx ---------------------- @@ -106,7 +142,8 @@ Guides on using sphinx Hello world =========== - This is.. (etc.) + This is a reference to :ref:`hello_world` and will work even if we + move the target to another file or change the title of the section. ``` The ``_hello_world:`` will make it possible to link to this position From 569ad8a8f45b92f553999b710c02780e5dc16915 Mon Sep 17 00:00:00 2001 From: Thomas Schroeter Date: Wed, 19 Feb 2014 10:50:11 +0000 Subject: [PATCH 151/403] Correct port range in OS X installation doc Docker-DCO-1.1-Signed-off-by: Thomas Schroeter (github: thschroeter) Upstream-commit: 8a289a8a54480b968a6924536e084d749e567cf0 Component: engine --- components/engine/docs/sources/installation/mac.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/installation/mac.rst b/components/engine/docs/sources/installation/mac.rst index 75e788b597..5139324d0b 100644 --- a/components/engine/docs/sources/installation/mac.rst +++ b/components/engine/docs/sources/installation/mac.rst @@ -126,7 +126,7 @@ with our containers as if they were running locally: .. code-block:: bash # vm must be powered off - for i in {4900..49900}; do + for i in {49000..49900}; do VBoxManage modifyvm "boot2docker-vm" --natpf1 "tcp-port$i,tcp,,$i,,$i"; VBoxManage modifyvm "boot2docker-vm" --natpf1 "udp-port$i,udp,,$i,,$i"; done From 33370133022b9ca05e6c2c813d129b4e3cdb537d Mon Sep 17 00:00:00 2001 From: Fabio Falci Date: Mon, 10 Feb 2014 23:21:20 +0000 Subject: [PATCH 152/403] Package coverage when running integration tests If coverpkg is missing on `go test` command, only the current package will be covered. That's the case of unit tests. For integration tests we need to explicitly declare each package. Docker-DCO-1.1-Signed-off-by: Fabio Falci (github: fabiofalci) Upstream-commit: b3d5e9527a050a0c7252f0125a8a26a6fc5175f1 Component: engine --- components/engine/hack/make.sh | 15 +++++++++++++-- components/engine/hack/make/test | 13 +------------ components/engine/hack/make/test-integration | 3 ++- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index c0092b106f..59bd716022 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -16,7 +16,7 @@ set -e # in the Dockerfile at the root of the source. In other words: # DO NOT CALL THIS SCRIPT DIRECTLY. # - The right way to call this script is to invoke "make" from -# your checkout of the Docker repository. +# your checkout of the Docker repository. # the Makefile will do a "docker build -t docker ." and then # "docker run hack/make.sh" in the resulting container image. # @@ -101,13 +101,14 @@ fi # go_test_dir() { dir=$1 + coverpkg=$2 testcover=() if [ "$HAVE_GO_TEST_COVER" ]; then # if our current go install has -cover, we want to use it :) mkdir -p "$DEST/coverprofiles" coverprofile="docker${dir#.}" coverprofile="$DEST/coverprofiles/${coverprofile//\//-}" - testcover=( -cover -coverprofile "$coverprofile" ) + testcover=( -cover -coverprofile "$coverprofile" $coverpkg ) fi ( set -x @@ -116,6 +117,16 @@ go_test_dir() { ) } +# This helper function walks the current directory looking for directories +# holding certain files ($1 parameter), and prints their paths on standard +# output, one per line. +find_dirs() { + find -not \( \ + \( -wholename './vendor' -o -wholename './integration' -o -wholename './contrib' -o -wholename './pkg/mflag/example' \) \ + -prune \ + \) -name "$1" -print0 | xargs -0n1 dirname | sort -u +} + bundle() { bundlescript=$1 bundle=$(basename $bundlescript) diff --git a/components/engine/hack/make/test b/components/engine/hack/make/test index 760c5a5fc6..68b4c52202 100644 --- a/components/engine/hack/make/test +++ b/components/engine/hack/make/test @@ -19,7 +19,7 @@ bundle_test() { date TESTS_FAILED=() - for test_dir in $(find_test_dirs); do + for test_dir in $(find_dirs '*_test.go'); do echo if ! LDFLAGS="$LDFLAGS $LDFLAGS_STATIC" go_test_dir "$test_dir"; then @@ -48,15 +48,4 @@ bundle_test() { } 2>&1 | tee $DEST/test.log } - -# This helper function walks the current directory looking for directories -# holding Go test files, and prints their paths on standard output, one per -# line. -find_test_dirs() { - find -not \( \ - \( -wholename './vendor' -o -wholename './integration' \) \ - -prune \ - \) -name '*_test.go' -print0 | xargs -0n1 dirname | sort -u -} - bundle_test diff --git a/components/engine/hack/make/test-integration b/components/engine/hack/make/test-integration index f1ab0b99c3..93d63ad595 100644 --- a/components/engine/hack/make/test-integration +++ b/components/engine/hack/make/test-integration @@ -5,7 +5,8 @@ DEST=$1 set -e bundle_test_integration() { - LDFLAGS="$LDFLAGS $LDFLAGS_STATIC" go_test_dir ./integration + LDFLAGS="$LDFLAGS $LDFLAGS_STATIC" go_test_dir ./integration \ + "-coverpkg $(find_dirs '*.go' | sed 's,^\.,github.com/dotcloud/docker,g' | paste -d, -s)" } bundle_test_integration 2>&1 | tee $DEST/test.log From 98d4bbe0f67907e4757ba43a771ddaa73c138df9 Mon Sep 17 00:00:00 2001 From: Andy Rothfusz Date: Tue, 18 Feb 2014 19:25:43 -0800 Subject: [PATCH 153/403] update links Docker-DCO-1.1-Signed-off-by: Andy Rothfusz (github: metalivedev) Upstream-commit: 196702fbf9905426e590c0dbcb6101e72c62aac6 Component: engine --- components/engine/docs/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/docs/README.md b/components/engine/docs/README.md index 71b3a09995..9379d86870 100644 --- a/components/engine/docs/README.md +++ b/components/engine/docs/README.md @@ -81,8 +81,8 @@ starts clean. ## Normal Case: * Follow the contribution guidelines ([see - ``../CONTRIBUTING.md``](../CONTRIBUTING)). -* Remember to sign your work! + ``../CONTRIBUTING.md``](../CONTRIBUTING.md)). +* [Remember to sign your work!](../CONTRIBUTING.md#sign-your-work) * Work in your own fork of the code, we accept pull requests. * Change the ``.rst`` files with your favorite editor -- try to keep the lines short and respect RST and Sphinx conventions. @@ -115,6 +115,7 @@ Alternatively, for small changes and typos you might want to use GitHub's built in file editor. It allows you to preview your changes right online (though there can be some differences between GitHub markdown and Sphinx RST). Just be careful not to create many commits. +And you must still [sign your work!](../CONTRIBUTING.md#sign-your-work) Images ------ From 785171a902869769f2872bf09d1370cee1402c79 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Wed, 19 Feb 2014 08:53:31 -0500 Subject: [PATCH 154/403] FIX: Linux mint troubleshooting tip Package is called 'cgroup-lite' not 'cgroups-lite'. Verified on Linux Mint 16. Ref http://packages.ubuntu.com/search?keywords=cgroup-lite Docker-DCO-1.1-Signed-off-by: Dan Williams (github: deedubs) Upstream-commit: 6073b450e83a67a27ffe8c11ebda36791a7923b4 Component: engine --- components/engine/docs/sources/installation/ubuntulinux.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index f37be90d7d..dc8832a982 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -220,12 +220,12 @@ To install the latest version of docker, use the standard ``apt-get`` method: Troubleshooting ^^^^^^^^^^^^^^^ -On Linux Mint, the ``cgroups-lite`` package is not installed by default. +On Linux Mint, the ``cgroup-lite`` package is not installed by default. Before Docker will work correctly, you will need to install this via: .. code-block:: bash - sudo apt-get update && sudo apt-get install cgroups-lite + sudo apt-get update && sudo apt-get install cgroup-lite .. _ufw: From 0802cd5e0ff11039ec9279caa767f6a92d2c4fa4 Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 20 Feb 2014 01:11:04 +0200 Subject: [PATCH 155/403] don't skip cert check for an example & use HTTPS This changes two URLs from http to https and it fixes a Dockerfile to stop skipping certificate validation. It also adds the ca-certificates package to that Dockerfile example. Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: 73a277eb2fd66b815b11e9d2caff2c31c18a0854 Component: engine --- components/engine/contrib/mkimage-arch.sh | 2 +- .../docs/sources/examples/cfengine_process_management.rst | 4 ++-- .../engine/docs/sources/reference/api/docker_remote_api.rst | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/engine/contrib/mkimage-arch.sh b/components/engine/contrib/mkimage-arch.sh index d178a1df3d..73a4173b11 100755 --- a/components/engine/contrib/mkimage-arch.sh +++ b/components/engine/contrib/mkimage-arch.sh @@ -39,7 +39,7 @@ arch-chroot $ROOTFS /bin/sh -c "haveged -w 1024; pacman-key --init; pkill havege arch-chroot $ROOTFS /bin/sh -c "ln -s /usr/share/zoneinfo/UTC /etc/localtime" echo 'en_US.UTF-8 UTF-8' > $ROOTFS/etc/locale.gen arch-chroot $ROOTFS locale-gen -arch-chroot $ROOTFS /bin/sh -c 'echo "Server = http://mirrors.kernel.org/archlinux/\$repo/os/\$arch" > /etc/pacman.d/mirrorlist' +arch-chroot $ROOTFS /bin/sh -c 'echo "Server = https://mirrors.kernel.org/archlinux/\$repo/os/\$arch" > /etc/pacman.d/mirrorlist' # udev doesn't work in containers, rebuild /dev DEV=$ROOTFS/dev diff --git a/components/engine/docs/sources/examples/cfengine_process_management.rst b/components/engine/docs/sources/examples/cfengine_process_management.rst index 9d7681f556..7ca2c35498 100644 --- a/components/engine/docs/sources/examples/cfengine_process_management.rst +++ b/components/engine/docs/sources/examples/cfengine_process_management.rst @@ -55,7 +55,7 @@ The first two steps can be done as part of a Dockerfile, as follows. FROM ubuntu MAINTAINER Eystein Måløy Stenberg - RUN apt-get -y install wget lsb-release unzip + RUN apt-get -y install wget lsb-release unzip ca-certificates # install latest CFEngine RUN wget -qO- http://cfengine.com/pub/gpg.key | apt-key add - @@ -64,7 +64,7 @@ The first two steps can be done as part of a Dockerfile, as follows. RUN apt-get install cfengine-community # install cfe-docker process management policy - RUN wget --no-check-certificate https://github.com/estenberg/cfe-docker/archive/master.zip -P /tmp/ && unzip /tmp/master.zip -d /tmp/ + RUN wget https://github.com/estenberg/cfe-docker/archive/master.zip -P /tmp/ && unzip /tmp/master.zip -d /tmp/ RUN cp /tmp/cfe-docker-master/cfengine/bin/* /var/cfengine/bin/ RUN cp /tmp/cfe-docker-master/cfengine/inputs/* /var/cfengine/inputs/ RUN rm -rf /tmp/cfe-docker-master /tmp/master.zip diff --git a/components/engine/docs/sources/reference/api/docker_remote_api.rst b/components/engine/docs/sources/reference/api/docker_remote_api.rst index f7cd7faf4f..69bbf71ec9 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api.rst @@ -2,7 +2,7 @@ :description: API Documentation for Docker :keywords: API, Docker, rcli, REST, documentation -.. COMMENT use http://pythonhosted.org/sphinxcontrib-httpdomain/ to +.. COMMENT use https://pythonhosted.org/sphinxcontrib-httpdomain/ to .. document the REST API. ================= From 852e62316ce21cedfea1fd54a61a5407e09dc6b2 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 19 Feb 2014 16:53:15 -0700 Subject: [PATCH 156/403] Add back the Ubuntu "Memory and Swap Accounting" section that was lost when the "Kernel" page was culled Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 42b97be441f5272ead365c3400b91fb756565b1a Component: engine --- .../docs/sources/installation/ubuntulinux.rst | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index f37be90d7d..55ea036522 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -217,6 +217,26 @@ To install the latest version of docker, use the standard ``apt-get`` method: # install the latest sudo apt-get install lxc-docker +Memory and Swap Accounting +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If want to enable memory and swap accounting, you must add the following +command-line parameters to your kernel:: + + cgroup_enable=memory swapaccount=1 + +On systems using GRUB (which is the default for Ubuntu), you can add those +parameters by editing ``/etc/default/grub`` and extending +``GRUB_CMDLINE_LINUX``. Look for the following line:: + + GRUB_CMDLINE_LINUX="" + +And replace it by the following one:: + + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +Then run ``update-grub``, and reboot. + Troubleshooting ^^^^^^^^^^^^^^^ From c8d7265499475d995b4d0b18064cbc2ba9d8e1b7 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 20 Feb 2014 13:49:15 +1000 Subject: [PATCH 157/403] lets talk about the other way to make base images Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 9f1c69ca7b1b2afd96b4d154431395ee48ada97d Component: engine --- .../docs/sources/articles/baseimages.rst | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/components/engine/docs/sources/articles/baseimages.rst b/components/engine/docs/sources/articles/baseimages.rst index 6fd1823f8d..61c8f7d9c5 100644 --- a/components/engine/docs/sources/articles/baseimages.rst +++ b/components/engine/docs/sources/articles/baseimages.rst @@ -13,8 +13,8 @@ The specific process will depend heavily on the Linux distribution you want to package. We have some examples below, and you are encouraged to submit pull requests to contribute new ones. -Getting Started -............... +Create a full image using tar +............................. In general, you'll want to start with a working machine that is running the distribution you'd like to package as a base image, though @@ -44,3 +44,22 @@ Docker GitHub Repo: `_ * `Debian / Ubuntu `_ + + +Creating a simple base image using ``scratch`` +.............................................. + +There is a special repository in the Docker registry called ``scratch``, which +was created using an empty tar file:: + + $ tar cv --files-from /dev/null | docker import - scratch + +which you can ``docker pull``. You can then use that image to base your new +minimal containers ``FROM``:: + + FROM scratch + ADD true-asm /true + CMD ["/true"] + +The Dockerfile above is from extremely minimal image - +`tianon/true `_. From 3d478c1043434edf347e8758f9ed4085bc70e6eb Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 14 Feb 2014 14:24:37 +0100 Subject: [PATCH 158/403] Archive: Add Add Lgetxattr and Lsetxattr implementations Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: d19e998e7a99d24122da3e98d9886e3b7b4b3501 Component: engine --- components/engine/archive/stat_linux.go | 53 +++++++++++++++++++ components/engine/archive/stat_unsupported.go | 8 +++ 2 files changed, 61 insertions(+) diff --git a/components/engine/archive/stat_linux.go b/components/engine/archive/stat_linux.go index f87a99c55a..2910ce5bab 100644 --- a/components/engine/archive/stat_linux.go +++ b/components/engine/archive/stat_linux.go @@ -37,3 +37,56 @@ func UtimesNano(path string, ts []syscall.Timespec) error { } return nil } + +// Returns a nil slice and nil error if the xattr is not set +func Lgetxattr(path string, attr string) ([]byte, error) { + pathBytes, err := syscall.BytePtrFromString(path) + if err != nil { + return nil, err + } + attrBytes, err := syscall.BytePtrFromString(attr) + if err != nil { + return nil, err + } + + dest := make([]byte, 128) + destBytes := unsafe.Pointer(&dest[0]) + sz, _, errno := syscall.Syscall6(syscall.SYS_LGETXATTR, uintptr(unsafe.Pointer(pathBytes)), uintptr(unsafe.Pointer(attrBytes)), uintptr(destBytes), uintptr(len(dest)), 0, 0) + if errno == syscall.ENODATA { + return nil, nil + } + if errno == syscall.ERANGE { + dest = make([]byte, sz) + destBytes := unsafe.Pointer(&dest[0]) + sz, _, errno = syscall.Syscall6(syscall.SYS_LGETXATTR, uintptr(unsafe.Pointer(pathBytes)), uintptr(unsafe.Pointer(attrBytes)), uintptr(destBytes), uintptr(len(dest)), 0, 0) + } + if errno != 0 { + return nil, errno + } + + return dest[:sz], nil +} + +var _zero uintptr + +func Lsetxattr(path string, attr string, data []byte, flags int) error { + pathBytes, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + attrBytes, err := syscall.BytePtrFromString(attr) + if err != nil { + return err + } + var dataBytes unsafe.Pointer + if len(data) > 0 { + dataBytes = unsafe.Pointer(&data[0]) + } else { + dataBytes = unsafe.Pointer(&_zero) + } + _, _, errno := syscall.Syscall6(syscall.SYS_LSETXATTR, uintptr(unsafe.Pointer(pathBytes)), uintptr(unsafe.Pointer(attrBytes)), uintptr(dataBytes), uintptr(len(data)), uintptr(flags), 0) + if errno != 0 { + return errno + } + return nil +} diff --git a/components/engine/archive/stat_unsupported.go b/components/engine/archive/stat_unsupported.go index 50ca461867..99f1fbc7da 100644 --- a/components/engine/archive/stat_unsupported.go +++ b/components/engine/archive/stat_unsupported.go @@ -19,3 +19,11 @@ func LUtimesNano(path string, ts []syscall.Timespec) error { func UtimesNano(path string, ts []syscall.Timespec) error { return ErrNotImplemented } + +func Lgetxattr(path string, attr string) ([]byte, error) { + return nil, ErrNotImplemented +} + +func Lsetxattr(path string, attr string, data []byte, flags int) error { + return ErrNotImplemented +} From 91e61d40222f63f379b04c537eda52b5969cb4b9 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 15 Jan 2014 15:05:30 +0100 Subject: [PATCH 159/403] archive: extract xattrs from tarfiles Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: c8428d77fdde41786aa5c0c4e64e0e762f873676 Component: engine --- components/engine/archive/archive.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index 16c01993b7..206580fa08 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -251,6 +251,12 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader) e return err } + for key, value := range hdr.Xattrs { + if err := Lsetxattr(path, key, []byte(value), 0); err != nil { + return err + } + } + // There is no LChmod, so ignore mode for symlink. Also, this // must happen after chown, as that can modify the file mode if hdr.Typeflag != tar.TypeSymlink { From 8cd94e7ac60018eddc04becd4352220a23ae315b Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Fri, 17 Jan 2014 10:51:59 +0100 Subject: [PATCH 160/403] archive: Detect file changes to capability bits Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 87ca750cdc3114a340af1c5bc9394cc5f6242677 Component: engine --- components/engine/archive/changes.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/components/engine/archive/changes.go b/components/engine/archive/changes.go index b46b13bbe7..a9eba8196f 100644 --- a/components/engine/archive/changes.go +++ b/components/engine/archive/changes.go @@ -1,6 +1,7 @@ package archive import ( + "bytes" "code.google.com/p/go/src/pkg/archive/tar" "fmt" "github.com/dotcloud/docker/utils" @@ -126,10 +127,11 @@ func Changes(layers []string, rw string) ([]Change, error) { } type FileInfo struct { - parent *FileInfo - name string - stat syscall.Stat_t - children map[string]*FileInfo + parent *FileInfo + name string + stat syscall.Stat_t + children map[string]*FileInfo + capability []byte } func (root *FileInfo) LookUp(path string) *FileInfo { @@ -200,7 +202,8 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) { oldStat.Rdev != newStat.Rdev || // Don't look at size for dirs, its not a good measure of change (oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) || - !sameFsTimeSpec(getLastModification(oldStat), getLastModification(newStat)) { + !sameFsTimeSpec(getLastModification(oldStat), getLastModification(newStat)) || + bytes.Compare(oldChild.capability, newChild.capability) != 0 { change := Change{ Path: newChild.path(), Kind: ChangeModify, @@ -275,6 +278,8 @@ func collectFileInfo(sourceDir string) (*FileInfo, error) { return err } + info.capability, _ = Lgetxattr(path, "security.capability") + parent.children[info.name] = info return nil From 7d5d3a982f55254efd748ba036046f092e0ceea8 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 20 Jan 2014 12:26:08 +0100 Subject: [PATCH 161/403] archive: Handle capabilities in tar files If a file has a security.capability set, we push this to the tar file. This is important to handle in e.g. layer files or when copying files to containers, as some distros (e.g. Fedora) use capability bits as a more finegrained version of setuid bits, and thus if the capabilites are stripped (and setuid is not set) the binaries will fail to work. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 3b9953903b12eaca76655311bd44533768f6f3da Component: engine --- components/engine/archive/archive.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index 206580fa08..8bb0a8ed1c 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -165,6 +165,13 @@ func addTarFile(path, name string, tw *tar.Writer) error { hdr.Devmajor = int64(major(uint64(stat.Rdev))) hdr.Devminor = int64(minor(uint64(stat.Rdev))) } + + } + + capability, _ := Lgetxattr(path, "security.capability") + if capability != nil { + hdr.Xattrs = make(map[string]string) + hdr.Xattrs["security.capability"] = string(capability) } if err := tw.WriteHeader(hdr); err != nil { From e100a3787851aed5b9ee7aa5d3fd56e40d6c263b Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Thu, 20 Feb 2014 08:51:39 -0500 Subject: [PATCH 162/403] Site ver for Webmaster tools Docker-DCO-1.1-Signed-off-by: James Turnbull (github: jamtur01) Upstream-commit: ef826621585a62d9b563e8ab2fd72f731b14817c Component: engine --- components/engine/docs/theme/docker/layout.html | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/docs/theme/docker/layout.html b/components/engine/docs/theme/docker/layout.html index a966556044..7d78fb9c3c 100755 --- a/components/engine/docs/theme/docker/layout.html +++ b/components/engine/docs/theme/docker/layout.html @@ -3,6 +3,7 @@ + {{ meta['title'] if meta and meta['title'] else title }} - Docker Documentation From 6ec2f023129d7a00849e56c8bd79d0b6b51d7aa0 Mon Sep 17 00:00:00 2001 From: Adrian Mouat Date: Thu, 20 Feb 2014 16:55:07 +0100 Subject: [PATCH 163/403] Fix misspelling Docker-DCO-1.1-Signed-off-by: Adrian Mouat (github: amouat) Upstream-commit: 88d24df6687d64bf0e19c23973da32eed32fda28 Component: engine --- components/engine/docs/sources/use/working_with_volumes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/use/working_with_volumes.rst b/components/engine/docs/sources/use/working_with_volumes.rst index 40e282c986..8ddd4196c7 100644 --- a/components/engine/docs/sources/use/working_with_volumes.rst +++ b/components/engine/docs/sources/use/working_with_volumes.rst @@ -80,7 +80,7 @@ similar to :ref:`ambassador_pattern_linking `. If you remove containers that mount volumes, including the initial DATA container, or the middleman, the volumes will not be deleted until there are no containers still -referencing those volumes. This allows you to upgrade, or effectivly migrate data volumes +referencing those volumes. This allows you to upgrade, or effectively migrate data volumes between containers. Mount a Host Directory as a Container Volume: From 1124129b254733cd73a2428081152145c6aaa373 Mon Sep 17 00:00:00 2001 From: Nolan Darilek Date: Thu, 20 Feb 2014 11:18:27 -0600 Subject: [PATCH 164/403] Don't call applyVolumesFrom on containers with volumes already configured (closes #2973) Docker-DCO-1.1-Signed-off-by: Nolan Darilek (github: ndarilek) Upstream-commit: 937ea1d3805771b0e5763f336f9d64e5996c2c95 Component: engine --- components/engine/volumes.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/engine/volumes.go b/components/engine/volumes.go index a53ef722e7..9f76e3698b 100644 --- a/components/engine/volumes.go +++ b/components/engine/volumes.go @@ -23,11 +23,11 @@ func prepareVolumesForContainer(container *Container) error { if container.Volumes == nil || len(container.Volumes) == 0 { container.Volumes = make(map[string]string) container.VolumesRW = make(map[string]bool) + if err := applyVolumesFrom(container); err != nil { + return err + } } - if err := applyVolumesFrom(container); err != nil { - return err - } if err := createVolumes(container); err != nil { return err } From c7ecc00d6b0a7c89973a8e6070c6421da3bfdacb Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 20 Feb 2014 19:43:00 +0200 Subject: [PATCH 165/403] docs: explain DNS warnings & how to fix them This explains how to fix the DNS warnings on Ubuntu and why they're shown. Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: cae554b32e8c3341767e0efe7b9a8977a499d826 Component: engine --- .../docs/sources/installation/ubuntulinux.rst | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index f37be90d7d..0fd24e9996 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -261,6 +261,64 @@ incoming connections on the Docker port (default 4243): .. _installmirrors: +Docker and local DNS server warnings +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Systems which are running Ubuntu or an Ubuntu derivative on the desktop will +use `127.0.0.1` as the default nameserver in `/etc/resolv.conf`. NetworkManager +sets up dnsmasq to use the real DNS servers of the connection and sets up +`nameserver 127.0.0.1` in `/etc/resolv.conf`. + +When starting containers on these desktop machines, users will see a warning: + +.. code-block:: bash + + WARNING: Local (127.0.0.1) DNS resolver found in resolv.conf and containers can't use it. Using default external servers : [8.8.8.8 8.8.4.4] + +This warning is shown because the containers can't use the local DNS nameserver +and Docker will default to using an external nameserver. + +This can be worked around by specifying a DNS server to be used by the Docker +daemon for the containers: + +.. code-block:: bash + + sudo nano /etc/default/docker + --- + # Add: + DOCKER_OPTS="-dns 8.8.8.8" + # 8.8.8.8 could be replaced with a local DNS server, such as 192.168.1.1 + # multiple DNS servers can be specified: -dns 8.8.8.8 -dns 192.168.1.1 + +The Docker daemon has to be restarted: + +.. code-block:: bash + + sudo restart docker + +.. warning:: If you're doing this on a laptop which connects to various networks, make sure to choose a public DNS server. + +An alternative solution involves disabling dnsmasq in NetworkManager by +following these steps: + +.. code-block:: bash + + sudo nano /etc/NetworkManager/NetworkManager.conf + ---- + # Change: + dns=dnsmasq + # to + #dns=dnsmasq + +NetworkManager and Docker need to be restarted afterwards: + +.. code-block:: bash + + sudo restart network-manager + sudo restart docker + +.. warning:: This might make DNS resolution slower on some networks. + Mirrors ^^^^^^^ From 87c3fb28d226f0fbc0f472eb36a2496130daa035 Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 20 Feb 2014 17:16:45 +0200 Subject: [PATCH 166/403] forbid chained onbuild, from & maintainer triggers This changes the way onbuild works: - forbids the chaining of onbuild instructions - forbids the use of `onbuild from` - forbids the use of `onbuild maintainer` It also makes docker throw errors when encountering such triggers when executing the triggers during `FROM`. Three tests have been added: - ensure that chained onbuild (`onbuild onbuild`) is forbidden - ensure that `onbuild from` is forbidden - ensure that `onbuild maintainer` is forbidden Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: b829e96cde4f798bc21f05a763b2f6a623dfe1eb Component: engine --- components/engine/buildfile.go | 16 +++++++ .../engine/docs/sources/reference/builder.rst | 2 + .../engine/integration/buildfile_test.go | 42 +++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index c7181b9146..b30a220ddc 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -117,6 +117,14 @@ func (b *buildFile) CmdFrom(name string) error { fmt.Fprintf(b.errStream, "# Executing %d build triggers\n", nTriggers) } for n, step := range b.config.OnBuild { + splitStep := strings.Split(step, " ") + stepInstruction := strings.ToUpper(strings.Trim(splitStep[0], " ")) + switch stepInstruction { + case "ONBUILD": + return fmt.Errorf("Source image contains forbidden chained `ONBUILD ONBUILD` trigger: %s", step) + case "MAINTAINER", "FROM": + return fmt.Errorf("Source image contains forbidden %s trigger: %s", stepInstruction, step) + } if err := b.BuildStep(fmt.Sprintf("onbuild-%d", n), step); err != nil { return err } @@ -128,6 +136,14 @@ func (b *buildFile) CmdFrom(name string) error { // The ONBUILD command declares a build instruction to be executed in any future build // using the current image as a base. func (b *buildFile) CmdOnbuild(trigger string) error { + splitTrigger := strings.Split(trigger, " ") + triggerInstruction := strings.ToUpper(strings.Trim(splitTrigger[0], " ")) + switch triggerInstruction { + case "ONBUILD": + return fmt.Errorf("Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed") + case "MAINTAINER", "FROM": + return fmt.Errorf("%s isn't allowed as an ONBUILD trigger", triggerInstruction) + } b.config.OnBuild = append(b.config.OnBuild, trigger) return b.commit("", b.config.Cmd, fmt.Sprintf("ONBUILD %s", trigger)) } diff --git a/components/engine/docs/sources/reference/builder.rst b/components/engine/docs/sources/reference/builder.rst index 4b6a151006..6fabd6629d 100644 --- a/components/engine/docs/sources/reference/builder.rst +++ b/components/engine/docs/sources/reference/builder.rst @@ -466,6 +466,8 @@ For example you might add something like this: ONBUILD RUN /usr/local/bin/python-build --dir /app/src [...] +.. warning:: Chaining ONBUILD instructions using `ONBUILD ONBUILD` isn't allowed. +.. warning:: ONBUILD may not trigger FROM or MAINTAINER instructions. .. _dockerfile_examples: diff --git a/components/engine/integration/buildfile_test.go b/components/engine/integration/buildfile_test.go index 805932b57a..efab9707ec 100644 --- a/components/engine/integration/buildfile_test.go +++ b/components/engine/integration/buildfile_test.go @@ -924,3 +924,45 @@ func TestBuildOnBuildTrigger(t *testing.T) { } // FIXME: test that the 'foobar' file was created in the final build. } + +func TestBuildOnBuildForbiddenChainedTrigger(t *testing.T) { + _, err := buildImage(testContextTemplate{` + from {IMAGE} + onbuild onbuild run echo test + `, + nil, nil, + }, + t, nil, true, + ) + if err == nil { + t.Fatal("Error should not be nil") + } +} + +func TestBuildOnBuildForbiddenFromTrigger(t *testing.T) { + _, err := buildImage(testContextTemplate{` + from {IMAGE} + onbuild from {IMAGE} + `, + nil, nil, + }, + t, nil, true, + ) + if err == nil { + t.Fatal("Error should not be nil") + } +} + +func TestBuildOnBuildForbiddenMaintainerTrigger(t *testing.T) { + _, err := buildImage(testContextTemplate{` + from {IMAGE} + onbuild maintainer test + `, + nil, nil, + }, + t, nil, true, + ) + if err == nil { + t.Fatal("Error should not be nil") + } +} From 733cdcbf96a61ac28c654bbd1fffbbc14bd30623 Mon Sep 17 00:00:00 2001 From: Jake Moshenko Date: Thu, 20 Feb 2014 17:57:58 -0500 Subject: [PATCH 167/403] Fix registry auth by storing the string passed on the command line, and allowing for credential selection by normalizing on hostname. Also, remove remote ping calls from CmdPush and CmdPull. Docker-DCO-1.1-Signed-off-by: Jake Moshenko (github: jakedt) Upstream-commit: 90b0cce07b4f68d8099903f7e1470f79541f45d0 Component: engine --- components/engine/api/client.go | 26 ++++----- components/engine/auth/auth.go | 59 +++++++++------------ components/engine/auth/auth_test.go | 7 +++ components/engine/registry/registry.go | 9 ++-- components/engine/registry/registry_test.go | 2 +- components/engine/server.go | 14 ++++- 6 files changed, 60 insertions(+), 57 deletions(-) diff --git a/components/engine/api/client.go b/components/engine/api/client.go index eb345ae40b..f421c6366c 100644 --- a/components/engine/api/client.go +++ b/components/engine/api/client.go @@ -977,13 +977,13 @@ func (cli *DockerCli) CmdPush(args ...string) error { cli.LoadConfigFile() - // Resolve the Repository name from fqn to endpoint + name - endpoint, _, err := registry.ResolveRepositoryName(name) + // Resolve the Repository name from fqn to hostname + name + hostname, _, err := registry.ResolveRepositoryName(name) if err != nil { return err } // Resolve the Auth config relevant for this server - authConfig := cli.configFile.ResolveAuthConfig(endpoint) + authConfig := cli.configFile.ResolveAuthConfig(hostname) // If we're not using a custom registry, we know the restrictions // applied to repository names and can warn the user in advance. // Custom repositories can have different rules, and we must also @@ -1014,10 +1014,10 @@ func (cli *DockerCli) CmdPush(args ...string) error { if err := push(authConfig); err != nil { if strings.Contains(err.Error(), "Status 401") { fmt.Fprintln(cli.out, "\nPlease login prior to push:") - if err := cli.CmdLogin(endpoint); err != nil { + if err := cli.CmdLogin(hostname); err != nil { return err } - authConfig := cli.configFile.ResolveAuthConfig(endpoint) + authConfig := cli.configFile.ResolveAuthConfig(hostname) return push(authConfig) } return err @@ -1042,8 +1042,8 @@ func (cli *DockerCli) CmdPull(args ...string) error { *tag = parsedTag } - // Resolve the Repository name from fqn to endpoint + name - endpoint, _, err := registry.ResolveRepositoryName(remote) + // Resolve the Repository name from fqn to hostname + name + hostname, _, err := registry.ResolveRepositoryName(remote) if err != nil { return err } @@ -1051,7 +1051,7 @@ func (cli *DockerCli) CmdPull(args ...string) error { cli.LoadConfigFile() // Resolve the Auth config relevant for this server - authConfig := cli.configFile.ResolveAuthConfig(endpoint) + authConfig := cli.configFile.ResolveAuthConfig(hostname) v := url.Values{} v.Set("fromImage", remote) v.Set("tag", *tag) @@ -1073,10 +1073,10 @@ func (cli *DockerCli) CmdPull(args ...string) error { if err := pull(authConfig); err != nil { if strings.Contains(err.Error(), "Status 401") { fmt.Fprintln(cli.out, "\nPlease login prior to pull:") - if err := cli.CmdLogin(endpoint); err != nil { + if err := cli.CmdLogin(hostname); err != nil { return err } - authConfig := cli.configFile.ResolveAuthConfig(endpoint) + authConfig := cli.configFile.ResolveAuthConfig(hostname) return pull(authConfig) } return err @@ -1753,8 +1753,8 @@ func (cli *DockerCli) CmdRun(args ...string) error { v.Set("fromImage", repos) v.Set("tag", tag) - // Resolve the Repository name from fqn to endpoint + name - endpoint, _, err := registry.ResolveRepositoryName(repos) + // Resolve the Repository name from fqn to hostname + name + hostname, _, err := registry.ResolveRepositoryName(repos) if err != nil { return err } @@ -1763,7 +1763,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { cli.LoadConfigFile() // Resolve the Auth config relevant for this server - authConfig := cli.configFile.ResolveAuthConfig(endpoint) + authConfig := cli.configFile.ResolveAuthConfig(hostname) buf, err := json.Marshal(authConfig) if err != nil { return err diff --git a/components/engine/auth/auth.go b/components/engine/auth/auth.go index cbca81f3e3..4417dd0f7a 100644 --- a/components/engine/auth/auth.go +++ b/components/engine/auth/auth.go @@ -252,50 +252,39 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e } // this method matches a auth configuration to a server address or a url -func (config *ConfigFile) ResolveAuthConfig(registry string) AuthConfig { - if registry == IndexServerAddress() || len(registry) == 0 { +func (config *ConfigFile) ResolveAuthConfig(hostname string) AuthConfig { + if hostname == IndexServerAddress() || len(hostname) == 0 { // default to the index server return config.Configs[IndexServerAddress()] } - // if it's not the index server there are three cases: - // - // 1. a full config url -> it should be used as is - // 2. a full url, but with the wrong protocol - // 3. a hostname, with an optional port - // - // as there is only one auth entry which is fully qualified we need to start - // parsing and matching - swapProtocol := func(url string) string { - if strings.HasPrefix(url, "http:") { - return strings.Replace(url, "http:", "https:", 1) - } - if strings.HasPrefix(url, "https:") { - return strings.Replace(url, "https:", "http:", 1) - } - return url + // First try the happy case + if c, found := config.Configs[hostname]; found { + return c } - resolveIgnoringProtocol := func(url string) AuthConfig { - if c, found := config.Configs[url]; found { - return c + convertToHostname := func(url string) string { + stripped := url + if strings.HasPrefix(url, "http://") { + stripped = strings.Replace(url, "http://", "", 1) + } else if strings.HasPrefix(url, "https://") { + stripped = strings.Replace(url, "https://", "", 1) } - registrySwappedProtocol := swapProtocol(url) - // now try to match with the different protocol - if c, found := config.Configs[registrySwappedProtocol]; found { - return c - } - return AuthConfig{} + + nameParts := strings.SplitN(stripped, "/", 2) + + return nameParts[0] } - // match both protocols as it could also be a server name like httpfoo - if strings.HasPrefix(registry, "http:") || strings.HasPrefix(registry, "https:") { - return resolveIgnoringProtocol(registry) + // Maybe they have a legacy config file, we will iterate the keys converting + // them to the new format and testing + normalizedHostename := convertToHostname(hostname) + for registry, config := range config.Configs { + if registryHostname := convertToHostname(registry); registryHostname == normalizedHostename { + return config + } } - url := "https://" + registry - if !strings.Contains(registry, "/") { - url = url + "/v1/" - } - return resolveIgnoringProtocol(url) + // When all else fails, return an empty auth config + return AuthConfig{} } diff --git a/components/engine/auth/auth_test.go b/components/engine/auth/auth_test.go index 5f2d3b85fd..2335072609 100644 --- a/components/engine/auth/auth_test.go +++ b/components/engine/auth/auth_test.go @@ -108,6 +108,7 @@ func TestResolveAuthConfigFullURL(t *testing.T) { } configFile.Configs["https://registry.example.com/v1/"] = registryAuth configFile.Configs["http://localhost:8000/v1/"] = localAuth + configFile.Configs["registry.com"] = registryAuth validRegistries := map[string][]string{ "https://registry.example.com/v1/": { @@ -122,6 +123,12 @@ func TestResolveAuthConfigFullURL(t *testing.T) { "localhost:8000", "localhost:8000/v1/", }, + "registry.com": { + "https://registry.com/v1/", + "http://registry.com/v1/", + "registry.com", + "registry.com/v1/", + }, } for configKey, registries := range validRegistries { diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index df94302305..37e107fad4 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -91,7 +91,7 @@ func validateRepositoryName(repositoryName string) error { return nil } -// Resolves a repository name to a endpoint + name +// Resolves a repository name to a hostname + name func ResolveRepositoryName(reposName string) (string, string, error) { if strings.Contains(reposName, "://") { // It cannot contain a scheme! @@ -117,11 +117,8 @@ func ResolveRepositoryName(reposName string) (string, string, error) { if err := validateRepositoryName(reposName); err != nil { return "", "", err } - endpoint, err := ExpandAndVerifyRegistryUrl(hostname) - if err != nil { - return "", "", err - } - return endpoint, reposName, err + + return hostname, reposName, nil } // this method expands the registry name as used in the prefix of a repo diff --git a/components/engine/registry/registry_test.go b/components/engine/registry/registry_test.go index 16bc431e55..5e398f9933 100644 --- a/components/engine/registry/registry_test.go +++ b/components/engine/registry/registry_test.go @@ -145,7 +145,7 @@ func TestResolveRepositoryName(t *testing.T) { if err != nil { t.Fatal(err) } - assertEqual(t, ep, "http://"+u+"/v1/", "Expected endpoint to be "+u) + assertEqual(t, ep, u, "Expected endpoint to be "+u) assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase") } diff --git a/components/engine/server.go b/components/engine/server.go index 0ed96fee31..7555830c0a 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -1336,7 +1336,12 @@ func (srv *Server) ImagePull(job *engine.Job) engine.Status { defer srv.poolRemove("pull", localName+":"+tag) // Resolve the Repository name from fqn to endpoint + name - endpoint, remoteName, err := registry.ResolveRepositoryName(localName) + hostname, remoteName, err := registry.ResolveRepositoryName(localName) + if err != nil { + return job.Error(err) + } + + endpoint, err := registry.ExpandAndVerifyRegistryUrl(hostname) if err != nil { return job.Error(err) } @@ -1538,7 +1543,12 @@ func (srv *Server) ImagePush(job *engine.Job) engine.Status { defer srv.poolRemove("push", localName) // Resolve the Repository name from fqn to endpoint + name - endpoint, remoteName, err := registry.ResolveRepositoryName(localName) + hostname, remoteName, err := registry.ResolveRepositoryName(localName) + if err != nil { + return job.Error(err) + } + + endpoint, err := registry.ExpandAndVerifyRegistryUrl(hostname) if err != nil { return job.Error(err) } From 5a22c079a1d1aa08b3836969e5d519a55a2340c5 Mon Sep 17 00:00:00 2001 From: jakedt Date: Thu, 20 Feb 2014 19:22:13 -0500 Subject: [PATCH 168/403] Fix build command auth by adding the auth config and registry objects to the build job environment. Docker-DCO-1.1-Signed-off-by: Jake Moshenko (github: jakedt) Upstream-commit: 7dcbae485cd46ad4b8af1ea758dd1d2c69f11fbe Component: engine --- components/engine/api/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/api/api.go b/components/engine/api/api.go index 8d9bae978f..fe965ddafa 100644 --- a/components/engine/api/api.go +++ b/components/engine/api/api.go @@ -912,6 +912,8 @@ func postBuild(eng *engine.Engine, version float64, w http.ResponseWriter, r *ht job.Setenv("q", r.FormValue("q")) job.Setenv("nocache", r.FormValue("nocache")) job.Setenv("rm", r.FormValue("rm")) + job.SetenvJson("authConfig", authConfig) + job.SetenvJson("configFile", configFile) if err := job.Run(); err != nil { if !job.Stdout.Used() { From 2524fdc1ab56c235b5a8f9d80ddc8d62fdade808 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 21 Feb 2014 13:51:08 +1000 Subject: [PATCH 169/403] reinforce that docker build --rm is a good default to use Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: e59485e148f5d1ab0cf5ec18ca3bb7c684e8cf7b Component: engine --- .../engine/docs/sources/reference/commandline/cli.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index 8ef0ab8b72..41cdfe7cbb 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -197,6 +197,8 @@ is given as ``URL``, then no context is set. When a Git repository is set as ``URL``, then the repository is used as the context. Git repositories are cloned with their submodules (`git clone --recursive`). +.. note:: ``docker build --rm`` does not affect the image cache which is used to accelerate builds, it only removes the duplicate writeable container layers. + .. _cli_build_examples: .. seealso:: :ref:`dockerbuilder`. @@ -206,7 +208,7 @@ Examples: .. code-block:: bash - $ sudo docker build . + $ sudo docker build --rm . Uploading context 10240 bytes Step 1 : FROM busybox Pulling repository busybox @@ -229,6 +231,9 @@ Examples: ---> Running in 02071fceb21b ---> f52f38b7823e Successfully built f52f38b7823e + Removing intermediate container 9c9e81692ae9 + Removing intermediate container 02071fceb21b + This example specifies that the ``PATH`` is ``.``, and so all the files in the local directory get tar'd and sent to the Docker daemon. The ``PATH`` @@ -243,6 +248,10 @@ The transfer of context from the local machine to the Docker daemon is what the ``docker`` client means when you see the "Uploading context" message. +The ``--rm`` option tells Docker to remove the intermediate containers and +layers that were used to create each image layer. Doing so has no impact on +the image build cache. + .. code-block:: bash From ecb0d90e1959185115c9cd6328d59435db0c8c9c Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Thu, 20 Feb 2014 23:42:31 -0700 Subject: [PATCH 170/403] Add newlines to the JSON stream functions This makes the JSON streams a _lot_ easier to parse in less well-baked JSON parsers, and no less so in better ones. Line-based JSON streams are very, very common, where simply chunk-based is not very common at all. Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 060da572d20dfeee4fe02ce6b975a6cb33e50dbe Component: engine --- components/engine/utils/streamformatter.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/components/engine/utils/streamformatter.go b/components/engine/utils/streamformatter.go index 9345c3cb16..8876fa5cb7 100644 --- a/components/engine/utils/streamformatter.go +++ b/components/engine/utils/streamformatter.go @@ -14,6 +14,10 @@ func NewStreamFormatter(json bool) *StreamFormatter { return &StreamFormatter{json, false} } +const streamNewline = "\r\n" + +var streamNewlineBytes = []byte(streamNewline) + func (sf *StreamFormatter) FormatStream(str string) []byte { sf.used = true if sf.json { @@ -21,7 +25,7 @@ func (sf *StreamFormatter) FormatStream(str string) []byte { if err != nil { return sf.FormatError(err) } - return b + return append(b, streamNewlineBytes...) } return []byte(str + "\r") } @@ -34,9 +38,9 @@ func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []b if err != nil { return sf.FormatError(err) } - return b + return append(b, streamNewlineBytes...) } - return []byte(str + "\r\n") + return []byte(str + streamNewline) } func (sf *StreamFormatter) FormatError(err error) []byte { @@ -47,11 +51,11 @@ func (sf *StreamFormatter) FormatError(err error) []byte { jsonError = &JSONError{Message: err.Error()} } if b, err := json.Marshal(&JSONMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil { - return b + return append(b, streamNewlineBytes...) } - return []byte("{\"error\":\"format error\"}") + return []byte("{\"error\":\"format error\"}" + streamNewline) } - return []byte("Error: " + err.Error() + "\r\n") + return []byte("Error: " + err.Error() + streamNewline) } func (sf *StreamFormatter) FormatProgress(id, action string, progress *JSONProgress) []byte { From 611dddcc478506d48abd381ea33558b9361a0ce9 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 21 Feb 2014 17:42:02 +1000 Subject: [PATCH 171/403] small update to API docs to go with #4276 Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: f83e553452db74b73e9de7fa770646e72e7af88f Component: engine --- .../docs/sources/reference/api/docker_remote_api_v1.9.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst index 47cdb46b28..a7372d5dc8 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst @@ -993,12 +993,12 @@ Search images 2.3 Misc -------- -Build an image from Dockerfile via stdin -**************************************** +Build an image from Dockerfile +****************************** .. http:post:: /build - Build an image from Dockerfile via stdin + Build an image from Dockerfile using a POST body. **Example request**: @@ -1032,6 +1032,7 @@ Build an image from Dockerfile via stdin :query t: repository name (and optionally a tag) to be applied to the resulting image in case of success :query q: suppress verbose build output :query nocache: do not use the cache when building the image + :query rm: Remove intermediate containers after a successful build :reqheader Content-type: should be set to ``"application/tar"``. :reqheader X-Registry-Config: base64-encoded ConfigFile object :statuscode 200: no error From 9b049ad74b2a9602b9ceb2a0b46d75cb8184978d Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Fri, 21 Feb 2014 03:02:06 -0700 Subject: [PATCH 172/403] Add MTU to lxc conf to make host and container MTU match If you are using jumbo frames the host side of the veth was being set to 1500. Docker would set the MTU of the container side of the veth to 9001. This would lead to a situation in which the two sides of the veth had different MTU sizes causing issues in network traffic. Docker-DCO-1.1-Signed-off-by: Darren Shepherd (github: ibuildthecloud) Upstream-commit: 0db53bd2ecba097c6ace1a1088e11458e139390a Component: engine --- components/engine/execdriver/lxc/lxc_template.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/execdriver/lxc/lxc_template.go b/components/engine/execdriver/lxc/lxc_template.go index 639780f5d8..1181396a18 100644 --- a/components/engine/execdriver/lxc/lxc_template.go +++ b/components/engine/execdriver/lxc/lxc_template.go @@ -12,6 +12,7 @@ const LxcTemplate = ` lxc.network.type = veth lxc.network.link = {{.Network.Bridge}} lxc.network.name = eth0 +lxc.network.mtu = {{.Network.Mtu}} {{else}} # network is disabled (-n=false) lxc.network.type = empty From 1ddffb74450914ec32a16191ed75e14ef3f3bc52 Mon Sep 17 00:00:00 2001 From: Danny Berger Date: Fri, 21 Feb 2014 10:51:51 -0500 Subject: [PATCH 173/403] Fix cli argument usage typo for docker --mtu Docker-DCO-1.1-Signed-off-by: Danny Berger (github: dpb587) Upstream-commit: 8c4160fe61151f38f471e93e7bf98dcbf5406a9e Component: engine --- components/engine/docker/docker.go | 2 +- components/engine/docs/sources/reference/commandline/cli.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 02c99b9316..9e93b71767 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -40,7 +40,7 @@ func main() { flInterContainerComm = flag.Bool([]string{"#icc", "-icc"}, true, "Enable inter-container communication") flGraphDriver = flag.String([]string{"s", "-storage-driver"}, "", "Force the docker runtime to use a specific storage driver") flHosts = opts.NewListOpts(api.ValidateHost) - flMtu = flag.Int([]string{"#mtu", "-mtu"}, 0, "Set the containers network MTU; if no value is provided: default to the default route MTU or 1500 if not default route is available") + flMtu = flag.Int([]string{"#mtu", "-mtu"}, 0, "Set the containers network MTU; if no value is provided: default to the default route MTU or 1500 if no default route is available") ) flag.Var(&flDns, []string{"#dns", "-dns"}, "Force docker to use specific DNS servers") flag.Var(&flHosts, []string{"H", "-host"}, "tcp://host:port, unix://path/to/socket, fd://* or fd://socketfd to use in daemon mode. Multiple sockets can be specified") diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index 41cdfe7cbb..d5267cd2d0 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -80,7 +80,7 @@ Commands -r, --restart=true: Restart previously running containers -s, --storage-driver="": Force the docker runtime to use a specific storage driver -v, --version=false: Print version information and quit - -mtu, --mtu=0: Set the containers network MTU; if no value is provided: default to the default route MTU or 1500 if not default route is available + -mtu, --mtu=0: Set the containers network MTU; if no value is provided: default to the default route MTU or 1500 if no default route is available The Docker daemon is the persistent process that manages containers. Docker uses the same binary for both the daemon and client. To run the daemon you provide the ``-d`` flag. From 1377718158903c7807629a8c96b730eacd961e65 Mon Sep 17 00:00:00 2001 From: Manuel Woelker Date: Fri, 21 Feb 2014 19:25:49 +0100 Subject: [PATCH 174/403] docs: document some JSON parameters /containers/create and /containers/(id)/start in remote api (fixes #2948) Docker-DCO-1.1-Signed-off-by: Manuel Woelker (github: manuel-woelker) Upstream-commit: c3faab3955a6cfb5b36249889269dd8d272e9e8b Component: engine --- .../reference/api/docker_remote_api_v1.8.rst | 17 +++++++++++++++-- .../reference/api/docker_remote_api_v1.9.rst | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.8.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.8.rst index 5033f34210..b752f2f8a4 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.8.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.8.rst @@ -118,6 +118,7 @@ Create a container "User":"", "Memory":0, "MemorySwap":0, + "CpuShares":0, "AttachStdin":false, "AttachStdout":true, "AttachStderr":true, @@ -153,7 +154,15 @@ Create a container "Warnings":[] } - :jsonparam config: the container's configuration + :jsonparam Hostname: Container host name + :jsonparam User: Username or UID + :jsonparam Memory: Memory Limit in bytes + :jsonparam CpuShares: CPU shares (relative weight) + :jsonparam AttachStdin: 1/True/true or 0/False/false, attach to standard input. Default false + :jsonparam AttachStdout: 1/True/true or 0/False/false, attach to standard output. Default false + :jsonparam AttachStderr: 1/True/true or 0/False/false, attach to standard error. Default false + :jsonparam Tty: 1/True/true or 0/False/false, allocate a pseudo-tty. Default false + :jsonparam OpenStdin: 1/True/true or 0/False/false, keep stdin open even if not attached. Default false :query name: Assign the specified name to the container. Must match ``/?[a-zA-Z0-9_-]+``. :statuscode 201: no error :statuscode 404: no such container @@ -394,7 +403,11 @@ Start a container HTTP/1.1 204 No Content Content-Type: text/plain - :jsonparam hostConfig: the container's host configuration (optional) + :jsonparam Binds: Create a bind mount to a directory or file with [host-path]:[container-path]:[rw|ro]. If a directory "container-path" is missing, then docker creates a new volume. + :jsonparam LxcConf: Map of custom lxc options + :jsonparam PortBindings: Expose ports from the container, optionally publishing them via the HostPort flag + :jsonparam PublishAllPorts: 1/True/true or 0/False/false, publish all exposed ports to the host interfaces. Default false + :jsonparam Privileged: 1/True/true or 0/False/false, give extended privileges to this container. Default false :statuscode 204: no error :statuscode 404: no such container :statuscode 500: server error diff --git a/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst b/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst index 47cdb46b28..3ea4761530 100644 --- a/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst +++ b/components/engine/docs/sources/reference/api/docker_remote_api_v1.9.rst @@ -118,6 +118,7 @@ Create a container "User":"", "Memory":0, "MemorySwap":0, + "CpuShares":0, "AttachStdin":false, "AttachStdout":true, "AttachStderr":true, @@ -153,7 +154,15 @@ Create a container "Warnings":[] } - :jsonparam config: the container's configuration + :jsonparam Hostname: Container host name + :jsonparam User: Username or UID + :jsonparam Memory: Memory Limit in bytes + :jsonparam CpuShares: CPU shares (relative weight) + :jsonparam AttachStdin: 1/True/true or 0/False/false, attach to standard input. Default false + :jsonparam AttachStdout: 1/True/true or 0/False/false, attach to standard output. Default false + :jsonparam AttachStderr: 1/True/true or 0/False/false, attach to standard error. Default false + :jsonparam Tty: 1/True/true or 0/False/false, allocate a pseudo-tty. Default false + :jsonparam OpenStdin: 1/True/true or 0/False/false, keep stdin open even if not attached. Default false :query name: Assign the specified name to the container. Must match ``/?[a-zA-Z0-9_-]+``. :statuscode 201: no error :statuscode 404: no such container @@ -394,7 +403,11 @@ Start a container HTTP/1.1 204 No Content Content-Type: text/plain - :jsonparam hostConfig: the container's host configuration (optional) + :jsonparam Binds: Create a bind mount to a directory or file with [host-path]:[container-path]:[rw|ro]. If a directory "container-path" is missing, then docker creates a new volume. + :jsonparam LxcConf: Map of custom lxc options + :jsonparam PortBindings: Expose ports from the container, optionally publishing them via the HostPort flag + :jsonparam PublishAllPorts: 1/True/true or 0/False/false, publish all exposed ports to the host interfaces. Default false + :jsonparam Privileged: 1/True/true or 0/False/false, give extended privileges to this container. Default false :statuscode 204: no error :statuscode 404: no such container :statuscode 500: server error From 56684e6d5fd1e1ccc45b72d197afc7bcfcda9fca Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Fri, 21 Feb 2014 11:26:04 -0700 Subject: [PATCH 175/403] Add simple "grep" to hide the harmless "warning: no packages being tested depend on ..." in "go test -coverpkg ..." output Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 41b1f93bf72e4c012a159a013697090efdcc183b Component: engine --- components/engine/hack/make/test-integration | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/engine/hack/make/test-integration b/components/engine/hack/make/test-integration index 93d63ad595..fa8b8be719 100644 --- a/components/engine/hack/make/test-integration +++ b/components/engine/hack/make/test-integration @@ -9,4 +9,8 @@ bundle_test_integration() { "-coverpkg $(find_dirs '*.go' | sed 's,^\.,github.com/dotcloud/docker,g' | paste -d, -s)" } -bundle_test_integration 2>&1 | tee $DEST/test.log +# this "grep" hides some really irritating warnings that "go test -coverpkg" +# spews when it is given packages that aren't used +bundle_test_integration 2>&1 \ + | grep -v '^warning: no packages being tested depend on ' \ + | tee $DEST/test.log From 773b1a3574130ad52890d26d8953ad474bf642bc Mon Sep 17 00:00:00 2001 From: Danny Berger Date: Fri, 21 Feb 2014 13:43:03 -0500 Subject: [PATCH 176/403] Drop -mtu from docs in favor of --mtu only Docker-DCO-1.1-Signed-off-by: Danny Berger (github: dpb587) Upstream-commit: afb565e97c79b87d4563e8e728513ca5b50514c7 Component: engine --- components/engine/docs/sources/reference/commandline/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/reference/commandline/cli.rst b/components/engine/docs/sources/reference/commandline/cli.rst index d5267cd2d0..67794a41c5 100644 --- a/components/engine/docs/sources/reference/commandline/cli.rst +++ b/components/engine/docs/sources/reference/commandline/cli.rst @@ -80,7 +80,7 @@ Commands -r, --restart=true: Restart previously running containers -s, --storage-driver="": Force the docker runtime to use a specific storage driver -v, --version=false: Print version information and quit - -mtu, --mtu=0: Set the containers network MTU; if no value is provided: default to the default route MTU or 1500 if no default route is available + --mtu=0: Set the containers network MTU; if no value is provided: default to the default route MTU or 1500 if no default route is available The Docker daemon is the persistent process that manages containers. Docker uses the same binary for both the daemon and client. To run the daemon you provide the ``-d`` flag. From 9de10592bc44924ac2e9571871969eb669794299 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 11:47:53 -0800 Subject: [PATCH 177/403] Move console into execdriver Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 8c783c1c1336d8f2d1b08b9cbd8e2298d066750c Component: engine --- components/engine/container.go | 93 +++------------- components/engine/execdriver/console.go | 137 ++++++++++++++++++++++++ components/engine/utils/utils.go | 2 +- components/engine/utils/utils_test.go | 2 +- 4 files changed, 155 insertions(+), 79 deletions(-) create mode 100644 components/engine/execdriver/console.go diff --git a/components/engine/container.go b/components/engine/container.go index 9c7fc8ffd7..f38876ed24 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -10,10 +10,8 @@ import ( "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/links" "github.com/dotcloud/docker/nat" - "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" - "github.com/kr/pty" "io" "io/ioutil" "log" @@ -57,11 +55,11 @@ type Container struct { Driver string command *execdriver.Command + console execdriver.Console stdout *utils.WriteBroadcaster stderr *utils.WriteBroadcaster stdin io.ReadCloser stdinPipe io.WriteCloser - ptyMaster io.Closer runtime *Runtime @@ -213,56 +211,6 @@ func (container *Container) generateEnvConfig(env []string) error { return nil } -func (container *Container) setupPty() error { - ptyMaster, ptySlave, err := pty.Open() - if err != nil { - return err - } - container.ptyMaster = ptyMaster - container.command.Stdout = ptySlave - container.command.Stderr = ptySlave - container.command.Console = ptySlave.Name() - - // Copy the PTYs to our broadcasters - go func() { - defer container.stdout.CloseWriters() - utils.Debugf("startPty: begin of stdout pipe") - io.Copy(container.stdout, ptyMaster) - utils.Debugf("startPty: end of stdout pipe") - }() - - // stdin - if container.Config.OpenStdin { - container.command.Stdin = ptySlave - container.command.SysProcAttr.Setctty = true - go func() { - defer container.stdin.Close() - utils.Debugf("startPty: begin of stdin pipe") - io.Copy(ptyMaster, container.stdin) - utils.Debugf("startPty: end of stdin pipe") - }() - } - return nil -} - -func (container *Container) setupStd() error { - container.command.Stdout = container.stdout - container.command.Stderr = container.stderr - if container.Config.OpenStdin { - stdin, err := container.command.StdinPipe() - if err != nil { - return err - } - go func() { - defer stdin.Close() - utils.Debugf("start: begin of stdin pipe") - io.Copy(stdin, container.stdin) - utils.Debugf("start: end of stdin pipe") - }() - } - return nil -} - func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, stdout io.Writer, stderr io.Writer) chan error { var cStdout, cStderr io.ReadCloser @@ -593,14 +541,13 @@ func (container *Container) Start() (err error) { } container.waitLock = make(chan struct{}) - // Setuping pipes and/or Pty - var setup func() error - if container.Config.Tty { - setup = container.setupPty - } else { - setup = container.setupStd + container.console, err = execdriver.NewConsole( + container.stdin, container.stdout, container.stderr, + container.Config.OpenStdin, container.Config.Tty) + if err != nil { + return err } - if err := setup(); err != nil { + if err := container.console.AttachTo(container.command); err != nil { return err } @@ -887,22 +834,20 @@ func (container *Container) cleanup() { link.Disable() } } - if container.Config.OpenStdin { if err := container.stdin.Close(); err != nil { utils.Errorf("%s: Error close stdin: %s", container.ID, err) } } - if err := container.stdout.CloseWriters(); err != nil { + if err := container.stdout.Close(); err != nil { utils.Errorf("%s: Error close stdout: %s", container.ID, err) } - if err := container.stderr.CloseWriters(); err != nil { + if err := container.stderr.Close(); err != nil { utils.Errorf("%s: Error close stderr: %s", container.ID, err) } - - if container.ptyMaster != nil { - if err := container.ptyMaster.Close(); err != nil { - utils.Errorf("%s: Error closing Pty master: %s", container.ID, err) + if container.console != nil { + if err := container.console.Close(); err != nil { + utils.Errorf("%s: Error closing console: %s", container.ID, err) } } @@ -994,11 +939,7 @@ func (container *Container) Wait() int { } func (container *Container) Resize(h, w int) error { - pty, ok := container.ptyMaster.(*os.File) - if !ok { - return fmt.Errorf("ptyMaster does not have Fd() method") - } - return term.SetWinsize(pty.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) + return container.console.Resize(h, w) } func (container *Container) ExportRw() (archive.Archive, error) { @@ -1202,11 +1143,9 @@ func (container *Container) Exposes(p nat.Port) bool { } func (container *Container) GetPtyMaster() (*os.File, error) { - if container.ptyMaster == nil { + ttyConsole, ok := container.console.(*execdriver.TtyConsole) + if !ok { return nil, ErrNoTTY } - if pty, ok := container.ptyMaster.(*os.File); ok { - return pty, nil - } - return nil, ErrNotATTY + return ttyConsole.Master, nil } diff --git a/components/engine/execdriver/console.go b/components/engine/execdriver/console.go new file mode 100644 index 0000000000..01b45afb60 --- /dev/null +++ b/components/engine/execdriver/console.go @@ -0,0 +1,137 @@ +package execdriver + +import ( + "github.com/dotcloud/docker/pkg/term" + "github.com/kr/pty" + "io" + "os" +) + +type Console interface { + io.Closer + Resize(height, width int) error + AttachTo(command *Command) error +} + +type pipes struct { + Stdin io.ReadCloser + Stdout, Stderr io.WriteCloser +} + +func (p *pipes) Close() error { + if p.Stderr != nil { + p.Stdin.Close() + } + if p.Stdout != nil { + p.Stdout.Close() + } + if p.Stderr != nil { + p.Stderr.Close() + } + return nil +} + +func NewConsole(stdin io.ReadCloser, stdout, stderr io.WriteCloser, useStdin, tty bool) (Console, error) { + p := &pipes{ + Stdout: stdout, + Stderr: stderr, + } + if useStdin { + p.Stdin = stdin + } + if tty { + return NewTtyConsole(p) + } + return NewStdConsole(p) +} + +type TtyConsole struct { + Master *os.File + Slave *os.File + pipes *pipes +} + +func NewTtyConsole(p *pipes) (*TtyConsole, error) { + ptyMaster, ptySlave, err := pty.Open() + if err != nil { + return nil, err + } + tty := &TtyConsole{ + Master: ptyMaster, + Slave: ptySlave, + pipes: p, + } + return tty, nil +} + +func (t *TtyConsole) Resize(h, w int) error { + return term.SetWinsize(t.Master.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) + +} + +func (t *TtyConsole) AttachTo(command *Command) error { + command.Stdout = t.Slave + command.Stderr = t.Slave + + command.Console = t.Slave.Name() + + go func() { + defer t.pipes.Stdout.Close() + io.Copy(t.pipes.Stdout, t.Master) + }() + + if t.pipes.Stdin != nil { + command.Stdin = t.Slave + command.SysProcAttr.Setctty = true + + go func() { + defer t.pipes.Stdin.Close() + io.Copy(t.Master, t.pipes.Stdin) + }() + } + return nil +} + +func (t *TtyConsole) Close() error { + err := t.Slave.Close() + if merr := t.Master.Close(); err == nil { + err = merr + } + return err +} + +type StdConsole struct { + pipes *pipes +} + +func NewStdConsole(p *pipes) (*StdConsole, error) { + return &StdConsole{p}, nil +} + +func (s *StdConsole) AttachTo(command *Command) error { + command.Stdout = s.pipes.Stdout + command.Stderr = s.pipes.Stderr + + if s.pipes.Stdin != nil { + stdin, err := command.StdinPipe() + if err != nil { + return err + } + + go func() { + defer stdin.Close() + io.Copy(stdin, s.pipes.Stdin) + }() + } + return nil +} + +func (s *StdConsole) Resize(h, w int) error { + // we do not need to reside a non tty + return nil +} + +func (s *StdConsole) Close() error { + // nothing to close here + return nil +} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 1aba80ff41..3a87f76f5f 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -388,7 +388,7 @@ func (w *WriteBroadcaster) Write(p []byte) (n int, err error) { return len(p), nil } -func (w *WriteBroadcaster) CloseWriters() error { +func (w *WriteBroadcaster) Close() error { w.Lock() defer w.Unlock() for sw := range w.writers { diff --git a/components/engine/utils/utils_test.go b/components/engine/utils/utils_test.go index 7e63a45cf7..f8b3920548 100644 --- a/components/engine/utils/utils_test.go +++ b/components/engine/utils/utils_test.go @@ -110,7 +110,7 @@ func TestWriteBroadcaster(t *testing.T) { t.Errorf("Buffer contains %v", bufferC.String()) } - writer.CloseWriters() + writer.Close() } type devNullCloser int From a808fdbf65a64c8aba7f9e56b4df6c9818247e0b Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 12:32:14 -0800 Subject: [PATCH 178/403] Change Console to Terminal Move creation and attach to driver Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 1e742876988546efb876f67f83de6737ee5e9cf6 Component: engine --- components/engine/container.go | 27 ++-- components/engine/execdriver/chroot/driver.go | 2 +- components/engine/execdriver/console.go | 137 ------------------ components/engine/execdriver/driver.go | 6 +- components/engine/execdriver/lxc/driver.go | 7 +- components/engine/execdriver/term.go | 137 ++++++++++++++++++ components/engine/runtime.go | 4 +- 7 files changed, 159 insertions(+), 161 deletions(-) delete mode 100644 components/engine/execdriver/console.go create mode 100644 components/engine/execdriver/term.go diff --git a/components/engine/container.go b/components/engine/container.go index f38876ed24..025479f240 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -55,7 +55,6 @@ type Container struct { Driver string command *execdriver.Command - console execdriver.Console stdout *utils.WriteBroadcaster stderr *utils.WriteBroadcaster stdin io.ReadCloser @@ -531,6 +530,9 @@ func (container *Container) Start() (err error) { } populateCommand(container) + if err := execdriver.NewTerminal(container.command); err != nil { + return err + } // Setup logging of stdout and stderr to disk if err := container.runtime.LogToDisk(container.stdout, container.logPath("json"), "stdout"); err != nil { @@ -541,16 +543,6 @@ func (container *Container) Start() (err error) { } container.waitLock = make(chan struct{}) - container.console, err = execdriver.NewConsole( - container.stdin, container.stdout, container.stderr, - container.Config.OpenStdin, container.Config.Tty) - if err != nil { - return err - } - if err := container.console.AttachTo(container.command); err != nil { - return err - } - callbackLock := make(chan struct{}) callback := func(command *execdriver.Command) { container.State.SetRunning(command.Pid()) @@ -790,7 +782,8 @@ func (container *Container) monitor(callback execdriver.StartCallback) error { populateCommand(container) err = container.runtime.RestoreCommand(container) } else { - exitCode, err = container.runtime.Run(container, callback) + pipes := execdriver.NewPipes(container.stdin, container.stdout, container.stderr, container.Config.OpenStdin) + exitCode, err = container.runtime.Run(container, pipes, callback) } if err != nil { @@ -845,9 +838,9 @@ func (container *Container) cleanup() { if err := container.stderr.Close(); err != nil { utils.Errorf("%s: Error close stderr: %s", container.ID, err) } - if container.console != nil { - if err := container.console.Close(); err != nil { - utils.Errorf("%s: Error closing console: %s", container.ID, err) + if container.command.Terminal != nil { + if err := container.command.Terminal.Close(); err != nil { + utils.Errorf("%s: Error closing terminal: %s", container.ID, err) } } @@ -939,7 +932,7 @@ func (container *Container) Wait() int { } func (container *Container) Resize(h, w int) error { - return container.console.Resize(h, w) + return container.command.Terminal.Resize(h, w) } func (container *Container) ExportRw() (archive.Archive, error) { @@ -1143,7 +1136,7 @@ func (container *Container) Exposes(p nat.Port) bool { } func (container *Container) GetPtyMaster() (*os.File, error) { - ttyConsole, ok := container.console.(*execdriver.TtyConsole) + ttyConsole, ok := container.command.Terminal.(*execdriver.TtyConsole) if !ok { return nil, ErrNoTTY } diff --git a/components/engine/execdriver/chroot/driver.go b/components/engine/execdriver/chroot/driver.go index 396df87bad..dfec680d84 100644 --- a/components/engine/execdriver/chroot/driver.go +++ b/components/engine/execdriver/chroot/driver.go @@ -37,7 +37,7 @@ func NewDriver() (*driver, error) { return &driver{}, nil } -func (d *driver) Run(c *execdriver.Command, startCallback execdriver.StartCallback) (int, error) { +func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) { params := []string{ "chroot", c.Rootfs, diff --git a/components/engine/execdriver/console.go b/components/engine/execdriver/console.go deleted file mode 100644 index 01b45afb60..0000000000 --- a/components/engine/execdriver/console.go +++ /dev/null @@ -1,137 +0,0 @@ -package execdriver - -import ( - "github.com/dotcloud/docker/pkg/term" - "github.com/kr/pty" - "io" - "os" -) - -type Console interface { - io.Closer - Resize(height, width int) error - AttachTo(command *Command) error -} - -type pipes struct { - Stdin io.ReadCloser - Stdout, Stderr io.WriteCloser -} - -func (p *pipes) Close() error { - if p.Stderr != nil { - p.Stdin.Close() - } - if p.Stdout != nil { - p.Stdout.Close() - } - if p.Stderr != nil { - p.Stderr.Close() - } - return nil -} - -func NewConsole(stdin io.ReadCloser, stdout, stderr io.WriteCloser, useStdin, tty bool) (Console, error) { - p := &pipes{ - Stdout: stdout, - Stderr: stderr, - } - if useStdin { - p.Stdin = stdin - } - if tty { - return NewTtyConsole(p) - } - return NewStdConsole(p) -} - -type TtyConsole struct { - Master *os.File - Slave *os.File - pipes *pipes -} - -func NewTtyConsole(p *pipes) (*TtyConsole, error) { - ptyMaster, ptySlave, err := pty.Open() - if err != nil { - return nil, err - } - tty := &TtyConsole{ - Master: ptyMaster, - Slave: ptySlave, - pipes: p, - } - return tty, nil -} - -func (t *TtyConsole) Resize(h, w int) error { - return term.SetWinsize(t.Master.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) - -} - -func (t *TtyConsole) AttachTo(command *Command) error { - command.Stdout = t.Slave - command.Stderr = t.Slave - - command.Console = t.Slave.Name() - - go func() { - defer t.pipes.Stdout.Close() - io.Copy(t.pipes.Stdout, t.Master) - }() - - if t.pipes.Stdin != nil { - command.Stdin = t.Slave - command.SysProcAttr.Setctty = true - - go func() { - defer t.pipes.Stdin.Close() - io.Copy(t.Master, t.pipes.Stdin) - }() - } - return nil -} - -func (t *TtyConsole) Close() error { - err := t.Slave.Close() - if merr := t.Master.Close(); err == nil { - err = merr - } - return err -} - -type StdConsole struct { - pipes *pipes -} - -func NewStdConsole(p *pipes) (*StdConsole, error) { - return &StdConsole{p}, nil -} - -func (s *StdConsole) AttachTo(command *Command) error { - command.Stdout = s.pipes.Stdout - command.Stderr = s.pipes.Stderr - - if s.pipes.Stdin != nil { - stdin, err := command.StdinPipe() - if err != nil { - return err - } - - go func() { - defer stdin.Close() - io.Copy(stdin, s.pipes.Stdin) - }() - } - return nil -} - -func (s *StdConsole) Resize(h, w int) error { - // we do not need to reside a non tty - return nil -} - -func (s *StdConsole) Close() error { - // nothing to close here - return nil -} diff --git a/components/engine/execdriver/driver.go b/components/engine/execdriver/driver.go index 32b39771b6..c9f62f500e 100644 --- a/components/engine/execdriver/driver.go +++ b/components/engine/execdriver/driver.go @@ -58,7 +58,7 @@ type Info interface { } type Driver interface { - Run(c *Command, startCallback StartCallback) (int, error) // Run executes the process and blocks until the process exits and returns the exit code + Run(c *Command, pipes *Pipes, startCallback StartCallback) (int, error) // Run executes the process and blocks until the process exits and returns the exit code Kill(c *Command, sig int) error Restore(c *Command) error // Wait and try to re-attach on an out of process command Name() string // Driver name @@ -82,7 +82,6 @@ type Resources struct { } // Process wrapps an os/exec.Cmd to add more metadata -// TODO: Rename to Command type Command struct { exec.Cmd `json:"-"` @@ -100,7 +99,8 @@ type Command struct { Config []string `json:"config"` // generic values that specific drivers can consume Resources *Resources `json:"resources"` - Console string `json:"-"` + Terminal Term `json:"-"` + Console string `json:"-"` } // Return the pid of the process diff --git a/components/engine/execdriver/lxc/driver.go b/components/engine/execdriver/lxc/driver.go index ee4d02a6b6..da3bc1ec7c 100644 --- a/components/engine/execdriver/lxc/driver.go +++ b/components/engine/execdriver/lxc/driver.go @@ -76,7 +76,12 @@ func (d *driver) Name() string { return fmt.Sprintf("%s-%s", DriverName, version) } -func (d *driver) Run(c *execdriver.Command, startCallback execdriver.StartCallback) (int, error) { +func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) { + if c.Terminal != nil { + if err := c.Terminal.Attach(pipes); err != nil { + return -1, err + } + } configPath, err := d.generateLXCConfig(c) if err != nil { return -1, err diff --git a/components/engine/execdriver/term.go b/components/engine/execdriver/term.go new file mode 100644 index 0000000000..e7ec46653c --- /dev/null +++ b/components/engine/execdriver/term.go @@ -0,0 +1,137 @@ +package execdriver + +import ( + "github.com/dotcloud/docker/pkg/term" + "github.com/kr/pty" + "io" + "os" +) + +type Term interface { + io.Closer + Resize(height, width int) error + Attach(pipes *Pipes) error +} + +type Pipes struct { + Stdin io.ReadCloser + Stdout, Stderr io.WriteCloser +} + +func NewPipes(stdin io.ReadCloser, stdout, stderr io.WriteCloser, useStdin bool) *Pipes { + p := &Pipes{ + Stdout: stdout, + Stderr: stderr, + } + if useStdin { + p.Stdin = stdin + } + return p +} + +func NewTerminal(command *Command) error { + var ( + term Term + err error + ) + if command.Tty { + term, err = NewTtyConsole(command) + } else { + term, err = NewStdConsole(command) + } + if err != nil { + return err + } + command.Terminal = term + return nil +} + +type TtyConsole struct { + command *Command + Master *os.File + Slave *os.File +} + +func NewTtyConsole(command *Command) (*TtyConsole, error) { + ptyMaster, ptySlave, err := pty.Open() + if err != nil { + return nil, err + } + tty := &TtyConsole{ + Master: ptyMaster, + Slave: ptySlave, + command: command, + } + return tty, nil +} + +func (t *TtyConsole) Resize(h, w int) error { + return term.SetWinsize(t.Master.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) +} + +func (t *TtyConsole) Attach(pipes *Pipes) error { + t.command.Stdout = t.Slave + t.command.Stderr = t.Slave + + t.command.Console = t.Slave.Name() + + go func() { + defer pipes.Stdout.Close() + io.Copy(pipes.Stdout, t.Master) + }() + + if pipes.Stdin != nil { + t.command.Stdin = t.Slave + t.command.SysProcAttr.Setctty = true + + go func() { + defer pipes.Stdin.Close() + io.Copy(t.Master, pipes.Stdin) + }() + } + return nil +} + +func (t *TtyConsole) Close() error { + err := t.Slave.Close() + if merr := t.Master.Close(); err == nil { + err = merr + } + return err +} + +type StdConsole struct { + command *Command +} + +func NewStdConsole(command *Command) (*StdConsole, error) { + return &StdConsole{command}, nil +} + +func (s *StdConsole) Attach(pipes *Pipes) error { + s.command.Stdout = pipes.Stdout + s.command.Stderr = pipes.Stderr + + if pipes.Stdin != nil { + stdin, err := s.command.StdinPipe() + if err != nil { + return err + } + + go func() { + defer stdin.Close() + io.Copy(stdin, pipes.Stdin) + }() + } + return nil +} + +func (s *StdConsole) Resize(h, w int) error { + // we do not need to reside a non tty + return nil +} + +func (s *StdConsole) Close() error { + // nothing to close here + return nil +} diff --git a/components/engine/runtime.go b/components/engine/runtime.go index eed28f92ab..a38109cca0 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -812,8 +812,8 @@ func (runtime *Runtime) Diff(container *Container) (archive.Archive, error) { }), nil } -func (runtime *Runtime) Run(c *Container, startCallback execdriver.StartCallback) (int, error) { - return runtime.execDriver.Run(c.command, startCallback) +func (runtime *Runtime) Run(c *Container, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) { + return runtime.execDriver.Run(c.command, pipes, startCallback) } func (runtime *Runtime) Kill(c *Container, sig int) error { From b0992a0e9909521c8229a94c4d78c8b414a326cc Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 12:42:37 -0800 Subject: [PATCH 179/403] Move term creation into driver Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 592c2f6f9a472bda227a03c819f73b8edc7c3320 Component: engine --- components/engine/container.go | 3 -- components/engine/execdriver/lxc/driver.go | 6 +-- components/engine/execdriver/term.go | 61 +++++++++++----------- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index 025479f240..733a31d5a2 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -530,9 +530,6 @@ func (container *Container) Start() (err error) { } populateCommand(container) - if err := execdriver.NewTerminal(container.command); err != nil { - return err - } // Setup logging of stdout and stderr to disk if err := container.runtime.LogToDisk(container.stdout, container.logPath("json"), "stdout"); err != nil { diff --git a/components/engine/execdriver/lxc/driver.go b/components/engine/execdriver/lxc/driver.go index da3bc1ec7c..c18b2e6ab4 100644 --- a/components/engine/execdriver/lxc/driver.go +++ b/components/engine/execdriver/lxc/driver.go @@ -77,10 +77,8 @@ func (d *driver) Name() string { } func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) { - if c.Terminal != nil { - if err := c.Terminal.Attach(pipes); err != nil { - return -1, err - } + if err := execdriver.SetTerminal(c, pipes); err != nil { + return -1, err } configPath, err := d.generateLXCConfig(c) if err != nil { diff --git a/components/engine/execdriver/term.go b/components/engine/execdriver/term.go index e7ec46653c..ab399e8337 100644 --- a/components/engine/execdriver/term.go +++ b/components/engine/execdriver/term.go @@ -10,7 +10,6 @@ import ( type Term interface { io.Closer Resize(height, width int) error - Attach(pipes *Pipes) error } type Pipes struct { @@ -29,15 +28,15 @@ func NewPipes(stdin io.ReadCloser, stdout, stderr io.WriteCloser, useStdin bool) return p } -func NewTerminal(command *Command) error { +func SetTerminal(command *Command, pipes *Pipes) error { var ( term Term err error ) if command.Tty { - term, err = NewTtyConsole(command) + term, err = NewTtyConsole(command, pipes) } else { - term, err = NewStdConsole(command) + term, err = NewStdConsole(command, pipes) } if err != nil { return err @@ -47,20 +46,22 @@ func NewTerminal(command *Command) error { } type TtyConsole struct { - command *Command - Master *os.File - Slave *os.File + Master *os.File + Slave *os.File } -func NewTtyConsole(command *Command) (*TtyConsole, error) { +func NewTtyConsole(command *Command, pipes *Pipes) (*TtyConsole, error) { ptyMaster, ptySlave, err := pty.Open() if err != nil { return nil, err } tty := &TtyConsole{ - Master: ptyMaster, - Slave: ptySlave, - command: command, + Master: ptyMaster, + Slave: ptySlave, + } + if err := tty.attach(command, pipes); err != nil { + tty.Close() + return nil, err } return tty, nil } @@ -69,11 +70,10 @@ func (t *TtyConsole) Resize(h, w int) error { return term.SetWinsize(t.Master.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) } -func (t *TtyConsole) Attach(pipes *Pipes) error { - t.command.Stdout = t.Slave - t.command.Stderr = t.Slave - - t.command.Console = t.Slave.Name() +func (t *TtyConsole) attach(command *Command, pipes *Pipes) error { + command.Stdout = t.Slave + command.Stderr = t.Slave + command.Console = t.Slave.Name() go func() { defer pipes.Stdout.Close() @@ -81,8 +81,8 @@ func (t *TtyConsole) Attach(pipes *Pipes) error { }() if pipes.Stdin != nil { - t.command.Stdin = t.Slave - t.command.SysProcAttr.Setctty = true + command.Stdin = t.Slave + command.SysProcAttr.Setctty = true go func() { defer pipes.Stdin.Close() @@ -93,27 +93,28 @@ func (t *TtyConsole) Attach(pipes *Pipes) error { } func (t *TtyConsole) Close() error { - err := t.Slave.Close() - if merr := t.Master.Close(); err == nil { - err = merr - } - return err + t.Slave.Close() + return t.Master.Close() } type StdConsole struct { - command *Command } -func NewStdConsole(command *Command) (*StdConsole, error) { - return &StdConsole{command}, nil +func NewStdConsole(command *Command, pipes *Pipes) (*StdConsole, error) { + std := &StdConsole{} + + if err := std.attach(command, pipes); err != nil { + return nil, err + } + return std, nil } -func (s *StdConsole) Attach(pipes *Pipes) error { - s.command.Stdout = pipes.Stdout - s.command.Stderr = pipes.Stderr +func (s *StdConsole) attach(command *Command, pipes *Pipes) error { + command.Stdout = pipes.Stdout + command.Stderr = pipes.Stderr if pipes.Stdin != nil { - stdin, err := s.command.StdinPipe() + stdin, err := command.StdinPipe() if err != nil { return err } From a3562ba47c04923c3e1a77c1d5bf0e64c08de52f Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 12:52:18 -0800 Subject: [PATCH 180/403] Add CloseWriters back and do an interface cast Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 8e2284aaa2364a8e47a8058e65d60813f9cd5089 Component: engine --- components/engine/container.go | 4 ++-- components/engine/execdriver/term.go | 10 +++++++--- components/engine/utils/utils.go | 2 +- components/engine/utils/utils_test.go | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/components/engine/container.go b/components/engine/container.go index 733a31d5a2..218ae1e4a8 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -829,10 +829,10 @@ func (container *Container) cleanup() { utils.Errorf("%s: Error close stdin: %s", container.ID, err) } } - if err := container.stdout.Close(); err != nil { + if err := container.stdout.CloseWriters(); err != nil { utils.Errorf("%s: Error close stdout: %s", container.ID, err) } - if err := container.stderr.Close(); err != nil { + if err := container.stderr.CloseWriters(); err != nil { utils.Errorf("%s: Error close stderr: %s", container.ID, err) } if container.command.Terminal != nil { diff --git a/components/engine/execdriver/term.go b/components/engine/execdriver/term.go index ab399e8337..ec3c368012 100644 --- a/components/engine/execdriver/term.go +++ b/components/engine/execdriver/term.go @@ -14,10 +14,10 @@ type Term interface { type Pipes struct { Stdin io.ReadCloser - Stdout, Stderr io.WriteCloser + Stdout, Stderr io.Writer } -func NewPipes(stdin io.ReadCloser, stdout, stderr io.WriteCloser, useStdin bool) *Pipes { +func NewPipes(stdin io.ReadCloser, stdout, stderr io.Writer, useStdin bool) *Pipes { p := &Pipes{ Stdout: stdout, Stderr: stderr, @@ -76,7 +76,11 @@ func (t *TtyConsole) attach(command *Command, pipes *Pipes) error { command.Console = t.Slave.Name() go func() { - defer pipes.Stdout.Close() + if wb, ok := pipes.Stdout.(interface { + CloseWriters() error + }); ok { + defer wb.CloseWriters() + } io.Copy(pipes.Stdout, t.Master) }() diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 3a87f76f5f..1aba80ff41 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -388,7 +388,7 @@ func (w *WriteBroadcaster) Write(p []byte) (n int, err error) { return len(p), nil } -func (w *WriteBroadcaster) Close() error { +func (w *WriteBroadcaster) CloseWriters() error { w.Lock() defer w.Unlock() for sw := range w.writers { diff --git a/components/engine/utils/utils_test.go b/components/engine/utils/utils_test.go index f8b3920548..7e63a45cf7 100644 --- a/components/engine/utils/utils_test.go +++ b/components/engine/utils/utils_test.go @@ -110,7 +110,7 @@ func TestWriteBroadcaster(t *testing.T) { t.Errorf("Buffer contains %v", bufferC.String()) } - writer.Close() + writer.CloseWriters() } type devNullCloser int From 28cc135001feb3e5bfb593e39308fdd224622de9 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 13:27:15 -0800 Subject: [PATCH 181/403] Move current tty and pipe impl to lxc driver Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: aac702727ea02b3974a2848748b0713ac583fc84 Component: engine --- components/engine/container.go | 4 +- components/engine/execdriver/driver.go | 18 ++++- components/engine/execdriver/lxc/driver.go | 2 +- .../engine/execdriver/{ => lxc}/term.go | 68 +++++++------------ components/engine/execdriver/pipes.go | 23 +++++++ 5 files changed, 68 insertions(+), 47 deletions(-) rename components/engine/execdriver/{ => lxc}/term.go (58%) create mode 100644 components/engine/execdriver/pipes.go diff --git a/components/engine/container.go b/components/engine/container.go index 218ae1e4a8..ca53bb57c7 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -1133,9 +1133,9 @@ func (container *Container) Exposes(p nat.Port) bool { } func (container *Container) GetPtyMaster() (*os.File, error) { - ttyConsole, ok := container.command.Terminal.(*execdriver.TtyConsole) + ttyConsole, ok := container.command.Terminal.(execdriver.TtyTerminal) if !ok { return nil, ErrNoTTY } - return ttyConsole.Master, nil + return ttyConsole.Master(), nil } diff --git a/components/engine/execdriver/driver.go b/components/engine/execdriver/driver.go index c9f62f500e..a6d865caf3 100644 --- a/components/engine/execdriver/driver.go +++ b/components/engine/execdriver/driver.go @@ -2,6 +2,8 @@ package execdriver import ( "errors" + "io" + "os" "os/exec" ) @@ -57,6 +59,18 @@ type Info interface { IsRunning() bool } +// Terminal in an interface for drivers to implement +// if they want to support Close and Resize calls from +// the core +type Terminal interface { + io.Closer + Resize(height, width int) error +} + +type TtyTerminal interface { + Master() *os.File +} + type Driver interface { Run(c *Command, pipes *Pipes, startCallback StartCallback) (int, error) // Run executes the process and blocks until the process exits and returns the exit code Kill(c *Command, sig int) error @@ -99,8 +113,8 @@ type Command struct { Config []string `json:"config"` // generic values that specific drivers can consume Resources *Resources `json:"resources"` - Terminal Term `json:"-"` - Console string `json:"-"` + Terminal Terminal `json:"-"` // standard or tty terminal + Console string `json:"-"` // dev/console path } // Return the pid of the process diff --git a/components/engine/execdriver/lxc/driver.go b/components/engine/execdriver/lxc/driver.go index c18b2e6ab4..5be7ad2219 100644 --- a/components/engine/execdriver/lxc/driver.go +++ b/components/engine/execdriver/lxc/driver.go @@ -77,7 +77,7 @@ func (d *driver) Name() string { } func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) { - if err := execdriver.SetTerminal(c, pipes); err != nil { + if err := SetTerminal(c, pipes); err != nil { return -1, err } configPath, err := d.generateLXCConfig(c) diff --git a/components/engine/execdriver/term.go b/components/engine/execdriver/lxc/term.go similarity index 58% rename from components/engine/execdriver/term.go rename to components/engine/execdriver/lxc/term.go index ec3c368012..d772f60972 100644 --- a/components/engine/execdriver/term.go +++ b/components/engine/execdriver/lxc/term.go @@ -1,36 +1,16 @@ -package execdriver +package lxc import ( + "github.com/dotcloud/docker/execdriver" "github.com/dotcloud/docker/pkg/term" "github.com/kr/pty" "io" "os" ) -type Term interface { - io.Closer - Resize(height, width int) error -} - -type Pipes struct { - Stdin io.ReadCloser - Stdout, Stderr io.Writer -} - -func NewPipes(stdin io.ReadCloser, stdout, stderr io.Writer, useStdin bool) *Pipes { - p := &Pipes{ - Stdout: stdout, - Stderr: stderr, - } - if useStdin { - p.Stdin = stdin - } - return p -} - -func SetTerminal(command *Command, pipes *Pipes) error { +func SetTerminal(command *execdriver.Command, pipes *execdriver.Pipes) error { var ( - term Term + term execdriver.Terminal err error ) if command.Tty { @@ -46,18 +26,18 @@ func SetTerminal(command *Command, pipes *Pipes) error { } type TtyConsole struct { - Master *os.File - Slave *os.File + master *os.File + slave *os.File } -func NewTtyConsole(command *Command, pipes *Pipes) (*TtyConsole, error) { +func NewTtyConsole(command *execdriver.Command, pipes *execdriver.Pipes) (*TtyConsole, error) { ptyMaster, ptySlave, err := pty.Open() if err != nil { return nil, err } tty := &TtyConsole{ - Master: ptyMaster, - Slave: ptySlave, + master: ptyMaster, + slave: ptySlave, } if err := tty.attach(command, pipes); err != nil { tty.Close() @@ -66,14 +46,18 @@ func NewTtyConsole(command *Command, pipes *Pipes) (*TtyConsole, error) { return tty, nil } -func (t *TtyConsole) Resize(h, w int) error { - return term.SetWinsize(t.Master.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) +func (t *TtyConsole) Master() *os.File { + return t.master } -func (t *TtyConsole) attach(command *Command, pipes *Pipes) error { - command.Stdout = t.Slave - command.Stderr = t.Slave - command.Console = t.Slave.Name() +func (t *TtyConsole) Resize(h, w int) error { + return term.SetWinsize(t.master.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) +} + +func (t *TtyConsole) attach(command *execdriver.Command, pipes *execdriver.Pipes) error { + command.Stdout = t.slave + command.Stderr = t.slave + command.Console = t.slave.Name() go func() { if wb, ok := pipes.Stdout.(interface { @@ -81,30 +65,30 @@ func (t *TtyConsole) attach(command *Command, pipes *Pipes) error { }); ok { defer wb.CloseWriters() } - io.Copy(pipes.Stdout, t.Master) + io.Copy(pipes.Stdout, t.master) }() if pipes.Stdin != nil { - command.Stdin = t.Slave + command.Stdin = t.slave command.SysProcAttr.Setctty = true go func() { defer pipes.Stdin.Close() - io.Copy(t.Master, pipes.Stdin) + io.Copy(t.master, pipes.Stdin) }() } return nil } func (t *TtyConsole) Close() error { - t.Slave.Close() - return t.Master.Close() + t.slave.Close() + return t.master.Close() } type StdConsole struct { } -func NewStdConsole(command *Command, pipes *Pipes) (*StdConsole, error) { +func NewStdConsole(command *execdriver.Command, pipes *execdriver.Pipes) (*StdConsole, error) { std := &StdConsole{} if err := std.attach(command, pipes); err != nil { @@ -113,7 +97,7 @@ func NewStdConsole(command *Command, pipes *Pipes) (*StdConsole, error) { return std, nil } -func (s *StdConsole) attach(command *Command, pipes *Pipes) error { +func (s *StdConsole) attach(command *execdriver.Command, pipes *execdriver.Pipes) error { command.Stdout = pipes.Stdout command.Stderr = pipes.Stderr diff --git a/components/engine/execdriver/pipes.go b/components/engine/execdriver/pipes.go new file mode 100644 index 0000000000..158219f0c5 --- /dev/null +++ b/components/engine/execdriver/pipes.go @@ -0,0 +1,23 @@ +package execdriver + +import ( + "io" +) + +// Pipes is a wrapper around a containers output for +// stdin, stdout, stderr +type Pipes struct { + Stdin io.ReadCloser + Stdout, Stderr io.Writer +} + +func NewPipes(stdin io.ReadCloser, stdout, stderr io.Writer, useStdin bool) *Pipes { + p := &Pipes{ + Stdout: stdout, + Stderr: stderr, + } + if useStdin { + p.Stdin = stdin + } + return p +} From a326ecddb84f4c120f26bc682685199c720dbd23 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 18 Feb 2014 16:56:11 -0800 Subject: [PATCH 182/403] Initial commit of libcontainer Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: e8abaf217b887fcd6a157b4f905156bd245f8f0a Component: engine --- .../engine/pkg/libcontainer/MAINTAINERS | 2 + components/engine/pkg/libcontainer/README.md | 63 +++++ .../libcontainer/capabilities/capabilities.go | 49 ++++ .../engine/pkg/libcontainer/cli/main.go | 171 +++++++++++ .../engine/pkg/libcontainer/container.go | 27 ++ .../engine/pkg/libcontainer/container.json | 38 +++ components/engine/pkg/libcontainer/errors.go | 9 + .../libcontainer/namespaces/calls_linux.go | 164 +++++++++++ .../pkg/libcontainer/namespaces/exec.go | 266 ++++++++++++++++++ .../libcontainer/namespaces/linux_x86_64.go | 7 + .../pkg/libcontainer/namespaces/mount.go | 207 ++++++++++++++ .../pkg/libcontainer/namespaces/namespaces.go | 70 +++++ .../pkg/libcontainer/namespaces/ns_linux.go | 35 +++ .../pkg/libcontainer/namespaces/utils.go | 108 +++++++ .../pkg/libcontainer/network/network.go | 104 +++++++ .../engine/pkg/libcontainer/network/veth.go | 85 ++++++ .../engine/pkg/libcontainer/privileged.json | 22 ++ components/engine/pkg/libcontainer/types.go | 49 ++++ .../engine/pkg/libcontainer/ubuntu.json | 22 ++ .../engine/pkg/libcontainer/utils/utils.go | 33 +++ 20 files changed, 1531 insertions(+) create mode 100644 components/engine/pkg/libcontainer/MAINTAINERS create mode 100644 components/engine/pkg/libcontainer/README.md create mode 100644 components/engine/pkg/libcontainer/capabilities/capabilities.go create mode 100644 components/engine/pkg/libcontainer/cli/main.go create mode 100644 components/engine/pkg/libcontainer/container.go create mode 100644 components/engine/pkg/libcontainer/container.json create mode 100644 components/engine/pkg/libcontainer/errors.go create mode 100644 components/engine/pkg/libcontainer/namespaces/calls_linux.go create mode 100644 components/engine/pkg/libcontainer/namespaces/exec.go create mode 100644 components/engine/pkg/libcontainer/namespaces/linux_x86_64.go create mode 100644 components/engine/pkg/libcontainer/namespaces/mount.go create mode 100644 components/engine/pkg/libcontainer/namespaces/namespaces.go create mode 100644 components/engine/pkg/libcontainer/namespaces/ns_linux.go create mode 100644 components/engine/pkg/libcontainer/namespaces/utils.go create mode 100644 components/engine/pkg/libcontainer/network/network.go create mode 100644 components/engine/pkg/libcontainer/network/veth.go create mode 100644 components/engine/pkg/libcontainer/privileged.json create mode 100644 components/engine/pkg/libcontainer/types.go create mode 100644 components/engine/pkg/libcontainer/ubuntu.json create mode 100644 components/engine/pkg/libcontainer/utils/utils.go diff --git a/components/engine/pkg/libcontainer/MAINTAINERS b/components/engine/pkg/libcontainer/MAINTAINERS new file mode 100644 index 0000000000..e53d933d47 --- /dev/null +++ b/components/engine/pkg/libcontainer/MAINTAINERS @@ -0,0 +1,2 @@ +Michael Crosby (@crosbymichael) +Guillaume Charmes (@creack) diff --git a/components/engine/pkg/libcontainer/README.md b/components/engine/pkg/libcontainer/README.md new file mode 100644 index 0000000000..91d747863c --- /dev/null +++ b/components/engine/pkg/libcontainer/README.md @@ -0,0 +1,63 @@ +## libcontainer - reference implementation for containers + +#### playground + + +Use the cli package to test out functionality + +First setup a container configuration. You will need a root fs, better go the path to a +stopped docker container and use that. + + +```json +{ + "id": "koye", + "namespace_pid": 12265, + "command": { + "args": [ + "/bin/bash" + ], + "environment": [ + "HOME=/", + "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", + "container=docker", + "TERM=xterm" + ] + }, + "rootfs": "/root/development/gocode/src/github.com/docker/libcontainer/namespaces/ubuntu", + "network": null, + "user": "", + "working_dir": "", + "namespaces": [ + "NEWNET", + "NEWIPC", + "NEWNS", + "NEWPID", + "NEWUTS" + ], + "capabilities": [ + "SETPCAP", + "SYS_MODULE", + "SYS_RAWIO", + "SYS_PACCT", + "SYS_ADMIN", + "SYS_NICE", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "MKNOD", + "AUDIT_WRITE", + "AUDIT_CONTROL", + "MAC_OVERRIDE", + "MAC_ADMIN" + ] +} +``` + +After you have a json file and a rootfs path to use just run: +`./cli exec container.json` + + +If you want to attach to an existing namespace just use the same json +file with the container still running and do: +`./cli execin container.json` diff --git a/components/engine/pkg/libcontainer/capabilities/capabilities.go b/components/engine/pkg/libcontainer/capabilities/capabilities.go new file mode 100644 index 0000000000..3301e10f7f --- /dev/null +++ b/components/engine/pkg/libcontainer/capabilities/capabilities.go @@ -0,0 +1,49 @@ +package capabilities + +import ( + "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/syndtr/gocapability/capability" + "os" +) + +var capMap = map[libcontainer.Capability]capability.Cap{ + libcontainer.CAP_SETPCAP: capability.CAP_SETPCAP, + libcontainer.CAP_SYS_MODULE: capability.CAP_SYS_MODULE, + libcontainer.CAP_SYS_RAWIO: capability.CAP_SYS_RAWIO, + libcontainer.CAP_SYS_PACCT: capability.CAP_SYS_PACCT, + libcontainer.CAP_SYS_ADMIN: capability.CAP_SYS_ADMIN, + libcontainer.CAP_SYS_NICE: capability.CAP_SYS_NICE, + libcontainer.CAP_SYS_RESOURCE: capability.CAP_SYS_RESOURCE, + libcontainer.CAP_SYS_TIME: capability.CAP_SYS_TIME, + libcontainer.CAP_SYS_TTY_CONFIG: capability.CAP_SYS_TTY_CONFIG, + libcontainer.CAP_MKNOD: capability.CAP_MKNOD, + libcontainer.CAP_AUDIT_WRITE: capability.CAP_AUDIT_WRITE, + libcontainer.CAP_AUDIT_CONTROL: capability.CAP_AUDIT_CONTROL, + libcontainer.CAP_MAC_OVERRIDE: capability.CAP_MAC_OVERRIDE, + libcontainer.CAP_MAC_ADMIN: capability.CAP_MAC_ADMIN, +} + +// DropCapabilities drops capabilities for the current process based +// on the container's configuration. +func DropCapabilities(container *libcontainer.Container) error { + if drop := getCapabilities(container); len(drop) > 0 { + c, err := capability.NewPid(os.Getpid()) + if err != nil { + return err + } + c.Unset(capability.CAPS|capability.BOUNDS, drop...) + + if err := c.Apply(capability.CAPS | capability.BOUNDS); err != nil { + return err + } + } + return nil +} + +func getCapabilities(container *libcontainer.Container) []capability.Cap { + drop := []capability.Cap{} + for _, c := range container.Capabilities { + drop = append(drop, capMap[c]) + } + return drop +} diff --git a/components/engine/pkg/libcontainer/cli/main.go b/components/engine/pkg/libcontainer/cli/main.go new file mode 100644 index 0000000000..490135ef5a --- /dev/null +++ b/components/engine/pkg/libcontainer/cli/main.go @@ -0,0 +1,171 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/libcontainer/namespaces" + "github.com/dotcloud/docker/pkg/libcontainer/network" + "github.com/dotcloud/docker/pkg/libcontainer/utils" + "os" +) + +var ( + displayPid bool + newCommand string + usrNet bool +) + +func init() { + flag.BoolVar(&displayPid, "pid", false, "display the pid before waiting") + flag.StringVar(&newCommand, "cmd", "/bin/bash", "command to run in the existing namespace") + flag.BoolVar(&usrNet, "net", false, "user a net namespace") + flag.Parse() +} + +func exec(container *libcontainer.Container) error { + var ( + netFile *os.File + err error + ) + container.NetNsFd = 0 + + if usrNet { + netFile, err = os.Open("/root/nsroot/test") + if err != nil { + return err + } + container.NetNsFd = netFile.Fd() + } + + pid, err := namespaces.Exec(container) + if err != nil { + return fmt.Errorf("error exec container %s", err) + } + + if displayPid { + fmt.Println(pid) + } + + exitcode, err := utils.WaitOnPid(pid) + if err != nil { + return fmt.Errorf("error waiting on child %s", err) + } + fmt.Println(exitcode) + if usrNet { + netFile.Close() + if err := network.DeleteNetworkNamespace("/root/nsroot/test"); err != nil { + return err + } + } + os.Exit(exitcode) + return nil +} + +func execIn(container *libcontainer.Container) error { + f, err := os.Open("/root/nsroot/test") + if err != nil { + return err + } + container.NetNsFd = f.Fd() + pid, err := namespaces.ExecIn(container, &libcontainer.Command{ + Env: container.Command.Env, + Args: []string{ + newCommand, + }, + }) + if err != nil { + return fmt.Errorf("error exexin container %s", err) + } + exitcode, err := utils.WaitOnPid(pid) + if err != nil { + return fmt.Errorf("error waiting on child %s", err) + } + os.Exit(exitcode) + return nil +} + +func createNet(config *libcontainer.Network) error { + root := "/root/nsroot" + if err := network.SetupNamespaceMountDir(root); err != nil { + return err + } + + nspath := root + "/test" + if err := network.CreateNetworkNamespace(nspath); err != nil { + return nil + } + if err := network.CreateVethPair("veth0", config.TempVethName); err != nil { + return err + } + if err := network.SetInterfaceMaster("veth0", config.Bridge); err != nil { + return err + } + if err := network.InterfaceUp("veth0"); err != nil { + return err + } + + f, err := os.Open(nspath) + if err != nil { + return err + } + defer f.Close() + + if err := network.SetInterfaceInNamespaceFd("veth1", int(f.Fd())); err != nil { + return err + } + + /* + if err := network.SetupVethInsideNamespace(f.Fd(), config); err != nil { + return err + } + */ + return nil +} + +func printErr(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} + +func main() { + var ( + err error + cliCmd = flag.Arg(0) + config = flag.Arg(1) + ) + f, err := os.Open(config) + if err != nil { + printErr(err) + } + + dec := json.NewDecoder(f) + var container *libcontainer.Container + + if err := dec.Decode(&container); err != nil { + printErr(err) + } + f.Close() + + switch cliCmd { + case "exec": + err = exec(container) + case "execin": + err = execIn(container) + case "net": + err = createNet(&libcontainer.Network{ + TempVethName: "veth1", + IP: "172.17.0.100/16", + Gateway: "172.17.42.1", + Mtu: 1500, + Bridge: "docker0", + }) + default: + err = fmt.Errorf("command not supported: %s", cliCmd) + } + + if err != nil { + printErr(err) + } +} diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go new file mode 100644 index 0000000000..b77890fb5c --- /dev/null +++ b/components/engine/pkg/libcontainer/container.go @@ -0,0 +1,27 @@ +package libcontainer + +type Container struct { + ID string `json:"id,omitempty"` + NsPid int `json:"namespace_pid,omitempty"` + Command *Command `json:"command,omitempty"` + RootFs string `json:"rootfs,omitempty"` + ReadonlyFs bool `json:"readonly_fs,omitempty"` + NetNsFd uintptr `json:"network_namespace_fd,omitempty"` + User string `json:"user,omitempty"` + WorkingDir string `json:"working_dir,omitempty"` + Namespaces Namespaces `json:"namespaces,omitempty"` + Capabilities Capabilities `json:"capabilities,omitempty"` +} + +type Command struct { + Args []string `json:"args,omitempty"` + Env []string `json:"environment,omitempty"` +} + +type Network struct { + TempVethName string `json:"temp_veth,omitempty"` + IP string `json:"ip,omitempty"` + Gateway string `json:"gateway,omitempty"` + Bridge string `json:"bridge,omitempty"` + Mtu int `json:"mtu,omitempty"` +} diff --git a/components/engine/pkg/libcontainer/container.json b/components/engine/pkg/libcontainer/container.json new file mode 100644 index 0000000000..ed8eb1bd78 --- /dev/null +++ b/components/engine/pkg/libcontainer/container.json @@ -0,0 +1,38 @@ +{ + "id": "koye", + "namespace_pid": 3117, + "command": { + "args": [ + "/bin/bash" + ], + "environment": [ + "HOME=/", + "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", + "container=docker", + "TERM=xterm" + ] + }, + "rootfs": "/root/main/mycontainer", + "namespaces": [ + "NEWIPC", + "NEWNS", + "NEWPID", + "NEWUTS" + ], + "capabilities": [ + "SETPCAP", + "SYS_MODULE", + "SYS_RAWIO", + "SYS_PACCT", + "SYS_ADMIN", + "SYS_NICE", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "MKNOD", + "AUDIT_WRITE", + "AUDIT_CONTROL", + "MAC_OVERRIDE", + "MAC_ADMIN" + ] +} diff --git a/components/engine/pkg/libcontainer/errors.go b/components/engine/pkg/libcontainer/errors.go new file mode 100644 index 0000000000..c6964ee8e6 --- /dev/null +++ b/components/engine/pkg/libcontainer/errors.go @@ -0,0 +1,9 @@ +package libcontainer + +import ( + "errors" +) + +var ( + ErrInvalidPid = errors.New("no ns pid found") +) diff --git a/components/engine/pkg/libcontainer/namespaces/calls_linux.go b/components/engine/pkg/libcontainer/namespaces/calls_linux.go new file mode 100644 index 0000000000..793e940b6e --- /dev/null +++ b/components/engine/pkg/libcontainer/namespaces/calls_linux.go @@ -0,0 +1,164 @@ +package namespaces + +import ( + "fmt" + "os" + "syscall" + "unsafe" +) + +const ( + TIOCGPTN = 0x80045430 + TIOCSPTLCK = 0x40045431 +) + +func chroot(dir string) error { + return syscall.Chroot(dir) +} + +func chdir(dir string) error { + return syscall.Chdir(dir) +} + +func exec(cmd string, args []string, env []string) error { + return syscall.Exec(cmd, args, env) +} + +func fork() (int, error) { + syscall.ForkLock.Lock() + pid, _, err := syscall.Syscall(syscall.SYS_FORK, 0, 0, 0) + syscall.ForkLock.Unlock() + if err != 0 { + return -1, err + } + return int(pid), nil +} + +func vfork() (int, error) { + syscall.ForkLock.Lock() + pid, _, err := syscall.Syscall(syscall.SYS_VFORK, 0, 0, 0) + syscall.ForkLock.Unlock() + if err != 0 { + return -1, err + } + return int(pid), nil +} + +func mount(source, target, fstype string, flags uintptr, data string) error { + return syscall.Mount(source, target, fstype, flags, data) +} + +func unmount(target string, flags int) error { + return syscall.Unmount(target, flags) +} + +func pivotroot(newroot, putold string) error { + return syscall.PivotRoot(newroot, putold) +} + +func unshare(flags int) error { + return syscall.Unshare(flags) +} + +func clone(flags uintptr) (int, error) { + syscall.ForkLock.Lock() + pid, _, err := syscall.RawSyscall(syscall.SYS_CLONE, flags, 0, 0) + syscall.ForkLock.Unlock() + if err != 0 { + return -1, err + } + return int(pid), nil +} + +func setns(fd uintptr, flags uintptr) error { + _, _, err := syscall.RawSyscall(SYS_SETNS, fd, flags, 0) + if err != 0 { + return err + } + return nil +} + +func usetCloseOnExec(fd uintptr) error { + if _, _, err := syscall.Syscall(syscall.SYS_FCNTL, fd, syscall.F_SETFD, 0); err != 0 { + return err + } + return nil +} + +func setgroups(gids []int) error { + return syscall.Setgroups(gids) +} + +func setresgid(rgid, egid, sgid int) error { + return syscall.Setresgid(rgid, egid, sgid) +} + +func setresuid(ruid, euid, suid int) error { + return syscall.Setresuid(ruid, euid, suid) +} + +func sethostname(name string) error { + return syscall.Sethostname([]byte(name)) +} + +func setsid() (int, error) { + return syscall.Setsid() +} + +func ioctl(fd uintptr, flag, data uintptr) error { + if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, flag, data); err != 0 { + return err + } + return nil +} + +func openpmtx() (*os.File, error) { + return os.OpenFile("/dev/ptmx", syscall.O_RDONLY|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) +} + +func unlockpt(f *os.File) error { + var u int + return ioctl(f.Fd(), TIOCSPTLCK, uintptr(unsafe.Pointer(&u))) +} + +func ptsname(f *os.File) (string, error) { + var n int + if err := ioctl(f.Fd(), TIOCGPTN, uintptr(unsafe.Pointer(&n))); err != nil { + return "", err + } + return fmt.Sprintf("/dev/pts/%d", n), nil +} + +func closefd(fd uintptr) error { + return syscall.Close(int(fd)) +} + +func dup2(fd1, fd2 uintptr) error { + return syscall.Dup2(int(fd1), int(fd2)) +} + +func mknod(path string, mode uint32, dev int) error { + return syscall.Mknod(path, mode, dev) +} + +func parentDeathSignal() error { + if _, _, err := syscall.RawSyscall6(syscall.SYS_PRCTL, syscall.PR_SET_PDEATHSIG, uintptr(syscall.SIGKILL), 0, 0, 0, 0); err != 0 { + return err + } + return nil +} + +func setctty() error { + if _, _, err := syscall.RawSyscall(syscall.SYS_IOCTL, 0, uintptr(syscall.TIOCSCTTY), 0); err != 0 { + return err + } + return nil +} + +func mkfifo(name string, mode uint32) error { + return syscall.Mkfifo(name, mode) +} + +func umask(mask int) int { + return syscall.Umask(mask) +} diff --git a/components/engine/pkg/libcontainer/namespaces/exec.go b/components/engine/pkg/libcontainer/namespaces/exec.go new file mode 100644 index 0000000000..893b302887 --- /dev/null +++ b/components/engine/pkg/libcontainer/namespaces/exec.go @@ -0,0 +1,266 @@ +/* + Higher level convience functions for setting up a container +*/ + +package namespaces + +import ( + "errors" + "fmt" + "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/libcontainer/capabilities" + "github.com/dotcloud/docker/pkg/libcontainer/utils" + "io" + "log" + "os" + "path/filepath" + "syscall" +) + +var ( + ErrExistingNetworkNamespace = errors.New("specified both CLONE_NEWNET and an existing network namespace") +) + +// Exec will spawn new namespaces with the specified Container configuration +// in the RootFs path and return the pid of the new containerized process. +// +// If an existing network namespace is specified the container +// will join that namespace. If an existing network namespace is not specified but CLONE_NEWNET is, +// the container will be spawned with a new network namespace with no configuration. Omiting an +// existing network namespace and the CLONE_NEWNET option in the container configuration will allow +// the container to the the host's networking options and configuration. +func Exec(container *libcontainer.Container) (pid int, err error) { + // a user cannot pass CLONE_NEWNET and an existing net namespace fd to join + if container.NetNsFd > 0 && container.Namespaces.Contains(libcontainer.CLONE_NEWNET) { + return -1, ErrExistingNetworkNamespace + } + + rootfs, err := resolveRootfs(container) + if err != nil { + return -1, err + } + + master, console, err := createMasterAndConsole() + if err != nil { + return -1, err + } + + logger, err := os.OpenFile("/root/logs", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0755) + if err != nil { + return -1, err + } + log.SetOutput(logger) + + // we need CLONE_VFORK so we can wait on the child + flag := getNamespaceFlags(container.Namespaces) | CLONE_VFORK + + if pid, err = clone(uintptr(flag | SIGCHLD)); err != nil { + return -1, fmt.Errorf("error cloning process: %s", err) + } + + if pid == 0 { + // welcome to your new namespace ;) + // + // any errors encoutered inside the namespace we should write + // out to a log or a pipe to our parent and exit(1) + // because writing to stderr will not work after we close + if err := closeMasterAndStd(master); err != nil { + writeError("close master and std %s", err) + } + slave, err := openTerminal(console, syscall.O_RDWR) + if err != nil { + writeError("open terminal %s", err) + } + if err := dupSlave(slave); err != nil { + writeError("dup2 slave %s", err) + } + + if container.NetNsFd > 0 { + if err := JoinExistingNamespace(container.NetNsFd, libcontainer.CLONE_NEWNET); err != nil { + writeError("join existing net namespace %s", err) + } + } + + if _, err := setsid(); err != nil { + writeError("setsid %s", err) + } + if err := setctty(); err != nil { + writeError("setctty %s", err) + } + if err := parentDeathSignal(); err != nil { + writeError("parent deth signal %s", err) + } + if err := SetupNewMountNamespace(rootfs, console, container.ReadonlyFs); err != nil { + writeError("setup mount namespace %s", err) + } + if err := sethostname(container.ID); err != nil { + writeError("sethostname %s", err) + } + if err := capabilities.DropCapabilities(container); err != nil { + writeError("drop capabilities %s", err) + } + if err := setupUser(container); err != nil { + writeError("setup user %s", err) + } + if container.WorkingDir != "" { + if err := chdir(container.WorkingDir); err != nil { + writeError("chdir to %s %s", container.WorkingDir, err) + } + } + if err := exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { + writeError("exec %s", err) + } + panic("unreachable") + } + + go func() { + if _, err := io.Copy(os.Stdout, master); err != nil { + log.Println(err) + } + }() + go func() { + if _, err := io.Copy(master, os.Stdin); err != nil { + log.Println(err) + } + }() + return pid, nil +} + +// ExecIn will spawn a new command inside an existing container's namespaces. The existing container's +// pid and namespace configuration is needed along with the specific capabilities that should +// be dropped once inside the namespace. +func ExecIn(container *libcontainer.Container, cmd *libcontainer.Command) (int, error) { + if container.NsPid <= 0 { + return -1, libcontainer.ErrInvalidPid + } + + fds, err := getNsFds(container) + if err != nil { + return -1, err + } + + if container.NetNsFd > 0 { + fds = append(fds, container.NetNsFd) + } + + pid, err := fork() + if err != nil { + for _, fd := range fds { + syscall.Close(int(fd)) + } + return -1, err + } + + if pid == 0 { + for _, fd := range fds { + if fd > 0 { + if err := JoinExistingNamespace(fd, ""); err != nil { + for _, fd := range fds { + syscall.Close(int(fd)) + } + writeError("join existing namespace for %d %s", fd, err) + } + } + syscall.Close(int(fd)) + } + + if container.Namespaces.Contains(libcontainer.CLONE_NEWNS) && + container.Namespaces.Contains(libcontainer.CLONE_NEWPID) { + // important: + // + // we need to fork and unshare so that re can remount proc and sys within + // the namespace so the CLONE_NEWPID namespace will take effect + // if we don't fork we would end up unmounting proc and sys for the entire + // namespace + child, err := fork() + if err != nil { + writeError("fork child %s", err) + } + + if child == 0 { + if err := unshare(CLONE_NEWNS); err != nil { + writeError("unshare newns %s", err) + } + if err := remountProc(); err != nil { + writeError("remount proc %s", err) + } + if err := remountSys(); err != nil { + writeError("remount sys %s", err) + } + if err := capabilities.DropCapabilities(container); err != nil { + writeError("drop caps %s", err) + } + if err := exec(cmd.Args[0], cmd.Args[0:], cmd.Env); err != nil { + writeError("exec %s", err) + } + panic("unreachable") + } + exit, err := utils.WaitOnPid(child) + if err != nil { + writeError("wait on child %s", err) + } + os.Exit(exit) + } + if err := exec(cmd.Args[0], cmd.Args[0:], cmd.Env); err != nil { + writeError("exec %s", err) + } + panic("unreachable") + } + return pid, err +} + +func resolveRootfs(container *libcontainer.Container) (string, error) { + rootfs, err := filepath.Abs(container.RootFs) + if err != nil { + return "", err + } + return filepath.EvalSymlinks(rootfs) +} + +func createMasterAndConsole() (*os.File, string, error) { + master, err := os.OpenFile("/dev/ptmx", syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) + if err != nil { + return nil, "", err + } + + console, err := ptsname(master) + if err != nil { + return nil, "", err + } + + if err := unlockpt(master); err != nil { + return nil, "", err + } + return master, console, nil +} + +func closeMasterAndStd(master *os.File) error { + closefd(master.Fd()) + closefd(0) + closefd(1) + closefd(2) + + return nil +} + +func dupSlave(slave *os.File) error { + // we close Stdin,etc so our pty slave should have fd 0 + if slave.Fd() != 0 { + return fmt.Errorf("slave fd not 0 %d", slave.Fd()) + } + if err := dup2(slave.Fd(), 1); err != nil { + return err + } + if err := dup2(slave.Fd(), 2); err != nil { + return err + } + return nil +} + +func openTerminal(name string, flag int) (*os.File, error) { + r, e := syscall.Open(name, flag, 0) + if e != nil { + return nil, &os.PathError{"open", name, e} + } + return os.NewFile(uintptr(r), name), nil +} diff --git a/components/engine/pkg/libcontainer/namespaces/linux_x86_64.go b/components/engine/pkg/libcontainer/namespaces/linux_x86_64.go new file mode 100644 index 0000000000..ac9a014763 --- /dev/null +++ b/components/engine/pkg/libcontainer/namespaces/linux_x86_64.go @@ -0,0 +1,7 @@ +// +build linux,x86_64 +package namespaces + +// Via http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=7b21fddd087678a70ad64afc0f632e0f1071b092 +const ( + SYS_SETNS = 308 +) diff --git a/components/engine/pkg/libcontainer/namespaces/mount.go b/components/engine/pkg/libcontainer/namespaces/mount.go new file mode 100644 index 0000000000..6d867c91ec --- /dev/null +++ b/components/engine/pkg/libcontainer/namespaces/mount.go @@ -0,0 +1,207 @@ +package namespaces + +import ( + "fmt" + "log" + "os" + "path/filepath" + "syscall" +) + +var ( + // default mount point options + defaults = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV +) + +func SetupNewMountNamespace(rootfs, console string, readonly bool) error { + if err := mount("", "/", "", syscall.MS_SLAVE|syscall.MS_REC, ""); err != nil { + return fmt.Errorf("mounting / as slave %s", err) + } + + if err := mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil { + return fmt.Errorf("mouting %s as bind %s", rootfs, err) + } + + if readonly { + if err := mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_REC, ""); err != nil { + return fmt.Errorf("mounting %s as readonly %s", rootfs, err) + } + } + + if err := mountSystem(rootfs); err != nil { + return fmt.Errorf("mount system %s", err) + } + + if err := copyDevNodes(rootfs); err != nil { + return fmt.Errorf("copy dev nodes %s", err) + } + + ptmx := filepath.Join(rootfs, "dev/ptmx") + if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) { + return err + } + if err := os.Symlink(filepath.Join(rootfs, "pts/ptmx"), ptmx); err != nil { + return fmt.Errorf("symlink dev ptmx %s", err) + } + + if err := setupDev(rootfs); err != nil { + return err + } + + if err := setupConsole(rootfs, console); err != nil { + return err + } + + if err := chdir(rootfs); err != nil { + return fmt.Errorf("chdir into %s %s", rootfs, err) + } + + if err := mount(rootfs, "/", "", syscall.MS_MOVE, ""); err != nil { + return fmt.Errorf("mount move %s into / %s", rootfs, err) + } + + if err := chroot("."); err != nil { + return fmt.Errorf("chroot . %s", err) + } + + if err := chdir("/"); err != nil { + return fmt.Errorf("chdir / %s", err) + } + + umask(0022) + + return nil +} + +func copyDevNodes(rootfs string) error { + umask(0000) + + for _, node := range []string{ + "null", + "zero", + "full", + "random", + "urandom", + "tty", + } { + stat, err := os.Stat(filepath.Join("/dev", node)) + if err != nil { + return err + } + + var ( + dest = filepath.Join(rootfs, "dev", node) + st = stat.Sys().(*syscall.Stat_t) + ) + + log.Printf("copy %s to %s %d\n", node, dest, st.Rdev) + if err := mknod(dest, st.Mode, int(st.Rdev)); err != nil && !os.IsExist(err) { + return fmt.Errorf("copy %s %s", node, err) + } + } + return nil +} + +func setupDev(rootfs string) error { + for _, link := range []struct { + from string + to string + }{ + {"/proc/kcore", "/dev/core"}, + {"/proc/self/fd", "/dev/fd"}, + {"/proc/self/fd/0", "/dev/stdin"}, + {"/proc/self/fd/1", "/dev/stdout"}, + {"/proc/self/fd/2", "/dev/stderr"}, + } { + dest := filepath.Join(rootfs, link.to) + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove %s %s", dest, err) + } + if err := os.Symlink(link.from, dest); err != nil { + return fmt.Errorf("symlink %s %s", dest, err) + } + } + return nil +} + +func setupConsole(rootfs, console string) error { + umask(0000) + + stat, err := os.Stat(console) + if err != nil { + return fmt.Errorf("stat console %s %s", console, err) + } + st := stat.Sys().(*syscall.Stat_t) + + dest := filepath.Join(rootfs, "dev/console") + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove %s %s", dest, err) + } + + if err := os.Chmod(console, 0600); err != nil { + return err + } + if err := os.Chown(console, 0, 0); err != nil { + return err + } + + if err := mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil { + return fmt.Errorf("mknod %s %s", dest, err) + } + + if err := mount(console, dest, "bind", syscall.MS_BIND, ""); err != nil { + return fmt.Errorf("bind %s to %s %s", console, dest, err) + } + return nil +} + +// mountSystem sets up linux specific system mounts like sys, proc, shm, and devpts +// inside the mount namespace +func mountSystem(rootfs string) error { + mounts := []struct { + source string + path string + device string + flags int + data string + }{ + {source: "proc", path: filepath.Join(rootfs, "proc"), device: "proc", flags: defaults}, + {source: "sysfs", path: filepath.Join(rootfs, "sys"), device: "sysfs", flags: defaults}, + {source: "tmpfs", path: filepath.Join(rootfs, "dev"), device: "tmpfs", flags: syscall.MS_NOSUID | syscall.MS_STRICTATIME, data: "mode=755"}, + {source: "shm", path: filepath.Join(rootfs, "dev", "shm"), device: "tmpfs", flags: defaults, data: "mode=1777"}, + {source: "devpts", path: filepath.Join(rootfs, "dev", "pts"), device: "devpts", flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, data: "newinstance,ptmxmode=0666,mode=620,gid=5"}, + {source: "tmpfs", path: filepath.Join(rootfs, "run"), device: "tmpfs", flags: syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_STRICTATIME, data: "mode=755"}, + } + for _, m := range mounts { + if err := os.MkdirAll(m.path, 0755); err != nil && !os.IsExist(err) { + return fmt.Errorf("mkdirall %s %s", m.path, err) + } + if err := mount(m.source, m.path, m.device, uintptr(m.flags), m.data); err != nil { + return fmt.Errorf("mounting %s into %s %s", m.source, m.path, err) + } + } + return nil +} + +func remountProc() error { + if err := unmount("/proc", syscall.MNT_DETACH); err != nil { + return err + } + if err := mount("proc", "/proc", "proc", uintptr(defaults), ""); err != nil { + return err + } + return nil +} + +func remountSys() error { + if err := unmount("/sys", syscall.MNT_DETACH); err != nil { + if err != syscall.EINVAL { + return err + } + } else { + if err := mount("sysfs", "/sys", "sysfs", uintptr(defaults), ""); err != nil { + return err + } + } + return nil +} diff --git a/components/engine/pkg/libcontainer/namespaces/namespaces.go b/components/engine/pkg/libcontainer/namespaces/namespaces.go new file mode 100644 index 0000000000..2a50847015 --- /dev/null +++ b/components/engine/pkg/libcontainer/namespaces/namespaces.go @@ -0,0 +1,70 @@ +/* + TODO + pivot root + cgroups + more mount stuff that I probably am forgetting + apparmor +*/ + +package namespaces + +import ( + "fmt" + "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/libcontainer/utils" + "os" + "path/filepath" + "syscall" +) + +// CreateNewNamespace creates a new namespace and binds it's fd to the specified path +func CreateNewNamespace(namespace libcontainer.Namespace, bindTo string) error { + var ( + flag = namespaceMap[namespace] + name = namespaceFileMap[namespace] + nspath = filepath.Join("/proc/self/ns", name) + ) + // TODO: perform validation on name and flag + + pid, err := fork() + if err != nil { + return err + } + + if pid == 0 { + if err := unshare(flag); err != nil { + writeError("unshare %s", err) + } + if err := mount(nspath, bindTo, "none", syscall.MS_BIND, ""); err != nil { + writeError("bind mount %s", err) + } + os.Exit(0) + } + exit, err := utils.WaitOnPid(pid) + if err != nil { + return err + } + if exit != 0 { + return fmt.Errorf("exit status %d", exit) + } + return err +} + +// JoinExistingNamespace uses the fd of an existing linux namespace and +// has the current process join that namespace or the spacespace specified by ns +func JoinExistingNamespace(fd uintptr, ns libcontainer.Namespace) error { + flag := namespaceMap[ns] + if err := setns(fd, uintptr(flag)); err != nil { + return err + } + return nil +} + +// getNamespaceFlags parses the container's Namespaces options to set the correct +// flags on clone, unshare, and setns +func getNamespaceFlags(namespaces libcontainer.Namespaces) (flag int) { + for _, ns := range namespaces { + flag |= namespaceMap[ns] + } + return +} diff --git a/components/engine/pkg/libcontainer/namespaces/ns_linux.go b/components/engine/pkg/libcontainer/namespaces/ns_linux.go new file mode 100644 index 0000000000..b0e5119130 --- /dev/null +++ b/components/engine/pkg/libcontainer/namespaces/ns_linux.go @@ -0,0 +1,35 @@ +package namespaces + +import ( + "github.com/dotcloud/docker/pkg/libcontainer" +) + +const ( + SIGCHLD = 0x14 + CLONE_VFORK = 0x00004000 + CLONE_NEWNS = 0x00020000 + CLONE_NEWUTS = 0x04000000 + CLONE_NEWIPC = 0x08000000 + CLONE_NEWUSER = 0x10000000 + CLONE_NEWPID = 0x20000000 + CLONE_NEWNET = 0x40000000 +) + +var namespaceMap = map[libcontainer.Namespace]int{ + "": 0, + libcontainer.CLONE_NEWNS: CLONE_NEWNS, + libcontainer.CLONE_NEWUTS: CLONE_NEWUTS, + libcontainer.CLONE_NEWIPC: CLONE_NEWIPC, + libcontainer.CLONE_NEWUSER: CLONE_NEWUSER, + libcontainer.CLONE_NEWPID: CLONE_NEWPID, + libcontainer.CLONE_NEWNET: CLONE_NEWNET, +} + +var namespaceFileMap = map[libcontainer.Namespace]string{ + libcontainer.CLONE_NEWNS: "mnt", + libcontainer.CLONE_NEWUTS: "uts", + libcontainer.CLONE_NEWIPC: "ipc", + libcontainer.CLONE_NEWUSER: "user", + libcontainer.CLONE_NEWPID: "pid", + libcontainer.CLONE_NEWNET: "net", +} diff --git a/components/engine/pkg/libcontainer/namespaces/utils.go b/components/engine/pkg/libcontainer/namespaces/utils.go new file mode 100644 index 0000000000..438d896484 --- /dev/null +++ b/components/engine/pkg/libcontainer/namespaces/utils.go @@ -0,0 +1,108 @@ +package namespaces + +import ( + "fmt" + "github.com/dotcloud/docker/pkg/libcontainer" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" +) + +func addEnvIfNotSet(container *libcontainer.Container, key, value string) { + jv := fmt.Sprintf("%s=%s", key, value) + if len(container.Command.Env) == 0 { + container.Command.Env = []string{jv} + return + } + + for _, v := range container.Command.Env { + parts := strings.Split(v, "=") + if parts[0] == key { + return + } + } + container.Command.Env = append(container.Command.Env, jv) +} + +// print and error to stderr and exit(1) +func writeError(format string, v ...interface{}) { + fmt.Fprintf(os.Stderr, format, v...) + os.Exit(1) +} + +// getNsFds inspects the container's namespace configuration and opens the fds to +// each of the namespaces. +func getNsFds(container *libcontainer.Container) ([]uintptr, error) { + var ( + namespaces = []string{} + fds = []uintptr{} + ) + + for _, ns := range container.Namespaces { + namespaces = append(namespaces, namespaceFileMap[ns]) + } + + for _, ns := range namespaces { + fd, err := getNsFd(container.NsPid, ns) + if err != nil { + for _, fd = range fds { + syscall.Close(int(fd)) + } + return nil, err + } + fds = append(fds, fd) + } + return fds, nil +} + +// getNsFd returns the fd for a specific pid and namespace option +func getNsFd(pid int, ns string) (uintptr, error) { + nspath := filepath.Join("/proc", strconv.Itoa(pid), "ns", ns) + // OpenFile adds closOnExec + f, err := os.OpenFile(nspath, os.O_RDONLY, 0666) + if err != nil { + return 0, err + } + return f.Fd(), nil +} + +// setupEnvironment adds additional environment variables to the container's +// Command such as USER, LOGNAME, container, and TERM +func setupEnvironment(container *libcontainer.Container) { + addEnvIfNotSet(container, "container", "docker") + // TODO: check if pty + addEnvIfNotSet(container, "TERM", "xterm") + // TODO: get username from container + addEnvIfNotSet(container, "USER", "root") + addEnvIfNotSet(container, "LOGNAME", "root") +} + +func setupUser(container *libcontainer.Container) error { + // TODO: honor user passed on container + if err := setgroups(nil); err != nil { + return err + } + if err := setresgid(0, 0, 0); err != nil { + return err + } + if err := setresuid(0, 0, 0); err != nil { + return err + } + return nil +} + +func getMasterAndConsole(container *libcontainer.Container) (string, *os.File, error) { + master, err := openpmtx() + if err != nil { + return "", nil, err + } + + console, err := ptsname(master) + if err != nil { + master.Close() + return "", nil, err + } + return console, master, nil +} diff --git a/components/engine/pkg/libcontainer/network/network.go b/components/engine/pkg/libcontainer/network/network.go new file mode 100644 index 0000000000..31c5d32492 --- /dev/null +++ b/components/engine/pkg/libcontainer/network/network.go @@ -0,0 +1,104 @@ +package network + +import ( + "errors" + "github.com/dotcloud/docker/pkg/netlink" + "net" +) + +var ( + ErrNoDefaultRoute = errors.New("no default network route found") +) + +func InterfaceUp(name string) error { + iface, err := net.InterfaceByName(name) + if err != nil { + return err + } + return netlink.NetworkLinkUp(iface) +} + +func InterfaceDown(name string) error { + iface, err := net.InterfaceByName(name) + if err != nil { + return err + } + return netlink.NetworkLinkDown(iface) +} + +func ChangeInterfaceName(old, newName string) error { + iface, err := net.InterfaceByName(old) + if err != nil { + return err + } + return netlink.NetworkChangeName(iface, newName) +} + +func CreateVethPair(name1, name2 string) error { + return netlink.NetworkCreateVethPair(name1, name2) +} + +func SetInterfaceInNamespacePid(name string, nsPid int) error { + iface, err := net.InterfaceByName(name) + if err != nil { + return err + } + return netlink.NetworkSetNsPid(iface, nsPid) +} + +func SetInterfaceInNamespaceFd(name string, fd int) error { + iface, err := net.InterfaceByName(name) + if err != nil { + return err + } + return netlink.NetworkSetNsFd(iface, fd) +} + +func SetInterfaceMaster(name, master string) error { + iface, err := net.InterfaceByName(name) + if err != nil { + return err + } + masterIface, err := net.InterfaceByName(master) + if err != nil { + return err + } + return netlink.NetworkSetMaster(iface, masterIface) +} + +func SetDefaultGateway(ip string) error { + return netlink.AddDefaultGw(net.ParseIP(ip)) +} + +func SetInterfaceIp(name string, rawIp string) error { + iface, err := net.InterfaceByName(name) + if err != nil { + return err + } + ip, ipNet, err := net.ParseCIDR(rawIp) + if err != nil { + return err + } + return netlink.NetworkLinkAddIp(iface, ip, ipNet) +} + +func SetMtu(name string, mtu int) error { + iface, err := net.InterfaceByName(name) + if err != nil { + return err + } + return netlink.NetworkSetMTU(iface, mtu) +} + +func GetDefaultMtu() (int, error) { + routes, err := netlink.NetworkGetRoutes() + if err != nil { + return -1, err + } + for _, r := range routes { + if r.Default { + return r.Iface.MTU, nil + } + } + return -1, ErrNoDefaultRoute +} diff --git a/components/engine/pkg/libcontainer/network/veth.go b/components/engine/pkg/libcontainer/network/veth.go new file mode 100644 index 0000000000..dc207b3394 --- /dev/null +++ b/components/engine/pkg/libcontainer/network/veth.go @@ -0,0 +1,85 @@ +package network + +import ( + "fmt" + "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/libcontainer/namespaces" + "os" + "syscall" +) + +// SetupVeth sets up an existing network namespace with the specified +// network configuration. +func SetupVeth(config *libcontainer.Network) error { + if err := InterfaceDown(config.TempVethName); err != nil { + return fmt.Errorf("interface down %s %s", config.TempVethName, err) + } + if err := ChangeInterfaceName(config.TempVethName, "eth0"); err != nil { + return fmt.Errorf("change %s to eth0 %s", config.TempVethName, err) + } + if err := SetInterfaceIp("eth0", config.IP); err != nil { + return fmt.Errorf("set eth0 ip %s", err) + } + + if err := SetMtu("eth0", config.Mtu); err != nil { + return fmt.Errorf("set eth0 mtu to %d %s", config.Mtu, err) + } + if err := InterfaceUp("eth0"); err != nil { + return fmt.Errorf("eth0 up %s", err) + } + + if err := SetMtu("lo", config.Mtu); err != nil { + return fmt.Errorf("set lo mtu to %d %s", config.Mtu, err) + } + if err := InterfaceUp("lo"); err != nil { + return fmt.Errorf("lo up %s", err) + } + + if config.Gateway != "" { + if err := SetDefaultGateway(config.Gateway); err != nil { + return fmt.Errorf("set gateway to %s %s", config.Gateway, err) + } + } + return nil +} + +// SetupNamespaceMountDir prepares a new root for use as a mount +// source for bind mounting namespace fd to an outside path +func SetupNamespaceMountDir(root string) error { + if err := os.MkdirAll(root, 0666); err != nil { + return err + } + // make sure mounts are not unmounted by other mnt namespaces + if err := syscall.Mount("", root, "none", syscall.MS_SHARED|syscall.MS_REC, ""); err != nil && err != syscall.EINVAL { + return err + } + if err := syscall.Mount(root, root, "none", syscall.MS_BIND, ""); err != nil { + return err + } + return nil +} + +// CreateNetworkNamespace creates a new network namespace and binds it's fd +// at the binding path +func CreateNetworkNamespace(bindingPath string) error { + f, err := os.OpenFile(bindingPath, os.O_RDONLY|os.O_CREATE|os.O_EXCL, 0) + if err != nil { + return err + } + f.Close() + + if err := namespaces.CreateNewNamespace(libcontainer.CLONE_NEWNET, bindingPath); err != nil { + return err + } + return nil +} + +// DeleteNetworkNamespace unmounts the binding path and removes the +// file so that no references to the fd are present and the network +// namespace is automatically cleaned up +func DeleteNetworkNamespace(bindingPath string) error { + if err := syscall.Unmount(bindingPath, 0); err != nil { + return err + } + return os.Remove(bindingPath) +} diff --git a/components/engine/pkg/libcontainer/privileged.json b/components/engine/pkg/libcontainer/privileged.json new file mode 100644 index 0000000000..be877ad335 --- /dev/null +++ b/components/engine/pkg/libcontainer/privileged.json @@ -0,0 +1,22 @@ +{ + "id": "koye", + "namespace_pid": 3745, + "command": { + "args": [ + "/usr/lib/systemd/systemd" + ], + "environment": [ + "HOME=/", + "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", + "container=docker", + "TERM=" + ] + }, + "rootfs": "/root/main/mycontainer", + "namespaces": [ + "NEWIPC", + "NEWNS", + "NEWPID", + "NEWUTS" + ] +} diff --git a/components/engine/pkg/libcontainer/types.go b/components/engine/pkg/libcontainer/types.go new file mode 100644 index 0000000000..db1c3b9738 --- /dev/null +++ b/components/engine/pkg/libcontainer/types.go @@ -0,0 +1,49 @@ +package libcontainer + +type Namespace string +type Namespaces []Namespace + +func (n Namespaces) Contains(ns Namespace) bool { + for _, nns := range n { + if nns == ns { + return true + } + } + return false +} + +type Capability string +type Capabilities []Capability + +func (c Capabilities) Contains(capp Capability) bool { + for _, cc := range c { + if cc == capp { + return true + } + } + return false +} + +const ( + CAP_SETPCAP Capability = "SETPCAP" + CAP_SYS_MODULE Capability = "SYS_MODULE" + CAP_SYS_RAWIO Capability = "SYS_RAWIO" + CAP_SYS_PACCT Capability = "SYS_PACCT" + CAP_SYS_ADMIN Capability = "SYS_ADMIN" + CAP_SYS_NICE Capability = "SYS_NICE" + CAP_SYS_RESOURCE Capability = "SYS_RESOURCE" + CAP_SYS_TIME Capability = "SYS_TIME" + CAP_SYS_TTY_CONFIG Capability = "SYS_TTY_CONFIG" + CAP_MKNOD Capability = "MKNOD" + CAP_AUDIT_WRITE Capability = "AUDIT_WRITE" + CAP_AUDIT_CONTROL Capability = "AUDIT_CONTROL" + CAP_MAC_OVERRIDE Capability = "MAC_OVERRIDE" + CAP_MAC_ADMIN Capability = "MAC_ADMIN" + + CLONE_NEWNS Namespace = "NEWNS" // mount + CLONE_NEWUTS Namespace = "NEWUTS" // utsname + CLONE_NEWIPC Namespace = "NEWIPC" // ipc + CLONE_NEWUSER Namespace = "NEWUSER" // user + CLONE_NEWPID Namespace = "NEWPID" // pid + CLONE_NEWNET Namespace = "NEWNET" // network +) diff --git a/components/engine/pkg/libcontainer/ubuntu.json b/components/engine/pkg/libcontainer/ubuntu.json new file mode 100644 index 0000000000..0a450ae066 --- /dev/null +++ b/components/engine/pkg/libcontainer/ubuntu.json @@ -0,0 +1,22 @@ +{ + "id": "koye", + "namespace_pid": 3745, + "command": { + "args": [ + "/sbin/init" + ], + "environment": [ + "HOME=/", + "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", + "container=docker", + "TERM=xterm" + ] + }, + "rootfs": "/var/lib/docker/btrfs/subvolumes/7c0f15df1ad2e2fe04d7a6e079aec17406e9465a6a37dd16cb0dd754fc0167b3", + "namespaces": [ + "NEWIPC", + "NEWNS", + "NEWPID", + "NEWUTS" + ] +} diff --git a/components/engine/pkg/libcontainer/utils/utils.go b/components/engine/pkg/libcontainer/utils/utils.go new file mode 100644 index 0000000000..7289fecf2e --- /dev/null +++ b/components/engine/pkg/libcontainer/utils/utils.go @@ -0,0 +1,33 @@ +package utils + +import ( + "crypto/rand" + "encoding/hex" + "io" + "os" + "syscall" +) + +func WaitOnPid(pid int) (exitcode int, err error) { + child, err := os.FindProcess(pid) + if err != nil { + return -1, err + } + state, err := child.Wait() + if err != nil { + return -1, err + } + return getExitCode(state), nil +} + +func getExitCode(state *os.ProcessState) int { + return state.Sys().(syscall.WaitStatus).ExitStatus() +} + +func GenerateRandomName(size int) (string, error) { + id := make([]byte, size) + if _, err := io.ReadFull(rand.Reader, id); err != nil { + return "", err + } + return hex.EncodeToString(id), nil +} From d60eb0a2ddaac37967f25c57d7c3f8d48361af08 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 18 Feb 2014 17:52:06 -0800 Subject: [PATCH 183/403] Make separate nsinit pkg for a dockerinit like init Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 68b049aed4663eb5f6f53241390f7602c1b40c12 Component: engine --- .../engine/pkg/libcontainer/container.go | 2 + .../libcontainer/namespaces/calls_linux.go | 80 +++--- .../pkg/libcontainer/namespaces/exec.go | 270 ++++++------------ .../pkg/libcontainer/namespaces/mount.go | 36 +-- .../pkg/libcontainer/namespaces/namespaces.go | 40 +-- .../libcontainer/namespaces/nsinit/init.go | 140 +++++++++ .../pkg/libcontainer/namespaces/utils.go | 24 +- 7 files changed, 285 insertions(+), 307 deletions(-) create mode 100644 components/engine/pkg/libcontainer/namespaces/nsinit/init.go diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index b77890fb5c..dd5e728e68 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -11,6 +11,8 @@ type Container struct { WorkingDir string `json:"working_dir,omitempty"` Namespaces Namespaces `json:"namespaces,omitempty"` Capabilities Capabilities `json:"capabilities,omitempty"` + Master uintptr `json:"master"` + Console string `json:"console"` } type Command struct { diff --git a/components/engine/pkg/libcontainer/namespaces/calls_linux.go b/components/engine/pkg/libcontainer/namespaces/calls_linux.go index 793e940b6e..f006d56da6 100644 --- a/components/engine/pkg/libcontainer/namespaces/calls_linux.go +++ b/components/engine/pkg/libcontainer/namespaces/calls_linux.go @@ -12,19 +12,19 @@ const ( TIOCSPTLCK = 0x40045431 ) -func chroot(dir string) error { +func Chroot(dir string) error { return syscall.Chroot(dir) } -func chdir(dir string) error { +func Chdir(dir string) error { return syscall.Chdir(dir) } -func exec(cmd string, args []string, env []string) error { +func Exec(cmd string, args []string, env []string) error { return syscall.Exec(cmd, args, env) } -func fork() (int, error) { +func Fork() (int, error) { syscall.ForkLock.Lock() pid, _, err := syscall.Syscall(syscall.SYS_FORK, 0, 0, 0) syscall.ForkLock.Unlock() @@ -34,33 +34,23 @@ func fork() (int, error) { return int(pid), nil } -func vfork() (int, error) { - syscall.ForkLock.Lock() - pid, _, err := syscall.Syscall(syscall.SYS_VFORK, 0, 0, 0) - syscall.ForkLock.Unlock() - if err != 0 { - return -1, err - } - return int(pid), nil -} - -func mount(source, target, fstype string, flags uintptr, data string) error { +func Mount(source, target, fstype string, flags uintptr, data string) error { return syscall.Mount(source, target, fstype, flags, data) } -func unmount(target string, flags int) error { +func Unmount(target string, flags int) error { return syscall.Unmount(target, flags) } -func pivotroot(newroot, putold string) error { +func Pivotroot(newroot, putold string) error { return syscall.PivotRoot(newroot, putold) } -func unshare(flags int) error { +func Unshare(flags int) error { return syscall.Unshare(flags) } -func clone(flags uintptr) (int, error) { +func Clone(flags uintptr) (int, error) { syscall.ForkLock.Lock() pid, _, err := syscall.RawSyscall(syscall.SYS_CLONE, flags, 0, 0) syscall.ForkLock.Unlock() @@ -70,7 +60,7 @@ func clone(flags uintptr) (int, error) { return int(pid), nil } -func setns(fd uintptr, flags uintptr) error { +func Setns(fd uintptr, flags uintptr) error { _, _, err := syscall.RawSyscall(SYS_SETNS, fd, flags, 0) if err != 0 { return err @@ -78,87 +68,87 @@ func setns(fd uintptr, flags uintptr) error { return nil } -func usetCloseOnExec(fd uintptr) error { +func UsetCloseOnExec(fd uintptr) error { if _, _, err := syscall.Syscall(syscall.SYS_FCNTL, fd, syscall.F_SETFD, 0); err != 0 { return err } return nil } -func setgroups(gids []int) error { +func Setgroups(gids []int) error { return syscall.Setgroups(gids) } -func setresgid(rgid, egid, sgid int) error { +func Setresgid(rgid, egid, sgid int) error { return syscall.Setresgid(rgid, egid, sgid) } -func setresuid(ruid, euid, suid int) error { +func Setresuid(ruid, euid, suid int) error { return syscall.Setresuid(ruid, euid, suid) } -func sethostname(name string) error { +func Sethostname(name string) error { return syscall.Sethostname([]byte(name)) } -func setsid() (int, error) { +func Setsid() (int, error) { return syscall.Setsid() } -func ioctl(fd uintptr, flag, data uintptr) error { +func Unlockpt(f *os.File) error { + var u int + return Ioctl(f.Fd(), TIOCSPTLCK, uintptr(unsafe.Pointer(&u))) +} + +func Ioctl(fd uintptr, flag, data uintptr) error { if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, flag, data); err != 0 { return err } return nil } -func openpmtx() (*os.File, error) { - return os.OpenFile("/dev/ptmx", syscall.O_RDONLY|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) -} - -func unlockpt(f *os.File) error { - var u int - return ioctl(f.Fd(), TIOCSPTLCK, uintptr(unsafe.Pointer(&u))) -} - -func ptsname(f *os.File) (string, error) { +func Ptsname(f *os.File) (string, error) { var n int - if err := ioctl(f.Fd(), TIOCGPTN, uintptr(unsafe.Pointer(&n))); err != nil { + if err := Ioctl(f.Fd(), TIOCGPTN, uintptr(unsafe.Pointer(&n))); err != nil { return "", err } return fmt.Sprintf("/dev/pts/%d", n), nil } -func closefd(fd uintptr) error { +func Openpmtx() (*os.File, error) { + return os.OpenFile("/dev/ptmx", syscall.O_RDONLY|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) +} + +func Closefd(fd uintptr) error { return syscall.Close(int(fd)) } -func dup2(fd1, fd2 uintptr) error { +func Dup2(fd1, fd2 uintptr) error { return syscall.Dup2(int(fd1), int(fd2)) } -func mknod(path string, mode uint32, dev int) error { +func Mknod(path string, mode uint32, dev int) error { return syscall.Mknod(path, mode, dev) } -func parentDeathSignal() error { +func ParentDeathSignal() error { if _, _, err := syscall.RawSyscall6(syscall.SYS_PRCTL, syscall.PR_SET_PDEATHSIG, uintptr(syscall.SIGKILL), 0, 0, 0, 0); err != 0 { return err } return nil } -func setctty() error { +func Setctty() error { if _, _, err := syscall.RawSyscall(syscall.SYS_IOCTL, 0, uintptr(syscall.TIOCSCTTY), 0); err != 0 { return err } return nil } -func mkfifo(name string, mode uint32) error { +func Mkfifo(name string, mode uint32) error { return syscall.Mkfifo(name, mode) } -func umask(mask int) int { +func Umask(mask int) int { return syscall.Umask(mask) } diff --git a/components/engine/pkg/libcontainer/namespaces/exec.go b/components/engine/pkg/libcontainer/namespaces/exec.go index 893b302887..0077a0b16c 100644 --- a/components/engine/pkg/libcontainer/namespaces/exec.go +++ b/components/engine/pkg/libcontainer/namespaces/exec.go @@ -8,12 +8,10 @@ import ( "errors" "fmt" "github.com/dotcloud/docker/pkg/libcontainer" - "github.com/dotcloud/docker/pkg/libcontainer/capabilities" - "github.com/dotcloud/docker/pkg/libcontainer/utils" "io" "log" "os" - "path/filepath" + "os/exec" "syscall" ) @@ -29,89 +27,31 @@ var ( // the container will be spawned with a new network namespace with no configuration. Omiting an // existing network namespace and the CLONE_NEWNET option in the container configuration will allow // the container to the the host's networking options and configuration. -func Exec(container *libcontainer.Container) (pid int, err error) { +func ExecContainer(container *libcontainer.Container) (pid int, err error) { // a user cannot pass CLONE_NEWNET and an existing net namespace fd to join if container.NetNsFd > 0 && container.Namespaces.Contains(libcontainer.CLONE_NEWNET) { return -1, ErrExistingNetworkNamespace } - rootfs, err := resolveRootfs(container) - if err != nil { - return -1, err - } - master, console, err := createMasterAndConsole() if err != nil { return -1, err } - - logger, err := os.OpenFile("/root/logs", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0755) - if err != nil { - return -1, err - } - log.SetOutput(logger) + container.Console = console + container.Master = master.Fd() // we need CLONE_VFORK so we can wait on the child - flag := getNamespaceFlags(container.Namespaces) | CLONE_VFORK + flag := uintptr(getNamespaceFlags(container.Namespaces) | CLONE_VFORK) - if pid, err = clone(uintptr(flag | SIGCHLD)); err != nil { - return -1, fmt.Errorf("error cloning process: %s", err) - } - - if pid == 0 { - // welcome to your new namespace ;) - // - // any errors encoutered inside the namespace we should write - // out to a log or a pipe to our parent and exit(1) - // because writing to stderr will not work after we close - if err := closeMasterAndStd(master); err != nil { - writeError("close master and std %s", err) - } - slave, err := openTerminal(console, syscall.O_RDWR) - if err != nil { - writeError("open terminal %s", err) - } - if err := dupSlave(slave); err != nil { - writeError("dup2 slave %s", err) - } - - if container.NetNsFd > 0 { - if err := JoinExistingNamespace(container.NetNsFd, libcontainer.CLONE_NEWNET); err != nil { - writeError("join existing net namespace %s", err) - } - } - - if _, err := setsid(); err != nil { - writeError("setsid %s", err) - } - if err := setctty(); err != nil { - writeError("setctty %s", err) - } - if err := parentDeathSignal(); err != nil { - writeError("parent deth signal %s", err) - } - if err := SetupNewMountNamespace(rootfs, console, container.ReadonlyFs); err != nil { - writeError("setup mount namespace %s", err) - } - if err := sethostname(container.ID); err != nil { - writeError("sethostname %s", err) - } - if err := capabilities.DropCapabilities(container); err != nil { - writeError("drop capabilities %s", err) - } - if err := setupUser(container); err != nil { - writeError("setup user %s", err) - } - if container.WorkingDir != "" { - if err := chdir(container.WorkingDir); err != nil { - writeError("chdir to %s %s", container.WorkingDir, err) - } - } - if err := exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { - writeError("exec %s", err) - } - panic("unreachable") + command := exec.Command("/.nsinit") + command.SysProcAttr = &syscall.SysProcAttr{} + command.SysProcAttr.Cloneflags = flag + command.SysProcAttr.Setctty = true + + if err := command.Start(); err != nil { + return -1, err } + pid = command.Process.Pid go func() { if _, err := io.Copy(os.Stdout, master); err != nil { @@ -130,91 +70,86 @@ func Exec(container *libcontainer.Container) (pid int, err error) { // pid and namespace configuration is needed along with the specific capabilities that should // be dropped once inside the namespace. func ExecIn(container *libcontainer.Container, cmd *libcontainer.Command) (int, error) { - if container.NsPid <= 0 { - return -1, libcontainer.ErrInvalidPid - } - - fds, err := getNsFds(container) - if err != nil { - return -1, err - } - - if container.NetNsFd > 0 { - fds = append(fds, container.NetNsFd) - } - - pid, err := fork() - if err != nil { - for _, fd := range fds { - syscall.Close(int(fd)) + return -1, fmt.Errorf("not implemented") + /* + if container.NsPid <= 0 { + return -1, libcontainer.ErrInvalidPid } - return -1, err - } - if pid == 0 { - for _, fd := range fds { - if fd > 0 { - if err := JoinExistingNamespace(fd, ""); err != nil { - for _, fd := range fds { - syscall.Close(int(fd)) + fds, err := getNsFds(container) + if err != nil { + return -1, err + } + + if container.NetNsFd > 0 { + fds = append(fds, container.NetNsFd) + } + + pid, err := fork() + if err != nil { + for _, fd := range fds { + syscall.Close(int(fd)) + } + return -1, err + } + + if pid == 0 { + for _, fd := range fds { + if fd > 0 { + if err := JoinExistingNamespace(fd, ""); err != nil { + for _, fd := range fds { + syscall.Close(int(fd)) + } + writeError("join existing namespace for %d %s", fd, err) } - writeError("join existing namespace for %d %s", fd, err) } - } - syscall.Close(int(fd)) - } - - if container.Namespaces.Contains(libcontainer.CLONE_NEWNS) && - container.Namespaces.Contains(libcontainer.CLONE_NEWPID) { - // important: - // - // we need to fork and unshare so that re can remount proc and sys within - // the namespace so the CLONE_NEWPID namespace will take effect - // if we don't fork we would end up unmounting proc and sys for the entire - // namespace - child, err := fork() - if err != nil { - writeError("fork child %s", err) + syscall.Close(int(fd)) } - if child == 0 { - if err := unshare(CLONE_NEWNS); err != nil { - writeError("unshare newns %s", err) + if container.Namespaces.Contains(libcontainer.CLONE_NEWNS) && + container.Namespaces.Contains(libcontainer.CLONE_NEWPID) { + // important: + // + // we need to fork and unshare so that re can remount proc and sys within + // the namespace so the CLONE_NEWPID namespace will take effect + // if we don't fork we would end up unmounting proc and sys for the entire + // namespace + child, err := fork() + if err != nil { + writeError("fork child %s", err) } - if err := remountProc(); err != nil { - writeError("remount proc %s", err) - } - if err := remountSys(); err != nil { - writeError("remount sys %s", err) - } - if err := capabilities.DropCapabilities(container); err != nil { - writeError("drop caps %s", err) - } - if err := exec(cmd.Args[0], cmd.Args[0:], cmd.Env); err != nil { - writeError("exec %s", err) - } - panic("unreachable") - } - exit, err := utils.WaitOnPid(child) - if err != nil { - writeError("wait on child %s", err) - } - os.Exit(exit) - } - if err := exec(cmd.Args[0], cmd.Args[0:], cmd.Env); err != nil { - writeError("exec %s", err) - } - panic("unreachable") - } - return pid, err -} -func resolveRootfs(container *libcontainer.Container) (string, error) { - rootfs, err := filepath.Abs(container.RootFs) - if err != nil { - return "", err - } - return filepath.EvalSymlinks(rootfs) + if child == 0 { + if err := unshare(CLONE_NEWNS); err != nil { + writeError("unshare newns %s", err) + } + if err := remountProc(); err != nil { + writeError("remount proc %s", err) + } + if err := remountSys(); err != nil { + writeError("remount sys %s", err) + } + if err := capabilities.DropCapabilities(container); err != nil { + writeError("drop caps %s", err) + } + if err := exec(cmd.Args[0], cmd.Args[0:], cmd.Env); err != nil { + writeError("exec %s", err) + } + panic("unreachable") + } + exit, err := utils.WaitOnPid(child) + if err != nil { + writeError("wait on child %s", err) + } + os.Exit(exit) + } + if err := exec(cmd.Args[0], cmd.Args[0:], cmd.Env); err != nil { + writeError("exec %s", err) + } + panic("unreachable") + } + return pid, err + */ } func createMasterAndConsole() (*os.File, string, error) { @@ -223,44 +158,13 @@ func createMasterAndConsole() (*os.File, string, error) { return nil, "", err } - console, err := ptsname(master) + console, err := Ptsname(master) if err != nil { return nil, "", err } - if err := unlockpt(master); err != nil { + if err := Unlockpt(master); err != nil { return nil, "", err } return master, console, nil } - -func closeMasterAndStd(master *os.File) error { - closefd(master.Fd()) - closefd(0) - closefd(1) - closefd(2) - - return nil -} - -func dupSlave(slave *os.File) error { - // we close Stdin,etc so our pty slave should have fd 0 - if slave.Fd() != 0 { - return fmt.Errorf("slave fd not 0 %d", slave.Fd()) - } - if err := dup2(slave.Fd(), 1); err != nil { - return err - } - if err := dup2(slave.Fd(), 2); err != nil { - return err - } - return nil -} - -func openTerminal(name string, flag int) (*os.File, error) { - r, e := syscall.Open(name, flag, 0) - if e != nil { - return nil, &os.PathError{"open", name, e} - } - return os.NewFile(uintptr(r), name), nil -} diff --git a/components/engine/pkg/libcontainer/namespaces/mount.go b/components/engine/pkg/libcontainer/namespaces/mount.go index 6d867c91ec..8e7c54b046 100644 --- a/components/engine/pkg/libcontainer/namespaces/mount.go +++ b/components/engine/pkg/libcontainer/namespaces/mount.go @@ -14,16 +14,16 @@ var ( ) func SetupNewMountNamespace(rootfs, console string, readonly bool) error { - if err := mount("", "/", "", syscall.MS_SLAVE|syscall.MS_REC, ""); err != nil { + if err := Mount("", "/", "", syscall.MS_SLAVE|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mounting / as slave %s", err) } - if err := mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil { + if err := Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mouting %s as bind %s", rootfs, err) } if readonly { - if err := mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_REC, ""); err != nil { + if err := Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mounting %s as readonly %s", rootfs, err) } } @@ -52,29 +52,29 @@ func SetupNewMountNamespace(rootfs, console string, readonly bool) error { return err } - if err := chdir(rootfs); err != nil { + if err := Chdir(rootfs); err != nil { return fmt.Errorf("chdir into %s %s", rootfs, err) } - if err := mount(rootfs, "/", "", syscall.MS_MOVE, ""); err != nil { + if err := Mount(rootfs, "/", "", syscall.MS_MOVE, ""); err != nil { return fmt.Errorf("mount move %s into / %s", rootfs, err) } - if err := chroot("."); err != nil { + if err := Chroot("."); err != nil { return fmt.Errorf("chroot . %s", err) } - if err := chdir("/"); err != nil { + if err := Chdir("/"); err != nil { return fmt.Errorf("chdir / %s", err) } - umask(0022) + Umask(0022) return nil } func copyDevNodes(rootfs string) error { - umask(0000) + Umask(0000) for _, node := range []string{ "null", @@ -95,7 +95,7 @@ func copyDevNodes(rootfs string) error { ) log.Printf("copy %s to %s %d\n", node, dest, st.Rdev) - if err := mknod(dest, st.Mode, int(st.Rdev)); err != nil && !os.IsExist(err) { + if err := Mknod(dest, st.Mode, int(st.Rdev)); err != nil && !os.IsExist(err) { return fmt.Errorf("copy %s %s", node, err) } } @@ -125,7 +125,7 @@ func setupDev(rootfs string) error { } func setupConsole(rootfs, console string) error { - umask(0000) + Umask(0000) stat, err := os.Stat(console) if err != nil { @@ -145,11 +145,11 @@ func setupConsole(rootfs, console string) error { return err } - if err := mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil { + if err := Mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil { return fmt.Errorf("mknod %s %s", dest, err) } - if err := mount(console, dest, "bind", syscall.MS_BIND, ""); err != nil { + if err := Mount(console, dest, "bind", syscall.MS_BIND, ""); err != nil { return fmt.Errorf("bind %s to %s %s", console, dest, err) } return nil @@ -176,7 +176,7 @@ func mountSystem(rootfs string) error { if err := os.MkdirAll(m.path, 0755); err != nil && !os.IsExist(err) { return fmt.Errorf("mkdirall %s %s", m.path, err) } - if err := mount(m.source, m.path, m.device, uintptr(m.flags), m.data); err != nil { + if err := Mount(m.source, m.path, m.device, uintptr(m.flags), m.data); err != nil { return fmt.Errorf("mounting %s into %s %s", m.source, m.path, err) } } @@ -184,22 +184,22 @@ func mountSystem(rootfs string) error { } func remountProc() error { - if err := unmount("/proc", syscall.MNT_DETACH); err != nil { + if err := Unmount("/proc", syscall.MNT_DETACH); err != nil { return err } - if err := mount("proc", "/proc", "proc", uintptr(defaults), ""); err != nil { + if err := Mount("proc", "/proc", "proc", uintptr(defaults), ""); err != nil { return err } return nil } func remountSys() error { - if err := unmount("/sys", syscall.MNT_DETACH); err != nil { + if err := Unmount("/sys", syscall.MNT_DETACH); err != nil { if err != syscall.EINVAL { return err } } else { - if err := mount("sysfs", "/sys", "sysfs", uintptr(defaults), ""); err != nil { + if err := Mount("sysfs", "/sys", "sysfs", uintptr(defaults), ""); err != nil { return err } } diff --git a/components/engine/pkg/libcontainer/namespaces/namespaces.go b/components/engine/pkg/libcontainer/namespaces/namespaces.go index 2a50847015..05ef0ac7a9 100644 --- a/components/engine/pkg/libcontainer/namespaces/namespaces.go +++ b/components/engine/pkg/libcontainer/namespaces/namespaces.go @@ -9,52 +9,14 @@ package namespaces import ( - "fmt" "github.com/dotcloud/docker/pkg/libcontainer" - "github.com/dotcloud/docker/pkg/libcontainer/utils" - "os" - "path/filepath" - "syscall" ) -// CreateNewNamespace creates a new namespace and binds it's fd to the specified path -func CreateNewNamespace(namespace libcontainer.Namespace, bindTo string) error { - var ( - flag = namespaceMap[namespace] - name = namespaceFileMap[namespace] - nspath = filepath.Join("/proc/self/ns", name) - ) - // TODO: perform validation on name and flag - - pid, err := fork() - if err != nil { - return err - } - - if pid == 0 { - if err := unshare(flag); err != nil { - writeError("unshare %s", err) - } - if err := mount(nspath, bindTo, "none", syscall.MS_BIND, ""); err != nil { - writeError("bind mount %s", err) - } - os.Exit(0) - } - exit, err := utils.WaitOnPid(pid) - if err != nil { - return err - } - if exit != 0 { - return fmt.Errorf("exit status %d", exit) - } - return err -} - // JoinExistingNamespace uses the fd of an existing linux namespace and // has the current process join that namespace or the spacespace specified by ns func JoinExistingNamespace(fd uintptr, ns libcontainer.Namespace) error { flag := namespaceMap[ns] - if err := setns(fd, uintptr(flag)); err != nil { + if err := Setns(fd, uintptr(flag)); err != nil { return err } return nil diff --git a/components/engine/pkg/libcontainer/namespaces/nsinit/init.go b/components/engine/pkg/libcontainer/namespaces/nsinit/init.go new file mode 100644 index 0000000000..9a7563642c --- /dev/null +++ b/components/engine/pkg/libcontainer/namespaces/nsinit/init.go @@ -0,0 +1,140 @@ +package nsinit + +import ( + "fmt" + "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/libcontainer/capabilities" + "github.com/dotcloud/docker/pkg/libcontainer/namespaces" + "log" + "os" + "path/filepath" + "syscall" +) + +// InitNamespace should be run inside an existing namespace to setup +// common mounts, drop capabilities, and setup network interfaces +func InitNamespace(container *libcontainer.Container) error { + rootfs, err := resolveRootfs(container) + if err != nil { + return err + } + + // any errors encoutered inside the namespace we should write + // out to a log or a pipe to our parent and exit(1) + // because writing to stderr will not work after we close + if err := closeMasterAndStd(container.Master); err != nil { + log.Fatalf("close master and std %s", err) + return err + } + + slave, err := openTerminal(container.Console, syscall.O_RDWR) + if err != nil { + log.Fatalf("open terminal %s", err) + return err + } + if err := dupSlave(slave); err != nil { + log.Fatalf("dup2 slave %s", err) + return err + } + + /* + if container.NetNsFd > 0 { + if err := joinExistingNamespace(container.NetNsFd, libcontainer.CLONE_NEWNET); err != nil { + log.Fatalf("join existing net namespace %s", err) + } + } + */ + + if _, err := namespaces.Setsid(); err != nil { + log.Fatalf("setsid %s", err) + return err + } + if err := namespaces.Setctty(); err != nil { + log.Fatalf("setctty %s", err) + return err + } + if err := namespaces.ParentDeathSignal(); err != nil { + log.Fatalf("parent deth signal %s", err) + return err + } + if err := namespaces.SetupNewMountNamespace(rootfs, container.Console, container.ReadonlyFs); err != nil { + log.Fatalf("setup mount namespace %s", err) + return err + } + if err := namespaces.Sethostname(container.ID); err != nil { + log.Fatalf("sethostname %s", err) + return err + } + if err := capabilities.DropCapabilities(container); err != nil { + log.Fatalf("drop capabilities %s", err) + return err + } + if err := setupUser(container); err != nil { + log.Fatalf("setup user %s", err) + return err + } + if container.WorkingDir != "" { + if err := namespaces.Chdir(container.WorkingDir); err != nil { + log.Fatalf("chdir to %s %s", container.WorkingDir, err) + return err + } + } + if err := namespaces.Exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { + log.Fatalf("exec %s", err) + return err + } + panic("unreachable") +} + +func resolveRootfs(container *libcontainer.Container) (string, error) { + rootfs, err := filepath.Abs(container.RootFs) + if err != nil { + return "", err + } + return filepath.EvalSymlinks(rootfs) +} + +func closeMasterAndStd(master uintptr) error { + namespaces.Closefd(master) + namespaces.Closefd(0) + namespaces.Closefd(1) + namespaces.Closefd(2) + + return nil +} + +func setupUser(container *libcontainer.Container) error { + // TODO: honor user passed on container + if err := namespaces.Setgroups(nil); err != nil { + return err + } + if err := namespaces.Setresgid(0, 0, 0); err != nil { + return err + } + if err := namespaces.Setresuid(0, 0, 0); err != nil { + return err + } + return nil +} + +func dupSlave(slave *os.File) error { + // we close Stdin,etc so our pty slave should have fd 0 + if slave.Fd() != 0 { + return fmt.Errorf("slave fd not 0 %d", slave.Fd()) + } + if err := namespaces.Dup2(slave.Fd(), 1); err != nil { + return err + } + if err := namespaces.Dup2(slave.Fd(), 2); err != nil { + return err + } + return nil +} + +func openTerminal(name string, flag int) (*os.File, error) { + r, e := syscall.Open(name, flag, 0) + if e != nil { + return nil, &os.PathError{"open", name, e} + } + return os.NewFile(uintptr(r), name), nil +} diff --git a/components/engine/pkg/libcontainer/namespaces/utils.go b/components/engine/pkg/libcontainer/namespaces/utils.go index 438d896484..fd195c0ad1 100644 --- a/components/engine/pkg/libcontainer/namespaces/utils.go +++ b/components/engine/pkg/libcontainer/namespaces/utils.go @@ -26,12 +26,6 @@ func addEnvIfNotSet(container *libcontainer.Container, key, value string) { container.Command.Env = append(container.Command.Env, jv) } -// print and error to stderr and exit(1) -func writeError(format string, v ...interface{}) { - fmt.Fprintf(os.Stderr, format, v...) - os.Exit(1) -} - // getNsFds inspects the container's namespace configuration and opens the fds to // each of the namespaces. func getNsFds(container *libcontainer.Container) ([]uintptr, error) { @@ -79,27 +73,13 @@ func setupEnvironment(container *libcontainer.Container) { addEnvIfNotSet(container, "LOGNAME", "root") } -func setupUser(container *libcontainer.Container) error { - // TODO: honor user passed on container - if err := setgroups(nil); err != nil { - return err - } - if err := setresgid(0, 0, 0); err != nil { - return err - } - if err := setresuid(0, 0, 0); err != nil { - return err - } - return nil -} - func getMasterAndConsole(container *libcontainer.Container) (string, *os.File, error) { - master, err := openpmtx() + master, err := Openpmtx() if err != nil { return "", nil, err } - console, err := ptsname(master) + console, err := Ptsname(master) if err != nil { master.Close() return "", nil, err From e746c16a002caa2a031c3f40445aa5cac12876cb Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 18 Feb 2014 18:15:41 -0800 Subject: [PATCH 184/403] WIP moving to nsini Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 72e65b654b75d5087e50cc6366e78bd2f8318bae Component: engine --- .../engine/pkg/libcontainer/cli/main.go | 78 +++++++++++-------- .../pkg/libcontainer/namespaces/exec.go | 10 ++- .../engine/pkg/libcontainer/network/veth.go | 16 ---- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/components/engine/pkg/libcontainer/cli/main.go b/components/engine/pkg/libcontainer/cli/main.go index 490135ef5a..0430e29430 100644 --- a/components/engine/pkg/libcontainer/cli/main.go +++ b/components/engine/pkg/libcontainer/cli/main.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/namespaces" + "github.com/dotcloud/docker/pkg/libcontainer/namespaces/nsinit" "github.com/dotcloud/docker/pkg/libcontainer/network" "github.com/dotcloud/docker/pkg/libcontainer/utils" "os" @@ -15,15 +16,26 @@ var ( displayPid bool newCommand string usrNet bool + masterFd int + console string ) func init() { flag.BoolVar(&displayPid, "pid", false, "display the pid before waiting") flag.StringVar(&newCommand, "cmd", "/bin/bash", "command to run in the existing namespace") flag.BoolVar(&usrNet, "net", false, "user a net namespace") + flag.IntVar(&masterFd, "master", 0, "master fd") + flag.StringVar(&console, "console", "", "console path") flag.Parse() } +func nsinitFunc(container *libcontainer.Container) error { + container.Master = uintptr(masterFd) + container.Console = console + + return nsinit.InitNamespace(container) +} + func exec(container *libcontainer.Container) error { var ( netFile *os.File @@ -39,7 +51,7 @@ func exec(container *libcontainer.Container) error { container.NetNsFd = netFile.Fd() } - pid, err := namespaces.Exec(container) + pid, err := namespaces.ExecContainer(container) if err != nil { return fmt.Errorf("error exec container %s", err) } @@ -87,39 +99,39 @@ func execIn(container *libcontainer.Container) error { } func createNet(config *libcontainer.Network) error { - root := "/root/nsroot" - if err := network.SetupNamespaceMountDir(root); err != nil { - return err - } - - nspath := root + "/test" - if err := network.CreateNetworkNamespace(nspath); err != nil { - return nil - } - if err := network.CreateVethPair("veth0", config.TempVethName); err != nil { - return err - } - if err := network.SetInterfaceMaster("veth0", config.Bridge); err != nil { - return err - } - if err := network.InterfaceUp("veth0"); err != nil { - return err - } - - f, err := os.Open(nspath) - if err != nil { - return err - } - defer f.Close() - - if err := network.SetInterfaceInNamespaceFd("veth1", int(f.Fd())); err != nil { - return err - } - /* - if err := network.SetupVethInsideNamespace(f.Fd(), config); err != nil { + root := "/root/nsroot" + if err := network.SetupNamespaceMountDir(root); err != nil { return err } + + nspath := root + "/test" + if err := network.CreateNetworkNamespace(nspath); err != nil { + return nil + } + if err := network.CreateVethPair("veth0", config.TempVethName); err != nil { + return err + } + if err := network.SetInterfaceMaster("veth0", config.Bridge); err != nil { + return err + } + if err := network.InterfaceUp("veth0"); err != nil { + return err + } + + f, err := os.Open(nspath) + if err != nil { + return err + } + defer f.Close() + + if err := network.SetInterfaceInNamespaceFd("veth1", int(f.Fd())); err != nil { + return err + } + + if err := network.SetupVethInsideNamespace(f.Fd(), config); err != nil { + return err + } */ return nil } @@ -133,7 +145,7 @@ func main() { var ( err error cliCmd = flag.Arg(0) - config = flag.Arg(1) + config = "/root/development/gocode/src/github.com/dotcloud/docker/pkg/libcontainer/container.json" //flag.Arg(1) ) f, err := os.Open(config) if err != nil { @@ -149,6 +161,8 @@ func main() { f.Close() switch cliCmd { + case "init": + err = nsinitFunc(container) case "exec": err = exec(container) case "execin": diff --git a/components/engine/pkg/libcontainer/namespaces/exec.go b/components/engine/pkg/libcontainer/namespaces/exec.go index 0077a0b16c..93b155ba24 100644 --- a/components/engine/pkg/libcontainer/namespaces/exec.go +++ b/components/engine/pkg/libcontainer/namespaces/exec.go @@ -12,6 +12,8 @@ import ( "log" "os" "os/exec" + "path/filepath" + "strconv" "syscall" ) @@ -37,16 +39,15 @@ func ExecContainer(container *libcontainer.Container) (pid int, err error) { if err != nil { return -1, err } - container.Console = console - container.Master = master.Fd() + nsinit := filepath.Join(container.RootFs, ".nsinit") // we need CLONE_VFORK so we can wait on the child flag := uintptr(getNamespaceFlags(container.Namespaces) | CLONE_VFORK) - command := exec.Command("/.nsinit") + command := exec.Command(nsinit, "init", "-master", strconv.Itoa(int(master.Fd())), "-console", console) command.SysProcAttr = &syscall.SysProcAttr{} command.SysProcAttr.Cloneflags = flag - command.SysProcAttr.Setctty = true + // command.SysProcAttr.Setctty = true if err := command.Start(); err != nil { return -1, err @@ -63,6 +64,7 @@ func ExecContainer(container *libcontainer.Container) (pid int, err error) { log.Println(err) } }() + command.Wait() return pid, nil } diff --git a/components/engine/pkg/libcontainer/network/veth.go b/components/engine/pkg/libcontainer/network/veth.go index dc207b3394..2ecce22c3e 100644 --- a/components/engine/pkg/libcontainer/network/veth.go +++ b/components/engine/pkg/libcontainer/network/veth.go @@ -3,7 +3,6 @@ package network import ( "fmt" "github.com/dotcloud/docker/pkg/libcontainer" - "github.com/dotcloud/docker/pkg/libcontainer/namespaces" "os" "syscall" ) @@ -59,21 +58,6 @@ func SetupNamespaceMountDir(root string) error { return nil } -// CreateNetworkNamespace creates a new network namespace and binds it's fd -// at the binding path -func CreateNetworkNamespace(bindingPath string) error { - f, err := os.OpenFile(bindingPath, os.O_RDONLY|os.O_CREATE|os.O_EXCL, 0) - if err != nil { - return err - } - f.Close() - - if err := namespaces.CreateNewNamespace(libcontainer.CLONE_NEWNET, bindingPath); err != nil { - return err - } - return nil -} - // DeleteNetworkNamespace unmounts the binding path and removes the // file so that no references to the fd are present and the network // namespace is automatically cleaned up From 1da80198b4a225c47f859c8fd41bd51850750761 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 10:44:29 -0800 Subject: [PATCH 185/403] Use nsinit for setting up namespace Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 11429457691be3b009c6d9f4cc9fce9150d4e810 Component: engine --- components/engine/pkg/libcontainer/cli/main.go | 1 + components/engine/pkg/libcontainer/container.go | 1 + .../engine/pkg/libcontainer/namespaces/exec.go | 4 ++-- .../pkg/libcontainer/namespaces/nsinit/init.go | 13 +++++++++++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/components/engine/pkg/libcontainer/cli/main.go b/components/engine/pkg/libcontainer/cli/main.go index 0430e29430..ac0ea29924 100644 --- a/components/engine/pkg/libcontainer/cli/main.go +++ b/components/engine/pkg/libcontainer/cli/main.go @@ -32,6 +32,7 @@ func init() { func nsinitFunc(container *libcontainer.Container) error { container.Master = uintptr(masterFd) container.Console = console + container.LogFile = "/root/logs" return nsinit.InitNamespace(container) } diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index dd5e728e68..c9a3f2e902 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -13,6 +13,7 @@ type Container struct { Capabilities Capabilities `json:"capabilities,omitempty"` Master uintptr `json:"master"` Console string `json:"console"` + LogFile string `json:"log_file"` } type Command struct { diff --git a/components/engine/pkg/libcontainer/namespaces/exec.go b/components/engine/pkg/libcontainer/namespaces/exec.go index 93b155ba24..7f4b4a609a 100644 --- a/components/engine/pkg/libcontainer/namespaces/exec.go +++ b/components/engine/pkg/libcontainer/namespaces/exec.go @@ -44,9 +44,10 @@ func ExecContainer(container *libcontainer.Container) (pid int, err error) { // we need CLONE_VFORK so we can wait on the child flag := uintptr(getNamespaceFlags(container.Namespaces) | CLONE_VFORK) - command := exec.Command(nsinit, "init", "-master", strconv.Itoa(int(master.Fd())), "-console", console) + command := exec.Command(nsinit, "-master", strconv.Itoa(int(master.Fd())), "-console", console, "init") command.SysProcAttr = &syscall.SysProcAttr{} command.SysProcAttr.Cloneflags = flag + command.ExtraFiles = []*os.File{master} // command.SysProcAttr.Setctty = true if err := command.Start(); err != nil { @@ -64,7 +65,6 @@ func ExecContainer(container *libcontainer.Container) (pid int, err error) { log.Println(err) } }() - command.Wait() return pid, nil } diff --git a/components/engine/pkg/libcontainer/namespaces/nsinit/init.go b/components/engine/pkg/libcontainer/namespaces/nsinit/init.go index 9a7563642c..ae6159b45a 100644 --- a/components/engine/pkg/libcontainer/namespaces/nsinit/init.go +++ b/components/engine/pkg/libcontainer/namespaces/nsinit/init.go @@ -14,6 +14,10 @@ import ( // InitNamespace should be run inside an existing namespace to setup // common mounts, drop capabilities, and setup network interfaces func InitNamespace(container *libcontainer.Container) error { + if err := setLogFile(container); err != nil { + return err + } + rootfs, err := resolveRootfs(container) if err != nil { return err @@ -138,3 +142,12 @@ func openTerminal(name string, flag int) (*os.File, error) { } return os.NewFile(uintptr(r), name), nil } + +func setLogFile(container *libcontainer.Container) error { + f, err := os.OpenFile(container.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0655) + if err != nil { + return err + } + log.SetOutput(f) + return nil +} From 0ffa3c9f04a6679696dfe7655a14dc41c77736ec Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Tue, 18 Feb 2014 23:13:36 -0800 Subject: [PATCH 186/403] Improve general quality of libcontainer Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 93d41e53ae64fd77340c31d3bc5531864ed28779 Component: engine --- .../engine/pkg/libcontainer/cli/main.go | 65 ++++++----- .../engine/pkg/libcontainer/container.json | 2 +- .../pkg/libcontainer/namespaces/exec.go | 101 ++---------------- .../pkg/libcontainer/namespaces/mount.go | 44 ++++---- .../pkg/libcontainer/namespaces/namespaces.go | 32 ------ .../pkg/libcontainer/namespaces/ns_linux.go | 9 ++ .../libcontainer/namespaces/nsinit/init.go | 41 +++---- .../pkg/libcontainer/namespaces/utils.go | 14 --- .../namespaces => system}/calls_linux.go | 37 +------ components/engine/pkg/system/pty_linux.go | 31 ++++++ components/engine/pkg/system/setns_linux.go | 13 +++ .../engine/pkg/system/setns_linux_amd64.go | 8 ++ 12 files changed, 159 insertions(+), 238 deletions(-) delete mode 100644 components/engine/pkg/libcontainer/namespaces/namespaces.go rename components/engine/pkg/{libcontainer/namespaces => system}/calls_linux.go (74%) create mode 100644 components/engine/pkg/system/pty_linux.go create mode 100644 components/engine/pkg/system/setns_linux.go create mode 100644 components/engine/pkg/system/setns_linux_amd64.go diff --git a/components/engine/pkg/libcontainer/cli/main.go b/components/engine/pkg/libcontainer/cli/main.go index ac0ea29924..93bb0399f0 100644 --- a/components/engine/pkg/libcontainer/cli/main.go +++ b/components/engine/pkg/libcontainer/cli/main.go @@ -10,6 +10,9 @@ import ( "github.com/dotcloud/docker/pkg/libcontainer/network" "github.com/dotcloud/docker/pkg/libcontainer/utils" "os" + exec_ "os/exec" + "path" + "path/filepath" ) var ( @@ -52,6 +55,18 @@ func exec(container *libcontainer.Container) error { container.NetNsFd = netFile.Fd() } + self, err := exec_.LookPath(os.Args[0]) + if err != nil { + return err + } + if output, err := exec_.Command("cp", self, path.Join(container.RootFs, ".nsinit")).CombinedOutput(); err != nil { + return fmt.Errorf("Error exec cp: %s, (%s)", err, output) + } else { + println(self, container.RootFs) + fmt.Printf("-----> %s\n", output) + } + println("----") + pid, err := namespaces.ExecContainer(container) if err != nil { return fmt.Errorf("error exec container %s", err) @@ -77,25 +92,25 @@ func exec(container *libcontainer.Container) error { } func execIn(container *libcontainer.Container) error { - f, err := os.Open("/root/nsroot/test") - if err != nil { - return err - } - container.NetNsFd = f.Fd() - pid, err := namespaces.ExecIn(container, &libcontainer.Command{ - Env: container.Command.Env, - Args: []string{ - newCommand, - }, - }) - if err != nil { - return fmt.Errorf("error exexin container %s", err) - } - exitcode, err := utils.WaitOnPid(pid) - if err != nil { - return fmt.Errorf("error waiting on child %s", err) - } - os.Exit(exitcode) + // f, err := os.Open("/root/nsroot/test") + // if err != nil { + // return err + // } + // container.NetNsFd = f.Fd() + // pid, err := namespaces.ExecIn(container, &libcontainer.Command{ + // Env: container.Command.Env, + // Args: []string{ + // newCommand, + // }, + // }) + // if err != nil { + // return fmt.Errorf("error exexin container %s", err) + // } + // exitcode, err := utils.WaitOnPid(pid) + // if err != nil { + // return fmt.Errorf("error waiting on child %s", err) + // } + // os.Exit(exitcode) return nil } @@ -143,11 +158,13 @@ func printErr(err error) { } func main() { - var ( - err error - cliCmd = flag.Arg(0) - config = "/root/development/gocode/src/github.com/dotcloud/docker/pkg/libcontainer/container.json" //flag.Arg(1) - ) + cliCmd := flag.Arg(0) + + config, err := filepath.Abs(flag.Arg(1)) + if err != nil { + printErr(err) + } + println("cli:", cliCmd, "config:", config) f, err := os.Open(config) if err != nil { printErr(err) diff --git a/components/engine/pkg/libcontainer/container.json b/components/engine/pkg/libcontainer/container.json index ed8eb1bd78..6e4fda54c8 100644 --- a/components/engine/pkg/libcontainer/container.json +++ b/components/engine/pkg/libcontainer/container.json @@ -12,7 +12,7 @@ "TERM=xterm" ] }, - "rootfs": "/root/main/mycontainer", + "rootfs": "/var/lib/docker/containers/ee76122136d691d63e09d24168a91ddb2ef9fdcf210b4de5c50aa76354892f4b/root", "namespaces": [ "NEWIPC", "NEWNS", diff --git a/components/engine/pkg/libcontainer/namespaces/exec.go b/components/engine/pkg/libcontainer/namespaces/exec.go index 7f4b4a609a..ea3d2caa70 100644 --- a/components/engine/pkg/libcontainer/namespaces/exec.go +++ b/components/engine/pkg/libcontainer/namespaces/exec.go @@ -6,8 +6,8 @@ package namespaces import ( "errors" - "fmt" "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/system" "io" "log" "os" @@ -44,12 +44,15 @@ func ExecContainer(container *libcontainer.Container) (pid int, err error) { // we need CLONE_VFORK so we can wait on the child flag := uintptr(getNamespaceFlags(container.Namespaces) | CLONE_VFORK) - command := exec.Command(nsinit, "-master", strconv.Itoa(int(master.Fd())), "-console", console, "init") + command := exec.Command(nsinit, "-master", strconv.Itoa(int(master.Fd())), "-console", console, "init", "container.json") + // command.Stdin = os.Stdin + // command.Stdout = os.Stdout + // command.Stderr = os.Stderr command.SysProcAttr = &syscall.SysProcAttr{} command.SysProcAttr.Cloneflags = flag - command.ExtraFiles = []*os.File{master} - // command.SysProcAttr.Setctty = true + //command.ExtraFiles = []*os.File{master} + println("vvvvvvvvv") if err := command.Start(); err != nil { return -1, err } @@ -68,104 +71,18 @@ func ExecContainer(container *libcontainer.Container) (pid int, err error) { return pid, nil } -// ExecIn will spawn a new command inside an existing container's namespaces. The existing container's -// pid and namespace configuration is needed along with the specific capabilities that should -// be dropped once inside the namespace. -func ExecIn(container *libcontainer.Container, cmd *libcontainer.Command) (int, error) { - return -1, fmt.Errorf("not implemented") - /* - if container.NsPid <= 0 { - return -1, libcontainer.ErrInvalidPid - } - - fds, err := getNsFds(container) - if err != nil { - return -1, err - } - - if container.NetNsFd > 0 { - fds = append(fds, container.NetNsFd) - } - - pid, err := fork() - if err != nil { - for _, fd := range fds { - syscall.Close(int(fd)) - } - return -1, err - } - - if pid == 0 { - for _, fd := range fds { - if fd > 0 { - if err := JoinExistingNamespace(fd, ""); err != nil { - for _, fd := range fds { - syscall.Close(int(fd)) - } - writeError("join existing namespace for %d %s", fd, err) - } - } - syscall.Close(int(fd)) - } - - if container.Namespaces.Contains(libcontainer.CLONE_NEWNS) && - container.Namespaces.Contains(libcontainer.CLONE_NEWPID) { - // important: - // - // we need to fork and unshare so that re can remount proc and sys within - // the namespace so the CLONE_NEWPID namespace will take effect - // if we don't fork we would end up unmounting proc and sys for the entire - // namespace - child, err := fork() - if err != nil { - writeError("fork child %s", err) - } - - if child == 0 { - if err := unshare(CLONE_NEWNS); err != nil { - writeError("unshare newns %s", err) - } - if err := remountProc(); err != nil { - writeError("remount proc %s", err) - } - if err := remountSys(); err != nil { - writeError("remount sys %s", err) - } - if err := capabilities.DropCapabilities(container); err != nil { - writeError("drop caps %s", err) - } - if err := exec(cmd.Args[0], cmd.Args[0:], cmd.Env); err != nil { - writeError("exec %s", err) - } - panic("unreachable") - } - exit, err := utils.WaitOnPid(child) - if err != nil { - writeError("wait on child %s", err) - } - os.Exit(exit) - } - if err := exec(cmd.Args[0], cmd.Args[0:], cmd.Env); err != nil { - writeError("exec %s", err) - } - panic("unreachable") - } - return pid, err - */ -} - func createMasterAndConsole() (*os.File, string, error) { master, err := os.OpenFile("/dev/ptmx", syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) if err != nil { return nil, "", err } - console, err := Ptsname(master) + console, err := system.Ptsname(master) if err != nil { return nil, "", err } - if err := Unlockpt(master); err != nil { + if err := system.Unlockpt(master); err != nil { return nil, "", err } return master, console, nil diff --git a/components/engine/pkg/libcontainer/namespaces/mount.go b/components/engine/pkg/libcontainer/namespaces/mount.go index 8e7c54b046..a9b981ecd9 100644 --- a/components/engine/pkg/libcontainer/namespaces/mount.go +++ b/components/engine/pkg/libcontainer/namespaces/mount.go @@ -2,6 +2,7 @@ package namespaces import ( "fmt" + "github.com/dotcloud/docker/pkg/system" "log" "os" "path/filepath" @@ -14,16 +15,16 @@ var ( ) func SetupNewMountNamespace(rootfs, console string, readonly bool) error { - if err := Mount("", "/", "", syscall.MS_SLAVE|syscall.MS_REC, ""); err != nil { + if err := system.Mount("", "/", "", syscall.MS_SLAVE|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mounting / as slave %s", err) } - if err := Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil { + if err := system.Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mouting %s as bind %s", rootfs, err) } if readonly { - if err := Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_REC, ""); err != nil { + if err := system.Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mounting %s as readonly %s", rootfs, err) } } @@ -52,29 +53,30 @@ func SetupNewMountNamespace(rootfs, console string, readonly bool) error { return err } - if err := Chdir(rootfs); err != nil { + if err := system.Chdir(rootfs); err != nil { return fmt.Errorf("chdir into %s %s", rootfs, err) } - if err := Mount(rootfs, "/", "", syscall.MS_MOVE, ""); err != nil { + if err := system.Mount(rootfs, "/", "", syscall.MS_MOVE, ""); err != nil { return fmt.Errorf("mount move %s into / %s", rootfs, err) } - if err := Chroot("."); err != nil { + if err := system.Chroot("."); err != nil { return fmt.Errorf("chroot . %s", err) } - if err := Chdir("/"); err != nil { + if err := system.Chdir("/"); err != nil { return fmt.Errorf("chdir / %s", err) } - Umask(0022) + system.Umask(0022) return nil } func copyDevNodes(rootfs string) error { - Umask(0000) + oldMask := system.Umask(0000) + defer system.Umask(oldMask) for _, node := range []string{ "null", @@ -95,7 +97,7 @@ func copyDevNodes(rootfs string) error { ) log.Printf("copy %s to %s %d\n", node, dest, st.Rdev) - if err := Mknod(dest, st.Mode, int(st.Rdev)); err != nil && !os.IsExist(err) { + if err := system.Mknod(dest, st.Mode, int(st.Rdev)); err != nil && !os.IsExist(err) { return fmt.Errorf("copy %s %s", node, err) } } @@ -125,7 +127,8 @@ func setupDev(rootfs string) error { } func setupConsole(rootfs, console string) error { - Umask(0000) + oldMask := system.Umask(0000) + defer system.Umask(oldMask) stat, err := os.Stat(console) if err != nil { @@ -145,11 +148,11 @@ func setupConsole(rootfs, console string) error { return err } - if err := Mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil { + if err := system.Mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil { return fmt.Errorf("mknod %s %s", dest, err) } - if err := Mount(console, dest, "bind", syscall.MS_BIND, ""); err != nil { + if err := system.Mount(console, dest, "bind", syscall.MS_BIND, ""); err != nil { return fmt.Errorf("bind %s to %s %s", console, dest, err) } return nil @@ -158,7 +161,7 @@ func setupConsole(rootfs, console string) error { // mountSystem sets up linux specific system mounts like sys, proc, shm, and devpts // inside the mount namespace func mountSystem(rootfs string) error { - mounts := []struct { + for _, m := range []struct { source string path string device string @@ -171,12 +174,11 @@ func mountSystem(rootfs string) error { {source: "shm", path: filepath.Join(rootfs, "dev", "shm"), device: "tmpfs", flags: defaults, data: "mode=1777"}, {source: "devpts", path: filepath.Join(rootfs, "dev", "pts"), device: "devpts", flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, data: "newinstance,ptmxmode=0666,mode=620,gid=5"}, {source: "tmpfs", path: filepath.Join(rootfs, "run"), device: "tmpfs", flags: syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_STRICTATIME, data: "mode=755"}, - } - for _, m := range mounts { + } { if err := os.MkdirAll(m.path, 0755); err != nil && !os.IsExist(err) { return fmt.Errorf("mkdirall %s %s", m.path, err) } - if err := Mount(m.source, m.path, m.device, uintptr(m.flags), m.data); err != nil { + if err := system.Mount(m.source, m.path, m.device, uintptr(m.flags), m.data); err != nil { return fmt.Errorf("mounting %s into %s %s", m.source, m.path, err) } } @@ -184,22 +186,22 @@ func mountSystem(rootfs string) error { } func remountProc() error { - if err := Unmount("/proc", syscall.MNT_DETACH); err != nil { + if err := system.Unmount("/proc", syscall.MNT_DETACH); err != nil { return err } - if err := Mount("proc", "/proc", "proc", uintptr(defaults), ""); err != nil { + if err := system.Mount("proc", "/proc", "proc", uintptr(defaults), ""); err != nil { return err } return nil } func remountSys() error { - if err := Unmount("/sys", syscall.MNT_DETACH); err != nil { + if err := system.Unmount("/sys", syscall.MNT_DETACH); err != nil { if err != syscall.EINVAL { return err } } else { - if err := Mount("sysfs", "/sys", "sysfs", uintptr(defaults), ""); err != nil { + if err := system.Mount("sysfs", "/sys", "sysfs", uintptr(defaults), ""); err != nil { return err } } diff --git a/components/engine/pkg/libcontainer/namespaces/namespaces.go b/components/engine/pkg/libcontainer/namespaces/namespaces.go deleted file mode 100644 index 05ef0ac7a9..0000000000 --- a/components/engine/pkg/libcontainer/namespaces/namespaces.go +++ /dev/null @@ -1,32 +0,0 @@ -/* - TODO - pivot root - cgroups - more mount stuff that I probably am forgetting - apparmor -*/ - -package namespaces - -import ( - "github.com/dotcloud/docker/pkg/libcontainer" -) - -// JoinExistingNamespace uses the fd of an existing linux namespace and -// has the current process join that namespace or the spacespace specified by ns -func JoinExistingNamespace(fd uintptr, ns libcontainer.Namespace) error { - flag := namespaceMap[ns] - if err := Setns(fd, uintptr(flag)); err != nil { - return err - } - return nil -} - -// getNamespaceFlags parses the container's Namespaces options to set the correct -// flags on clone, unshare, and setns -func getNamespaceFlags(namespaces libcontainer.Namespaces) (flag int) { - for _, ns := range namespaces { - flag |= namespaceMap[ns] - } - return -} diff --git a/components/engine/pkg/libcontainer/namespaces/ns_linux.go b/components/engine/pkg/libcontainer/namespaces/ns_linux.go index b0e5119130..f61279334d 100644 --- a/components/engine/pkg/libcontainer/namespaces/ns_linux.go +++ b/components/engine/pkg/libcontainer/namespaces/ns_linux.go @@ -33,3 +33,12 @@ var namespaceFileMap = map[libcontainer.Namespace]string{ libcontainer.CLONE_NEWPID: "pid", libcontainer.CLONE_NEWNET: "net", } + +// getNamespaceFlags parses the container's Namespaces options to set the correct +// flags on clone, unshare, and setns +func getNamespaceFlags(namespaces libcontainer.Namespaces) (flag int) { + for _, ns := range namespaces { + flag |= namespaceMap[ns] + } + return +} diff --git a/components/engine/pkg/libcontainer/namespaces/nsinit/init.go b/components/engine/pkg/libcontainer/namespaces/nsinit/init.go index ae6159b45a..7f85ebacdb 100644 --- a/components/engine/pkg/libcontainer/namespaces/nsinit/init.go +++ b/components/engine/pkg/libcontainer/namespaces/nsinit/init.go @@ -5,6 +5,7 @@ import ( "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/capabilities" "github.com/dotcloud/docker/pkg/libcontainer/namespaces" + "github.com/dotcloud/docker/pkg/system" "log" "os" "path/filepath" @@ -14,10 +15,12 @@ import ( // InitNamespace should be run inside an existing namespace to setup // common mounts, drop capabilities, and setup network interfaces func InitNamespace(container *libcontainer.Container) error { + println("|||||||||||||") if err := setLogFile(container); err != nil { return err } - + println(container.LogFile) + log.Printf("--------->") rootfs, err := resolveRootfs(container) if err != nil { return err @@ -26,7 +29,7 @@ func InitNamespace(container *libcontainer.Container) error { // any errors encoutered inside the namespace we should write // out to a log or a pipe to our parent and exit(1) // because writing to stderr will not work after we close - if err := closeMasterAndStd(container.Master); err != nil { + if err := closeMasterAndStd(os.NewFile(container.Master, "/dev/ptmx")); err != nil { log.Fatalf("close master and std %s", err) return err } @@ -49,15 +52,15 @@ func InitNamespace(container *libcontainer.Container) error { } */ - if _, err := namespaces.Setsid(); err != nil { + if _, err := system.Setsid(); err != nil { log.Fatalf("setsid %s", err) return err } - if err := namespaces.Setctty(); err != nil { + if err := system.Setctty(); err != nil { log.Fatalf("setctty %s", err) return err } - if err := namespaces.ParentDeathSignal(); err != nil { + if err := system.ParentDeathSignal(); err != nil { log.Fatalf("parent deth signal %s", err) return err } @@ -65,7 +68,7 @@ func InitNamespace(container *libcontainer.Container) error { log.Fatalf("setup mount namespace %s", err) return err } - if err := namespaces.Sethostname(container.ID); err != nil { + if err := system.Sethostname(container.ID); err != nil { log.Fatalf("sethostname %s", err) return err } @@ -78,12 +81,12 @@ func InitNamespace(container *libcontainer.Container) error { return err } if container.WorkingDir != "" { - if err := namespaces.Chdir(container.WorkingDir); err != nil { + if err := system.Chdir(container.WorkingDir); err != nil { log.Fatalf("chdir to %s %s", container.WorkingDir, err) return err } } - if err := namespaces.Exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { + if err := system.Exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { log.Fatalf("exec %s", err) return err } @@ -98,24 +101,23 @@ func resolveRootfs(container *libcontainer.Container) (string, error) { return filepath.EvalSymlinks(rootfs) } -func closeMasterAndStd(master uintptr) error { - namespaces.Closefd(master) - namespaces.Closefd(0) - namespaces.Closefd(1) - namespaces.Closefd(2) - +func closeMasterAndStd(master *os.File) error { + master.Close() + os.Stdin.Close() + os.Stdout.Close() + os.Stderr.Close() return nil } func setupUser(container *libcontainer.Container) error { // TODO: honor user passed on container - if err := namespaces.Setgroups(nil); err != nil { + if err := system.Setgroups(nil); err != nil { return err } - if err := namespaces.Setresgid(0, 0, 0); err != nil { + if err := system.Setresgid(0, 0, 0); err != nil { return err } - if err := namespaces.Setresuid(0, 0, 0); err != nil { + if err := system.Setresuid(0, 0, 0); err != nil { return err } return nil @@ -126,15 +128,16 @@ func dupSlave(slave *os.File) error { if slave.Fd() != 0 { return fmt.Errorf("slave fd not 0 %d", slave.Fd()) } - if err := namespaces.Dup2(slave.Fd(), 1); err != nil { + if err := system.Dup2(slave.Fd(), 1); err != nil { return err } - if err := namespaces.Dup2(slave.Fd(), 2); err != nil { + if err := system.Dup2(slave.Fd(), 2); err != nil { return err } return nil } +// openTerminal is a clone of os.OpenFile without the O_CLOEXEC addition. func openTerminal(name string, flag int) (*os.File, error) { r, e := syscall.Open(name, flag, 0) if e != nil { diff --git a/components/engine/pkg/libcontainer/namespaces/utils.go b/components/engine/pkg/libcontainer/namespaces/utils.go index fd195c0ad1..a5d677c7b3 100644 --- a/components/engine/pkg/libcontainer/namespaces/utils.go +++ b/components/engine/pkg/libcontainer/namespaces/utils.go @@ -72,17 +72,3 @@ func setupEnvironment(container *libcontainer.Container) { addEnvIfNotSet(container, "USER", "root") addEnvIfNotSet(container, "LOGNAME", "root") } - -func getMasterAndConsole(container *libcontainer.Container) (string, *os.File, error) { - master, err := Openpmtx() - if err != nil { - return "", nil, err - } - - console, err := Ptsname(master) - if err != nil { - master.Close() - return "", nil, err - } - return console, master, nil -} diff --git a/components/engine/pkg/libcontainer/namespaces/calls_linux.go b/components/engine/pkg/system/calls_linux.go similarity index 74% rename from components/engine/pkg/libcontainer/namespaces/calls_linux.go rename to components/engine/pkg/system/calls_linux.go index f006d56da6..42afa349c2 100644 --- a/components/engine/pkg/libcontainer/namespaces/calls_linux.go +++ b/components/engine/pkg/system/calls_linux.go @@ -1,15 +1,7 @@ -package namespaces +package system import ( - "fmt" - "os" "syscall" - "unsafe" -) - -const ( - TIOCGPTN = 0x80045430 - TIOCSPTLCK = 0x40045431 ) func Chroot(dir string) error { @@ -60,14 +52,6 @@ func Clone(flags uintptr) (int, error) { return int(pid), nil } -func Setns(fd uintptr, flags uintptr) error { - _, _, err := syscall.RawSyscall(SYS_SETNS, fd, flags, 0) - if err != 0 { - return err - } - return nil -} - func UsetCloseOnExec(fd uintptr) error { if _, _, err := syscall.Syscall(syscall.SYS_FCNTL, fd, syscall.F_SETFD, 0); err != 0 { return err @@ -95,11 +79,6 @@ func Setsid() (int, error) { return syscall.Setsid() } -func Unlockpt(f *os.File) error { - var u int - return Ioctl(f.Fd(), TIOCSPTLCK, uintptr(unsafe.Pointer(&u))) -} - func Ioctl(fd uintptr, flag, data uintptr) error { if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, flag, data); err != 0 { return err @@ -107,18 +86,6 @@ func Ioctl(fd uintptr, flag, data uintptr) error { return nil } -func Ptsname(f *os.File) (string, error) { - var n int - if err := Ioctl(f.Fd(), TIOCGPTN, uintptr(unsafe.Pointer(&n))); err != nil { - return "", err - } - return fmt.Sprintf("/dev/pts/%d", n), nil -} - -func Openpmtx() (*os.File, error) { - return os.OpenFile("/dev/ptmx", syscall.O_RDONLY|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) -} - func Closefd(fd uintptr) error { return syscall.Close(int(fd)) } @@ -132,7 +99,7 @@ func Mknod(path string, mode uint32, dev int) error { } func ParentDeathSignal() error { - if _, _, err := syscall.RawSyscall6(syscall.SYS_PRCTL, syscall.PR_SET_PDEATHSIG, uintptr(syscall.SIGKILL), 0, 0, 0, 0); err != 0 { + if _, _, err := syscall.RawSyscall(syscall.SYS_PRCTL, syscall.PR_SET_PDEATHSIG, uintptr(syscall.SIGKILL), 0); err != 0 { return err } return nil diff --git a/components/engine/pkg/system/pty_linux.go b/components/engine/pkg/system/pty_linux.go new file mode 100644 index 0000000000..b281b719fb --- /dev/null +++ b/components/engine/pkg/system/pty_linux.go @@ -0,0 +1,31 @@ +package system + +import ( + "fmt" + "os" + "syscall" + "unsafe" +) + +// Unlockpt unlocks the slave pseudoterminal device corresponding to the master pseudoterminal referred to by f. +// Unlockpt should be called before opening the slave side of a pseudoterminal. +func Unlockpt(f *os.File) error { + var u int + return Ioctl(f.Fd(), syscall.TIOCSPTLCK, uintptr(unsafe.Pointer(&u))) +} + +// Ptsname retrieves the name of the first available pts for the given master. +func Ptsname(f *os.File) (string, error) { + var n int + + if err := Ioctl(f.Fd(), syscall.TIOCGPTN, uintptr(unsafe.Pointer(&n))); err != nil { + return "", err + } + return fmt.Sprintf("/dev/pts/%d", n), nil +} + +// OpenPtmx opens /dev/ptmx, i.e. the PTY master. +func OpenPtmx() (*os.File, error) { + // O_NOCTTY and O_CLOEXEC are not present in os package so we use the syscall's one for all. + return os.OpenFile("/dev/ptmx", syscall.O_RDONLY|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) +} diff --git a/components/engine/pkg/system/setns_linux.go b/components/engine/pkg/system/setns_linux.go new file mode 100644 index 0000000000..be6f3edb30 --- /dev/null +++ b/components/engine/pkg/system/setns_linux.go @@ -0,0 +1,13 @@ +package system + +import ( + "syscall" +) + +func Setns(fd uintptr, flags uintptr) error { + _, _, err := syscall.RawSyscall(SYS_SETNS, fd, flags, 0) + if err != 0 { + return err + } + return nil +} diff --git a/components/engine/pkg/system/setns_linux_amd64.go b/components/engine/pkg/system/setns_linux_amd64.go new file mode 100644 index 0000000000..4e306253d9 --- /dev/null +++ b/components/engine/pkg/system/setns_linux_amd64.go @@ -0,0 +1,8 @@ +// +build linux,amd64 + +package system + +// Via http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=7b21fddd087678a70ad64afc0f632e0f1071b092 +const ( + SYS_SETNS = 308 +) From 570aeb08e3a0c44385fef227495a2e00c46c26e1 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 19 Feb 2014 12:47:01 -0800 Subject: [PATCH 187/403] Fix ptmx issue on libcontainer Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 18f06b8d16c475568fd023e97eecc138ab052c2d Component: engine --- components/engine/pkg/libcontainer/namespaces/exec.go | 3 ++- components/engine/pkg/libcontainer/namespaces/mount.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/pkg/libcontainer/namespaces/exec.go b/components/engine/pkg/libcontainer/namespaces/exec.go index ea3d2caa70..77550d6089 100644 --- a/components/engine/pkg/libcontainer/namespaces/exec.go +++ b/components/engine/pkg/libcontainer/namespaces/exec.go @@ -50,7 +50,8 @@ func ExecContainer(container *libcontainer.Container) (pid int, err error) { // command.Stderr = os.Stderr command.SysProcAttr = &syscall.SysProcAttr{} command.SysProcAttr.Cloneflags = flag - //command.ExtraFiles = []*os.File{master} + + command.ExtraFiles = []*os.File{master} println("vvvvvvvvv") if err := command.Start(); err != nil { diff --git a/components/engine/pkg/libcontainer/namespaces/mount.go b/components/engine/pkg/libcontainer/namespaces/mount.go index a9b981ecd9..5c0b8ead16 100644 --- a/components/engine/pkg/libcontainer/namespaces/mount.go +++ b/components/engine/pkg/libcontainer/namespaces/mount.go @@ -41,7 +41,7 @@ func SetupNewMountNamespace(rootfs, console string, readonly bool) error { if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) { return err } - if err := os.Symlink(filepath.Join(rootfs, "pts/ptmx"), ptmx); err != nil { + if err := os.Symlink("pts/ptmx", ptmx); err != nil { return fmt.Errorf("symlink dev ptmx %s", err) } From a92b2b9a1bb82e84885cf1d42e52a442ae7149a3 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 14:33:25 -0800 Subject: [PATCH 188/403] Use nsinit as app Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: e25065a6b1df09771598d77cc698e4fcf1159bd4 Component: engine --- .../engine/pkg/libcontainer/container.go | 10 +- .../pkg/libcontainer/namespaces/exec.go | 40 ++----- .../libcontainer/namespaces/linux_x86_64.go | 7 -- .../pkg/libcontainer/namespaces/ns_linux.go | 2 +- .../libcontainer/namespaces/nsinit/init.go | 112 ++++++++++-------- .../namespaces/{ => nsinit}/mount.go | 4 +- .../pkg/libcontainer/namespaces/utils.go | 26 ---- 7 files changed, 82 insertions(+), 119 deletions(-) delete mode 100644 components/engine/pkg/libcontainer/namespaces/linux_x86_64.go rename components/engine/pkg/libcontainer/namespaces/{ => nsinit}/mount.go (98%) diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index c9a3f2e902..c2885447fd 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -2,18 +2,14 @@ package libcontainer type Container struct { ID string `json:"id,omitempty"` - NsPid int `json:"namespace_pid,omitempty"` Command *Command `json:"command,omitempty"` - RootFs string `json:"rootfs,omitempty"` ReadonlyFs bool `json:"readonly_fs,omitempty"` - NetNsFd uintptr `json:"network_namespace_fd,omitempty"` User string `json:"user,omitempty"` WorkingDir string `json:"working_dir,omitempty"` Namespaces Namespaces `json:"namespaces,omitempty"` Capabilities Capabilities `json:"capabilities,omitempty"` - Master uintptr `json:"master"` - Console string `json:"console"` - LogFile string `json:"log_file"` + LogFile string `json:"log_file,omitempty"` + Network *Network `json:"network,omitempty"` } type Command struct { @@ -22,9 +18,9 @@ type Command struct { } type Network struct { - TempVethName string `json:"temp_veth,omitempty"` IP string `json:"ip,omitempty"` Gateway string `json:"gateway,omitempty"` Bridge string `json:"bridge,omitempty"` Mtu int `json:"mtu,omitempty"` + TempVethName string `json:"temp_veth,omitempty"` } diff --git a/components/engine/pkg/libcontainer/namespaces/exec.go b/components/engine/pkg/libcontainer/namespaces/exec.go index 77550d6089..8e5bf68aef 100644 --- a/components/engine/pkg/libcontainer/namespaces/exec.go +++ b/components/engine/pkg/libcontainer/namespaces/exec.go @@ -1,27 +1,17 @@ -/* - Higher level convience functions for setting up a container -*/ - package namespaces import ( - "errors" "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/system" + "github.com/dotcloud/docker/pkg/term" "io" "log" "os" "os/exec" - "path/filepath" - "strconv" "syscall" ) -var ( - ErrExistingNetworkNamespace = errors.New("specified both CLONE_NEWNET and an existing network namespace") -) - -// Exec will spawn new namespaces with the specified Container configuration +// ExecContainer will spawn new namespaces with the specified Container configuration // in the RootFs path and return the pid of the new containerized process. // // If an existing network namespace is specified the container @@ -30,30 +20,19 @@ var ( // existing network namespace and the CLONE_NEWNET option in the container configuration will allow // the container to the the host's networking options and configuration. func ExecContainer(container *libcontainer.Container) (pid int, err error) { - // a user cannot pass CLONE_NEWNET and an existing net namespace fd to join - if container.NetNsFd > 0 && container.Namespaces.Contains(libcontainer.CLONE_NEWNET) { - return -1, ErrExistingNetworkNamespace - } - master, console, err := createMasterAndConsole() if err != nil { return -1, err } - nsinit := filepath.Join(container.RootFs, ".nsinit") // we need CLONE_VFORK so we can wait on the child flag := uintptr(getNamespaceFlags(container.Namespaces) | CLONE_VFORK) - command := exec.Command(nsinit, "-master", strconv.Itoa(int(master.Fd())), "-console", console, "init", "container.json") - // command.Stdin = os.Stdin - // command.Stdout = os.Stdout - // command.Stderr = os.Stderr - command.SysProcAttr = &syscall.SysProcAttr{} - command.SysProcAttr.Cloneflags = flag + command := exec.Command("nsinit", console) + command.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: flag, + } - command.ExtraFiles = []*os.File{master} - - println("vvvvvvvvv") if err := command.Start(); err != nil { return -1, err } @@ -64,11 +43,18 @@ func ExecContainer(container *libcontainer.Container) (pid int, err error) { log.Println(err) } }() + go func() { if _, err := io.Copy(master, os.Stdin); err != nil { log.Println(err) } }() + + term.SetRawTerminal(os.Stdin.Fd()) + + if err := command.Wait(); err != nil { + return pid, err + } return pid, nil } diff --git a/components/engine/pkg/libcontainer/namespaces/linux_x86_64.go b/components/engine/pkg/libcontainer/namespaces/linux_x86_64.go deleted file mode 100644 index ac9a014763..0000000000 --- a/components/engine/pkg/libcontainer/namespaces/linux_x86_64.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build linux,x86_64 -package namespaces - -// Via http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=7b21fddd087678a70ad64afc0f632e0f1071b092 -const ( - SYS_SETNS = 308 -) diff --git a/components/engine/pkg/libcontainer/namespaces/ns_linux.go b/components/engine/pkg/libcontainer/namespaces/ns_linux.go index f61279334d..2c73e08e58 100644 --- a/components/engine/pkg/libcontainer/namespaces/ns_linux.go +++ b/components/engine/pkg/libcontainer/namespaces/ns_linux.go @@ -40,5 +40,5 @@ func getNamespaceFlags(namespaces libcontainer.Namespaces) (flag int) { for _, ns := range namespaces { flag |= namespaceMap[ns] } - return + return flag } diff --git a/components/engine/pkg/libcontainer/namespaces/nsinit/init.go b/components/engine/pkg/libcontainer/namespaces/nsinit/init.go index 7f85ebacdb..523854e5d6 100644 --- a/components/engine/pkg/libcontainer/namespaces/nsinit/init.go +++ b/components/engine/pkg/libcontainer/namespaces/nsinit/init.go @@ -1,6 +1,7 @@ -package nsinit +package main import ( + "encoding/json" "fmt" "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/capabilities" @@ -12,103 +13,112 @@ import ( "syscall" ) -// InitNamespace should be run inside an existing namespace to setup -// common mounts, drop capabilities, and setup network interfaces -func InitNamespace(container *libcontainer.Container) error { - println("|||||||||||||") - if err := setLogFile(container); err != nil { - return err - } - println(container.LogFile) - log.Printf("--------->") - rootfs, err := resolveRootfs(container) +func loadContainer() (*libcontainer.Container, error) { + f, err := os.Open("container.json") if err != nil { - return err + return nil, err + } + defer f.Close() + + var container *libcontainer.Container + if err := json.NewDecoder(f).Decode(&container); err != nil { + return nil, err + } + return container, nil +} + +func main() { + container, err := loadContainer() + if err != nil { + log.Fatal(err) } - // any errors encoutered inside the namespace we should write - // out to a log or a pipe to our parent and exit(1) - // because writing to stderr will not work after we close - if err := closeMasterAndStd(os.NewFile(container.Master, "/dev/ptmx")); err != nil { - log.Fatalf("close master and std %s", err) - return err + if os.Args[1] == "exec" { + _, err := namespaces.ExecContainer(container) + if err != nil { + log.Fatal(err) + } + os.Exit(0) + } + console := os.Args[1] + + if err := setLogFile(container); err != nil { + log.Fatal(err) } - slave, err := openTerminal(container.Console, syscall.O_RDWR) + rootfs, err := resolveRootfs() + if err != nil { + log.Fatal(err) + } + + // close pipes so that we can replace it with the pty + os.Stdin.Close() + os.Stdout.Close() + os.Stderr.Close() + + slave, err := openTerminal(console, syscall.O_RDWR) if err != nil { log.Fatalf("open terminal %s", err) - return err + } + if slave.Fd() != 0 { + log.Fatalf("slave fd should be 0") } if err := dupSlave(slave); err != nil { log.Fatalf("dup2 slave %s", err) - return err } - /* - if container.NetNsFd > 0 { - if err := joinExistingNamespace(container.NetNsFd, libcontainer.CLONE_NEWNET); err != nil { - log.Fatalf("join existing net namespace %s", err) - } - } - */ - if _, err := system.Setsid(); err != nil { log.Fatalf("setsid %s", err) - return err } if err := system.Setctty(); err != nil { log.Fatalf("setctty %s", err) - return err } if err := system.ParentDeathSignal(); err != nil { log.Fatalf("parent deth signal %s", err) - return err } - if err := namespaces.SetupNewMountNamespace(rootfs, container.Console, container.ReadonlyFs); err != nil { + + if err := setupNewMountNamespace(rootfs, console, container.ReadonlyFs); err != nil { log.Fatalf("setup mount namespace %s", err) - return err } + + if container.Network != nil { + if err := setupNetworking(container); err != nil { + log.Fatalf("setup networking %s", err) + } + } + if err := system.Sethostname(container.ID); err != nil { log.Fatalf("sethostname %s", err) - return err } if err := capabilities.DropCapabilities(container); err != nil { log.Fatalf("drop capabilities %s", err) - return err } if err := setupUser(container); err != nil { log.Fatalf("setup user %s", err) - return err } if container.WorkingDir != "" { if err := system.Chdir(container.WorkingDir); err != nil { log.Fatalf("chdir to %s %s", container.WorkingDir, err) - return err } } if err := system.Exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { log.Fatalf("exec %s", err) - return err } panic("unreachable") } -func resolveRootfs(container *libcontainer.Container) (string, error) { - rootfs, err := filepath.Abs(container.RootFs) +func resolveRootfs() (string, error) { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + rootfs, err := filepath.Abs(cwd) if err != nil { return "", err } return filepath.EvalSymlinks(rootfs) } -func closeMasterAndStd(master *os.File) error { - master.Close() - os.Stdin.Close() - os.Stdout.Close() - os.Stderr.Close() - return nil -} - func setupUser(container *libcontainer.Container) error { // TODO: honor user passed on container if err := system.Setgroups(nil); err != nil { @@ -154,3 +164,7 @@ func setLogFile(container *libcontainer.Container) error { log.SetOutput(f) return nil } + +func setupNetworking(conatiner *libcontainer.Container) error { + return nil +} diff --git a/components/engine/pkg/libcontainer/namespaces/mount.go b/components/engine/pkg/libcontainer/namespaces/nsinit/mount.go similarity index 98% rename from components/engine/pkg/libcontainer/namespaces/mount.go rename to components/engine/pkg/libcontainer/namespaces/nsinit/mount.go index 5c0b8ead16..f9ee969636 100644 --- a/components/engine/pkg/libcontainer/namespaces/mount.go +++ b/components/engine/pkg/libcontainer/namespaces/nsinit/mount.go @@ -1,4 +1,4 @@ -package namespaces +package main import ( "fmt" @@ -14,7 +14,7 @@ var ( defaults = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV ) -func SetupNewMountNamespace(rootfs, console string, readonly bool) error { +func setupNewMountNamespace(rootfs, console string, readonly bool) error { if err := system.Mount("", "/", "", syscall.MS_SLAVE|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mounting / as slave %s", err) } diff --git a/components/engine/pkg/libcontainer/namespaces/utils.go b/components/engine/pkg/libcontainer/namespaces/utils.go index a5d677c7b3..edc3ab52cc 100644 --- a/components/engine/pkg/libcontainer/namespaces/utils.go +++ b/components/engine/pkg/libcontainer/namespaces/utils.go @@ -7,7 +7,6 @@ import ( "path/filepath" "strconv" "strings" - "syscall" ) func addEnvIfNotSet(container *libcontainer.Container, key, value string) { @@ -26,31 +25,6 @@ func addEnvIfNotSet(container *libcontainer.Container, key, value string) { container.Command.Env = append(container.Command.Env, jv) } -// getNsFds inspects the container's namespace configuration and opens the fds to -// each of the namespaces. -func getNsFds(container *libcontainer.Container) ([]uintptr, error) { - var ( - namespaces = []string{} - fds = []uintptr{} - ) - - for _, ns := range container.Namespaces { - namespaces = append(namespaces, namespaceFileMap[ns]) - } - - for _, ns := range namespaces { - fd, err := getNsFd(container.NsPid, ns) - if err != nil { - for _, fd = range fds { - syscall.Close(int(fd)) - } - return nil, err - } - fds = append(fds, fd) - } - return fds, nil -} - // getNsFd returns the fd for a specific pid and namespace option func getNsFd(pid int, ns string) (uintptr, error) { nspath := filepath.Join("/proc", strconv.Itoa(pid), "ns", ns) From 05ed3c19b6329a06dad19d5bc3b14fc9fb1b5c4b Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 14:55:34 -0800 Subject: [PATCH 189/403] Simplify namespaces with only nsinit Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 7bc3c012507edcfc5e8ab8523b240ac2bb03fe19 Component: engine --- .../engine/pkg/libcontainer/cli/main.go | 203 ------------------ .../pkg/libcontainer/namespaces/utils.go | 48 ----- .../{namespaces => nsinit}/exec.go | 26 ++- .../{namespaces => }/nsinit/init.go | 13 +- .../{namespaces => }/nsinit/mount.go | 0 .../{namespaces => nsinit}/ns_linux.go | 11 +- 6 files changed, 23 insertions(+), 278 deletions(-) delete mode 100644 components/engine/pkg/libcontainer/cli/main.go delete mode 100644 components/engine/pkg/libcontainer/namespaces/utils.go rename components/engine/pkg/libcontainer/{namespaces => nsinit}/exec.go (63%) rename components/engine/pkg/libcontainer/{namespaces => }/nsinit/init.go (93%) rename components/engine/pkg/libcontainer/{namespaces => }/nsinit/mount.go (100%) rename components/engine/pkg/libcontainer/{namespaces => nsinit}/ns_linux.go (74%) diff --git a/components/engine/pkg/libcontainer/cli/main.go b/components/engine/pkg/libcontainer/cli/main.go deleted file mode 100644 index 93bb0399f0..0000000000 --- a/components/engine/pkg/libcontainer/cli/main.go +++ /dev/null @@ -1,203 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "github.com/dotcloud/docker/pkg/libcontainer" - "github.com/dotcloud/docker/pkg/libcontainer/namespaces" - "github.com/dotcloud/docker/pkg/libcontainer/namespaces/nsinit" - "github.com/dotcloud/docker/pkg/libcontainer/network" - "github.com/dotcloud/docker/pkg/libcontainer/utils" - "os" - exec_ "os/exec" - "path" - "path/filepath" -) - -var ( - displayPid bool - newCommand string - usrNet bool - masterFd int - console string -) - -func init() { - flag.BoolVar(&displayPid, "pid", false, "display the pid before waiting") - flag.StringVar(&newCommand, "cmd", "/bin/bash", "command to run in the existing namespace") - flag.BoolVar(&usrNet, "net", false, "user a net namespace") - flag.IntVar(&masterFd, "master", 0, "master fd") - flag.StringVar(&console, "console", "", "console path") - flag.Parse() -} - -func nsinitFunc(container *libcontainer.Container) error { - container.Master = uintptr(masterFd) - container.Console = console - container.LogFile = "/root/logs" - - return nsinit.InitNamespace(container) -} - -func exec(container *libcontainer.Container) error { - var ( - netFile *os.File - err error - ) - container.NetNsFd = 0 - - if usrNet { - netFile, err = os.Open("/root/nsroot/test") - if err != nil { - return err - } - container.NetNsFd = netFile.Fd() - } - - self, err := exec_.LookPath(os.Args[0]) - if err != nil { - return err - } - if output, err := exec_.Command("cp", self, path.Join(container.RootFs, ".nsinit")).CombinedOutput(); err != nil { - return fmt.Errorf("Error exec cp: %s, (%s)", err, output) - } else { - println(self, container.RootFs) - fmt.Printf("-----> %s\n", output) - } - println("----") - - pid, err := namespaces.ExecContainer(container) - if err != nil { - return fmt.Errorf("error exec container %s", err) - } - - if displayPid { - fmt.Println(pid) - } - - exitcode, err := utils.WaitOnPid(pid) - if err != nil { - return fmt.Errorf("error waiting on child %s", err) - } - fmt.Println(exitcode) - if usrNet { - netFile.Close() - if err := network.DeleteNetworkNamespace("/root/nsroot/test"); err != nil { - return err - } - } - os.Exit(exitcode) - return nil -} - -func execIn(container *libcontainer.Container) error { - // f, err := os.Open("/root/nsroot/test") - // if err != nil { - // return err - // } - // container.NetNsFd = f.Fd() - // pid, err := namespaces.ExecIn(container, &libcontainer.Command{ - // Env: container.Command.Env, - // Args: []string{ - // newCommand, - // }, - // }) - // if err != nil { - // return fmt.Errorf("error exexin container %s", err) - // } - // exitcode, err := utils.WaitOnPid(pid) - // if err != nil { - // return fmt.Errorf("error waiting on child %s", err) - // } - // os.Exit(exitcode) - return nil -} - -func createNet(config *libcontainer.Network) error { - /* - root := "/root/nsroot" - if err := network.SetupNamespaceMountDir(root); err != nil { - return err - } - - nspath := root + "/test" - if err := network.CreateNetworkNamespace(nspath); err != nil { - return nil - } - if err := network.CreateVethPair("veth0", config.TempVethName); err != nil { - return err - } - if err := network.SetInterfaceMaster("veth0", config.Bridge); err != nil { - return err - } - if err := network.InterfaceUp("veth0"); err != nil { - return err - } - - f, err := os.Open(nspath) - if err != nil { - return err - } - defer f.Close() - - if err := network.SetInterfaceInNamespaceFd("veth1", int(f.Fd())); err != nil { - return err - } - - if err := network.SetupVethInsideNamespace(f.Fd(), config); err != nil { - return err - } - */ - return nil -} - -func printErr(err error) { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) -} - -func main() { - cliCmd := flag.Arg(0) - - config, err := filepath.Abs(flag.Arg(1)) - if err != nil { - printErr(err) - } - println("cli:", cliCmd, "config:", config) - f, err := os.Open(config) - if err != nil { - printErr(err) - } - - dec := json.NewDecoder(f) - var container *libcontainer.Container - - if err := dec.Decode(&container); err != nil { - printErr(err) - } - f.Close() - - switch cliCmd { - case "init": - err = nsinitFunc(container) - case "exec": - err = exec(container) - case "execin": - err = execIn(container) - case "net": - err = createNet(&libcontainer.Network{ - TempVethName: "veth1", - IP: "172.17.0.100/16", - Gateway: "172.17.42.1", - Mtu: 1500, - Bridge: "docker0", - }) - default: - err = fmt.Errorf("command not supported: %s", cliCmd) - } - - if err != nil { - printErr(err) - } -} diff --git a/components/engine/pkg/libcontainer/namespaces/utils.go b/components/engine/pkg/libcontainer/namespaces/utils.go deleted file mode 100644 index edc3ab52cc..0000000000 --- a/components/engine/pkg/libcontainer/namespaces/utils.go +++ /dev/null @@ -1,48 +0,0 @@ -package namespaces - -import ( - "fmt" - "github.com/dotcloud/docker/pkg/libcontainer" - "os" - "path/filepath" - "strconv" - "strings" -) - -func addEnvIfNotSet(container *libcontainer.Container, key, value string) { - jv := fmt.Sprintf("%s=%s", key, value) - if len(container.Command.Env) == 0 { - container.Command.Env = []string{jv} - return - } - - for _, v := range container.Command.Env { - parts := strings.Split(v, "=") - if parts[0] == key { - return - } - } - container.Command.Env = append(container.Command.Env, jv) -} - -// getNsFd returns the fd for a specific pid and namespace option -func getNsFd(pid int, ns string) (uintptr, error) { - nspath := filepath.Join("/proc", strconv.Itoa(pid), "ns", ns) - // OpenFile adds closOnExec - f, err := os.OpenFile(nspath, os.O_RDONLY, 0666) - if err != nil { - return 0, err - } - return f.Fd(), nil -} - -// setupEnvironment adds additional environment variables to the container's -// Command such as USER, LOGNAME, container, and TERM -func setupEnvironment(container *libcontainer.Container) { - addEnvIfNotSet(container, "container", "docker") - // TODO: check if pty - addEnvIfNotSet(container, "TERM", "xterm") - // TODO: get username from container - addEnvIfNotSet(container, "USER", "root") - addEnvIfNotSet(container, "LOGNAME", "root") -} diff --git a/components/engine/pkg/libcontainer/namespaces/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go similarity index 63% rename from components/engine/pkg/libcontainer/namespaces/exec.go rename to components/engine/pkg/libcontainer/nsinit/exec.go index 8e5bf68aef..ef81b0ef87 100644 --- a/components/engine/pkg/libcontainer/namespaces/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -1,4 +1,4 @@ -package namespaces +package main import ( "github.com/dotcloud/docker/pkg/libcontainer" @@ -11,15 +11,7 @@ import ( "syscall" ) -// ExecContainer will spawn new namespaces with the specified Container configuration -// in the RootFs path and return the pid of the new containerized process. -// -// If an existing network namespace is specified the container -// will join that namespace. If an existing network namespace is not specified but CLONE_NEWNET is, -// the container will be spawned with a new network namespace with no configuration. Omiting an -// existing network namespace and the CLONE_NEWNET option in the container configuration will allow -// the container to the the host's networking options and configuration. -func ExecContainer(container *libcontainer.Container) (pid int, err error) { +func execCommand(container *libcontainer.Container) (pid int, err error) { master, console, err := createMasterAndConsole() if err != nil { return -1, err @@ -50,7 +42,19 @@ func ExecContainer(container *libcontainer.Container) (pid int, err error) { } }() - term.SetRawTerminal(os.Stdin.Fd()) + ws, err := term.GetWinsize(os.Stdin.Fd()) + if err != nil { + return -1, err + } + if err := term.SetWinsize(master.Fd(), ws); err != nil { + return -1, err + } + state, err := term.SetRawTerminal(os.Stdin.Fd()) + if err != nil { + command.Process.Kill() + return -1, err + } + defer term.RestoreTerminal(os.Stdin.Fd(), state) if err := command.Wait(); err != nil { return pid, err diff --git a/components/engine/pkg/libcontainer/namespaces/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go similarity index 93% rename from components/engine/pkg/libcontainer/namespaces/nsinit/init.go rename to components/engine/pkg/libcontainer/nsinit/init.go index 523854e5d6..b4b7de410c 100644 --- a/components/engine/pkg/libcontainer/namespaces/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/capabilities" - "github.com/dotcloud/docker/pkg/libcontainer/namespaces" "github.com/dotcloud/docker/pkg/system" "log" "os" @@ -34,7 +33,7 @@ func main() { } if os.Args[1] == "exec" { - _, err := namespaces.ExecContainer(container) + _, err := execCommand(container) if err != nil { log.Fatal(err) } @@ -157,11 +156,13 @@ func openTerminal(name string, flag int) (*os.File, error) { } func setLogFile(container *libcontainer.Container) error { - f, err := os.OpenFile(container.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0655) - if err != nil { - return err + if container.LogFile != "" { + f, err := os.OpenFile(container.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0655) + if err != nil { + return err + } + log.SetOutput(f) } - log.SetOutput(f) return nil } diff --git a/components/engine/pkg/libcontainer/namespaces/nsinit/mount.go b/components/engine/pkg/libcontainer/nsinit/mount.go similarity index 100% rename from components/engine/pkg/libcontainer/namespaces/nsinit/mount.go rename to components/engine/pkg/libcontainer/nsinit/mount.go diff --git a/components/engine/pkg/libcontainer/namespaces/ns_linux.go b/components/engine/pkg/libcontainer/nsinit/ns_linux.go similarity index 74% rename from components/engine/pkg/libcontainer/namespaces/ns_linux.go rename to components/engine/pkg/libcontainer/nsinit/ns_linux.go index 2c73e08e58..b54bc2b993 100644 --- a/components/engine/pkg/libcontainer/namespaces/ns_linux.go +++ b/components/engine/pkg/libcontainer/nsinit/ns_linux.go @@ -1,4 +1,4 @@ -package namespaces +package main import ( "github.com/dotcloud/docker/pkg/libcontainer" @@ -25,15 +25,6 @@ var namespaceMap = map[libcontainer.Namespace]int{ libcontainer.CLONE_NEWNET: CLONE_NEWNET, } -var namespaceFileMap = map[libcontainer.Namespace]string{ - libcontainer.CLONE_NEWNS: "mnt", - libcontainer.CLONE_NEWUTS: "uts", - libcontainer.CLONE_NEWIPC: "ipc", - libcontainer.CLONE_NEWUSER: "user", - libcontainer.CLONE_NEWPID: "pid", - libcontainer.CLONE_NEWNET: "net", -} - // getNamespaceFlags parses the container's Namespaces options to set the correct // flags on clone, unshare, and setns func getNamespaceFlags(namespaces libcontainer.Namespaces) (flag int) { From 3f67fdb8edbda0a56c2350264383422ab3df8182 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 15:33:44 -0800 Subject: [PATCH 190/403] Implement init veth creation Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 34671f20103fb975fed31a03705e04bc65aed239 Component: engine --- .../engine/pkg/libcontainer/container.go | 9 ++--- .../engine/pkg/libcontainer/container.json | 14 +++++-- .../engine/pkg/libcontainer/network/veth.go | 38 +++---------------- .../engine/pkg/libcontainer/nsinit/exec.go | 33 ++++++++++++++++ .../engine/pkg/libcontainer/nsinit/init.go | 14 +++++-- .../engine/pkg/libcontainer/ubuntu.json | 22 ----------- 6 files changed, 63 insertions(+), 67 deletions(-) delete mode 100644 components/engine/pkg/libcontainer/ubuntu.json diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index c2885447fd..3f3961d496 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -18,9 +18,8 @@ type Command struct { } type Network struct { - IP string `json:"ip,omitempty"` - Gateway string `json:"gateway,omitempty"` - Bridge string `json:"bridge,omitempty"` - Mtu int `json:"mtu,omitempty"` - TempVethName string `json:"temp_veth,omitempty"` + IP string `json:"ip,omitempty"` + Gateway string `json:"gateway,omitempty"` + Bridge string `json:"bridge,omitempty"` + Mtu int `json:"mtu,omitempty"` } diff --git a/components/engine/pkg/libcontainer/container.json b/components/engine/pkg/libcontainer/container.json index 6e4fda54c8..8731170c2a 100644 --- a/components/engine/pkg/libcontainer/container.json +++ b/components/engine/pkg/libcontainer/container.json @@ -1,6 +1,6 @@ { "id": "koye", - "namespace_pid": 3117, + "log_file": "/root/logs", "command": { "args": [ "/bin/bash" @@ -12,12 +12,12 @@ "TERM=xterm" ] }, - "rootfs": "/var/lib/docker/containers/ee76122136d691d63e09d24168a91ddb2ef9fdcf210b4de5c50aa76354892f4b/root", "namespaces": [ "NEWIPC", "NEWNS", "NEWPID", - "NEWUTS" + "NEWUTS", + "NEWNET" ], "capabilities": [ "SETPCAP", @@ -34,5 +34,11 @@ "AUDIT_CONTROL", "MAC_OVERRIDE", "MAC_ADMIN" - ] + ], + "network": { + "ip": "172.17.0.100/16", + "gateway": "172.17.42.1", + "bridge": "docker0", + "mtu": 1500 + } } diff --git a/components/engine/pkg/libcontainer/network/veth.go b/components/engine/pkg/libcontainer/network/veth.go index 2ecce22c3e..05512e63c8 100644 --- a/components/engine/pkg/libcontainer/network/veth.go +++ b/components/engine/pkg/libcontainer/network/veth.go @@ -3,18 +3,16 @@ package network import ( "fmt" "github.com/dotcloud/docker/pkg/libcontainer" - "os" - "syscall" ) // SetupVeth sets up an existing network namespace with the specified // network configuration. -func SetupVeth(config *libcontainer.Network) error { - if err := InterfaceDown(config.TempVethName); err != nil { - return fmt.Errorf("interface down %s %s", config.TempVethName, err) +func SetupVeth(config *libcontainer.Network, tempVethName string) error { + if err := InterfaceDown(tempVethName); err != nil { + return fmt.Errorf("interface down %s %s", tempVethName, err) } - if err := ChangeInterfaceName(config.TempVethName, "eth0"); err != nil { - return fmt.Errorf("change %s to eth0 %s", config.TempVethName, err) + if err := ChangeInterfaceName(tempVethName, "eth0"); err != nil { + return fmt.Errorf("change %s to eth0 %s", tempVethName, err) } if err := SetInterfaceIp("eth0", config.IP); err != nil { return fmt.Errorf("set eth0 ip %s", err) @@ -41,29 +39,3 @@ func SetupVeth(config *libcontainer.Network) error { } return nil } - -// SetupNamespaceMountDir prepares a new root for use as a mount -// source for bind mounting namespace fd to an outside path -func SetupNamespaceMountDir(root string) error { - if err := os.MkdirAll(root, 0666); err != nil { - return err - } - // make sure mounts are not unmounted by other mnt namespaces - if err := syscall.Mount("", root, "none", syscall.MS_SHARED|syscall.MS_REC, ""); err != nil && err != syscall.EINVAL { - return err - } - if err := syscall.Mount(root, root, "none", syscall.MS_BIND, ""); err != nil { - return err - } - return nil -} - -// DeleteNetworkNamespace unmounts the binding path and removes the -// file so that no references to the fd are present and the network -// namespace is automatically cleaned up -func DeleteNetworkNamespace(bindingPath string) error { - if err := syscall.Unmount(bindingPath, 0); err != nil { - return err - } - return os.Remove(bindingPath) -} diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index ef81b0ef87..9cd1741706 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -1,7 +1,9 @@ package main import ( + "fmt" "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/libcontainer/network" "github.com/dotcloud/docker/pkg/system" "github.com/dotcloud/docker/pkg/term" "io" @@ -25,11 +27,34 @@ func execCommand(container *libcontainer.Container) (pid int, err error) { Cloneflags: flag, } + inPipe, err := command.StdinPipe() + if err != nil { + return -1, err + } + if err := command.Start(); err != nil { return -1, err } pid = command.Process.Pid + if container.Network != nil { + name1, name2, err := createVethPair() + if err != nil { + log.Fatal(err) + } + if err := network.SetInterfaceMaster(name1, container.Network.Bridge); err != nil { + log.Fatal(err) + } + if err := network.InterfaceUp(name1); err != nil { + log.Fatal(err) + } + if err := network.SetInterfaceInNamespacePid(name2, pid); err != nil { + log.Fatal(err) + } + fmt.Fprint(inPipe, name2) + inPipe.Close() + } + go func() { if _, err := io.Copy(os.Stdout, master); err != nil { log.Println(err) @@ -78,3 +103,11 @@ func createMasterAndConsole() (*os.File, string, error) { } return master, console, nil } + +func createVethPair() (name1 string, name2 string, err error) { + name1, name2 = "veth001", "veth002" + if err = network.CreateVethPair(name1, name2); err != nil { + return + } + return +} diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index b4b7de410c..2804f01e5f 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -5,7 +5,9 @@ import ( "fmt" "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/capabilities" + "github.com/dotcloud/docker/pkg/libcontainer/network" "github.com/dotcloud/docker/pkg/system" + "io/ioutil" "log" "os" "path/filepath" @@ -50,6 +52,12 @@ func main() { log.Fatal(err) } + data, err := ioutil.ReadAll(os.Stdin) + if err != nil { + log.Fatalf("error reading from stdin %s", err) + } + tempVethName := string(data) + // close pipes so that we can replace it with the pty os.Stdin.Close() os.Stdout.Close() @@ -81,7 +89,7 @@ func main() { } if container.Network != nil { - if err := setupNetworking(container); err != nil { + if err := setupNetworking(container, tempVethName); err != nil { log.Fatalf("setup networking %s", err) } } @@ -166,6 +174,6 @@ func setLogFile(container *libcontainer.Container) error { return nil } -func setupNetworking(conatiner *libcontainer.Container) error { - return nil +func setupNetworking(container *libcontainer.Container, tempVethName string) error { + return network.SetupVeth(container.Network, tempVethName) } diff --git a/components/engine/pkg/libcontainer/ubuntu.json b/components/engine/pkg/libcontainer/ubuntu.json deleted file mode 100644 index 0a450ae066..0000000000 --- a/components/engine/pkg/libcontainer/ubuntu.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "id": "koye", - "namespace_pid": 3745, - "command": { - "args": [ - "/sbin/init" - ], - "environment": [ - "HOME=/", - "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", - "container=docker", - "TERM=xterm" - ] - }, - "rootfs": "/var/lib/docker/btrfs/subvolumes/7c0f15df1ad2e2fe04d7a6e079aec17406e9465a6a37dd16cb0dd754fc0167b3", - "namespaces": [ - "NEWIPC", - "NEWNS", - "NEWPID", - "NEWUTS" - ] -} From 10d46be0eadf313eb59c1e0dba33bae9f4952fc9 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 15:54:53 -0800 Subject: [PATCH 191/403] Add dynamic veth name Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 5428964400ece4cd79cc5d482307df5e8913469f Component: engine --- .../engine/pkg/libcontainer/network/veth.go | 41 ----------------- .../engine/pkg/libcontainer/nsinit/exec.go | 10 +++- .../engine/pkg/libcontainer/nsinit/init.go | 46 +++++++++++++++---- .../engine/pkg/libcontainer/utils/utils.go | 24 ++-------- 4 files changed, 48 insertions(+), 73 deletions(-) delete mode 100644 components/engine/pkg/libcontainer/network/veth.go diff --git a/components/engine/pkg/libcontainer/network/veth.go b/components/engine/pkg/libcontainer/network/veth.go deleted file mode 100644 index 05512e63c8..0000000000 --- a/components/engine/pkg/libcontainer/network/veth.go +++ /dev/null @@ -1,41 +0,0 @@ -package network - -import ( - "fmt" - "github.com/dotcloud/docker/pkg/libcontainer" -) - -// SetupVeth sets up an existing network namespace with the specified -// network configuration. -func SetupVeth(config *libcontainer.Network, tempVethName string) error { - if err := InterfaceDown(tempVethName); err != nil { - return fmt.Errorf("interface down %s %s", tempVethName, err) - } - if err := ChangeInterfaceName(tempVethName, "eth0"); err != nil { - return fmt.Errorf("change %s to eth0 %s", tempVethName, err) - } - if err := SetInterfaceIp("eth0", config.IP); err != nil { - return fmt.Errorf("set eth0 ip %s", err) - } - - if err := SetMtu("eth0", config.Mtu); err != nil { - return fmt.Errorf("set eth0 mtu to %d %s", config.Mtu, err) - } - if err := InterfaceUp("eth0"); err != nil { - return fmt.Errorf("eth0 up %s", err) - } - - if err := SetMtu("lo", config.Mtu); err != nil { - return fmt.Errorf("set lo mtu to %d %s", config.Mtu, err) - } - if err := InterfaceUp("lo"); err != nil { - return fmt.Errorf("lo up %s", err) - } - - if config.Gateway != "" { - if err := SetDefaultGateway(config.Gateway); err != nil { - return fmt.Errorf("set gateway to %s %s", config.Gateway, err) - } - } - return nil -} diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 9cd1741706..e0324074c5 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/network" + "github.com/dotcloud/docker/pkg/libcontainer/utils" "github.com/dotcloud/docker/pkg/system" "github.com/dotcloud/docker/pkg/term" "io" @@ -105,7 +106,14 @@ func createMasterAndConsole() (*os.File, string, error) { } func createVethPair() (name1 string, name2 string, err error) { - name1, name2 = "veth001", "veth002" + name1, err = utils.GenerateRandomName("dock", 4) + if err != nil { + return + } + name2, err = utils.GenerateRandomName("dock", 4) + if err != nil { + return + } if err = network.CreateVethPair(name1, name2); err != nil { return } diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index 2804f01e5f..fe8fd4b4db 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -52,11 +52,14 @@ func main() { log.Fatal(err) } - data, err := ioutil.ReadAll(os.Stdin) - if err != nil { - log.Fatalf("error reading from stdin %s", err) + var tempVethName string + if container.Network != nil { + data, err := ioutil.ReadAll(os.Stdin) + if err != nil { + log.Fatalf("error reading from stdin %s", err) + } + tempVethName = string(data) } - tempVethName := string(data) // close pipes so that we can replace it with the pty os.Stdin.Close() @@ -73,7 +76,6 @@ func main() { if err := dupSlave(slave); err != nil { log.Fatalf("dup2 slave %s", err) } - if _, err := system.Setsid(); err != nil { log.Fatalf("setsid %s", err) } @@ -83,13 +85,11 @@ func main() { if err := system.ParentDeathSignal(); err != nil { log.Fatalf("parent deth signal %s", err) } - if err := setupNewMountNamespace(rootfs, console, container.ReadonlyFs); err != nil { log.Fatalf("setup mount namespace %s", err) } - if container.Network != nil { - if err := setupNetworking(container, tempVethName); err != nil { + if err := setupNetworking(container.Network, tempVethName); err != nil { log.Fatalf("setup networking %s", err) } } @@ -174,6 +174,32 @@ func setLogFile(container *libcontainer.Container) error { return nil } -func setupNetworking(container *libcontainer.Container, tempVethName string) error { - return network.SetupVeth(container.Network, tempVethName) +func setupNetworking(config *libcontainer.Network, tempVethName string) error { + if err := network.InterfaceDown(tempVethName); err != nil { + return fmt.Errorf("interface down %s %s", tempVethName, err) + } + if err := network.ChangeInterfaceName(tempVethName, "eth0"); err != nil { + return fmt.Errorf("change %s to eth0 %s", tempVethName, err) + } + if err := network.SetInterfaceIp("eth0", config.IP); err != nil { + return fmt.Errorf("set eth0 ip %s", err) + } + if err := network.SetMtu("eth0", config.Mtu); err != nil { + return fmt.Errorf("set eth0 mtu to %d %s", config.Mtu, err) + } + if err := network.InterfaceUp("eth0"); err != nil { + return fmt.Errorf("eth0 up %s", err) + } + if err := network.SetMtu("lo", config.Mtu); err != nil { + return fmt.Errorf("set lo mtu to %d %s", config.Mtu, err) + } + if err := network.InterfaceUp("lo"); err != nil { + return fmt.Errorf("lo up %s", err) + } + if config.Gateway != "" { + if err := network.SetDefaultGateway(config.Gateway); err != nil { + return fmt.Errorf("set gateway to %s %s", config.Gateway, err) + } + } + return nil } diff --git a/components/engine/pkg/libcontainer/utils/utils.go b/components/engine/pkg/libcontainer/utils/utils.go index 7289fecf2e..d3223c3e4d 100644 --- a/components/engine/pkg/libcontainer/utils/utils.go +++ b/components/engine/pkg/libcontainer/utils/utils.go @@ -4,30 +4,12 @@ import ( "crypto/rand" "encoding/hex" "io" - "os" - "syscall" ) -func WaitOnPid(pid int) (exitcode int, err error) { - child, err := os.FindProcess(pid) - if err != nil { - return -1, err - } - state, err := child.Wait() - if err != nil { - return -1, err - } - return getExitCode(state), nil -} - -func getExitCode(state *os.ProcessState) int { - return state.Sys().(syscall.WaitStatus).ExitStatus() -} - -func GenerateRandomName(size int) (string, error) { - id := make([]byte, size) +func GenerateRandomName(prefix string, size int) (string, error) { + id := make([]byte, 32) if _, err := io.ReadFull(rand.Reader, id); err != nil { return "", err } - return hex.EncodeToString(id), nil + return prefix + hex.EncodeToString(id)[:size], nil } From 3c99ad70330fba581ecb2880e1fb18d57c4854af Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 16:40:36 -0800 Subject: [PATCH 192/403] General cleanup of libcontainer Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 61a119220d88d20bb1cca111e9c8ba7cdb45d4f6 Component: engine --- components/engine/pkg/libcontainer/errors.go | 9 --- .../pkg/libcontainer/network/network.go | 26 -------- .../engine/pkg/libcontainer/nsinit/exec.go | 51 +++++++-------- .../engine/pkg/libcontainer/nsinit/init.go | 63 +++++-------------- .../engine/pkg/libcontainer/nsinit/main.go | 42 +++++++++++++ .../engine/pkg/libcontainer/nsinit/mount.go | 61 +++++++----------- .../pkg/libcontainer/nsinit/ns_linux.go | 25 +++----- 7 files changed, 111 insertions(+), 166 deletions(-) delete mode 100644 components/engine/pkg/libcontainer/errors.go create mode 100644 components/engine/pkg/libcontainer/nsinit/main.go diff --git a/components/engine/pkg/libcontainer/errors.go b/components/engine/pkg/libcontainer/errors.go deleted file mode 100644 index c6964ee8e6..0000000000 --- a/components/engine/pkg/libcontainer/errors.go +++ /dev/null @@ -1,9 +0,0 @@ -package libcontainer - -import ( - "errors" -) - -var ( - ErrInvalidPid = errors.New("no ns pid found") -) diff --git a/components/engine/pkg/libcontainer/network/network.go b/components/engine/pkg/libcontainer/network/network.go index 31c5d32492..8c7a4b618e 100644 --- a/components/engine/pkg/libcontainer/network/network.go +++ b/components/engine/pkg/libcontainer/network/network.go @@ -1,15 +1,10 @@ package network import ( - "errors" "github.com/dotcloud/docker/pkg/netlink" "net" ) -var ( - ErrNoDefaultRoute = errors.New("no default network route found") -) - func InterfaceUp(name string) error { iface, err := net.InterfaceByName(name) if err != nil { @@ -46,14 +41,6 @@ func SetInterfaceInNamespacePid(name string, nsPid int) error { return netlink.NetworkSetNsPid(iface, nsPid) } -func SetInterfaceInNamespaceFd(name string, fd int) error { - iface, err := net.InterfaceByName(name) - if err != nil { - return err - } - return netlink.NetworkSetNsFd(iface, fd) -} - func SetInterfaceMaster(name, master string) error { iface, err := net.InterfaceByName(name) if err != nil { @@ -89,16 +76,3 @@ func SetMtu(name string, mtu int) error { } return netlink.NetworkSetMTU(iface, mtu) } - -func GetDefaultMtu() (int, error) { - routes, err := netlink.NetworkGetRoutes() - if err != nil { - return -1, err - } - for _, r := range routes { - if r.Default { - return r.Iface.MTU, nil - } - } - return -1, ErrNoDefaultRoute -} diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index e0324074c5..4ac070db08 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -8,65 +8,54 @@ import ( "github.com/dotcloud/docker/pkg/system" "github.com/dotcloud/docker/pkg/term" "io" - "log" + "io/ioutil" "os" "os/exec" "syscall" ) -func execCommand(container *libcontainer.Container) (pid int, err error) { +func execCommand(container *libcontainer.Container) (int, error) { master, console, err := createMasterAndConsole() if err != nil { return -1, err } - // we need CLONE_VFORK so we can wait on the child - flag := uintptr(getNamespaceFlags(container.Namespaces) | CLONE_VFORK) - - command := exec.Command("nsinit", console) + command := exec.Command("nsinit", "init", console) command.SysProcAttr = &syscall.SysProcAttr{ - Cloneflags: flag, + Cloneflags: uintptr(getNamespaceFlags(container.Namespaces) | syscall.CLONE_VFORK), // we need CLONE_VFORK so we can wait on the child } inPipe, err := command.StdinPipe() if err != nil { return -1, err } - if err := command.Start(); err != nil { return -1, err } - pid = command.Process.Pid + if err := writePidFile(command); err != nil { + return -1, err + } if container.Network != nil { name1, name2, err := createVethPair() if err != nil { - log.Fatal(err) + return -1, err } if err := network.SetInterfaceMaster(name1, container.Network.Bridge); err != nil { - log.Fatal(err) + return -1, err } if err := network.InterfaceUp(name1); err != nil { - log.Fatal(err) + return -1, err } - if err := network.SetInterfaceInNamespacePid(name2, pid); err != nil { - log.Fatal(err) + if err := network.SetInterfaceInNamespacePid(name2, command.Process.Pid); err != nil { + return -1, err } fmt.Fprint(inPipe, name2) inPipe.Close() } - go func() { - if _, err := io.Copy(os.Stdout, master); err != nil { - log.Println(err) - } - }() - - go func() { - if _, err := io.Copy(master, os.Stdin); err != nil { - log.Println(err) - } - }() + go io.Copy(os.Stdout, master) + go io.Copy(master, os.Stdin) ws, err := term.GetWinsize(os.Stdin.Fd()) if err != nil { @@ -83,9 +72,11 @@ func execCommand(container *libcontainer.Container) (pid int, err error) { defer term.RestoreTerminal(os.Stdin.Fd(), state) if err := command.Wait(); err != nil { - return pid, err + if _, ok := err.(*exec.ExitError); !ok { + return -1, err + } } - return pid, nil + return command.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil } func createMasterAndConsole() (*os.File, string, error) { @@ -93,12 +84,10 @@ func createMasterAndConsole() (*os.File, string, error) { if err != nil { return nil, "", err } - console, err := system.Ptsname(master) if err != nil { return nil, "", err } - if err := system.Unlockpt(master); err != nil { return nil, "", err } @@ -119,3 +108,7 @@ func createVethPair() (name1 string, name2 string, err error) { } return } + +func writePidFile(command *exec.Cmd) error { + return ioutil.WriteFile(".nspid", []byte(fmt.Sprint(command.Process.Pid)), 0655) +} diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index fe8fd4b4db..16a30812f9 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "fmt" "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/capabilities" @@ -14,49 +13,21 @@ import ( "syscall" ) -func loadContainer() (*libcontainer.Container, error) { - f, err := os.Open("container.json") - if err != nil { - return nil, err - } - defer f.Close() - - var container *libcontainer.Container - if err := json.NewDecoder(f).Decode(&container); err != nil { - return nil, err - } - return container, nil -} - -func main() { - container, err := loadContainer() - if err != nil { - log.Fatal(err) - } - - if os.Args[1] == "exec" { - _, err := execCommand(container) - if err != nil { - log.Fatal(err) - } - os.Exit(0) - } - console := os.Args[1] - +func initCommand(container *libcontainer.Container, console string) error { if err := setLogFile(container); err != nil { - log.Fatal(err) + return err } rootfs, err := resolveRootfs() if err != nil { - log.Fatal(err) + return err } var tempVethName string if container.Network != nil { data, err := ioutil.ReadAll(os.Stdin) if err != nil { - log.Fatalf("error reading from stdin %s", err) + return fmt.Errorf("error reading from stdin %s", err) } tempVethName = string(data) } @@ -68,48 +39,48 @@ func main() { slave, err := openTerminal(console, syscall.O_RDWR) if err != nil { - log.Fatalf("open terminal %s", err) + return fmt.Errorf("open terminal %s", err) } if slave.Fd() != 0 { - log.Fatalf("slave fd should be 0") + return fmt.Errorf("slave fd should be 0") } if err := dupSlave(slave); err != nil { - log.Fatalf("dup2 slave %s", err) + return fmt.Errorf("dup2 slave %s", err) } if _, err := system.Setsid(); err != nil { - log.Fatalf("setsid %s", err) + return fmt.Errorf("setsid %s", err) } if err := system.Setctty(); err != nil { - log.Fatalf("setctty %s", err) + return fmt.Errorf("setctty %s", err) } if err := system.ParentDeathSignal(); err != nil { - log.Fatalf("parent deth signal %s", err) + return fmt.Errorf("parent deth signal %s", err) } if err := setupNewMountNamespace(rootfs, console, container.ReadonlyFs); err != nil { - log.Fatalf("setup mount namespace %s", err) + return fmt.Errorf("setup mount namespace %s", err) } if container.Network != nil { if err := setupNetworking(container.Network, tempVethName); err != nil { - log.Fatalf("setup networking %s", err) + return fmt.Errorf("setup networking %s", err) } } if err := system.Sethostname(container.ID); err != nil { - log.Fatalf("sethostname %s", err) + return fmt.Errorf("sethostname %s", err) } if err := capabilities.DropCapabilities(container); err != nil { - log.Fatalf("drop capabilities %s", err) + return fmt.Errorf("drop capabilities %s", err) } if err := setupUser(container); err != nil { - log.Fatalf("setup user %s", err) + return fmt.Errorf("setup user %s", err) } if container.WorkingDir != "" { if err := system.Chdir(container.WorkingDir); err != nil { - log.Fatalf("chdir to %s %s", container.WorkingDir, err) + return fmt.Errorf("chdir to %s %s", container.WorkingDir, err) } } if err := system.Exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { - log.Fatalf("exec %s", err) + return fmt.Errorf("exec %s", err) } panic("unreachable") } diff --git a/components/engine/pkg/libcontainer/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/main.go new file mode 100644 index 0000000000..47abcce0c5 --- /dev/null +++ b/components/engine/pkg/libcontainer/nsinit/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "encoding/json" + "github.com/dotcloud/docker/pkg/libcontainer" + "log" + "os" +) + +func main() { + container, err := loadContainer() + if err != nil { + log.Fatal(err) + } + + switch os.Args[1] { + case "exec": + exitCode, err := execCommand(container) + if err != nil { + log.Fatal(err) + } + os.Exit(exitCode) + case "init": + if err := initCommand(container, os.Args[2]); err != nil { + log.Fatal(err) + } + } +} + +func loadContainer() (*libcontainer.Container, error) { + f, err := os.Open("container.json") + if err != nil { + return nil, err + } + defer f.Close() + + var container *libcontainer.Container + if err := json.NewDecoder(f).Decode(&container); err != nil { + return nil, err + } + return container, nil +} diff --git a/components/engine/pkg/libcontainer/nsinit/mount.go b/components/engine/pkg/libcontainer/nsinit/mount.go index f9ee969636..13ee13e001 100644 --- a/components/engine/pkg/libcontainer/nsinit/mount.go +++ b/components/engine/pkg/libcontainer/nsinit/mount.go @@ -3,68 +3,47 @@ package main import ( "fmt" "github.com/dotcloud/docker/pkg/system" - "log" "os" "path/filepath" "syscall" ) -var ( - // default mount point options - defaults = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV -) +// default mount point options +const defaultMountFlags = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV func setupNewMountNamespace(rootfs, console string, readonly bool) error { if err := system.Mount("", "/", "", syscall.MS_SLAVE|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mounting / as slave %s", err) } - if err := system.Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mouting %s as bind %s", rootfs, err) } - if readonly { if err := system.Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mounting %s as readonly %s", rootfs, err) } } - if err := mountSystem(rootfs); err != nil { return fmt.Errorf("mount system %s", err) } - if err := copyDevNodes(rootfs); err != nil { return fmt.Errorf("copy dev nodes %s", err) } - - ptmx := filepath.Join(rootfs, "dev/ptmx") - if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) { - return err - } - if err := os.Symlink("pts/ptmx", ptmx); err != nil { - return fmt.Errorf("symlink dev ptmx %s", err) - } - if err := setupDev(rootfs); err != nil { return err } - - if err := setupConsole(rootfs, console); err != nil { + if err := setupPtmx(rootfs, console); err != nil { return err } - if err := system.Chdir(rootfs); err != nil { return fmt.Errorf("chdir into %s %s", rootfs, err) } - if err := system.Mount(rootfs, "/", "", syscall.MS_MOVE, ""); err != nil { return fmt.Errorf("mount move %s into / %s", rootfs, err) } - if err := system.Chroot("."); err != nil { return fmt.Errorf("chroot . %s", err) } - if err := system.Chdir("/"); err != nil { return fmt.Errorf("chdir / %s", err) } @@ -90,13 +69,10 @@ func copyDevNodes(rootfs string) error { if err != nil { return err } - var ( dest = filepath.Join(rootfs, "dev", node) st = stat.Sys().(*syscall.Stat_t) ) - - log.Printf("copy %s to %s %d\n", node, dest, st.Rdev) if err := system.Mknod(dest, st.Mode, int(st.Rdev)); err != nil && !os.IsExist(err) { return fmt.Errorf("copy %s %s", node, err) } @@ -134,24 +110,22 @@ func setupConsole(rootfs, console string) error { if err != nil { return fmt.Errorf("stat console %s %s", console, err) } - st := stat.Sys().(*syscall.Stat_t) - - dest := filepath.Join(rootfs, "dev/console") + var ( + st = stat.Sys().(*syscall.Stat_t) + dest = filepath.Join(rootfs, "dev/console") + ) if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { return fmt.Errorf("remove %s %s", dest, err) } - if err := os.Chmod(console, 0600); err != nil { return err } if err := os.Chown(console, 0, 0); err != nil { return err } - if err := system.Mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil { return fmt.Errorf("mknod %s %s", dest, err) } - if err := system.Mount(console, dest, "bind", syscall.MS_BIND, ""); err != nil { return fmt.Errorf("bind %s to %s %s", console, dest, err) } @@ -168,10 +142,10 @@ func mountSystem(rootfs string) error { flags int data string }{ - {source: "proc", path: filepath.Join(rootfs, "proc"), device: "proc", flags: defaults}, - {source: "sysfs", path: filepath.Join(rootfs, "sys"), device: "sysfs", flags: defaults}, + {source: "proc", path: filepath.Join(rootfs, "proc"), device: "proc", flags: defaultMountFlags}, + {source: "sysfs", path: filepath.Join(rootfs, "sys"), device: "sysfs", flags: defaultMountFlags}, {source: "tmpfs", path: filepath.Join(rootfs, "dev"), device: "tmpfs", flags: syscall.MS_NOSUID | syscall.MS_STRICTATIME, data: "mode=755"}, - {source: "shm", path: filepath.Join(rootfs, "dev", "shm"), device: "tmpfs", flags: defaults, data: "mode=1777"}, + {source: "shm", path: filepath.Join(rootfs, "dev", "shm"), device: "tmpfs", flags: defaultMountFlags, data: "mode=1777"}, {source: "devpts", path: filepath.Join(rootfs, "dev", "pts"), device: "devpts", flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, data: "newinstance,ptmxmode=0666,mode=620,gid=5"}, {source: "tmpfs", path: filepath.Join(rootfs, "run"), device: "tmpfs", flags: syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_STRICTATIME, data: "mode=755"}, } { @@ -189,7 +163,7 @@ func remountProc() error { if err := system.Unmount("/proc", syscall.MNT_DETACH); err != nil { return err } - if err := system.Mount("proc", "/proc", "proc", uintptr(defaults), ""); err != nil { + if err := system.Mount("proc", "/proc", "proc", uintptr(defaultMountFlags), ""); err != nil { return err } return nil @@ -201,9 +175,20 @@ func remountSys() error { return err } } else { - if err := system.Mount("sysfs", "/sys", "sysfs", uintptr(defaults), ""); err != nil { + if err := system.Mount("sysfs", "/sys", "sysfs", uintptr(defaultMountFlags), ""); err != nil { return err } } return nil } + +func setupPtmx(rootfs, console string) error { + ptmx := filepath.Join(rootfs, "dev/ptmx") + if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) { + return err + } + if err := os.Symlink("pts/ptmx", ptmx); err != nil { + return fmt.Errorf("symlink dev ptmx %s", err) + } + return setupConsole(rootfs, console) +} diff --git a/components/engine/pkg/libcontainer/nsinit/ns_linux.go b/components/engine/pkg/libcontainer/nsinit/ns_linux.go index b54bc2b993..2392ffd770 100644 --- a/components/engine/pkg/libcontainer/nsinit/ns_linux.go +++ b/components/engine/pkg/libcontainer/nsinit/ns_linux.go @@ -2,27 +2,16 @@ package main import ( "github.com/dotcloud/docker/pkg/libcontainer" -) - -const ( - SIGCHLD = 0x14 - CLONE_VFORK = 0x00004000 - CLONE_NEWNS = 0x00020000 - CLONE_NEWUTS = 0x04000000 - CLONE_NEWIPC = 0x08000000 - CLONE_NEWUSER = 0x10000000 - CLONE_NEWPID = 0x20000000 - CLONE_NEWNET = 0x40000000 + "syscall" ) var namespaceMap = map[libcontainer.Namespace]int{ - "": 0, - libcontainer.CLONE_NEWNS: CLONE_NEWNS, - libcontainer.CLONE_NEWUTS: CLONE_NEWUTS, - libcontainer.CLONE_NEWIPC: CLONE_NEWIPC, - libcontainer.CLONE_NEWUSER: CLONE_NEWUSER, - libcontainer.CLONE_NEWPID: CLONE_NEWPID, - libcontainer.CLONE_NEWNET: CLONE_NEWNET, + libcontainer.CLONE_NEWNS: syscall.CLONE_NEWNS, + libcontainer.CLONE_NEWUTS: syscall.CLONE_NEWUTS, + libcontainer.CLONE_NEWIPC: syscall.CLONE_NEWIPC, + libcontainer.CLONE_NEWUSER: syscall.CLONE_NEWUSER, + libcontainer.CLONE_NEWPID: syscall.CLONE_NEWPID, + libcontainer.CLONE_NEWNET: syscall.CLONE_NEWNET, } // getNamespaceFlags parses the container's Namespaces options to set the correct From 92c44382e1fd2b4b594eb0764459a393e80a169f Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Wed, 19 Feb 2014 16:50:10 -0800 Subject: [PATCH 193/403] OSX compilation Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: f3c48ec584707a9acaf9d79c2e161dadf1d2c99b Component: engine --- components/engine/pkg/libcontainer/nsinit/exec.go | 2 ++ components/engine/pkg/libcontainer/nsinit/init.go | 2 ++ components/engine/pkg/libcontainer/nsinit/main.go | 13 +++++++++++++ components/engine/pkg/libcontainer/nsinit/mount.go | 2 ++ 4 files changed, 19 insertions(+) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 4ac070db08..5b53be259c 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -1,3 +1,5 @@ +// +build linux + package main import ( diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index 16a30812f9..1c90ecca23 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -1,3 +1,5 @@ +// +build linux + package main import ( diff --git a/components/engine/pkg/libcontainer/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/main.go index 47abcce0c5..c9f9d7bc8f 100644 --- a/components/engine/pkg/libcontainer/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/main.go @@ -2,17 +2,27 @@ package main import ( "encoding/json" + "errors" "github.com/dotcloud/docker/pkg/libcontainer" "log" "os" ) +var ( + ErrUnsupported = errors.New("Unsupported method") + ErrWrongArguments = errors.New("Wrong argument count") +) + func main() { container, err := loadContainer() if err != nil { log.Fatal(err) } + argc := len(os.Args) + if argc < 2 { + log.Fatal(ErrWrongArguments) + } switch os.Args[1] { case "exec": exitCode, err := execCommand(container) @@ -21,6 +31,9 @@ func main() { } os.Exit(exitCode) case "init": + if argc != 3 { + log.Fatal(ErrWrongArguments) + } if err := initCommand(container, os.Args[2]); err != nil { log.Fatal(err) } diff --git a/components/engine/pkg/libcontainer/nsinit/mount.go b/components/engine/pkg/libcontainer/nsinit/mount.go index 13ee13e001..baa850f0fb 100644 --- a/components/engine/pkg/libcontainer/nsinit/mount.go +++ b/components/engine/pkg/libcontainer/nsinit/mount.go @@ -1,3 +1,5 @@ +// +build linux + package main import ( From ec270e1bfb22e5425b5c3520292d28f8edd64fbf Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 19:14:31 -0800 Subject: [PATCH 194/403] Refactor large funcs Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 5d62916c48cb97320b37640592805d97badfd8ff Component: engine --- .../engine/pkg/libcontainer/container.go | 5 +- .../engine/pkg/libcontainer/container.json | 3 +- .../engine/pkg/libcontainer/nsinit/exec.go | 60 ++++++++----- .../engine/pkg/libcontainer/nsinit/init.go | 87 +++++++++---------- components/engine/pkg/libcontainer/types.go | 48 +++++----- 5 files changed, 107 insertions(+), 96 deletions(-) diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index 3f3961d496..c8dbdd668f 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -1,14 +1,13 @@ package libcontainer type Container struct { - ID string `json:"id,omitempty"` - Command *Command `json:"command,omitempty"` + Hostname string `json:"hostname,omitempty"` ReadonlyFs bool `json:"readonly_fs,omitempty"` User string `json:"user,omitempty"` WorkingDir string `json:"working_dir,omitempty"` + Command *Command `json:"command,omitempty"` Namespaces Namespaces `json:"namespaces,omitempty"` Capabilities Capabilities `json:"capabilities,omitempty"` - LogFile string `json:"log_file,omitempty"` Network *Network `json:"network,omitempty"` } diff --git a/components/engine/pkg/libcontainer/container.json b/components/engine/pkg/libcontainer/container.json index 8731170c2a..2abf01adb9 100644 --- a/components/engine/pkg/libcontainer/container.json +++ b/components/engine/pkg/libcontainer/container.json @@ -1,6 +1,5 @@ { "id": "koye", - "log_file": "/root/logs", "command": { "args": [ "/bin/bash" @@ -9,7 +8,7 @@ "HOME=/", "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", "container=docker", - "TERM=xterm" + "TERM=xterm-256color" ] }, "namespaces": [ diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 5b53be259c..4abebd2941 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -27,6 +27,8 @@ func execCommand(container *libcontainer.Container) (int, error) { Cloneflags: uintptr(getNamespaceFlags(container.Namespaces) | syscall.CLONE_VFORK), // we need CLONE_VFORK so we can wait on the child } + // create a pipe so that we can syncronize with the namespaced process and + // pass the veth name to the child inPipe, err := command.StdinPipe() if err != nil { return -1, err @@ -39,34 +41,17 @@ func execCommand(container *libcontainer.Container) (int, error) { } if container.Network != nil { - name1, name2, err := createVethPair() + vethPair, err := setupVeth(container.Network.Bridge, command.Process.Pid) if err != nil { return -1, err } - if err := network.SetInterfaceMaster(name1, container.Network.Bridge); err != nil { - return -1, err - } - if err := network.InterfaceUp(name1); err != nil { - return -1, err - } - if err := network.SetInterfaceInNamespacePid(name2, command.Process.Pid); err != nil { - return -1, err - } - fmt.Fprint(inPipe, name2) - inPipe.Close() + sendVethName(vethPair, inPipe) } go io.Copy(os.Stdout, master) go io.Copy(master, os.Stdin) - ws, err := term.GetWinsize(os.Stdin.Fd()) - if err != nil { - return -1, err - } - if err := term.SetWinsize(master.Fd(), ws); err != nil { - return -1, err - } - state, err := term.SetRawTerminal(os.Stdin.Fd()) + state, err := setupWindow(master) if err != nil { command.Process.Kill() return -1, err @@ -81,6 +66,41 @@ func execCommand(container *libcontainer.Container) (int, error) { return command.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil } +func sendVethName(name string, pipe io.WriteCloser) { + // write the veth pair name to the child's stdin then close the + // pipe so that the child stops waiting + fmt.Fprint(pipe, name) + pipe.Close() +} + +func setupVeth(bridge string, nspid int) (string, error) { + name1, name2, err := createVethPair() + if err != nil { + return "", err + } + if err := network.SetInterfaceMaster(name1, bridge); err != nil { + return "", err + } + if err := network.InterfaceUp(name1); err != nil { + return "", err + } + if err := network.SetInterfaceInNamespacePid(name2, nspid); err != nil { + return "", err + } + return name2, nil +} + +func setupWindow(master *os.File) (*term.State, error) { + ws, err := term.GetWinsize(os.Stdin.Fd()) + if err != nil { + return nil, err + } + if err := term.SetWinsize(master.Fd(), ws); err != nil { + return nil, err + } + return term.SetRawTerminal(os.Stdin.Fd()) +} + func createMasterAndConsole() (*os.File, string, error) { master, err := os.OpenFile("/dev/ptmx", syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) if err != nil { diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index 1c90ecca23..d853a32d03 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -9,17 +9,12 @@ import ( "github.com/dotcloud/docker/pkg/libcontainer/network" "github.com/dotcloud/docker/pkg/system" "io/ioutil" - "log" "os" "path/filepath" "syscall" ) func initCommand(container *libcontainer.Container, console string) error { - if err := setLogFile(container); err != nil { - return err - } - rootfs, err := resolveRootfs() if err != nil { return err @@ -27,11 +22,10 @@ func initCommand(container *libcontainer.Container, console string) error { var tempVethName string if container.Network != nil { - data, err := ioutil.ReadAll(os.Stdin) + tempVethName, err = getVethName() if err != nil { - return fmt.Errorf("error reading from stdin %s", err) + return err } - tempVethName = string(data) } // close pipes so that we can replace it with the pty @@ -61,13 +55,10 @@ func initCommand(container *libcontainer.Container, console string) error { if err := setupNewMountNamespace(rootfs, console, container.ReadonlyFs); err != nil { return fmt.Errorf("setup mount namespace %s", err) } - if container.Network != nil { - if err := setupNetworking(container.Network, tempVethName); err != nil { - return fmt.Errorf("setup networking %s", err) - } + if err := setupNetworking(container.Network, tempVethName); err != nil { + return fmt.Errorf("setup networking %s", err) } - - if err := system.Sethostname(container.ID); err != nil { + if err := system.Sethostname(container.Hostname); err != nil { return fmt.Errorf("sethostname %s", err) } if err := capabilities.DropCapabilities(container); err != nil { @@ -136,43 +127,45 @@ func openTerminal(name string, flag int) (*os.File, error) { return os.NewFile(uintptr(r), name), nil } -func setLogFile(container *libcontainer.Container) error { - if container.LogFile != "" { - f, err := os.OpenFile(container.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0655) - if err != nil { - return err +func setupNetworking(config *libcontainer.Network, tempVethName string) error { + if config != nil { + if err := network.InterfaceDown(tempVethName); err != nil { + return fmt.Errorf("interface down %s %s", tempVethName, err) + } + if err := network.ChangeInterfaceName(tempVethName, "eth0"); err != nil { + return fmt.Errorf("change %s to eth0 %s", tempVethName, err) + } + if err := network.SetInterfaceIp("eth0", config.IP); err != nil { + return fmt.Errorf("set eth0 ip %s", err) + } + if err := network.SetMtu("eth0", config.Mtu); err != nil { + return fmt.Errorf("set eth0 mtu to %d %s", config.Mtu, err) + } + if err := network.InterfaceUp("eth0"); err != nil { + return fmt.Errorf("eth0 up %s", err) + } + if err := network.SetMtu("lo", config.Mtu); err != nil { + return fmt.Errorf("set lo mtu to %d %s", config.Mtu, err) + } + if err := network.InterfaceUp("lo"); err != nil { + return fmt.Errorf("lo up %s", err) + } + if config.Gateway != "" { + if err := network.SetDefaultGateway(config.Gateway); err != nil { + return fmt.Errorf("set gateway to %s %s", config.Gateway, err) + } } - log.SetOutput(f) } return nil } -func setupNetworking(config *libcontainer.Network, tempVethName string) error { - if err := network.InterfaceDown(tempVethName); err != nil { - return fmt.Errorf("interface down %s %s", tempVethName, err) +// getVethName reads from Stdin the temp veth name +// sent by the parent processes after the veth pair +// has been created and setup +func getVethName() (string, error) { + data, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return "", fmt.Errorf("error reading from stdin %s", err) } - if err := network.ChangeInterfaceName(tempVethName, "eth0"); err != nil { - return fmt.Errorf("change %s to eth0 %s", tempVethName, err) - } - if err := network.SetInterfaceIp("eth0", config.IP); err != nil { - return fmt.Errorf("set eth0 ip %s", err) - } - if err := network.SetMtu("eth0", config.Mtu); err != nil { - return fmt.Errorf("set eth0 mtu to %d %s", config.Mtu, err) - } - if err := network.InterfaceUp("eth0"); err != nil { - return fmt.Errorf("eth0 up %s", err) - } - if err := network.SetMtu("lo", config.Mtu); err != nil { - return fmt.Errorf("set lo mtu to %d %s", config.Mtu, err) - } - if err := network.InterfaceUp("lo"); err != nil { - return fmt.Errorf("lo up %s", err) - } - if config.Gateway != "" { - if err := network.SetDefaultGateway(config.Gateway); err != nil { - return fmt.Errorf("set gateway to %s %s", config.Gateway, err) - } - } - return nil + return string(data), nil } diff --git a/components/engine/pkg/libcontainer/types.go b/components/engine/pkg/libcontainer/types.go index db1c3b9738..b5d9932671 100644 --- a/components/engine/pkg/libcontainer/types.go +++ b/components/engine/pkg/libcontainer/types.go @@ -1,29 +1,5 @@ package libcontainer -type Namespace string -type Namespaces []Namespace - -func (n Namespaces) Contains(ns Namespace) bool { - for _, nns := range n { - if nns == ns { - return true - } - } - return false -} - -type Capability string -type Capabilities []Capability - -func (c Capabilities) Contains(capp Capability) bool { - for _, cc := range c { - if cc == capp { - return true - } - } - return false -} - const ( CAP_SETPCAP Capability = "SETPCAP" CAP_SYS_MODULE Capability = "SYS_MODULE" @@ -47,3 +23,27 @@ const ( CLONE_NEWPID Namespace = "NEWPID" // pid CLONE_NEWNET Namespace = "NEWNET" // network ) + +type Namespace string +type Namespaces []Namespace + +func (n Namespaces) Contains(ns Namespace) bool { + for _, nns := range n { + if nns == ns { + return true + } + } + return false +} + +type Capability string +type Capabilities []Capability + +func (c Capabilities) Contains(capp Capability) bool { + for _, cc := range c { + if cc == capp { + return true + } + } + return false +} From eba71238f8171d46f1ab9c21e8e6a9365360447e Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 19:53:25 -0800 Subject: [PATCH 195/403] Add execin function to running a process in a namespace Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 420b5eb211f877baac9622e7bedde2948c043619 Component: engine --- .../engine/pkg/libcontainer/container.json | 2 +- .../engine/pkg/libcontainer/nsinit/execin.go | 115 ++++++++++++++++++ .../engine/pkg/libcontainer/nsinit/main.go | 8 ++ .../pkg/libcontainer/nsinit/ns_linux.go | 9 ++ 4 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 components/engine/pkg/libcontainer/nsinit/execin.go diff --git a/components/engine/pkg/libcontainer/container.json b/components/engine/pkg/libcontainer/container.json index 2abf01adb9..c5807a7b28 100644 --- a/components/engine/pkg/libcontainer/container.json +++ b/components/engine/pkg/libcontainer/container.json @@ -1,5 +1,5 @@ { - "id": "koye", + "hostname": "koye", "command": { "args": [ "/bin/bash" diff --git a/components/engine/pkg/libcontainer/nsinit/execin.go b/components/engine/pkg/libcontainer/nsinit/execin.go new file mode 100644 index 0000000000..362cf5afd5 --- /dev/null +++ b/components/engine/pkg/libcontainer/nsinit/execin.go @@ -0,0 +1,115 @@ +package main + +import ( + "fmt" + "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/libcontainer/capabilities" + "github.com/dotcloud/docker/pkg/system" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "syscall" +) + +func execinCommand(container *libcontainer.Container) (int, error) { + nspid, err := readPid() + if err != nil { + return -1, err + } + + for _, ns := range container.Namespaces { + if err := system.Unshare(namespaceMap[ns]); err != nil { + return -1, err + } + } + fds, err := getNsFds(nspid, container) + closeFds := func() { + for _, f := range fds { + system.Closefd(f) + } + } + if err != nil { + closeFds() + return -1, err + } + + for _, fd := range fds { + if fd > 0 { + if err := system.Setns(fd, 0); err != nil { + closeFds() + return -1, fmt.Errorf("setns %s", err) + } + } + system.Closefd(fd) + } + + // if the container has a new pid and mount namespace we need to + // remount proc and sys to pick up the changes + if container.Namespaces.Contains(libcontainer.CLONE_NEWNS) && + container.Namespaces.Contains(libcontainer.CLONE_NEWPID) { + + pid, err := system.Fork() + if err != nil { + return -1, err + } + if pid == 0 { + // TODO: make all raw syscalls to be fork safe + if err := system.Unshare(syscall.CLONE_NEWNS); err != nil { + return -1, err + } + if err := remountProc(); err != nil { + return -1, fmt.Errorf("remount proc %s", err) + } + if err := remountSys(); err != nil { + return -1, fmt.Errorf("remount sys %s", err) + } + if err := capabilities.DropCapabilities(container); err != nil { + return -1, fmt.Errorf("drop capabilities %s", err) + } + if err := system.Exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { + return -1, err + } + } + proc, err := os.FindProcess(pid) + if err != nil { + return -1, err + } + state, err := proc.Wait() + if err != nil { + return -1, err + } + os.Exit(state.Sys().(syscall.WaitStatus).ExitStatus()) + } + if err := capabilities.DropCapabilities(container); err != nil { + return -1, fmt.Errorf("drop capabilities %s", err) + } + if err := system.Exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { + return -1, err + } + panic("unreachable") +} + +func readPid() (int, error) { + data, err := ioutil.ReadFile(".nspid") + if err != nil { + return -1, err + } + pid, err := strconv.Atoi(string(data)) + if err != nil { + return -1, err + } + return pid, nil +} + +func getNsFds(pid int, container *libcontainer.Container) ([]uintptr, error) { + fds := make([]uintptr, len(container.Namespaces)) + for i, ns := range container.Namespaces { + f, err := os.OpenFile(filepath.Join("/proc/", strconv.Itoa(pid), "ns", namespaceFileMap[ns]), os.O_RDONLY, 0) + if err != nil { + return fds, err + } + fds[i] = f.Fd() + } + return fds, nil +} diff --git a/components/engine/pkg/libcontainer/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/main.go index c9f9d7bc8f..8fe700e064 100644 --- a/components/engine/pkg/libcontainer/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/main.go @@ -37,6 +37,14 @@ func main() { if err := initCommand(container, os.Args[2]); err != nil { log.Fatal(err) } + case "execin": + exitCode, err := execinCommand(container) + if err != nil { + log.Fatal(err) + } + os.Exit(exitCode) + default: + log.Fatalf("command not supported for nsinit %s", os.Args[1]) } } diff --git a/components/engine/pkg/libcontainer/nsinit/ns_linux.go b/components/engine/pkg/libcontainer/nsinit/ns_linux.go index 2392ffd770..a2809eb199 100644 --- a/components/engine/pkg/libcontainer/nsinit/ns_linux.go +++ b/components/engine/pkg/libcontainer/nsinit/ns_linux.go @@ -14,6 +14,15 @@ var namespaceMap = map[libcontainer.Namespace]int{ libcontainer.CLONE_NEWNET: syscall.CLONE_NEWNET, } +var namespaceFileMap = map[libcontainer.Namespace]string{ + libcontainer.CLONE_NEWNS: "mnt", + libcontainer.CLONE_NEWUTS: "uts", + libcontainer.CLONE_NEWIPC: "ipc", + libcontainer.CLONE_NEWUSER: "user", + libcontainer.CLONE_NEWPID: "pid", + libcontainer.CLONE_NEWNET: "net", +} + // getNamespaceFlags parses the container's Namespaces options to set the correct // flags on clone, unshare, and setns func getNamespaceFlags(namespaces libcontainer.Namespaces) (flag int) { From e501c61ed362bc64550c8b645f8027b827e0ffb1 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 20:35:04 -0800 Subject: [PATCH 196/403] Refactor to remove cmd from container Pass the container's command via args Remove execin function and just look for an existing nspid file to join the namespace Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: d84feb8fe5e40838c81321249189f1f0a02825bb Component: engine --- .../engine/pkg/libcontainer/container.go | 7 +--- .../engine/pkg/libcontainer/container.json | 17 +++------ .../engine/pkg/libcontainer/nsinit/exec.go | 21 ++++++++--- .../engine/pkg/libcontainer/nsinit/execin.go | 24 ++---------- .../engine/pkg/libcontainer/nsinit/init.go | 4 +- .../engine/pkg/libcontainer/nsinit/main.go | 37 ++++++++++++++----- 6 files changed, 55 insertions(+), 55 deletions(-) diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index c8dbdd668f..763526f66b 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -5,17 +5,12 @@ type Container struct { ReadonlyFs bool `json:"readonly_fs,omitempty"` User string `json:"user,omitempty"` WorkingDir string `json:"working_dir,omitempty"` - Command *Command `json:"command,omitempty"` + Env []string `json:"environment,omitempty"` Namespaces Namespaces `json:"namespaces,omitempty"` Capabilities Capabilities `json:"capabilities,omitempty"` Network *Network `json:"network,omitempty"` } -type Command struct { - Args []string `json:"args,omitempty"` - Env []string `json:"environment,omitempty"` -} - type Network struct { IP string `json:"ip,omitempty"` Gateway string `json:"gateway,omitempty"` diff --git a/components/engine/pkg/libcontainer/container.json b/components/engine/pkg/libcontainer/container.json index c5807a7b28..ccc9abb041 100644 --- a/components/engine/pkg/libcontainer/container.json +++ b/components/engine/pkg/libcontainer/container.json @@ -1,16 +1,11 @@ { "hostname": "koye", - "command": { - "args": [ - "/bin/bash" - ], - "environment": [ - "HOME=/", - "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", - "container=docker", - "TERM=xterm-256color" - ] - }, + "environment": [ + "HOME=/", + "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", + "container=docker", + "TERM=xterm-256color" + ], "namespaces": [ "NEWIPC", "NEWNS", diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 4abebd2941..67f907af53 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -16,17 +16,13 @@ import ( "syscall" ) -func execCommand(container *libcontainer.Container) (int, error) { +func execCommand(container *libcontainer.Container, args []string) (int, error) { master, console, err := createMasterAndConsole() if err != nil { return -1, err } - command := exec.Command("nsinit", "init", console) - command.SysProcAttr = &syscall.SysProcAttr{ - Cloneflags: uintptr(getNamespaceFlags(container.Namespaces) | syscall.CLONE_VFORK), // we need CLONE_VFORK so we can wait on the child - } - + command := createCommand(container, console, args) // create a pipe so that we can syncronize with the namespaced process and // pass the veth name to the child inPipe, err := command.StdinPipe() @@ -39,6 +35,7 @@ func execCommand(container *libcontainer.Container) (int, error) { if err := writePidFile(command); err != nil { return -1, err } + defer deletePidFile() if container.Network != nil { vethPair, err := setupVeth(container.Network.Bridge, command.Process.Pid) @@ -134,3 +131,15 @@ func createVethPair() (name1 string, name2 string, err error) { func writePidFile(command *exec.Cmd) error { return ioutil.WriteFile(".nspid", []byte(fmt.Sprint(command.Process.Pid)), 0655) } + +func deletePidFile() error { + return os.Remove(".nspid") +} + +func createCommand(container *libcontainer.Container, console string, args []string) *exec.Cmd { + command := exec.Command("nsinit", append([]string{"init", console}, args...)...) + command.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: uintptr(getNamespaceFlags(container.Namespaces) | syscall.CLONE_VFORK), // we need CLONE_VFORK so we can wait on the child + } + return command +} diff --git a/components/engine/pkg/libcontainer/nsinit/execin.go b/components/engine/pkg/libcontainer/nsinit/execin.go index 362cf5afd5..7f32620cb8 100644 --- a/components/engine/pkg/libcontainer/nsinit/execin.go +++ b/components/engine/pkg/libcontainer/nsinit/execin.go @@ -5,19 +5,13 @@ import ( "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/capabilities" "github.com/dotcloud/docker/pkg/system" - "io/ioutil" "os" "path/filepath" "strconv" "syscall" ) -func execinCommand(container *libcontainer.Container) (int, error) { - nspid, err := readPid() - if err != nil { - return -1, err - } - +func execinCommand(container *libcontainer.Container, nspid int, args []string) (int, error) { for _, ns := range container.Namespaces { if err := system.Unshare(namespaceMap[ns]); err != nil { return -1, err @@ -67,7 +61,7 @@ func execinCommand(container *libcontainer.Container) (int, error) { if err := capabilities.DropCapabilities(container); err != nil { return -1, fmt.Errorf("drop capabilities %s", err) } - if err := system.Exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { + if err := system.Exec(args[0], args[0:], container.Env); err != nil { return -1, err } } @@ -84,24 +78,12 @@ func execinCommand(container *libcontainer.Container) (int, error) { if err := capabilities.DropCapabilities(container); err != nil { return -1, fmt.Errorf("drop capabilities %s", err) } - if err := system.Exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { + if err := system.Exec(args[0], args[0:], container.Env); err != nil { return -1, err } panic("unreachable") } -func readPid() (int, error) { - data, err := ioutil.ReadFile(".nspid") - if err != nil { - return -1, err - } - pid, err := strconv.Atoi(string(data)) - if err != nil { - return -1, err - } - return pid, nil -} - func getNsFds(pid int, container *libcontainer.Container) ([]uintptr, error) { fds := make([]uintptr, len(container.Namespaces)) for i, ns := range container.Namespaces { diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index d853a32d03..82706fdadd 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -14,7 +14,7 @@ import ( "syscall" ) -func initCommand(container *libcontainer.Container, console string) error { +func initCommand(container *libcontainer.Container, console string, args []string) error { rootfs, err := resolveRootfs() if err != nil { return err @@ -72,7 +72,7 @@ func initCommand(container *libcontainer.Container, console string) error { return fmt.Errorf("chdir to %s %s", container.WorkingDir, err) } } - if err := system.Exec(container.Command.Args[0], container.Command.Args[0:], container.Command.Env); err != nil { + if err := system.Exec(args[0], args[0:], container.Env); err != nil { return fmt.Errorf("exec %s", err) } panic("unreachable") diff --git a/components/engine/pkg/libcontainer/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/main.go index 8fe700e064..30c8b064e4 100644 --- a/components/engine/pkg/libcontainer/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/main.go @@ -4,8 +4,10 @@ import ( "encoding/json" "errors" "github.com/dotcloud/docker/pkg/libcontainer" + "io/ioutil" "log" "os" + "strconv" ) var ( @@ -25,24 +27,29 @@ func main() { } switch os.Args[1] { case "exec": - exitCode, err := execCommand(container) + var exitCode int + nspid, err := readPid() + if err != nil { + if !os.IsNotExist(err) { + log.Fatal(err) + } + } + if nspid > 0 { + exitCode, err = execinCommand(container, nspid, os.Args[2:]) + } else { + exitCode, err = execCommand(container, os.Args[2:]) + } if err != nil { log.Fatal(err) } os.Exit(exitCode) case "init": - if argc != 3 { + if argc < 3 { log.Fatal(ErrWrongArguments) } - if err := initCommand(container, os.Args[2]); err != nil { + if err := initCommand(container, os.Args[2], os.Args[3:]); err != nil { log.Fatal(err) } - case "execin": - exitCode, err := execinCommand(container) - if err != nil { - log.Fatal(err) - } - os.Exit(exitCode) default: log.Fatalf("command not supported for nsinit %s", os.Args[1]) } @@ -61,3 +68,15 @@ func loadContainer() (*libcontainer.Container, error) { } return container, nil } + +func readPid() (int, error) { + data, err := ioutil.ReadFile(".nspid") + if err != nil { + return -1, err + } + pid, err := strconv.Atoi(string(data)) + if err != nil { + return -1, err + } + return pid, nil +} From 60c5e9ec9384d577fd39622b3f01a721c5c89b15 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 21:15:44 -0800 Subject: [PATCH 197/403] Update readme and add TODO Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 3a97fe27d8f5a9bbcf4992cc9efe33880e73f274 Component: engine --- components/engine/pkg/libcontainer/README.md | 75 ++++++++++++-------- components/engine/pkg/libcontainer/TODO.md | 17 +++++ 2 files changed, 61 insertions(+), 31 deletions(-) create mode 100644 components/engine/pkg/libcontainer/TODO.md diff --git a/components/engine/pkg/libcontainer/README.md b/components/engine/pkg/libcontainer/README.md index 91d747863c..07fe4f7b2d 100644 --- a/components/engine/pkg/libcontainer/README.md +++ b/components/engine/pkg/libcontainer/README.md @@ -1,39 +1,34 @@ ## libcontainer - reference implementation for containers -#### playground +#### background + +libcontainer specifies configuration options for what a container is. It provides a native Go implementation +for using linux namespaces with no external dependencies. libcontainer provides many convience functions for working with namespaces, networking, and management. -Use the cli package to test out functionality - -First setup a container configuration. You will need a root fs, better go the path to a -stopped docker container and use that. - +#### container +A container is a self contained directory that is able to run one or more processes inside without +affecting the host system. The directory is usually a full system tree. Inside the directory +a `container.json` file just be placed with the runtime configuration for how the process +should be contained and run. Environment, networking, and different capabilities for the +process are specified in this file. +Sample `container.json` file: ```json { - "id": "koye", - "namespace_pid": 12265, - "command": { - "args": [ - "/bin/bash" - ], - "environment": [ - "HOME=/", - "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", - "container=docker", - "TERM=xterm" - ] - }, - "rootfs": "/root/development/gocode/src/github.com/docker/libcontainer/namespaces/ubuntu", - "network": null, - "user": "", - "working_dir": "", + "hostname": "koye", + "environment": [ + "HOME=/", + "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", + "container=docker", + "TERM=xterm-256color" + ], "namespaces": [ - "NEWNET", "NEWIPC", "NEWNS", "NEWPID", - "NEWUTS" + "NEWUTS", + "NEWNET" ], "capabilities": [ "SETPCAP", @@ -50,14 +45,32 @@ stopped docker container and use that. "AUDIT_CONTROL", "MAC_OVERRIDE", "MAC_ADMIN" - ] + ], + "network": { + "ip": "172.17.0.100/16", + "gateway": "172.17.42.1", + "bridge": "docker0", + "mtu": 1500 + } } ``` -After you have a json file and a rootfs path to use just run: -`./cli exec container.json` +Using this configuration and the current directory holding the rootfs for a process to live, one can se libcontainer to exec the container. Running the life of the namespace a `.nspid` file +is written to the current directory with the pid of the namespace'd process to the external word. A client can use this pid to wait, kill, or perform other operation with the container. If a user tries to run an new process inside an existing container with a live namespace with namespace will be joined by the new process. -If you want to attach to an existing namespace just use the same json -file with the container still running and do: -`./cli execin container.json` +#### nsinit + +`nsinit` is a cli application used as the reference implementation of libcontainer. It is able to +spawn or join new containers giving the current directory. To use `nsinit` cd into a linux +rootfs and copy a `container.json` file into the directory with your specified configuration. + +To execution `/bin/bash` in the current directory as a container just run: +```bash +nsinit exec /bin/bash +``` + +If you wish to spawn another process inside the container while your current bash session is +running just run the exact same command again to get another bash shell or change the command. If the original process dies, PID 1, all other processes spawned inside the container will also be killed and the namespace will be removed. + +You can identify if a process is running in a container by looking to see if `.nspid` is in the root of the directory. diff --git a/components/engine/pkg/libcontainer/TODO.md b/components/engine/pkg/libcontainer/TODO.md new file mode 100644 index 0000000000..f18c0b4c51 --- /dev/null +++ b/components/engine/pkg/libcontainer/TODO.md @@ -0,0 +1,17 @@ +#### goals +* small and simple - line count is not everything but less code is better +* clean lines between what we do in the pkg +* provide primitives for working with namespaces not cater to every option +* extend via configuration not by features - host networking, no networking, veth network can be accomplished via adjusting the container.json, nothing to do with code + +#### tasks +* proper tty for a new process in an existing container +* use exec or raw syscalls for new process in existing container +* setup proper user in namespace if specified +* implement hook or clean interface for cgroups +* example configs for different setups (host networking, boot init) +* improve pkg documentation with comments +* testing - this is hard in a low level pkg but we could do some, maybe +* pivot root +* selinux +* apparmor From 74894a63abe9443d1c8e2bf50c0e179082396b3a Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 21:21:49 -0800 Subject: [PATCH 198/403] Add CAP_NET_ADMIN Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: e0ff0f4dd6612e331459a2dec69adc728bc360fe Component: engine --- components/engine/pkg/libcontainer/capabilities/capabilities.go | 1 + components/engine/pkg/libcontainer/types.go | 1 + 2 files changed, 2 insertions(+) diff --git a/components/engine/pkg/libcontainer/capabilities/capabilities.go b/components/engine/pkg/libcontainer/capabilities/capabilities.go index 3301e10f7f..c19b719564 100644 --- a/components/engine/pkg/libcontainer/capabilities/capabilities.go +++ b/components/engine/pkg/libcontainer/capabilities/capabilities.go @@ -21,6 +21,7 @@ var capMap = map[libcontainer.Capability]capability.Cap{ libcontainer.CAP_AUDIT_CONTROL: capability.CAP_AUDIT_CONTROL, libcontainer.CAP_MAC_OVERRIDE: capability.CAP_MAC_OVERRIDE, libcontainer.CAP_MAC_ADMIN: capability.CAP_MAC_ADMIN, + libcontainer.CAP_NET_ADMIN: capability.CAP_NET_ADMIN, } // DropCapabilities drops capabilities for the current process based diff --git a/components/engine/pkg/libcontainer/types.go b/components/engine/pkg/libcontainer/types.go index b5d9932671..fcd00fd4f1 100644 --- a/components/engine/pkg/libcontainer/types.go +++ b/components/engine/pkg/libcontainer/types.go @@ -15,6 +15,7 @@ const ( CAP_AUDIT_CONTROL Capability = "AUDIT_CONTROL" CAP_MAC_OVERRIDE Capability = "MAC_OVERRIDE" CAP_MAC_ADMIN Capability = "MAC_ADMIN" + CAP_NET_ADMIN Capability = "NET_ADMIN" CLONE_NEWNS Namespace = "NEWNS" // mount CLONE_NEWUTS Namespace = "NEWUTS" // utsname From 4441df6975553c68d201ec375c2e4b5c30d267f1 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 22:43:40 -0800 Subject: [PATCH 199/403] Add comments to many functions Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 70593be139647cdedca0130250626ea6e0b8a277 Component: engine --- .../libcontainer/capabilities/capabilities.go | 1 + .../engine/pkg/libcontainer/container.go | 22 ++++++---- .../engine/pkg/libcontainer/nsinit/exec.go | 23 +++++++++-- .../engine/pkg/libcontainer/nsinit/execin.go | 10 ++--- .../engine/pkg/libcontainer/nsinit/init.go | 18 ++++---- .../engine/pkg/libcontainer/nsinit/main.go | 4 +- .../engine/pkg/libcontainer/nsinit/mount.go | 41 +++++++++++++------ .../pkg/libcontainer/nsinit/ns_linux.go | 3 ++ components/engine/pkg/libcontainer/types.go | 18 +++++--- .../engine/pkg/libcontainer/utils/utils.go | 2 + 10 files changed, 97 insertions(+), 45 deletions(-) diff --git a/components/engine/pkg/libcontainer/capabilities/capabilities.go b/components/engine/pkg/libcontainer/capabilities/capabilities.go index c19b719564..65fd455c26 100644 --- a/components/engine/pkg/libcontainer/capabilities/capabilities.go +++ b/components/engine/pkg/libcontainer/capabilities/capabilities.go @@ -41,6 +41,7 @@ func DropCapabilities(container *libcontainer.Container) error { return nil } +// getCapabilities returns the specific cap values for the libcontainer types func getCapabilities(container *libcontainer.Container) []capability.Cap { drop := []capability.Cap{} for _, c := range container.Capabilities { diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index 763526f66b..a6a57dab77 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -1,16 +1,22 @@ package libcontainer +// Container defines configuration options for how a +// container is setup inside a directory and how a process should be executed type Container struct { - Hostname string `json:"hostname,omitempty"` - ReadonlyFs bool `json:"readonly_fs,omitempty"` - User string `json:"user,omitempty"` - WorkingDir string `json:"working_dir,omitempty"` - Env []string `json:"environment,omitempty"` - Namespaces Namespaces `json:"namespaces,omitempty"` - Capabilities Capabilities `json:"capabilities,omitempty"` - Network *Network `json:"network,omitempty"` + Hostname string `json:"hostname,omitempty"` // hostname + ReadonlyFs bool `json:"readonly_fs,omitempty"` // set the containers rootfs as readonly + User string `json:"user,omitempty"` // user to execute the process as + WorkingDir string `json:"working_dir,omitempty"` // current working directory + Env []string `json:"environment,omitempty"` // environment to set + Namespaces Namespaces `json:"namespaces,omitempty"` // namespaces to apply + Capabilities Capabilities `json:"capabilities,omitempty"` // capabilities to drop + Network *Network `json:"network,omitempty"` // nil for host's network stack } +// Network defines configuration for a container's networking stack +// +// The network configuration can be omited from a container causing the +// container to be setup with the host's networking stack type Network struct { IP string `json:"ip,omitempty"` Gateway string `json:"gateway,omitempty"` diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 67f907af53..202cfcab5e 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -38,7 +38,7 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) defer deletePidFile() if container.Network != nil { - vethPair, err := setupVeth(container.Network.Bridge, command.Process.Pid) + vethPair, err := initializeContainerVeth(container.Network.Bridge, command.Process.Pid) if err != nil { return -1, err } @@ -63,14 +63,21 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) return command.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil } +// sendVethName writes the veth pair name to the child's stdin then closes the +// pipe so that the child stops waiting for more data func sendVethName(name string, pipe io.WriteCloser) { - // write the veth pair name to the child's stdin then close the - // pipe so that the child stops waiting fmt.Fprint(pipe, name) pipe.Close() } -func setupVeth(bridge string, nspid int) (string, error) { +// initializeContainerVeth will create a veth pair and setup the host's +// side of the pair by setting the specified bridge as the master and bringing +// up the interface. +// +// Then will with set the other side of the veth pair into the container's namespaced +// using the pid and returns the veth's interface name to provide to the container to +// finish setting up the interface inside the namespace +func initializeContainerVeth(bridge string, nspid int) (string, error) { name1, name2, err := createVethPair() if err != nil { return "", err @@ -98,6 +105,8 @@ func setupWindow(master *os.File) (*term.State, error) { return term.SetRawTerminal(os.Stdin.Fd()) } +// createMasterAndConsole will open /dev/ptmx on the host and retreive the +// pts name for use as the pty slave inside the container func createMasterAndConsole() (*os.File, string, error) { master, err := os.OpenFile("/dev/ptmx", syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) if err != nil { @@ -113,6 +122,8 @@ func createMasterAndConsole() (*os.File, string, error) { return master, console, nil } +// createVethPair will automatically generage two random names for +// the veth pair and ensure that they have been created func createVethPair() (name1 string, name2 string, err error) { name1, err = utils.GenerateRandomName("dock", 4) if err != nil { @@ -128,6 +139,7 @@ func createVethPair() (name1 string, name2 string, err error) { return } +// writePidFile writes the namespaced processes pid to .nspid in the rootfs for the container func writePidFile(command *exec.Cmd) error { return ioutil.WriteFile(".nspid", []byte(fmt.Sprint(command.Process.Pid)), 0655) } @@ -136,6 +148,9 @@ func deletePidFile() error { return os.Remove(".nspid") } +// createCommand will return an exec.Cmd with the Cloneflags set to the proper namespaces +// defined on the container's configuration and use the current binary as the init with the +// args provided func createCommand(container *libcontainer.Container, console string, args []string) *exec.Cmd { command := exec.Command("nsinit", append([]string{"init", console}, args...)...) command.SysProcAttr = &syscall.SysProcAttr{ diff --git a/components/engine/pkg/libcontainer/nsinit/execin.go b/components/engine/pkg/libcontainer/nsinit/execin.go index 7f32620cb8..d6224f95e6 100644 --- a/components/engine/pkg/libcontainer/nsinit/execin.go +++ b/components/engine/pkg/libcontainer/nsinit/execin.go @@ -28,6 +28,7 @@ func execinCommand(container *libcontainer.Container, nspid int, args []string) return -1, err } + // foreach namespace fd, use setns to join an existing container's namespaces for _, fd := range fds { if fd > 0 { if err := system.Setns(fd, 0); err != nil { @@ -42,7 +43,6 @@ func execinCommand(container *libcontainer.Container, nspid int, args []string) // remount proc and sys to pick up the changes if container.Namespaces.Contains(libcontainer.CLONE_NEWNS) && container.Namespaces.Contains(libcontainer.CLONE_NEWPID) { - pid, err := system.Fork() if err != nil { return -1, err @@ -58,12 +58,7 @@ func execinCommand(container *libcontainer.Container, nspid int, args []string) if err := remountSys(); err != nil { return -1, fmt.Errorf("remount sys %s", err) } - if err := capabilities.DropCapabilities(container); err != nil { - return -1, fmt.Errorf("drop capabilities %s", err) - } - if err := system.Exec(args[0], args[0:], container.Env); err != nil { - return -1, err - } + goto dropAndExec } proc, err := os.FindProcess(pid) if err != nil { @@ -75,6 +70,7 @@ func execinCommand(container *libcontainer.Container, nspid int, args []string) } os.Exit(state.Sys().(syscall.WaitStatus).ExitStatus()) } +dropAndExec: if err := capabilities.DropCapabilities(container); err != nil { return -1, fmt.Errorf("drop capabilities %s", err) } diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index 82706fdadd..c77fd90447 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -37,9 +37,6 @@ func initCommand(container *libcontainer.Container, console string, args []strin if err != nil { return fmt.Errorf("open terminal %s", err) } - if slave.Fd() != 0 { - return fmt.Errorf("slave fd should be 0") - } if err := dupSlave(slave); err != nil { return fmt.Errorf("dup2 slave %s", err) } @@ -55,7 +52,7 @@ func initCommand(container *libcontainer.Container, console string, args []strin if err := setupNewMountNamespace(rootfs, console, container.ReadonlyFs); err != nil { return fmt.Errorf("setup mount namespace %s", err) } - if err := setupNetworking(container.Network, tempVethName); err != nil { + if err := setupVethNetwork(container.Network, tempVethName); err != nil { return fmt.Errorf("setup networking %s", err) } if err := system.Sethostname(container.Hostname); err != nil { @@ -78,6 +75,8 @@ func initCommand(container *libcontainer.Container, console string, args []strin panic("unreachable") } +// resolveRootfs ensures that the current working directory is +// not a symlink and returns the absolute path to the rootfs func resolveRootfs() (string, error) { cwd, err := os.Getwd() if err != nil { @@ -104,8 +103,9 @@ func setupUser(container *libcontainer.Container) error { return nil } +// dupSlave dup2 the pty slave's fd into stdout and stdin and ensures that +// the slave's fd is 0, or stdin func dupSlave(slave *os.File) error { - // we close Stdin,etc so our pty slave should have fd 0 if slave.Fd() != 0 { return fmt.Errorf("slave fd not 0 %d", slave.Fd()) } @@ -118,7 +118,8 @@ func dupSlave(slave *os.File) error { return nil } -// openTerminal is a clone of os.OpenFile without the O_CLOEXEC addition. +// openTerminal is a clone of os.OpenFile without the O_CLOEXEC +// used to open the pty slave inside the container namespace func openTerminal(name string, flag int) (*os.File, error) { r, e := syscall.Open(name, flag, 0) if e != nil { @@ -127,7 +128,10 @@ func openTerminal(name string, flag int) (*os.File, error) { return os.NewFile(uintptr(r), name), nil } -func setupNetworking(config *libcontainer.Network, tempVethName string) error { +// setupVethNetwork uses the Network config if it is not nil to initialize +// the new veth interface inside the container for use by changing the name to eth0 +// setting the MTU and IP address along with the default gateway +func setupVethNetwork(config *libcontainer.Network, tempVethName string) error { if config != nil { if err := network.InterfaceDown(tempVethName); err != nil { return fmt.Errorf("interface down %s %s", tempVethName, err) diff --git a/components/engine/pkg/libcontainer/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/main.go index 30c8b064e4..f45fe55689 100644 --- a/components/engine/pkg/libcontainer/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/main.go @@ -26,7 +26,7 @@ func main() { log.Fatal(ErrWrongArguments) } switch os.Args[1] { - case "exec": + case "exec": // this is executed outside of the namespace in the cwd var exitCode int nspid, err := readPid() if err != nil { @@ -43,7 +43,7 @@ func main() { log.Fatal(err) } os.Exit(exitCode) - case "init": + case "init": // this is executed inside of the namespace to setup the container if argc < 3 { log.Fatal(ErrWrongArguments) } diff --git a/components/engine/pkg/libcontainer/nsinit/mount.go b/components/engine/pkg/libcontainer/nsinit/mount.go index baa850f0fb..6eb2e09060 100644 --- a/components/engine/pkg/libcontainer/nsinit/mount.go +++ b/components/engine/pkg/libcontainer/nsinit/mount.go @@ -10,10 +10,16 @@ import ( "syscall" ) -// default mount point options +// default mount point flags const defaultMountFlags = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV +// setupNewMountNamespace is used to initialize a new mount namespace for an new +// container in the rootfs that is specified. +// +// There is no need to unmount the new mounts because as soon as the mount namespace +// is no longer in use, the mounts will be removed automatically func setupNewMountNamespace(rootfs, console string, readonly bool) error { + // mount as slave so that the new mounts do not propagate to the host if err := system.Mount("", "/", "", syscall.MS_SLAVE|syscall.MS_REC, ""); err != nil { return fmt.Errorf("mounting / as slave %s", err) } @@ -55,6 +61,7 @@ func setupNewMountNamespace(rootfs, console string, readonly bool) error { return nil } +// copyDevNodes mknods the hosts devices so the new container has access to them func copyDevNodes(rootfs string) error { oldMask := system.Umask(0000) defer system.Umask(oldMask) @@ -82,6 +89,8 @@ func copyDevNodes(rootfs string) error { return nil } +// setupDev symlinks the current processes pipes into the +// appropriate destination on the containers rootfs func setupDev(rootfs string) error { for _, link := range []struct { from string @@ -104,6 +113,7 @@ func setupDev(rootfs string) error { return nil } +// setupConsole ensures that the container has a proper /dev/console setup func setupConsole(rootfs, console string) error { oldMask := system.Umask(0000) defer system.Umask(oldMask) @@ -161,6 +171,24 @@ func mountSystem(rootfs string) error { return nil } +// setupPtmx adds a symlink to pts/ptmx for /dev/ptmx and +// finishes setting up /dev/console +func setupPtmx(rootfs, console string) error { + ptmx := filepath.Join(rootfs, "dev/ptmx") + if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) { + return err + } + if err := os.Symlink("pts/ptmx", ptmx); err != nil { + return fmt.Errorf("symlink dev ptmx %s", err) + } + if err := setupConsole(rootfs, console); err != nil { + return err + } + return nil +} + +// remountProc is used to detach and remount the proc filesystem +// commonly needed with running a new process inside an existing container func remountProc() error { if err := system.Unmount("/proc", syscall.MNT_DETACH); err != nil { return err @@ -183,14 +211,3 @@ func remountSys() error { } return nil } - -func setupPtmx(rootfs, console string) error { - ptmx := filepath.Join(rootfs, "dev/ptmx") - if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) { - return err - } - if err := os.Symlink("pts/ptmx", ptmx); err != nil { - return fmt.Errorf("symlink dev ptmx %s", err) - } - return setupConsole(rootfs, console) -} diff --git a/components/engine/pkg/libcontainer/nsinit/ns_linux.go b/components/engine/pkg/libcontainer/nsinit/ns_linux.go index a2809eb199..481bdf79df 100644 --- a/components/engine/pkg/libcontainer/nsinit/ns_linux.go +++ b/components/engine/pkg/libcontainer/nsinit/ns_linux.go @@ -14,6 +14,9 @@ var namespaceMap = map[libcontainer.Namespace]int{ libcontainer.CLONE_NEWNET: syscall.CLONE_NEWNET, } +// namespaceFileMap is used to convert the libcontainer types +// into the names of the files located in /proc//ns/* for +// each namespace var namespaceFileMap = map[libcontainer.Namespace]string{ libcontainer.CLONE_NEWNS: "mnt", libcontainer.CLONE_NEWUTS: "uts", diff --git a/components/engine/pkg/libcontainer/types.go b/components/engine/pkg/libcontainer/types.go index fcd00fd4f1..bb54ff5130 100644 --- a/components/engine/pkg/libcontainer/types.go +++ b/components/engine/pkg/libcontainer/types.go @@ -1,5 +1,8 @@ package libcontainer +// These constants are defined as string types so that +// it is clear when adding the configuration in config files +// instead of using ints or other types const ( CAP_SETPCAP Capability = "SETPCAP" CAP_SYS_MODULE Capability = "SYS_MODULE" @@ -25,9 +28,15 @@ const ( CLONE_NEWNET Namespace = "NEWNET" // network ) -type Namespace string -type Namespaces []Namespace +type ( + Namespace string + Namespaces []Namespace + Capability string + Capabilities []Capability +) +// Contains returns true if the specified Namespace is +// in the slice func (n Namespaces) Contains(ns Namespace) bool { for _, nns := range n { if nns == ns { @@ -37,9 +46,8 @@ func (n Namespaces) Contains(ns Namespace) bool { return false } -type Capability string -type Capabilities []Capability - +// Contains returns true if the specified Capability is +// in the slice func (c Capabilities) Contains(capp Capability) bool { for _, cc := range c { if cc == capp { diff --git a/components/engine/pkg/libcontainer/utils/utils.go b/components/engine/pkg/libcontainer/utils/utils.go index d3223c3e4d..5050997ffd 100644 --- a/components/engine/pkg/libcontainer/utils/utils.go +++ b/components/engine/pkg/libcontainer/utils/utils.go @@ -6,6 +6,8 @@ import ( "io" ) +// GenerateRandomName returns a new name joined with a prefix. This size +// specified is used to truncate the randomly generated value func GenerateRandomName(prefix string, size int) (string, error) { id := make([]byte, 32) if _, err := io.ReadFull(rand.Reader, id); err != nil { From 99ce69e4d500bd1c46fcabf5f7be1b5c1e5e60c8 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Wed, 19 Feb 2014 22:46:02 -0800 Subject: [PATCH 200/403] Remove privileged.json config Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: e133d895a6934e650f64f391f9f26b29b0379457 Component: engine --- .../engine/pkg/libcontainer/privileged.json | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 components/engine/pkg/libcontainer/privileged.json diff --git a/components/engine/pkg/libcontainer/privileged.json b/components/engine/pkg/libcontainer/privileged.json deleted file mode 100644 index be877ad335..0000000000 --- a/components/engine/pkg/libcontainer/privileged.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "id": "koye", - "namespace_pid": 3745, - "command": { - "args": [ - "/usr/lib/systemd/systemd" - ], - "environment": [ - "HOME=/", - "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", - "container=docker", - "TERM=" - ] - }, - "rootfs": "/root/main/mycontainer", - "namespaces": [ - "NEWIPC", - "NEWNS", - "NEWPID", - "NEWUTS" - ] -} From f93f48e322dfcf8f1eadf10a02774ce06ea7e3fc Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 20 Feb 2014 12:00:54 -0800 Subject: [PATCH 201/403] WIP for setup kmsg Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: f0b4dd6e5883a65dc23121934b6eed7e70ac2515 Component: engine --- .../engine/pkg/libcontainer/nsinit/mount.go | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/components/engine/pkg/libcontainer/nsinit/mount.go b/components/engine/pkg/libcontainer/nsinit/mount.go index 6eb2e09060..67f9020350 100644 --- a/components/engine/pkg/libcontainer/nsinit/mount.go +++ b/components/engine/pkg/libcontainer/nsinit/mount.go @@ -43,6 +43,9 @@ func setupNewMountNamespace(rootfs, console string, readonly bool) error { if err := setupPtmx(rootfs, console); err != nil { return err } + if err := setupKmsg(rootfs); err != nil { + return err + } if err := system.Chdir(rootfs); err != nil { return fmt.Errorf("chdir into %s %s", rootfs, err) } @@ -211,3 +214,32 @@ func remountSys() error { } return nil } + +func setupKmsg(rootfs string) error { + oldMask := system.Umask(0000) + defer system.Umask(oldMask) + + var ( + source = filepath.Join(rootfs, "dev/kmsg") + dest = filepath.Join(rootfs, "proc/kmsg") + ) + + if err := system.Mkfifo(source, 0600); err != nil { + return err + } + + os.Chmod(source, 0600) + os.Chown(source, 0, 0) + + if err := system.Mount(source, dest, "bind", syscall.MS_BIND, ""); err != nil { + return err + } + _, err := os.OpenFile(source, syscall.O_RDWR|syscall.O_NDELAY|syscall.O_CLOEXEC, 0) + if err != nil { + return err + } + if err := syscall.Unlink(source); err != nil { + return err + } + return nil +} From d2ac05db40608a02a37c36118fd48259b2150c84 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Thu, 20 Feb 2014 23:12:08 +0100 Subject: [PATCH 202/403] libcontainer: Initial version of cgroups support This is a minimal version of raw cgroup support for libcontainer. It has only enough for what docker needs, and it has no support for systemd yet. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 664fc54e65ebc14ca9dd5bfc55e3dfe1796e51c8 Component: engine --- components/engine/pkg/cgroups/cgroups.go | 16 +- .../engine/pkg/libcontainer/cgroup/cgroup.go | 177 ++++++++++++++++++ .../engine/pkg/libcontainer/container.go | 7 + .../engine/pkg/libcontainer/container.json | 5 +- .../engine/pkg/libcontainer/nsinit/exec.go | 13 +- .../engine/pkg/libcontainer/nsinit/init.go | 10 +- 6 files changed, 218 insertions(+), 10 deletions(-) create mode 100644 components/engine/pkg/libcontainer/cgroup/cgroup.go diff --git a/components/engine/pkg/cgroups/cgroups.go b/components/engine/pkg/cgroups/cgroups.go index 91ac3842ac..b9318f99e7 100644 --- a/components/engine/pkg/cgroups/cgroups.go +++ b/components/engine/pkg/cgroups/cgroups.go @@ -40,6 +40,16 @@ func GetThisCgroupDir(subsystem string) (string, error) { return parseCgroupFile(subsystem, f) } +func GetInitCgroupDir(subsystem string) (string, error) { + f, err := os.Open("/proc/1/cgroup") + if err != nil { + return "", err + } + defer f.Close() + + return parseCgroupFile(subsystem, f) +} + func parseCgroupFile(subsystem string, r io.Reader) (string, error) { s := bufio.NewScanner(r) @@ -49,8 +59,10 @@ func parseCgroupFile(subsystem string, r io.Reader) (string, error) { } text := s.Text() parts := strings.Split(text, ":") - if parts[1] == subsystem { - return parts[2], nil + for _, subs := range strings.Split(parts[1], ",") { + if subs == subsystem { + return parts[2], nil + } } } return "", fmt.Errorf("cgroup '%s' not found in /proc/self/cgroup", subsystem) diff --git a/components/engine/pkg/libcontainer/cgroup/cgroup.go b/components/engine/pkg/libcontainer/cgroup/cgroup.go new file mode 100644 index 0000000000..e30262ca50 --- /dev/null +++ b/components/engine/pkg/libcontainer/cgroup/cgroup.go @@ -0,0 +1,177 @@ +package cgroup + +import ( + "fmt" + "github.com/dotcloud/docker/pkg/cgroups" + "github.com/dotcloud/docker/pkg/libcontainer" + "io/ioutil" + "os" + "path/filepath" + "strconv" +) + +// We have two implementation of cgroups support, one is based on +// systemd and the dbus api, and one is based on raw cgroup fs operations +// following the pre-single-writer model docs at: +// http://www.freedesktop.org/wiki/Software/systemd/PaxControlGroups/ +const ( + cgroupRoot = "/sys/fs/cgroup" +) + +func useSystemd() bool { + return false +} + +func applyCgroupSystemd(container *libcontainer.Container, pid int) error { + return fmt.Errorf("not supported yet") +} + +func writeFile(dir, file, data string) error { + return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) +} + +func getCgroup(subsystem string, container *libcontainer.Container) (string, error) { + cgroup := container.CgroupName + if container.CgroupParent != "" { + cgroup = filepath.Join(container.CgroupParent, cgroup) + } + + initPath, err := cgroups.GetInitCgroupDir(subsystem) + if err != nil { + return "", err + } + + path := filepath.Join(cgroupRoot, subsystem, initPath, cgroup) + + return path, nil +} + +func joinCgroup(subsystem string, container *libcontainer.Container, pid int) (string, error) { + path, err := getCgroup(subsystem, container) + if err != nil { + return "", err + } + + if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) { + return "", err + } + + if err := writeFile(path, "tasks", strconv.Itoa(pid)); err != nil { + return "", err + } + + return path, nil +} + +func applyCgroupRaw(container *libcontainer.Container, pid int) (retErr error) { + if _, err := os.Stat(cgroupRoot); err != nil { + return fmt.Errorf("cgroups fs not found") + } + + if !container.DeviceAccess { + dir, err := joinCgroup("devices", container, pid) + if err != nil { + return err + } + defer func() { + if retErr != nil { + os.RemoveAll(dir) + } + }() + + if err := writeFile(dir, "devices.deny", "a"); err != nil { + return err + } + + allow := []string{ + // /dev/null, zero, full + "c 1:3 rwm", + "c 1:5 rwm", + "c 1:7 rwm", + + // consoles + "c 5:1 rwm", + "c 5:0 rwm", + "c 4:0 rwm", + "c 4:1 rwm", + + // /dev/urandom,/dev/random + "c 1:9 rwm", + "c 1:8 rwm", + + // /dev/pts/ - pts namespaces are "coming soon" + "c 136:* rwm", + "c 5:2 rwm", + + // tuntap + "c 10:200 rwm", + } + + for _, val := range allow { + if err := writeFile(dir, "devices.allow", val); err != nil { + return err + } + } + } + + if container.Memory != 0 || container.MemorySwap != 0 { + dir, err := joinCgroup("memory", container, pid) + if err != nil { + return err + } + defer func() { + if retErr != nil { + os.RemoveAll(dir) + } + }() + + if container.Memory != 0 { + if err := writeFile(dir, "memory.limit_in_bytes", strconv.FormatInt(container.Memory, 10)); err != nil { + return err + } + if err := writeFile(dir, "memory.soft_limit_in_bytes", strconv.FormatInt(container.Memory, 10)); err != nil { + return err + } + } + if container.MemorySwap != 0 { + if err := writeFile(dir, "memory.memsw.limit_in_bytes", strconv.FormatInt(container.MemorySwap, 10)); err != nil { + return err + } + } + } + + // We always want to join the cpu group, to allow fair cpu scheduling + // on a container basis + dir, err := joinCgroup("cpu", container, pid) + if err != nil { + return err + } + if container.CpuShares != 0 { + if err := writeFile(dir, "cpu.shares", strconv.FormatInt(container.CpuShares, 10)); err != nil { + return err + } + } + return nil +} + +func CleanupCgroup(container *libcontainer.Container) error { + path, _ := getCgroup("memory", container) + os.RemoveAll(path) + path, _ = getCgroup("devices", container) + os.RemoveAll(path) + path, _ = getCgroup("cpu", container) + os.RemoveAll(path) + return nil +} + +func ApplyCgroup(container *libcontainer.Container, pid int) error { + if container.CgroupName == "" { + return nil + } + + if useSystemd() { + return applyCgroupSystemd(container, pid) + } else { + return applyCgroupRaw(container, pid) + } +} diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index a6a57dab77..b34ac8b351 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -11,6 +11,13 @@ type Container struct { Namespaces Namespaces `json:"namespaces,omitempty"` // namespaces to apply Capabilities Capabilities `json:"capabilities,omitempty"` // capabilities to drop Network *Network `json:"network,omitempty"` // nil for host's network stack + + CgroupName string `json:"cgroup_name,omitempty"` // name of cgroup + CgroupParent string `json:"cgroup_parent,omitempty"` // name of parent cgroup or slice + DeviceAccess bool `json:"device_access,omitempty"` // name of parent cgroup or slice + Memory int64 `json:"memory,omitempty"` // Memory limit (in bytes) + MemorySwap int64 `json:"memory_swap,omitempty"` // Total memory usage (memory + swap); set `-1' to disable swap + CpuShares int64 `json:"cpu_shares,omitempty"` // CPU shares (relative weight vs. other containers) } // Network defines configuration for a container's networking stack diff --git a/components/engine/pkg/libcontainer/container.json b/components/engine/pkg/libcontainer/container.json index ccc9abb041..3e23600630 100644 --- a/components/engine/pkg/libcontainer/container.json +++ b/components/engine/pkg/libcontainer/container.json @@ -34,5 +34,8 @@ "gateway": "172.17.42.1", "bridge": "docker0", "mtu": 1500 - } + }, + "cgroup_name": "docker-koye", + "cgroup_parent": "docker", + "memory": 524800 } diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 202cfcab5e..acff647c61 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -5,6 +5,7 @@ package main import ( "fmt" "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/libcontainer/cgroup" "github.com/dotcloud/docker/pkg/libcontainer/network" "github.com/dotcloud/docker/pkg/libcontainer/utils" "github.com/dotcloud/docker/pkg/system" @@ -33,10 +34,18 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) return -1, err } if err := writePidFile(command); err != nil { + command.Process.Kill() return -1, err } defer deletePidFile() + // Do this before syncing with child so that no children + // can escape the cgroup + if err := cgroup.ApplyCgroup(container, command.Process.Pid); err != nil { + command.Process.Kill() + return -1, err + } + if container.Network != nil { vethPair, err := initializeContainerVeth(container.Network.Bridge, command.Process.Pid) if err != nil { @@ -45,6 +54,9 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) sendVethName(vethPair, inPipe) } + // Sync with child + inPipe.Close() + go io.Copy(os.Stdout, master) go io.Copy(master, os.Stdin) @@ -67,7 +79,6 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) // pipe so that the child stops waiting for more data func sendVethName(name string, pipe io.WriteCloser) { fmt.Fprint(pipe, name) - pipe.Close() } // initializeContainerVeth will create a veth pair and setup the host's diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index c77fd90447..f619276e60 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -20,12 +20,10 @@ func initCommand(container *libcontainer.Container, console string, args []strin return err } - var tempVethName string - if container.Network != nil { - tempVethName, err = getVethName() - if err != nil { - return err - } + // We always read this as it is a way to sync with the parent as well + tempVethName, err := getVethName() + if err != nil { + return err } // close pipes so that we can replace it with the pty From 02c5334532f7ba7592547840563edec596ee3b61 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 20 Feb 2014 14:40:00 -0800 Subject: [PATCH 203/403] Revert "WIP for setup kmsg" This reverts commit 80db9a918337c4ae80ffa9a001da13bd24e848c8. Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 5f84738ef139f696e339afb8280eb74917f2167c Component: engine --- .../engine/pkg/libcontainer/nsinit/mount.go | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/components/engine/pkg/libcontainer/nsinit/mount.go b/components/engine/pkg/libcontainer/nsinit/mount.go index 67f9020350..6eb2e09060 100644 --- a/components/engine/pkg/libcontainer/nsinit/mount.go +++ b/components/engine/pkg/libcontainer/nsinit/mount.go @@ -43,9 +43,6 @@ func setupNewMountNamespace(rootfs, console string, readonly bool) error { if err := setupPtmx(rootfs, console); err != nil { return err } - if err := setupKmsg(rootfs); err != nil { - return err - } if err := system.Chdir(rootfs); err != nil { return fmt.Errorf("chdir into %s %s", rootfs, err) } @@ -214,32 +211,3 @@ func remountSys() error { } return nil } - -func setupKmsg(rootfs string) error { - oldMask := system.Umask(0000) - defer system.Umask(oldMask) - - var ( - source = filepath.Join(rootfs, "dev/kmsg") - dest = filepath.Join(rootfs, "proc/kmsg") - ) - - if err := system.Mkfifo(source, 0600); err != nil { - return err - } - - os.Chmod(source, 0600) - os.Chown(source, 0, 0) - - if err := system.Mount(source, dest, "bind", syscall.MS_BIND, ""); err != nil { - return err - } - _, err := os.OpenFile(source, syscall.O_RDWR|syscall.O_NDELAY|syscall.O_CLOEXEC, 0) - if err != nil { - return err - } - if err := syscall.Unlink(source); err != nil { - return err - } - return nil -} From dc9b9ecbccf5528a05e89e345e03ed051592ea9b Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 20 Feb 2014 14:40:36 -0800 Subject: [PATCH 204/403] Remove clone_vfork Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: f00f37413826e31e9eb87096b67c609fdfa457b9 Component: engine --- components/engine/pkg/libcontainer/nsinit/exec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index acff647c61..f73ad3281e 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -165,7 +165,7 @@ func deletePidFile() error { func createCommand(container *libcontainer.Container, console string, args []string) *exec.Cmd { command := exec.Command("nsinit", append([]string{"init", console}, args...)...) command.SysProcAttr = &syscall.SysProcAttr{ - Cloneflags: uintptr(getNamespaceFlags(container.Namespaces) | syscall.CLONE_VFORK), // we need CLONE_VFORK so we can wait on the child + Cloneflags: uintptr(getNamespaceFlags(container.Namespaces)), } return command } From 680db4b285f2b12e12bc25ed48536e728dcdddff Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 20 Feb 2014 15:48:48 -0800 Subject: [PATCH 205/403] Refactory cgroups into general pkg Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: c44258630575f70231b11fb55bc4edc3fb677cab Component: engine --- components/engine/pkg/cgroups/cgroups.go | 61 +++++++- .../engine/pkg/libcontainer/cgroup/cgroup.go | 131 +++++++----------- .../engine/pkg/libcontainer/container.go | 28 ++-- .../engine/pkg/libcontainer/container.json | 8 +- 4 files changed, 124 insertions(+), 104 deletions(-) diff --git a/components/engine/pkg/cgroups/cgroups.go b/components/engine/pkg/cgroups/cgroups.go index b9318f99e7..1e96caa7e3 100644 --- a/components/engine/pkg/cgroups/cgroups.go +++ b/components/engine/pkg/cgroups/cgroups.go @@ -5,10 +5,23 @@ import ( "fmt" "github.com/dotcloud/docker/pkg/mount" "io" + "io/ioutil" "os" + "path/filepath" + "strconv" "strings" ) +type Cgroup struct { + Name string `json:"name,omitempty"` + Parent string `json:"parent,omitempty"` + + DeviceAccess bool `json:"device_access,omitempty"` // name of parent cgroup or slice + Memory int64 `json:"memory,omitempty"` // Memory limit (in bytes) + MemorySwap int64 `json:"memory_swap,omitempty"` // Total memory usage (memory + swap); set `-1' to disable swap + CpuShares int64 `json:"cpu_shares,omitempty"` // CPU shares (relative weight vs. other containers) +} + // https://www.kernel.org/doc/Documentation/cgroups/cgroups.txt func FindCgroupMountpoint(subsystem string) (string, error) { mounts, err := mount.GetMounts() @@ -25,7 +38,6 @@ func FindCgroupMountpoint(subsystem string) (string, error) { } } } - return "", fmt.Errorf("cgroup mountpoint not found for %s", subsystem) } @@ -50,9 +62,50 @@ func GetInitCgroupDir(subsystem string) (string, error) { return parseCgroupFile(subsystem, f) } +func (c *Cgroup) Path(root, subsystem string) (string, error) { + cgroup := c.Name + if c.Parent != "" { + cgroup = filepath.Join(c.Parent, cgroup) + } + initPath, err := GetInitCgroupDir(subsystem) + if err != nil { + return "", err + } + return filepath.Join(root, subsystem, initPath, cgroup), nil +} + +func (c *Cgroup) Join(root, subsystem string, pid int) (string, error) { + path, err := c.Path(root, subsystem) + if err != nil { + return "", err + } + if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) { + return "", err + } + if err := writeFile(path, "tasks", strconv.Itoa(pid)); err != nil { + return "", err + } + return path, nil +} + +func (c *Cgroup) Cleanup(root string) error { + get := func(subsystem string) string { + path, _ := c.Path(root, subsystem) + return path + } + + for _, path := range []string{ + get("memory"), + get("devices"), + get("cpu"), + } { + os.RemoveAll(path) + } + return nil +} + func parseCgroupFile(subsystem string, r io.Reader) (string, error) { s := bufio.NewScanner(r) - for s.Scan() { if err := s.Err(); err != nil { return "", err @@ -67,3 +120,7 @@ func parseCgroupFile(subsystem string, r io.Reader) (string, error) { } return "", fmt.Errorf("cgroup '%s' not found in /proc/self/cgroup", subsystem) } + +func writeFile(dir, file, data string) error { + return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) +} diff --git a/components/engine/pkg/libcontainer/cgroup/cgroup.go b/components/engine/pkg/libcontainer/cgroup/cgroup.go index e30262ca50..5f27ac3ffb 100644 --- a/components/engine/pkg/libcontainer/cgroup/cgroup.go +++ b/components/engine/pkg/libcontainer/cgroup/cgroup.go @@ -10,71 +10,46 @@ import ( "strconv" ) -// We have two implementation of cgroups support, one is based on -// systemd and the dbus api, and one is based on raw cgroup fs operations -// following the pre-single-writer model docs at: -// http://www.freedesktop.org/wiki/Software/systemd/PaxControlGroups/ -const ( - cgroupRoot = "/sys/fs/cgroup" -) - -func useSystemd() bool { - return false -} - -func applyCgroupSystemd(container *libcontainer.Container, pid int) error { - return fmt.Errorf("not supported yet") -} - -func writeFile(dir, file, data string) error { - return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) -} - -func getCgroup(subsystem string, container *libcontainer.Container) (string, error) { - cgroup := container.CgroupName - if container.CgroupParent != "" { - cgroup = filepath.Join(container.CgroupParent, cgroup) +func ApplyCgroup(container *libcontainer.Container, pid int) (err error) { + if container.Cgroups == nil { + return nil } - initPath, err := cgroups.GetInitCgroupDir(subsystem) + // We have two implementation of cgroups support, one is based on + // systemd and the dbus api, and one is based on raw cgroup fs operations + // following the pre-single-writer model docs at: + // http://www.freedesktop.org/wiki/Software/systemd/PaxControlGroups/ + // + // we can pick any subsystem to find the root + cgroupRoot, err := cgroups.FindCgroupMountpoint("memory") if err != nil { - return "", err + return err } - - path := filepath.Join(cgroupRoot, subsystem, initPath, cgroup) - - return path, nil -} - -func joinCgroup(subsystem string, container *libcontainer.Container, pid int) (string, error) { - path, err := getCgroup(subsystem, container) - if err != nil { - return "", err - } - - if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) { - return "", err - } - - if err := writeFile(path, "tasks", strconv.Itoa(pid)); err != nil { - return "", err - } - - return path, nil -} - -func applyCgroupRaw(container *libcontainer.Container, pid int) (retErr error) { + cgroupRoot = filepath.Dir(cgroupRoot) if _, err := os.Stat(cgroupRoot); err != nil { return fmt.Errorf("cgroups fs not found") } + if err := setupDevices(container, cgroupRoot, pid); err != nil { + return err + } + if err := setupMemory(container, cgroupRoot, pid); err != nil { + return err + } + if err := setupCpu(container, cgroupRoot, pid); err != nil { + return err + } + return nil +} - if !container.DeviceAccess { - dir, err := joinCgroup("devices", container, pid) +func setupDevices(container *libcontainer.Container, cgroupRoot string, pid int) (err error) { + if !container.Cgroups.DeviceAccess { + dir, err := container.Cgroups.Join(cgroupRoot, "devices", pid) if err != nil { return err } + defer func() { - if retErr != nil { + if err != nil { os.RemoveAll(dir) } }() @@ -113,65 +88,53 @@ func applyCgroupRaw(container *libcontainer.Container, pid int) (retErr error) { } } } + return nil +} - if container.Memory != 0 || container.MemorySwap != 0 { - dir, err := joinCgroup("memory", container, pid) +func setupMemory(container *libcontainer.Container, cgroupRoot string, pid int) (err error) { + if container.Cgroups.Memory != 0 || container.Cgroups.MemorySwap != 0 { + dir, err := container.Cgroups.Join(cgroupRoot, "memory", pid) if err != nil { return err } defer func() { - if retErr != nil { + if err != nil { os.RemoveAll(dir) } }() - if container.Memory != 0 { - if err := writeFile(dir, "memory.limit_in_bytes", strconv.FormatInt(container.Memory, 10)); err != nil { + if container.Cgroups.Memory != 0 { + if err := writeFile(dir, "memory.limit_in_bytes", strconv.FormatInt(container.Cgroups.Memory, 10)); err != nil { return err } - if err := writeFile(dir, "memory.soft_limit_in_bytes", strconv.FormatInt(container.Memory, 10)); err != nil { + if err := writeFile(dir, "memory.soft_limit_in_bytes", strconv.FormatInt(container.Cgroups.Memory, 10)); err != nil { return err } } - if container.MemorySwap != 0 { - if err := writeFile(dir, "memory.memsw.limit_in_bytes", strconv.FormatInt(container.MemorySwap, 10)); err != nil { + if container.Cgroups.MemorySwap != 0 { + if err := writeFile(dir, "memory.memsw.limit_in_bytes", strconv.FormatInt(container.Cgroups.MemorySwap, 10)); err != nil { return err } } } + return nil +} +func setupCpu(container *libcontainer.Container, cgroupRoot string, pid int) (err error) { // We always want to join the cpu group, to allow fair cpu scheduling // on a container basis - dir, err := joinCgroup("cpu", container, pid) + dir, err := container.Cgroups.Join(cgroupRoot, "cpu", pid) if err != nil { return err } - if container.CpuShares != 0 { - if err := writeFile(dir, "cpu.shares", strconv.FormatInt(container.CpuShares, 10)); err != nil { + if container.Cgroups.CpuShares != 0 { + if err := writeFile(dir, "cpu.shares", strconv.FormatInt(container.Cgroups.CpuShares, 10)); err != nil { return err } } return nil } -func CleanupCgroup(container *libcontainer.Container) error { - path, _ := getCgroup("memory", container) - os.RemoveAll(path) - path, _ = getCgroup("devices", container) - os.RemoveAll(path) - path, _ = getCgroup("cpu", container) - os.RemoveAll(path) - return nil -} - -func ApplyCgroup(container *libcontainer.Container, pid int) error { - if container.CgroupName == "" { - return nil - } - - if useSystemd() { - return applyCgroupSystemd(container, pid) - } else { - return applyCgroupRaw(container, pid) - } +func writeFile(dir, file, data string) error { + return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) } diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index b34ac8b351..4c0e39a798 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -1,23 +1,21 @@ package libcontainer +import ( + "github.com/dotcloud/docker/pkg/cgroups" +) + // Container defines configuration options for how a // container is setup inside a directory and how a process should be executed type Container struct { - Hostname string `json:"hostname,omitempty"` // hostname - ReadonlyFs bool `json:"readonly_fs,omitempty"` // set the containers rootfs as readonly - User string `json:"user,omitempty"` // user to execute the process as - WorkingDir string `json:"working_dir,omitempty"` // current working directory - Env []string `json:"environment,omitempty"` // environment to set - Namespaces Namespaces `json:"namespaces,omitempty"` // namespaces to apply - Capabilities Capabilities `json:"capabilities,omitempty"` // capabilities to drop - Network *Network `json:"network,omitempty"` // nil for host's network stack - - CgroupName string `json:"cgroup_name,omitempty"` // name of cgroup - CgroupParent string `json:"cgroup_parent,omitempty"` // name of parent cgroup or slice - DeviceAccess bool `json:"device_access,omitempty"` // name of parent cgroup or slice - Memory int64 `json:"memory,omitempty"` // Memory limit (in bytes) - MemorySwap int64 `json:"memory_swap,omitempty"` // Total memory usage (memory + swap); set `-1' to disable swap - CpuShares int64 `json:"cpu_shares,omitempty"` // CPU shares (relative weight vs. other containers) + Hostname string `json:"hostname,omitempty"` // hostname + ReadonlyFs bool `json:"readonly_fs,omitempty"` // set the containers rootfs as readonly + User string `json:"user,omitempty"` // user to execute the process as + WorkingDir string `json:"working_dir,omitempty"` // current working directory + Env []string `json:"environment,omitempty"` // environment to set + Namespaces Namespaces `json:"namespaces,omitempty"` // namespaces to apply + Capabilities Capabilities `json:"capabilities,omitempty"` // capabilities to drop + Network *Network `json:"network,omitempty"` // nil for host's network stack + Cgroups *cgroups.Cgroup `json:"cgroups,omitempty"` } // Network defines configuration for a container's networking stack diff --git a/components/engine/pkg/libcontainer/container.json b/components/engine/pkg/libcontainer/container.json index 3e23600630..2207543bd0 100644 --- a/components/engine/pkg/libcontainer/container.json +++ b/components/engine/pkg/libcontainer/container.json @@ -35,7 +35,9 @@ "bridge": "docker0", "mtu": 1500 }, - "cgroup_name": "docker-koye", - "cgroup_parent": "docker", - "memory": 524800 + "cgroups": { + "name": "docker-koye", + "parent": "docker", + "memory": 524800 + } } From 9aba82e1b27effba65c72df740a584b9fda6b00a Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 20 Feb 2014 15:50:55 -0800 Subject: [PATCH 206/403] Change IP to address because it includes the subnet Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 3cb698125da7b55a7d7ec43b33858f35844a6143 Component: engine --- components/engine/pkg/libcontainer/README.md | 7 ++++++- components/engine/pkg/libcontainer/container.go | 2 +- components/engine/pkg/libcontainer/container.json | 2 +- components/engine/pkg/libcontainer/nsinit/init.go | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/components/engine/pkg/libcontainer/README.md b/components/engine/pkg/libcontainer/README.md index 07fe4f7b2d..163161c178 100644 --- a/components/engine/pkg/libcontainer/README.md +++ b/components/engine/pkg/libcontainer/README.md @@ -47,10 +47,15 @@ Sample `container.json` file: "MAC_ADMIN" ], "network": { - "ip": "172.17.0.100/16", + "address": "172.17.0.100/16", "gateway": "172.17.42.1", "bridge": "docker0", "mtu": 1500 + }, + "cgroups": { + "name": "docker-koye", + "parent": "docker", + "memory": 524800 } } ``` diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index 4c0e39a798..e6e4b4747e 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -23,7 +23,7 @@ type Container struct { // The network configuration can be omited from a container causing the // container to be setup with the host's networking stack type Network struct { - IP string `json:"ip,omitempty"` + Address string `json:"address,omitempty"` Gateway string `json:"gateway,omitempty"` Bridge string `json:"bridge,omitempty"` Mtu int `json:"mtu,omitempty"` diff --git a/components/engine/pkg/libcontainer/container.json b/components/engine/pkg/libcontainer/container.json index 2207543bd0..c1a07dc55b 100644 --- a/components/engine/pkg/libcontainer/container.json +++ b/components/engine/pkg/libcontainer/container.json @@ -30,7 +30,7 @@ "MAC_ADMIN" ], "network": { - "ip": "172.17.0.100/16", + "address": "172.17.0.100/16", "gateway": "172.17.42.1", "bridge": "docker0", "mtu": 1500 diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index f619276e60..f89e53982c 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -137,7 +137,7 @@ func setupVethNetwork(config *libcontainer.Network, tempVethName string) error { if err := network.ChangeInterfaceName(tempVethName, "eth0"); err != nil { return fmt.Errorf("change %s to eth0 %s", tempVethName, err) } - if err := network.SetInterfaceIp("eth0", config.IP); err != nil { + if err := network.SetInterfaceIp("eth0", config.Address); err != nil { return fmt.Errorf("set eth0 ip %s", err) } if err := network.SetMtu("eth0", config.Mtu); err != nil { From 3b343c063f3b536473776b856a4eb060ef512798 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 20 Feb 2014 16:11:22 -0800 Subject: [PATCH 207/403] Move rest of cgroups functions into cgroups pkg Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 7020e208c70dfca5ebc97d699553e4bf1c6ab0bb Component: engine --- components/engine/pkg/cgroups/cgroups.go | 122 +++++++++++++++ .../engine/pkg/libcontainer/cgroup/cgroup.go | 140 ------------------ .../engine/pkg/libcontainer/nsinit/exec.go | 9 +- 3 files changed, 127 insertions(+), 144 deletions(-) delete mode 100644 components/engine/pkg/libcontainer/cgroup/cgroup.go diff --git a/components/engine/pkg/cgroups/cgroups.go b/components/engine/pkg/cgroups/cgroups.go index 1e96caa7e3..96002f0af9 100644 --- a/components/engine/pkg/cgroups/cgroups.go +++ b/components/engine/pkg/cgroups/cgroups.go @@ -124,3 +124,125 @@ func parseCgroupFile(subsystem string, r io.Reader) (string, error) { func writeFile(dir, file, data string) error { return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) } + +func (c *Cgroup) Apply(pid int) error { + // We have two implementation of cgroups support, one is based on + // systemd and the dbus api, and one is based on raw cgroup fs operations + // following the pre-single-writer model docs at: + // http://www.freedesktop.org/wiki/Software/systemd/PaxControlGroups/ + // + // we can pick any subsystem to find the root + cgroupRoot, err := FindCgroupMountpoint("memory") + if err != nil { + return err + } + cgroupRoot = filepath.Dir(cgroupRoot) + + if _, err := os.Stat(cgroupRoot); err != nil { + return fmt.Errorf("cgroups fs not found") + } + if err := c.setupDevices(cgroupRoot, pid); err != nil { + return err + } + if err := c.setupMemory(cgroupRoot, pid); err != nil { + return err + } + if err := c.setupCpu(cgroupRoot, pid); err != nil { + return err + } + return nil +} + +func (c *Cgroup) setupDevices(cgroupRoot string, pid int) (err error) { + if !c.DeviceAccess { + dir, err := c.Join(cgroupRoot, "devices", pid) + if err != nil { + return err + } + + defer func() { + if err != nil { + os.RemoveAll(dir) + } + }() + + if err := writeFile(dir, "devices.deny", "a"); err != nil { + return err + } + + allow := []string{ + // /dev/null, zero, full + "c 1:3 rwm", + "c 1:5 rwm", + "c 1:7 rwm", + + // consoles + "c 5:1 rwm", + "c 5:0 rwm", + "c 4:0 rwm", + "c 4:1 rwm", + + // /dev/urandom,/dev/random + "c 1:9 rwm", + "c 1:8 rwm", + + // /dev/pts/ - pts namespaces are "coming soon" + "c 136:* rwm", + "c 5:2 rwm", + + // tuntap + "c 10:200 rwm", + } + + for _, val := range allow { + if err := writeFile(dir, "devices.allow", val); err != nil { + return err + } + } + } + return nil +} + +func (c *Cgroup) setupMemory(cgroupRoot string, pid int) (err error) { + if c.Memory != 0 || c.MemorySwap != 0 { + dir, err := c.Join(cgroupRoot, "memory", pid) + if err != nil { + return err + } + defer func() { + if err != nil { + os.RemoveAll(dir) + } + }() + + if c.Memory != 0 { + if err := writeFile(dir, "memory.limit_in_bytes", strconv.FormatInt(c.Memory, 10)); err != nil { + return err + } + if err := writeFile(dir, "memory.soft_limit_in_bytes", strconv.FormatInt(c.Memory, 10)); err != nil { + return err + } + } + if c.MemorySwap != 0 { + if err := writeFile(dir, "memory.memsw.limit_in_bytes", strconv.FormatInt(c.MemorySwap, 10)); err != nil { + return err + } + } + } + return nil +} + +func (c *Cgroup) setupCpu(cgroupRoot string, pid int) (err error) { + // We always want to join the cpu group, to allow fair cpu scheduling + // on a container basis + dir, err := c.Join(cgroupRoot, "cpu", pid) + if err != nil { + return err + } + if c.CpuShares != 0 { + if err := writeFile(dir, "cpu.shares", strconv.FormatInt(c.CpuShares, 10)); err != nil { + return err + } + } + return nil +} diff --git a/components/engine/pkg/libcontainer/cgroup/cgroup.go b/components/engine/pkg/libcontainer/cgroup/cgroup.go deleted file mode 100644 index 5f27ac3ffb..0000000000 --- a/components/engine/pkg/libcontainer/cgroup/cgroup.go +++ /dev/null @@ -1,140 +0,0 @@ -package cgroup - -import ( - "fmt" - "github.com/dotcloud/docker/pkg/cgroups" - "github.com/dotcloud/docker/pkg/libcontainer" - "io/ioutil" - "os" - "path/filepath" - "strconv" -) - -func ApplyCgroup(container *libcontainer.Container, pid int) (err error) { - if container.Cgroups == nil { - return nil - } - - // We have two implementation of cgroups support, one is based on - // systemd and the dbus api, and one is based on raw cgroup fs operations - // following the pre-single-writer model docs at: - // http://www.freedesktop.org/wiki/Software/systemd/PaxControlGroups/ - // - // we can pick any subsystem to find the root - cgroupRoot, err := cgroups.FindCgroupMountpoint("memory") - if err != nil { - return err - } - cgroupRoot = filepath.Dir(cgroupRoot) - if _, err := os.Stat(cgroupRoot); err != nil { - return fmt.Errorf("cgroups fs not found") - } - if err := setupDevices(container, cgroupRoot, pid); err != nil { - return err - } - if err := setupMemory(container, cgroupRoot, pid); err != nil { - return err - } - if err := setupCpu(container, cgroupRoot, pid); err != nil { - return err - } - return nil -} - -func setupDevices(container *libcontainer.Container, cgroupRoot string, pid int) (err error) { - if !container.Cgroups.DeviceAccess { - dir, err := container.Cgroups.Join(cgroupRoot, "devices", pid) - if err != nil { - return err - } - - defer func() { - if err != nil { - os.RemoveAll(dir) - } - }() - - if err := writeFile(dir, "devices.deny", "a"); err != nil { - return err - } - - allow := []string{ - // /dev/null, zero, full - "c 1:3 rwm", - "c 1:5 rwm", - "c 1:7 rwm", - - // consoles - "c 5:1 rwm", - "c 5:0 rwm", - "c 4:0 rwm", - "c 4:1 rwm", - - // /dev/urandom,/dev/random - "c 1:9 rwm", - "c 1:8 rwm", - - // /dev/pts/ - pts namespaces are "coming soon" - "c 136:* rwm", - "c 5:2 rwm", - - // tuntap - "c 10:200 rwm", - } - - for _, val := range allow { - if err := writeFile(dir, "devices.allow", val); err != nil { - return err - } - } - } - return nil -} - -func setupMemory(container *libcontainer.Container, cgroupRoot string, pid int) (err error) { - if container.Cgroups.Memory != 0 || container.Cgroups.MemorySwap != 0 { - dir, err := container.Cgroups.Join(cgroupRoot, "memory", pid) - if err != nil { - return err - } - defer func() { - if err != nil { - os.RemoveAll(dir) - } - }() - - if container.Cgroups.Memory != 0 { - if err := writeFile(dir, "memory.limit_in_bytes", strconv.FormatInt(container.Cgroups.Memory, 10)); err != nil { - return err - } - if err := writeFile(dir, "memory.soft_limit_in_bytes", strconv.FormatInt(container.Cgroups.Memory, 10)); err != nil { - return err - } - } - if container.Cgroups.MemorySwap != 0 { - if err := writeFile(dir, "memory.memsw.limit_in_bytes", strconv.FormatInt(container.Cgroups.MemorySwap, 10)); err != nil { - return err - } - } - } - return nil -} - -func setupCpu(container *libcontainer.Container, cgroupRoot string, pid int) (err error) { - // We always want to join the cpu group, to allow fair cpu scheduling - // on a container basis - dir, err := container.Cgroups.Join(cgroupRoot, "cpu", pid) - if err != nil { - return err - } - if container.Cgroups.CpuShares != 0 { - if err := writeFile(dir, "cpu.shares", strconv.FormatInt(container.Cgroups.CpuShares, 10)); err != nil { - return err - } - } - return nil -} - -func writeFile(dir, file, data string) error { - return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) -} diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index f73ad3281e..f04e9bee20 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -5,7 +5,6 @@ package main import ( "fmt" "github.com/dotcloud/docker/pkg/libcontainer" - "github.com/dotcloud/docker/pkg/libcontainer/cgroup" "github.com/dotcloud/docker/pkg/libcontainer/network" "github.com/dotcloud/docker/pkg/libcontainer/utils" "github.com/dotcloud/docker/pkg/system" @@ -41,9 +40,11 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) // Do this before syncing with child so that no children // can escape the cgroup - if err := cgroup.ApplyCgroup(container, command.Process.Pid); err != nil { - command.Process.Kill() - return -1, err + if container.Cgroups != nil { + if err := container.Cgroups.Apply(command.Process.Pid); err != nil { + command.Process.Kill() + return -1, err + } } if container.Network != nil { From ee2c282f60bef018aa86492d6fc522097e3ca1b1 Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 20 Feb 2014 17:53:50 -0800 Subject: [PATCH 208/403] Use flag for init Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: b519d3ea5a50ad7c15d576a89ec9846c4fc123fa Component: engine --- .../engine/pkg/libcontainer/nsinit/exec.go | 2 +- .../engine/pkg/libcontainer/nsinit/main.go | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index f04e9bee20..6d87f3b66d 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -164,7 +164,7 @@ func deletePidFile() error { // defined on the container's configuration and use the current binary as the init with the // args provided func createCommand(container *libcontainer.Container, console string, args []string) *exec.Cmd { - command := exec.Command("nsinit", append([]string{"init", console}, args...)...) + command := exec.Command("nsinit", append([]string{"-console", console, "init"}, args...)...) command.SysProcAttr = &syscall.SysProcAttr{ Cloneflags: uintptr(getNamespaceFlags(container.Namespaces)), } diff --git a/components/engine/pkg/libcontainer/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/main.go index f45fe55689..e7240df041 100644 --- a/components/engine/pkg/libcontainer/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/main.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "errors" + "flag" "github.com/dotcloud/docker/pkg/libcontainer" "io/ioutil" "log" @@ -16,16 +17,18 @@ var ( ) func main() { + console := flag.String("console", "", "Console (pty slave) name") + flag.Parse() + container, err := loadContainer() if err != nil { log.Fatal(err) } - argc := len(os.Args) - if argc < 2 { + if flag.NArg() < 1 { log.Fatal(ErrWrongArguments) } - switch os.Args[1] { + switch flag.Arg(0) { case "exec": // this is executed outside of the namespace in the cwd var exitCode int nspid, err := readPid() @@ -35,23 +38,23 @@ func main() { } } if nspid > 0 { - exitCode, err = execinCommand(container, nspid, os.Args[2:]) + exitCode, err = execinCommand(container, nspid, flag.Args()[1:]) } else { - exitCode, err = execCommand(container, os.Args[2:]) + exitCode, err = execCommand(container, flag.Args()[1:]) } if err != nil { log.Fatal(err) } os.Exit(exitCode) case "init": // this is executed inside of the namespace to setup the container - if argc < 3 { + if flag.NArg() < 2 { log.Fatal(ErrWrongArguments) } - if err := initCommand(container, os.Args[2], os.Args[3:]); err != nil { + if err := initCommand(container, *console, flag.Args()[1:]); err != nil { log.Fatal(err) } default: - log.Fatalf("command not supported for nsinit %s", os.Args[1]) + log.Fatalf("command not supported for nsinit %s", flag.Arg(0)) } } From 60d018051a703d45abe5f70576e5fbb35ec2363c Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 20 Feb 2014 17:58:13 -0800 Subject: [PATCH 209/403] Use a custom pipe instead of stdin for sync net namespace Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 8dec4adcb3fd905eb05f07678fa7f5bb47d8242f Component: engine --- .../engine/pkg/libcontainer/nsinit/exec.go | 16 ++++++++++------ .../engine/pkg/libcontainer/nsinit/init.go | 11 +++++++---- .../engine/pkg/libcontainer/nsinit/main.go | 3 ++- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 6d87f3b66d..8007ed4691 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -22,16 +22,20 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) return -1, err } - command := createCommand(container, console, args) // create a pipe so that we can syncronize with the namespaced process and // pass the veth name to the child - inPipe, err := command.StdinPipe() + r, w, err := os.Pipe() if err != nil { return -1, err } + system.UsetCloseOnExec(r.Fd()) + + command := createCommand(container, console, r.Fd(), args) + if err := command.Start(); err != nil { return -1, err } + if err := writePidFile(command); err != nil { command.Process.Kill() return -1, err @@ -52,11 +56,11 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) if err != nil { return -1, err } - sendVethName(vethPair, inPipe) + sendVethName(vethPair, w) } // Sync with child - inPipe.Close() + w.Close() go io.Copy(os.Stdout, master) go io.Copy(master, os.Stdin) @@ -163,8 +167,8 @@ func deletePidFile() error { // createCommand will return an exec.Cmd with the Cloneflags set to the proper namespaces // defined on the container's configuration and use the current binary as the init with the // args provided -func createCommand(container *libcontainer.Container, console string, args []string) *exec.Cmd { - command := exec.Command("nsinit", append([]string{"-console", console, "init"}, args...)...) +func createCommand(container *libcontainer.Container, console string, pipe uintptr, args []string) *exec.Cmd { + command := exec.Command("nsinit", append([]string{"-console", console, "-pipe", fmt.Sprint(pipe), "init"}, args...)...) command.SysProcAttr = &syscall.SysProcAttr{ Cloneflags: uintptr(getNamespaceFlags(container.Namespaces)), } diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index f89e53982c..a0815eef1b 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -8,20 +8,21 @@ import ( "github.com/dotcloud/docker/pkg/libcontainer/capabilities" "github.com/dotcloud/docker/pkg/libcontainer/network" "github.com/dotcloud/docker/pkg/system" + "io" "io/ioutil" "os" "path/filepath" "syscall" ) -func initCommand(container *libcontainer.Container, console string, args []string) error { +func initCommand(container *libcontainer.Container, console string, pipe io.ReadCloser, args []string) error { rootfs, err := resolveRootfs() if err != nil { return err } // We always read this as it is a way to sync with the parent as well - tempVethName, err := getVethName() + tempVethName, err := getVethName(pipe) if err != nil { return err } @@ -164,8 +165,10 @@ func setupVethNetwork(config *libcontainer.Network, tempVethName string) error { // getVethName reads from Stdin the temp veth name // sent by the parent processes after the veth pair // has been created and setup -func getVethName() (string, error) { - data, err := ioutil.ReadAll(os.Stdin) +func getVethName(pipe io.ReadCloser) (string, error) { + defer pipe.Close() + + data, err := ioutil.ReadAll(pipe) if err != nil { return "", fmt.Errorf("error reading from stdin %s", err) } diff --git a/components/engine/pkg/libcontainer/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/main.go index e7240df041..6f2825b25b 100644 --- a/components/engine/pkg/libcontainer/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/main.go @@ -18,6 +18,7 @@ var ( func main() { console := flag.String("console", "", "Console (pty slave) name") + pipeFd := flag.Int("pipe", 0, "sync pipe fd") flag.Parse() container, err := loadContainer() @@ -50,7 +51,7 @@ func main() { if flag.NArg() < 2 { log.Fatal(ErrWrongArguments) } - if err := initCommand(container, *console, flag.Args()[1:]); err != nil { + if err := initCommand(container, *console, os.NewFile(uintptr(*pipeFd), "pipe"), flag.Args()[1:]); err != nil { log.Fatal(err) } default: From 961a3fcf1357f28af6c179f3c30f877f0b3c500a Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 20 Feb 2014 17:59:08 -0800 Subject: [PATCH 210/403] Minor cleanup Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 83dfdd1d9587a7335bbf3a4656572baefae4f28d Component: engine --- components/engine/pkg/libcontainer/nsinit/exec.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 8007ed4691..7f552c2961 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -56,11 +56,12 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) if err != nil { return -1, err } - sendVethName(vethPair, w) + sendVethName(w, vethPair) } // Sync with child w.Close() + r.Close() go io.Copy(os.Stdout, master) go io.Copy(master, os.Stdin) @@ -82,7 +83,7 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) // sendVethName writes the veth pair name to the child's stdin then closes the // pipe so that the child stops waiting for more data -func sendVethName(name string, pipe io.WriteCloser) { +func sendVethName(pipe io.Writer, name string) { fmt.Fprint(pipe, name) } From 24e3f599c0019f72daeb90dd99ae61fafa0068cf Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 20 Feb 2014 18:05:40 -0800 Subject: [PATCH 211/403] Handle non-tty mode Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 1a4fb0921919720ab379bc82b7508580057770ee Component: engine --- .../engine/pkg/libcontainer/nsinit/exec.go | 55 +++++++++++++++---- .../engine/pkg/libcontainer/nsinit/init.go | 30 +++++----- .../engine/pkg/libcontainer/nsinit/main.go | 9 ++- .../engine/pkg/libcontainer/nsinit/mount.go | 6 +- 4 files changed, 70 insertions(+), 30 deletions(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 7f552c2961..b290ace3ce 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -16,10 +16,21 @@ import ( "syscall" ) -func execCommand(container *libcontainer.Container, args []string) (int, error) { - master, console, err := createMasterAndConsole() - if err != nil { - return -1, err +func execCommand(container *libcontainer.Container, tty bool, args []string) (int, error) { + var ( + master *os.File + console string + err error + + inPipe io.WriteCloser + outPipe, errPipe io.ReadCloser + ) + + if tty { + master, console, err = createMasterAndConsole() + if err != nil { + return -1, err + } } // create a pipe so that we can syncronize with the namespaced process and @@ -32,6 +43,21 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) command := createCommand(container, console, r.Fd(), args) + if !tty { + inPipe, err = command.StdinPipe() + if err != nil { + return -1, err + } + outPipe, err = command.StdoutPipe() + if err != nil { + return -1, err + } + errPipe, err = command.StderrPipe() + if err != nil { + return -1, err + } + } + if err := command.Start(); err != nil { return -1, err } @@ -63,15 +89,20 @@ func execCommand(container *libcontainer.Container, args []string) (int, error) w.Close() r.Close() - go io.Copy(os.Stdout, master) - go io.Copy(master, os.Stdin) - - state, err := setupWindow(master) - if err != nil { - command.Process.Kill() - return -1, err + if tty { + go io.Copy(os.Stdout, master) + go io.Copy(master, os.Stdin) + state, err := setupWindow(master) + if err != nil { + command.Process.Kill() + return -1, err + } + defer term.RestoreTerminal(os.Stdin.Fd(), state) + } else { + go io.Copy(inPipe, os.Stdin) + go io.Copy(os.Stdout, outPipe) + go io.Copy(os.Stderr, errPipe) } - defer term.RestoreTerminal(os.Stdin.Fd(), state) if err := command.Wait(); err != nil { if _, ok := err.(*exec.ExitError); !ok { diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index a0815eef1b..ef7fc4e44c 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -27,23 +27,27 @@ func initCommand(container *libcontainer.Container, console string, pipe io.Read return err } - // close pipes so that we can replace it with the pty - os.Stdin.Close() - os.Stdout.Close() - os.Stderr.Close() + if console != "" { + // close pipes so that we can replace it with the pty + os.Stdin.Close() + os.Stdout.Close() + os.Stderr.Close() + slave, err := openTerminal(console, syscall.O_RDWR) + if err != nil { + return fmt.Errorf("open terminal %s", err) + } + if err := dupSlave(slave); err != nil { + return fmt.Errorf("dup2 slave %s", err) + } + } - slave, err := openTerminal(console, syscall.O_RDWR) - if err != nil { - return fmt.Errorf("open terminal %s", err) - } - if err := dupSlave(slave); err != nil { - return fmt.Errorf("dup2 slave %s", err) - } if _, err := system.Setsid(); err != nil { return fmt.Errorf("setsid %s", err) } - if err := system.Setctty(); err != nil { - return fmt.Errorf("setctty %s", err) + if console != "" { + if err := system.Setctty(); err != nil { + return fmt.Errorf("setctty %s", err) + } } if err := system.ParentDeathSignal(); err != nil { return fmt.Errorf("parent deth signal %s", err) diff --git a/components/engine/pkg/libcontainer/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/main.go index 6f2825b25b..f66ff0d855 100644 --- a/components/engine/pkg/libcontainer/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/main.go @@ -17,8 +17,11 @@ var ( ) func main() { - console := flag.String("console", "", "Console (pty slave) name") - pipeFd := flag.Int("pipe", 0, "sync pipe fd") + var ( + console = flag.String("console", "", "Console (pty slave) name") + tty = flag.Bool("tty", false, "Create a tty") + pipeFd = flag.Int("pipe", 0, "sync pipe fd") + ) flag.Parse() container, err := loadContainer() @@ -41,7 +44,7 @@ func main() { if nspid > 0 { exitCode, err = execinCommand(container, nspid, flag.Args()[1:]) } else { - exitCode, err = execCommand(container, flag.Args()[1:]) + exitCode, err = execCommand(container, *tty, flag.Args()[1:]) } if err != nil { log.Fatal(err) diff --git a/components/engine/pkg/libcontainer/nsinit/mount.go b/components/engine/pkg/libcontainer/nsinit/mount.go index 6eb2e09060..9cf69f4184 100644 --- a/components/engine/pkg/libcontainer/nsinit/mount.go +++ b/components/engine/pkg/libcontainer/nsinit/mount.go @@ -40,8 +40,10 @@ func setupNewMountNamespace(rootfs, console string, readonly bool) error { if err := setupDev(rootfs); err != nil { return err } - if err := setupPtmx(rootfs, console); err != nil { - return err + if console != "" { + if err := setupPtmx(rootfs, console); err != nil { + return err + } } if err := system.Chdir(rootfs); err != nil { return fmt.Errorf("chdir into %s %s", rootfs, err) From a4597085bf1ab56eb15a074cee60e923a3ba828b Mon Sep 17 00:00:00 2001 From: "Guillaume J. Charmes" Date: Thu, 20 Feb 2014 18:10:30 -0800 Subject: [PATCH 212/403] Make sure to close the pipe upon ctrl-d Docker-DCO-1.1-Signed-off-by: Guillaume J. Charmes (github: creack) Upstream-commit: 66baa0653b636180b8b5c57c58f4bbc805aca8c5 Component: engine --- components/engine/pkg/libcontainer/nsinit/exec.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index b290ace3ce..44d9aff5f7 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -99,7 +99,10 @@ func execCommand(container *libcontainer.Container, tty bool, args []string) (in } defer term.RestoreTerminal(os.Stdin.Fd(), state) } else { - go io.Copy(inPipe, os.Stdin) + go func() { + defer inPipe.Close() + io.Copy(inPipe, os.Stdin) + }() go io.Copy(os.Stdout, outPipe) go io.Copy(os.Stderr, errPipe) } @@ -109,6 +112,7 @@ func execCommand(container *libcontainer.Container, tty bool, args []string) (in return -1, err } } + return command.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil } From 3ec79ee2525879ef8acdf9fd8a82300fd82bdd9f Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 20 Feb 2014 18:27:42 -0800 Subject: [PATCH 213/403] Make nsinit a proper go pkg and add the main in another dir Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 1316007e54e0c5a25f0d67675df7dec40286f5e8 Component: engine --- components/engine/pkg/libcontainer/README.md | 4 +++- components/engine/pkg/libcontainer/nsinit/exec.go | 6 ++++-- components/engine/pkg/libcontainer/nsinit/execin.go | 5 +++-- components/engine/pkg/libcontainer/nsinit/init.go | 6 ++++-- components/engine/pkg/libcontainer/nsinit/mount.go | 2 +- components/engine/pkg/libcontainer/nsinit/ns_linux.go | 2 +- .../engine/pkg/libcontainer/nsinit/{ => nsinit}/main.go | 7 ++++--- 7 files changed, 20 insertions(+), 12 deletions(-) rename components/engine/pkg/libcontainer/nsinit/{ => nsinit}/main.go (87%) diff --git a/components/engine/pkg/libcontainer/README.md b/components/engine/pkg/libcontainer/README.md index 163161c178..3a2a843b69 100644 --- a/components/engine/pkg/libcontainer/README.md +++ b/components/engine/pkg/libcontainer/README.md @@ -72,9 +72,11 @@ rootfs and copy a `container.json` file into the directory with your specified c To execution `/bin/bash` in the current directory as a container just run: ```bash -nsinit exec /bin/bash +nsinit -tty exec /bin/bash ``` +If you want a proper tty setup inside the new container you must use the `-tty` flag when running nsinit. + If you wish to spawn another process inside the container while your current bash session is running just run the exact same command again to get another bash shell or change the command. If the original process dies, PID 1, all other processes spawned inside the container will also be killed and the namespace will be removed. diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 44d9aff5f7..9d0f7fff4e 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -1,6 +1,6 @@ // +build linux -package main +package nsinit import ( "fmt" @@ -16,7 +16,9 @@ import ( "syscall" ) -func execCommand(container *libcontainer.Container, tty bool, args []string) (int, error) { +// Exec performes setup outside of a namespace so that a container can be +// executed. Exec is a high level function for working with container namespaces. +func Exec(container *libcontainer.Container, tty bool, args []string) (int, error) { var ( master *os.File console string diff --git a/components/engine/pkg/libcontainer/nsinit/execin.go b/components/engine/pkg/libcontainer/nsinit/execin.go index d6224f95e6..85a89905c1 100644 --- a/components/engine/pkg/libcontainer/nsinit/execin.go +++ b/components/engine/pkg/libcontainer/nsinit/execin.go @@ -1,4 +1,4 @@ -package main +package nsinit import ( "fmt" @@ -11,7 +11,8 @@ import ( "syscall" ) -func execinCommand(container *libcontainer.Container, nspid int, args []string) (int, error) { +// ExecIn uses an existing pid and joins the pid's namespaces with the new command. +func ExecIn(container *libcontainer.Container, nspid int, args []string) (int, error) { for _, ns := range container.Namespaces { if err := system.Unshare(namespaceMap[ns]); err != nil { return -1, err diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index ef7fc4e44c..f80d785bc4 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -1,6 +1,6 @@ // +build linux -package main +package nsinit import ( "fmt" @@ -15,7 +15,9 @@ import ( "syscall" ) -func initCommand(container *libcontainer.Container, console string, pipe io.ReadCloser, args []string) error { +// Init is the init process that first runs inside a new namespace to setup mounts, users, networking, +// and other options required for the new container. +func Init(container *libcontainer.Container, console string, pipe io.ReadCloser, args []string) error { rootfs, err := resolveRootfs() if err != nil { return err diff --git a/components/engine/pkg/libcontainer/nsinit/mount.go b/components/engine/pkg/libcontainer/nsinit/mount.go index 9cf69f4184..a73e97e375 100644 --- a/components/engine/pkg/libcontainer/nsinit/mount.go +++ b/components/engine/pkg/libcontainer/nsinit/mount.go @@ -1,6 +1,6 @@ // +build linux -package main +package nsinit import ( "fmt" diff --git a/components/engine/pkg/libcontainer/nsinit/ns_linux.go b/components/engine/pkg/libcontainer/nsinit/ns_linux.go index 481bdf79df..e42d4b88d7 100644 --- a/components/engine/pkg/libcontainer/nsinit/ns_linux.go +++ b/components/engine/pkg/libcontainer/nsinit/ns_linux.go @@ -1,4 +1,4 @@ -package main +package nsinit import ( "github.com/dotcloud/docker/pkg/libcontainer" diff --git a/components/engine/pkg/libcontainer/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go similarity index 87% rename from components/engine/pkg/libcontainer/nsinit/main.go rename to components/engine/pkg/libcontainer/nsinit/nsinit/main.go index f66ff0d855..9d3c201aa6 100644 --- a/components/engine/pkg/libcontainer/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/libcontainer/nsinit" "io/ioutil" "log" "os" @@ -42,9 +43,9 @@ func main() { } } if nspid > 0 { - exitCode, err = execinCommand(container, nspid, flag.Args()[1:]) + exitCode, err = nsinit.ExecIn(container, nspid, flag.Args()[1:]) } else { - exitCode, err = execCommand(container, *tty, flag.Args()[1:]) + exitCode, err = nsinit.Exec(container, *tty, flag.Args()[1:]) } if err != nil { log.Fatal(err) @@ -54,7 +55,7 @@ func main() { if flag.NArg() < 2 { log.Fatal(ErrWrongArguments) } - if err := initCommand(container, *console, os.NewFile(uintptr(*pipeFd), "pipe"), flag.Args()[1:]); err != nil { + if err := nsinit.Init(container, *console, os.NewFile(uintptr(*pipeFd), "pipe"), flag.Args()[1:]); err != nil { log.Fatal(err) } default: From 71f9b20db78b52f226fef6251531c8e0764dcd20 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Thu, 20 Feb 2014 18:38:28 -0800 Subject: [PATCH 214/403] Refactor the flag management for main Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 6b2e963ce0aef802e60eafe0e895f24abb294a07 Component: engine --- .../engine/pkg/libcontainer/nsinit/exec.go | 13 +++------ .../engine/pkg/libcontainer/nsinit/init.go | 1 - .../pkg/libcontainer/nsinit/nsinit/main.go | 27 ++++++++++++------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 9d0f7fff4e..d2b87b66ba 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -44,18 +44,14 @@ func Exec(container *libcontainer.Container, tty bool, args []string) (int, erro system.UsetCloseOnExec(r.Fd()) command := createCommand(container, console, r.Fd(), args) - if !tty { - inPipe, err = command.StdinPipe() - if err != nil { + if inPipe, err = command.StdinPipe(); err != nil { return -1, err } - outPipe, err = command.StdoutPipe() - if err != nil { + if outPipe, err = command.StdoutPipe(); err != nil { return -1, err } - errPipe, err = command.StderrPipe() - if err != nil { + if errPipe, err = command.StderrPipe(); err != nil { return -1, err } } @@ -63,7 +59,6 @@ func Exec(container *libcontainer.Container, tty bool, args []string) (int, erro if err := command.Start(); err != nil { return -1, err } - if err := writePidFile(command); err != nil { command.Process.Kill() return -1, err @@ -94,6 +89,7 @@ func Exec(container *libcontainer.Container, tty bool, args []string) (int, erro if tty { go io.Copy(os.Stdout, master) go io.Copy(master, os.Stdin) + state, err := setupWindow(master) if err != nil { command.Process.Kill() @@ -114,7 +110,6 @@ func Exec(container *libcontainer.Container, tty bool, args []string) (int, erro return -1, err } } - return command.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil } diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index f80d785bc4..88a5c3c5d5 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -173,7 +173,6 @@ func setupVethNetwork(config *libcontainer.Network, tempVethName string) error { // has been created and setup func getVethName(pipe io.ReadCloser) (string, error) { defer pipe.Close() - data, err := ioutil.ReadAll(pipe) if err != nil { return "", fmt.Errorf("error reading from stdin %s", err) diff --git a/components/engine/pkg/libcontainer/nsinit/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go index 9d3c201aa6..33a7747594 100644 --- a/components/engine/pkg/libcontainer/nsinit/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go @@ -12,27 +12,34 @@ import ( "strconv" ) +var ( + console string + tty bool + pipeFd int +) + var ( ErrUnsupported = errors.New("Unsupported method") ErrWrongArguments = errors.New("Wrong argument count") ) -func main() { - var ( - console = flag.String("console", "", "Console (pty slave) name") - tty = flag.Bool("tty", false, "Create a tty") - pipeFd = flag.Int("pipe", 0, "sync pipe fd") - ) - flag.Parse() +func init() { + flag.StringVar(&console, "console", "", "console (pty slave) path") + flag.BoolVar(&tty, "tty", false, "create a tty") + flag.IntVar(&pipeFd, "pipe", 0, "sync pipe fd") + flag.Parse() +} + +func main() { container, err := loadContainer() if err != nil { log.Fatal(err) } - if flag.NArg() < 1 { log.Fatal(ErrWrongArguments) } + switch flag.Arg(0) { case "exec": // this is executed outside of the namespace in the cwd var exitCode int @@ -45,7 +52,7 @@ func main() { if nspid > 0 { exitCode, err = nsinit.ExecIn(container, nspid, flag.Args()[1:]) } else { - exitCode, err = nsinit.Exec(container, *tty, flag.Args()[1:]) + exitCode, err = nsinit.Exec(container, tty, flag.Args()[1:]) } if err != nil { log.Fatal(err) @@ -55,7 +62,7 @@ func main() { if flag.NArg() < 2 { log.Fatal(ErrWrongArguments) } - if err := nsinit.Init(container, *console, os.NewFile(uintptr(*pipeFd), "pipe"), flag.Args()[1:]); err != nil { + if err := nsinit.Init(container, console, os.NewFile(uintptr(pipeFd), "pipe"), flag.Args()[1:]); err != nil { log.Fatal(err) } default: From 8be796c4b1fabe963d014208eaf0cc15125c102d Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 13:53:11 -0800 Subject: [PATCH 215/403] Move tty into container.json Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 7f247e7006761ac8922a58651a76b194a4655ffa Component: engine --- components/engine/pkg/libcontainer/README.md | 7 +++---- components/engine/pkg/libcontainer/container.go | 1 + components/engine/pkg/libcontainer/container.json | 3 ++- components/engine/pkg/libcontainer/nsinit/exec.go | 8 ++++---- components/engine/pkg/libcontainer/nsinit/nsinit/main.go | 4 +--- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/components/engine/pkg/libcontainer/README.md b/components/engine/pkg/libcontainer/README.md index 3a2a843b69..89a4ec0c48 100644 --- a/components/engine/pkg/libcontainer/README.md +++ b/components/engine/pkg/libcontainer/README.md @@ -17,6 +17,7 @@ Sample `container.json` file: ```json { "hostname": "koye", + "tty": true, "environment": [ "HOME=/", "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", @@ -55,7 +56,7 @@ Sample `container.json` file: "cgroups": { "name": "docker-koye", "parent": "docker", - "memory": 524800 + "memory": 5248000 } } ``` @@ -72,11 +73,9 @@ rootfs and copy a `container.json` file into the directory with your specified c To execution `/bin/bash` in the current directory as a container just run: ```bash -nsinit -tty exec /bin/bash +nsinit exec /bin/bash ``` -If you want a proper tty setup inside the new container you must use the `-tty` flag when running nsinit. - If you wish to spawn another process inside the container while your current bash session is running just run the exact same command again to get another bash shell or change the command. If the original process dies, PID 1, all other processes spawned inside the container will also be killed and the namespace will be removed. diff --git a/components/engine/pkg/libcontainer/container.go b/components/engine/pkg/libcontainer/container.go index e6e4b4747e..3c1b62b65a 100644 --- a/components/engine/pkg/libcontainer/container.go +++ b/components/engine/pkg/libcontainer/container.go @@ -12,6 +12,7 @@ type Container struct { User string `json:"user,omitempty"` // user to execute the process as WorkingDir string `json:"working_dir,omitempty"` // current working directory Env []string `json:"environment,omitempty"` // environment to set + Tty bool `json:"tty,omitempty"` // setup a proper tty or not Namespaces Namespaces `json:"namespaces,omitempty"` // namespaces to apply Capabilities Capabilities `json:"capabilities,omitempty"` // capabilities to drop Network *Network `json:"network,omitempty"` // nil for host's network stack diff --git a/components/engine/pkg/libcontainer/container.json b/components/engine/pkg/libcontainer/container.json index c1a07dc55b..07e52df428 100644 --- a/components/engine/pkg/libcontainer/container.json +++ b/components/engine/pkg/libcontainer/container.json @@ -1,5 +1,6 @@ { "hostname": "koye", + "tty": true, "environment": [ "HOME=/", "PATH=PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin", @@ -38,6 +39,6 @@ "cgroups": { "name": "docker-koye", "parent": "docker", - "memory": 524800 + "memory": 5248000 } } diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index d2b87b66ba..e2adf3d4f2 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -18,7 +18,7 @@ import ( // Exec performes setup outside of a namespace so that a container can be // executed. Exec is a high level function for working with container namespaces. -func Exec(container *libcontainer.Container, tty bool, args []string) (int, error) { +func Exec(container *libcontainer.Container, args []string) (int, error) { var ( master *os.File console string @@ -28,7 +28,7 @@ func Exec(container *libcontainer.Container, tty bool, args []string) (int, erro outPipe, errPipe io.ReadCloser ) - if tty { + if container.Tty { master, console, err = createMasterAndConsole() if err != nil { return -1, err @@ -44,7 +44,7 @@ func Exec(container *libcontainer.Container, tty bool, args []string) (int, erro system.UsetCloseOnExec(r.Fd()) command := createCommand(container, console, r.Fd(), args) - if !tty { + if !container.Tty { if inPipe, err = command.StdinPipe(); err != nil { return -1, err } @@ -86,7 +86,7 @@ func Exec(container *libcontainer.Container, tty bool, args []string) (int, erro w.Close() r.Close() - if tty { + if container.Tty { go io.Copy(os.Stdout, master) go io.Copy(master, os.Stdin) diff --git a/components/engine/pkg/libcontainer/nsinit/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go index 33a7747594..6508a3e9dd 100644 --- a/components/engine/pkg/libcontainer/nsinit/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go @@ -14,7 +14,6 @@ import ( var ( console string - tty bool pipeFd int ) @@ -25,7 +24,6 @@ var ( func init() { flag.StringVar(&console, "console", "", "console (pty slave) path") - flag.BoolVar(&tty, "tty", false, "create a tty") flag.IntVar(&pipeFd, "pipe", 0, "sync pipe fd") flag.Parse() @@ -52,7 +50,7 @@ func main() { if nspid > 0 { exitCode, err = nsinit.ExecIn(container, nspid, flag.Args()[1:]) } else { - exitCode, err = nsinit.Exec(container, tty, flag.Args()[1:]) + exitCode, err = nsinit.Exec(container, flag.Args()[1:]) } if err != nil { log.Fatal(err) From 2c3593d92d8736370f8a9333dbf33dd19228f0b7 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 14:49:55 -0800 Subject: [PATCH 216/403] Add good logging support to both sides Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 50c752fcb06497e9e597049a1007c53d77032d17 Component: engine --- .../engine/pkg/libcontainer/nsinit/exec.go | 35 ++++++++++++++--- .../engine/pkg/libcontainer/nsinit/init.go | 23 ++++++----- .../pkg/libcontainer/nsinit/nsinit/main.go | 39 ++++++++++++++++--- 3 files changed, 76 insertions(+), 21 deletions(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index e2adf3d4f2..24e722a22f 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -11,6 +11,7 @@ import ( "github.com/dotcloud/docker/pkg/term" "io" "io/ioutil" + "log" "os" "os/exec" "syscall" @@ -18,7 +19,7 @@ import ( // Exec performes setup outside of a namespace so that a container can be // executed. Exec is a high level function for working with container namespaces. -func Exec(container *libcontainer.Container, args []string) (int, error) { +func Exec(container *libcontainer.Container, logFile string, args []string) (int, error) { var ( master *os.File console string @@ -29,6 +30,7 @@ func Exec(container *libcontainer.Container, args []string) (int, error) { ) if container.Tty { + log.Printf("setting up master and console") master, console, err = createMasterAndConsole() if err != nil { return -1, err @@ -43,8 +45,9 @@ func Exec(container *libcontainer.Container, args []string) (int, error) { } system.UsetCloseOnExec(r.Fd()) - command := createCommand(container, console, r.Fd(), args) + command := createCommand(container, console, logFile, r.Fd(), args) if !container.Tty { + log.Printf("opening pipes on command") if inPipe, err = command.StdinPipe(); err != nil { return -1, err } @@ -56,9 +59,11 @@ func Exec(container *libcontainer.Container, args []string) (int, error) { } } + log.Printf("staring init") if err := command.Start(); err != nil { return -1, err } + log.Printf("writting state file") if err := writePidFile(command); err != nil { command.Process.Kill() return -1, err @@ -68,6 +73,7 @@ func Exec(container *libcontainer.Container, args []string) (int, error) { // Do this before syncing with child so that no children // can escape the cgroup if container.Cgroups != nil { + log.Printf("setting up cgroups") if err := container.Cgroups.Apply(command.Process.Pid); err != nil { command.Process.Kill() return -1, err @@ -75,18 +81,22 @@ func Exec(container *libcontainer.Container, args []string) (int, error) { } if container.Network != nil { - vethPair, err := initializeContainerVeth(container.Network.Bridge, command.Process.Pid) + log.Printf("creating veth pair") + vethPair, err := initializeContainerVeth(container.Network.Bridge, container.Network.Mtu, command.Process.Pid) if err != nil { return -1, err } + log.Printf("sending %s as veth pair name", vethPair) sendVethName(w, vethPair) } // Sync with child + log.Printf("closing sync pipes") w.Close() r.Close() if container.Tty { + log.Printf("starting copy for tty") go io.Copy(os.Stdout, master) go io.Copy(master, os.Stdin) @@ -97,6 +107,7 @@ func Exec(container *libcontainer.Container, args []string) (int, error) { } defer term.RestoreTerminal(os.Stdin.Fd(), state) } else { + log.Printf("starting copy for std pipes") go func() { defer inPipe.Close() io.Copy(inPipe, os.Stdin) @@ -105,11 +116,13 @@ func Exec(container *libcontainer.Container, args []string) (int, error) { go io.Copy(os.Stderr, errPipe) } + log.Printf("waiting on process") if err := command.Wait(); err != nil { if _, ok := err.(*exec.ExitError); !ok { return -1, err } } + log.Printf("process ended") return command.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil } @@ -126,17 +139,22 @@ func sendVethName(pipe io.Writer, name string) { // Then will with set the other side of the veth pair into the container's namespaced // using the pid and returns the veth's interface name to provide to the container to // finish setting up the interface inside the namespace -func initializeContainerVeth(bridge string, nspid int) (string, error) { +func initializeContainerVeth(bridge string, mtu, nspid int) (string, error) { name1, name2, err := createVethPair() if err != nil { return "", err } + log.Printf("veth pair created %s <> %s", name1, name2) if err := network.SetInterfaceMaster(name1, bridge); err != nil { return "", err } + if err := network.SetMtu(name1, mtu); err != nil { + return "", err + } if err := network.InterfaceUp(name1); err != nil { return "", err } + log.Printf("setting %s inside %d namespace", name2, nspid) if err := network.SetInterfaceInNamespacePid(name2, nspid); err != nil { return "", err } @@ -200,8 +218,13 @@ func deletePidFile() error { // createCommand will return an exec.Cmd with the Cloneflags set to the proper namespaces // defined on the container's configuration and use the current binary as the init with the // args provided -func createCommand(container *libcontainer.Container, console string, pipe uintptr, args []string) *exec.Cmd { - command := exec.Command("nsinit", append([]string{"-console", console, "-pipe", fmt.Sprint(pipe), "init"}, args...)...) +func createCommand(container *libcontainer.Container, console, logFile string, pipe uintptr, args []string) *exec.Cmd { + command := exec.Command("nsinit", append([]string{ + "-console", console, + "-pipe", fmt.Sprint(pipe), + "-log", logFile, + "init"}, args...)...) + command.SysProcAttr = &syscall.SysProcAttr{ Cloneflags: uintptr(getNamespaceFlags(container.Namespaces)), } diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index 88a5c3c5d5..8fc5f3d05c 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -10,6 +10,7 @@ import ( "github.com/dotcloud/docker/pkg/system" "io" "io/ioutil" + "log" "os" "path/filepath" "syscall" @@ -17,19 +18,23 @@ import ( // Init is the init process that first runs inside a new namespace to setup mounts, users, networking, // and other options required for the new container. -func Init(container *libcontainer.Container, console string, pipe io.ReadCloser, args []string) error { - rootfs, err := resolveRootfs() +func Init(container *libcontainer.Container, uncleanRootfs, console string, pipe io.ReadCloser, args []string) error { + rootfs, err := resolveRootfs(uncleanRootfs) if err != nil { return err } + log.Printf("initializing namespace at %s", rootfs) // We always read this as it is a way to sync with the parent as well tempVethName, err := getVethName(pipe) if err != nil { return err } - + if tempVethName != "" { + log.Printf("received veth name %s", tempVethName) + } if console != "" { + log.Printf("setting up console for %s", console) // close pipes so that we can replace it with the pty os.Stdin.Close() os.Stdout.Close() @@ -42,7 +47,6 @@ func Init(container *libcontainer.Container, console string, pipe io.ReadCloser, return fmt.Errorf("dup2 slave %s", err) } } - if _, err := system.Setsid(); err != nil { return fmt.Errorf("setsid %s", err) } @@ -63,9 +67,11 @@ func Init(container *libcontainer.Container, console string, pipe io.ReadCloser, if err := system.Sethostname(container.Hostname); err != nil { return fmt.Errorf("sethostname %s", err) } + log.Printf("dropping capabilities") if err := capabilities.DropCapabilities(container); err != nil { return fmt.Errorf("drop capabilities %s", err) } + log.Printf("setting user in namespace") if err := setupUser(container); err != nil { return fmt.Errorf("setup user %s", err) } @@ -74,6 +80,7 @@ func Init(container *libcontainer.Container, console string, pipe io.ReadCloser, return fmt.Errorf("chdir to %s %s", container.WorkingDir, err) } } + log.Printf("execing %s goodbye", args[0]) if err := system.Exec(args[0], args[0:], container.Env); err != nil { return fmt.Errorf("exec %s", err) } @@ -82,12 +89,8 @@ func Init(container *libcontainer.Container, console string, pipe io.ReadCloser, // resolveRootfs ensures that the current working directory is // not a symlink and returns the absolute path to the rootfs -func resolveRootfs() (string, error) { - cwd, err := os.Getwd() - if err != nil { - return "", err - } - rootfs, err := filepath.Abs(cwd) +func resolveRootfs(uncleanRootfs string) (string, error) { + rootfs, err := filepath.Abs(uncleanRootfs) if err != nil { return "", err } diff --git a/components/engine/pkg/libcontainer/nsinit/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go index 6508a3e9dd..0873c09fe0 100644 --- a/components/engine/pkg/libcontainer/nsinit/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go @@ -6,6 +6,7 @@ import ( "flag" "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/nsinit" + "io" "io/ioutil" "log" "os" @@ -15,6 +16,7 @@ import ( var ( console string pipeFd int + logFile string ) var ( @@ -24,22 +26,27 @@ var ( func init() { flag.StringVar(&console, "console", "", "console (pty slave) path") + flag.StringVar(&logFile, "log", "none", "log options (none, stderr, or a file path)") flag.IntVar(&pipeFd, "pipe", 0, "sync pipe fd") flag.Parse() } func main() { + if flag.NArg() < 1 { + log.Fatal(ErrWrongArguments) + } container, err := loadContainer() if err != nil { log.Fatal(err) } - if flag.NArg() < 1 { - log.Fatal(ErrWrongArguments) + if err := setupLogging(); err != nil { + log.Fatal(err) } - switch flag.Arg(0) { case "exec": // this is executed outside of the namespace in the cwd + log.SetPrefix("[nsinit exec] ") + var exitCode int nspid, err := readPid() if err != nil { @@ -50,17 +57,22 @@ func main() { if nspid > 0 { exitCode, err = nsinit.ExecIn(container, nspid, flag.Args()[1:]) } else { - exitCode, err = nsinit.Exec(container, flag.Args()[1:]) + exitCode, err = nsinit.Exec(container, logFile, flag.Args()[1:]) } if err != nil { log.Fatal(err) } os.Exit(exitCode) case "init": // this is executed inside of the namespace to setup the container + log.SetPrefix("[nsinit init] ") + cwd, err := os.Getwd() + if err != nil { + log.Fatal(err) + } if flag.NArg() < 2 { log.Fatal(ErrWrongArguments) } - if err := nsinit.Init(container, console, os.NewFile(uintptr(pipeFd), "pipe"), flag.Args()[1:]); err != nil { + if err := nsinit.Init(container, cwd, console, os.NewFile(uintptr(pipeFd), "pipe"), flag.Args()[1:]); err != nil { log.Fatal(err) } default: @@ -93,3 +105,20 @@ func readPid() (int, error) { } return pid, nil } + +func setupLogging() (err error) { + var writer io.Writer + switch logFile { + case "stderr": + writer = os.Stderr + case "none", "": + writer = ioutil.Discard + default: + writer, err = os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0755) + if err != nil { + return err + } + } + log.SetOutput(writer) + return nil +} From eb2bb513c8cb2c150fe29082a6dacd59a563539d Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 15:32:50 -0800 Subject: [PATCH 217/403] User os.Args[0] as name to reexec Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: ba025cb75cceaa8536d0d512889ea86f13349950 Component: engine --- components/engine/pkg/libcontainer/nsinit/exec.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 24e722a22f..ba548a2bd7 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -219,7 +219,9 @@ func deletePidFile() error { // defined on the container's configuration and use the current binary as the init with the // args provided func createCommand(container *libcontainer.Container, console, logFile string, pipe uintptr, args []string) *exec.Cmd { - command := exec.Command("nsinit", append([]string{ + // get our binary name so we can always reexec ourself + name := os.Args[0] + command := exec.Command(name, append([]string{ "-console", console, "-pipe", fmt.Sprint(pipe), "-log", logFile, From e210f44a119067f277023d6131ba96247a951a6b Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 16:17:18 -0800 Subject: [PATCH 218/403] Use lookup path for init Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: a352ecb01a788eff3446fe12191ca0434fce1eed Component: engine --- components/engine/pkg/libcontainer/nsinit/exec.go | 1 + components/engine/pkg/libcontainer/nsinit/init.go | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index ba548a2bd7..80fe8495ff 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -230,5 +230,6 @@ func createCommand(container *libcontainer.Container, console, logFile string, p command.SysProcAttr = &syscall.SysProcAttr{ Cloneflags: uintptr(getNamespaceFlags(container.Namespaces)), } + command.Env = container.Env return command } diff --git a/components/engine/pkg/libcontainer/nsinit/init.go b/components/engine/pkg/libcontainer/nsinit/init.go index 8fc5f3d05c..04716ba645 100644 --- a/components/engine/pkg/libcontainer/nsinit/init.go +++ b/components/engine/pkg/libcontainer/nsinit/init.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "log" "os" + "os/exec" "path/filepath" "syscall" ) @@ -80,8 +81,13 @@ func Init(container *libcontainer.Container, uncleanRootfs, console string, pipe return fmt.Errorf("chdir to %s %s", container.WorkingDir, err) } } - log.Printf("execing %s goodbye", args[0]) - if err := system.Exec(args[0], args[0:], container.Env); err != nil { + name, err := exec.LookPath(args[0]) + if err != nil { + return err + } + + log.Printf("execing %s goodbye", name) + if err := system.Exec(name, args[0:], container.Env); err != nil { return fmt.Errorf("exec %s", err) } panic("unreachable") From e38028e81a665105490376da8daa9f02456f0db8 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 16:28:43 -0800 Subject: [PATCH 219/403] Pass pipes into Exec function Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: c8fd81c27821576f339ccf4fd85c47375ba34042 Component: engine --- components/engine/pkg/libcontainer/nsinit/exec.go | 14 +++++++------- .../engine/pkg/libcontainer/nsinit/nsinit/main.go | 4 +++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 80fe8495ff..98f5209f03 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -19,7 +19,7 @@ import ( // Exec performes setup outside of a namespace so that a container can be // executed. Exec is a high level function for working with container namespaces. -func Exec(container *libcontainer.Container, logFile string, args []string) (int, error) { +func Exec(container *libcontainer.Container, stdin io.Reader, stdout, stderr io.Writer, logFile string, args []string) (int, error) { var ( master *os.File console string @@ -97,23 +97,23 @@ func Exec(container *libcontainer.Container, logFile string, args []string) (int if container.Tty { log.Printf("starting copy for tty") - go io.Copy(os.Stdout, master) - go io.Copy(master, os.Stdin) + go io.Copy(stdout, master) + go io.Copy(master, stdin) state, err := setupWindow(master) if err != nil { command.Process.Kill() return -1, err } - defer term.RestoreTerminal(os.Stdin.Fd(), state) + defer term.RestoreTerminal(uintptr(syscall.Stdin), state) } else { log.Printf("starting copy for std pipes") go func() { defer inPipe.Close() - io.Copy(inPipe, os.Stdin) + io.Copy(inPipe, stdin) }() - go io.Copy(os.Stdout, outPipe) - go io.Copy(os.Stderr, errPipe) + go io.Copy(stdout, outPipe) + go io.Copy(stderr, errPipe) } log.Printf("waiting on process") diff --git a/components/engine/pkg/libcontainer/nsinit/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go index 0873c09fe0..e6e3827713 100644 --- a/components/engine/pkg/libcontainer/nsinit/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go @@ -57,7 +57,9 @@ func main() { if nspid > 0 { exitCode, err = nsinit.ExecIn(container, nspid, flag.Args()[1:]) } else { - exitCode, err = nsinit.Exec(container, logFile, flag.Args()[1:]) + exitCode, err = nsinit.Exec(container, + os.Stdin, os.Stdout, os.Stderr, + logFile, flag.Args()[1:]) } if err != nil { log.Fatal(err) From dd4492ebc4c2fd42e280bd128d513b6294bd4d99 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 16:40:32 -0800 Subject: [PATCH 220/403] Pass tty master to Exec Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 332755b99d345a8ffbf4fb636ca8fed604a233c0 Component: engine --- components/engine/pkg/libcontainer/nsinit/exec.go | 3 +-- components/engine/pkg/libcontainer/nsinit/nsinit/main.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 98f5209f03..3622196b78 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -19,9 +19,8 @@ import ( // Exec performes setup outside of a namespace so that a container can be // executed. Exec is a high level function for working with container namespaces. -func Exec(container *libcontainer.Container, stdin io.Reader, stdout, stderr io.Writer, logFile string, args []string) (int, error) { +func Exec(container *libcontainer.Container, stdin io.Reader, stdout, stderr io.Writer, master *os.File, logFile string, args []string) (int, error) { var ( - master *os.File console string err error diff --git a/components/engine/pkg/libcontainer/nsinit/nsinit/main.go b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go index e6e3827713..28d42d4643 100644 --- a/components/engine/pkg/libcontainer/nsinit/nsinit/main.go +++ b/components/engine/pkg/libcontainer/nsinit/nsinit/main.go @@ -58,7 +58,7 @@ func main() { exitCode, err = nsinit.ExecIn(container, nspid, flag.Args()[1:]) } else { exitCode, err = nsinit.Exec(container, - os.Stdin, os.Stdout, os.Stderr, + os.Stdin, os.Stdout, os.Stderr, nil, logFile, flag.Args()[1:]) } if err != nil { From 4fbf234237ed624593b6625315d5439956b047f6 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 21 Feb 2014 17:11:57 -0800 Subject: [PATCH 221/403] Initial commit of libcontainer running docker Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) Upstream-commit: 2419e63d243255ef38f16799ffdc64084aa18fe4 Component: engine --- components/engine/container.go | 1 + .../execdriver/namespaces/default_template.go | 41 ++ .../engine/execdriver/namespaces/driver.go | 349 ++++++++++++++++++ .../engine/execdriver/namespaces/term.go | 26 ++ .../engine/pkg/libcontainer/nsinit/exec.go | 2 +- .../pkg/libcontainer/nsinit/ns_linux.go | 2 +- components/engine/runtime.go | 5 +- 7 files changed, 422 insertions(+), 4 deletions(-) create mode 100644 components/engine/execdriver/namespaces/default_template.go create mode 100644 components/engine/execdriver/namespaces/driver.go create mode 100644 components/engine/execdriver/namespaces/term.go diff --git a/components/engine/container.go b/components/engine/container.go index ca53bb57c7..76e51cdad3 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -530,6 +530,7 @@ func (container *Container) Start() (err error) { } populateCommand(container) + container.command.Env = env // Setup logging of stdout and stderr to disk if err := container.runtime.LogToDisk(container.stdout, container.logPath("json"), "stdout"); err != nil { diff --git a/components/engine/execdriver/namespaces/default_template.go b/components/engine/execdriver/namespaces/default_template.go new file mode 100644 index 0000000000..79b6ac1c11 --- /dev/null +++ b/components/engine/execdriver/namespaces/default_template.go @@ -0,0 +1,41 @@ +package namespaces + +import ( + "github.com/dotcloud/docker/pkg/cgroups" + "github.com/dotcloud/docker/pkg/libcontainer" +) + +// getDefaultTemplate returns the docker default for +// the libcontainer configuration file +func getDefaultTemplate() *libcontainer.Container { + return &libcontainer.Container{ + Capabilities: libcontainer.Capabilities{ + libcontainer.CAP_SETPCAP, + libcontainer.CAP_SYS_MODULE, + libcontainer.CAP_SYS_RAWIO, + libcontainer.CAP_SYS_PACCT, + libcontainer.CAP_SYS_ADMIN, + libcontainer.CAP_SYS_NICE, + libcontainer.CAP_SYS_RESOURCE, + libcontainer.CAP_SYS_TIME, + libcontainer.CAP_SYS_TTY_CONFIG, + libcontainer.CAP_MKNOD, + libcontainer.CAP_AUDIT_WRITE, + libcontainer.CAP_AUDIT_CONTROL, + libcontainer.CAP_MAC_ADMIN, + libcontainer.CAP_MAC_OVERRIDE, + libcontainer.CAP_NET_ADMIN, + }, + Namespaces: libcontainer.Namespaces{ + libcontainer.CLONE_NEWIPC, + libcontainer.CLONE_NEWNET, + libcontainer.CLONE_NEWNS, + libcontainer.CLONE_NEWPID, + libcontainer.CLONE_NEWUTS, + }, + Cgroups: &cgroups.Cgroup{ + Name: "docker", + DeviceAccess: false, + }, + } +} diff --git a/components/engine/execdriver/namespaces/driver.go b/components/engine/execdriver/namespaces/driver.go new file mode 100644 index 0000000000..e243c64703 --- /dev/null +++ b/components/engine/execdriver/namespaces/driver.go @@ -0,0 +1,349 @@ +package namespaces + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/dotcloud/docker/execdriver" + "github.com/dotcloud/docker/pkg/libcontainer" + "github.com/dotcloud/docker/pkg/libcontainer/network" + "github.com/dotcloud/docker/pkg/libcontainer/nsinit" + "github.com/dotcloud/docker/pkg/libcontainer/utils" + "github.com/dotcloud/docker/pkg/system" + "github.com/dotcloud/docker/pkg/term" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" +) + +const ( + DriverName = "namespaces" + Version = "0.1" +) + +var ( + ErrNotSupported = errors.New("not supported") +) + +func init() { + execdriver.RegisterInitFunc(DriverName, func(args *execdriver.InitArgs) error { + return nil + }) +} + +type driver struct { +} + +func NewDriver() (*driver, error) { + return &driver{}, nil +} + +func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) { + container := createContainer(c) + if err := writeContainerFile(container, c.Rootfs); err != nil { + return -1, err + } + + var ( + console string + master *os.File + err error + + inPipe io.WriteCloser + outPipe, errPipe io.ReadCloser + ) + + if container.Tty { + log.Printf("setting up master and console") + master, console, err = createMasterAndConsole() + if err != nil { + return -1, err + } + } + c.Terminal = NewTerm(pipes, master) + + // create a pipe so that we can syncronize with the namespaced process and + // pass the veth name to the child + r, w, err := os.Pipe() + if err != nil { + return -1, err + } + system.UsetCloseOnExec(r.Fd()) + + args := append([]string{c.Entrypoint}, c.Arguments...) + createCommand(c, container, console, "/nsinit.logs", r.Fd(), args) + command := c + + if !container.Tty { + log.Printf("opening pipes on command") + if inPipe, err = command.StdinPipe(); err != nil { + return -1, err + } + if outPipe, err = command.StdoutPipe(); err != nil { + return -1, err + } + if errPipe, err = command.StderrPipe(); err != nil { + return -1, err + } + } + + log.Printf("staring init") + if err := command.Start(); err != nil { + return -1, err + } + log.Printf("writting state file") + if err := writePidFile(c.Rootfs, command.Process.Pid); err != nil { + command.Process.Kill() + return -1, err + } + defer deletePidFile(c.Rootfs) + + // Do this before syncing with child so that no children + // can escape the cgroup + if container.Cgroups != nil { + log.Printf("setting up cgroups") + if err := container.Cgroups.Apply(command.Process.Pid); err != nil { + command.Process.Kill() + return -1, err + } + } + + if container.Network != nil { + log.Printf("creating veth pair") + vethPair, err := initializeContainerVeth(container.Network.Bridge, container.Network.Mtu, command.Process.Pid) + if err != nil { + return -1, err + } + log.Printf("sending %s as veth pair name", vethPair) + sendVethName(w, vethPair) + } + + // Sync with child + log.Printf("closing sync pipes") + w.Close() + r.Close() + + if container.Tty { + log.Printf("starting copy for tty") + go io.Copy(pipes.Stdout, master) + if pipes.Stdin != nil { + go io.Copy(master, pipes.Stdin) + } + + /* + state, err := setupWindow(master) + if err != nil { + command.Process.Kill() + return -1, err + } + defer term.RestoreTerminal(uintptr(syscall.Stdin), state) + */ + } else { + log.Printf("starting copy for std pipes") + if pipes.Stdin != nil { + go func() { + defer inPipe.Close() + io.Copy(inPipe, pipes.Stdin) + }() + } + go io.Copy(pipes.Stdout, outPipe) + go io.Copy(pipes.Stderr, errPipe) + } + + if startCallback != nil { + startCallback(c) + } + + log.Printf("waiting on process") + if err := command.Wait(); err != nil { + if _, ok := err.(*exec.ExitError); !ok { + return -1, err + } + } + log.Printf("process ended") + return command.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil +} + +func (d *driver) Kill(p *execdriver.Command, sig int) error { + return p.Process.Kill() +} + +func (d *driver) Restore(c *execdriver.Command) error { + return ErrNotSupported +} + +func (d *driver) Info(id string) execdriver.Info { + return nil +} + +func (d *driver) Name() string { + return fmt.Sprintf("%s-%s", DriverName, Version) +} + +func (d *driver) GetPidsForContainer(id string) ([]int, error) { + return nil, ErrNotSupported +} + +func writeContainerFile(container *libcontainer.Container, rootfs string) error { + data, err := json.Marshal(container) + if err != nil { + return err + } + return ioutil.WriteFile(filepath.Join(rootfs, "container.json"), data, 0755) +} + +func getEnv(key string, env []string) string { + for _, pair := range env { + parts := strings.Split(pair, "=") + if parts[0] == key { + return parts[1] + } + } + return "" +} + +// sendVethName writes the veth pair name to the child's stdin then closes the +// pipe so that the child stops waiting for more data +func sendVethName(pipe io.Writer, name string) { + fmt.Fprint(pipe, name) +} + +// initializeContainerVeth will create a veth pair and setup the host's +// side of the pair by setting the specified bridge as the master and bringing +// up the interface. +// +// Then will with set the other side of the veth pair into the container's namespaced +// using the pid and returns the veth's interface name to provide to the container to +// finish setting up the interface inside the namespace +func initializeContainerVeth(bridge string, mtu, nspid int) (string, error) { + name1, name2, err := createVethPair() + if err != nil { + return "", err + } + log.Printf("veth pair created %s <> %s", name1, name2) + if err := network.SetInterfaceMaster(name1, bridge); err != nil { + return "", err + } + if err := network.SetMtu(name1, mtu); err != nil { + return "", err + } + if err := network.InterfaceUp(name1); err != nil { + return "", err + } + log.Printf("setting %s inside %d namespace", name2, nspid) + if err := network.SetInterfaceInNamespacePid(name2, nspid); err != nil { + return "", err + } + return name2, nil +} + +func setupWindow(master *os.File) (*term.State, error) { + ws, err := term.GetWinsize(os.Stdin.Fd()) + if err != nil { + return nil, err + } + if err := term.SetWinsize(master.Fd(), ws); err != nil { + return nil, err + } + return term.SetRawTerminal(os.Stdin.Fd()) +} + +// createMasterAndConsole will open /dev/ptmx on the host and retreive the +// pts name for use as the pty slave inside the container +func createMasterAndConsole() (*os.File, string, error) { + master, err := os.OpenFile("/dev/ptmx", syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) + if err != nil { + return nil, "", err + } + console, err := system.Ptsname(master) + if err != nil { + return nil, "", err + } + if err := system.Unlockpt(master); err != nil { + return nil, "", err + } + return master, console, nil +} + +// createVethPair will automatically generage two random names for +// the veth pair and ensure that they have been created +func createVethPair() (name1 string, name2 string, err error) { + name1, err = utils.GenerateRandomName("dock", 4) + if err != nil { + return + } + name2, err = utils.GenerateRandomName("dock", 4) + if err != nil { + return + } + if err = network.CreateVethPair(name1, name2); err != nil { + return + } + return +} + +// writePidFile writes the namespaced processes pid to .nspid in the rootfs for the container +func writePidFile(rootfs string, pid int) error { + return ioutil.WriteFile(filepath.Join(rootfs, ".nspid"), []byte(fmt.Sprint(pid)), 0655) +} + +func deletePidFile(rootfs string) error { + return os.Remove(filepath.Join(rootfs, ".nspid")) +} + +// createCommand will return an exec.Cmd with the Cloneflags set to the proper namespaces +// defined on the container's configuration and use the current binary as the init with the +// args provided +func createCommand(c *execdriver.Command, container *libcontainer.Container, + console, logFile string, pipe uintptr, args []string) { + + aname, _ := exec.LookPath("nsinit") + c.Path = aname + c.Args = append([]string{ + aname, + "-console", console, + "-pipe", fmt.Sprint(pipe), + "-log", logFile, + "init", + }, args...) + c.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: uintptr(nsinit.GetNamespaceFlags(container.Namespaces)), + } + c.Env = container.Env + c.Dir = c.Rootfs +} + +func createContainer(c *execdriver.Command) *libcontainer.Container { + container := getDefaultTemplate() + + container.Hostname = getEnv("HOSTNAME", c.Env) + container.Tty = c.Tty + container.User = c.User + container.WorkingDir = c.WorkingDir + container.Env = c.Env + + container.Env = append(container.Env, "container=docker") + + if c.Network != nil { + container.Network = &libcontainer.Network{ + Mtu: c.Network.Mtu, + Address: fmt.Sprintf("%s/%d", c.Network.IPAddress, c.Network.IPPrefixLen), + Gateway: c.Network.Gateway, + Bridge: c.Network.Bridge, + } + } + if c.Privileged { + container.Capabilities = nil + } + if c.Resources != nil { + container.Cgroups.CpuShares = c.Resources.CpuShares + container.Cgroups.Memory = c.Resources.Memory + container.Cgroups.MemorySwap = c.Resources.MemorySwap + } + return container +} diff --git a/components/engine/execdriver/namespaces/term.go b/components/engine/execdriver/namespaces/term.go new file mode 100644 index 0000000000..682c6a27b1 --- /dev/null +++ b/components/engine/execdriver/namespaces/term.go @@ -0,0 +1,26 @@ +package namespaces + +import ( + "github.com/dotcloud/docker/execdriver" + "github.com/dotcloud/docker/pkg/term" + "os" +) + +type NsinitTerm struct { + master *os.File +} + +func NewTerm(pipes *execdriver.Pipes, master *os.File) *NsinitTerm { + return &NsinitTerm{master} +} + +func (t *NsinitTerm) Close() error { + return t.master.Close() +} + +func (t *NsinitTerm) Resize(h, w int) error { + if t.master != nil { + return term.SetWinsize(t.master.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) + } + return nil +} diff --git a/components/engine/pkg/libcontainer/nsinit/exec.go b/components/engine/pkg/libcontainer/nsinit/exec.go index 3622196b78..6671ebe129 100644 --- a/components/engine/pkg/libcontainer/nsinit/exec.go +++ b/components/engine/pkg/libcontainer/nsinit/exec.go @@ -227,7 +227,7 @@ func createCommand(container *libcontainer.Container, console, logFile string, p "init"}, args...)...) command.SysProcAttr = &syscall.SysProcAttr{ - Cloneflags: uintptr(getNamespaceFlags(container.Namespaces)), + Cloneflags: uintptr(GetNamespaceFlags(container.Namespaces)), } command.Env = container.Env return command diff --git a/components/engine/pkg/libcontainer/nsinit/ns_linux.go b/components/engine/pkg/libcontainer/nsinit/ns_linux.go index e42d4b88d7..58af24798f 100644 --- a/components/engine/pkg/libcontainer/nsinit/ns_linux.go +++ b/components/engine/pkg/libcontainer/nsinit/ns_linux.go @@ -28,7 +28,7 @@ var namespaceFileMap = map[libcontainer.Namespace]string{ // getNamespaceFlags parses the container's Namespaces options to set the correct // flags on clone, unshare, and setns -func getNamespaceFlags(namespaces libcontainer.Namespaces) (flag int) { +func GetNamespaceFlags(namespaces libcontainer.Namespaces) (flag int) { for _, ns := range namespaces { flag |= namespaceMap[ns] } diff --git a/components/engine/runtime.go b/components/engine/runtime.go index a38109cca0..9f16d6213b 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -7,7 +7,8 @@ import ( "github.com/dotcloud/docker/dockerversion" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/execdriver" - "github.com/dotcloud/docker/execdriver/lxc" + _ "github.com/dotcloud/docker/execdriver/lxc" + "github.com/dotcloud/docker/execdriver/namespaces" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/graphdriver/aufs" _ "github.com/dotcloud/docker/graphdriver/btrfs" @@ -703,7 +704,7 @@ func NewRuntimeFromDirectory(config *DaemonConfig, eng *engine.Engine) (*Runtime sysInfo := sysinfo.New(false) - ed, err := lxc.NewDriver(config.Root, sysInfo.AppArmor) + ed, err := namespaces.NewDriver() if err != nil { return nil, err } From 2cf8cc3b3a200ca369a9c3bd67daaefb1a6da30d Mon Sep 17 00:00:00 2001 From: unclejack Date: Fri, 21 Feb 2014 18:07:51 +0200 Subject: [PATCH 222/403] Remove Vagrantfile and remove it from all docs This removes the Vagrantfile and updates the documentation to remove the steps which explain how to install Docker in a VM via Vagrant. Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: 67d55860a52bec8b1a1327355b4f27674ec912aa Component: engine --- components/engine/.gitignore | 1 + components/engine/MAINTAINERS | 1 - components/engine/Vagrantfile | 206 ----------------- .../sources/contributing/devenvironment.rst | 8 - components/engine/docs/sources/faq.rst | 6 +- components/engine/docs/sources/index.rst | 2 +- .../docs/sources/installation/amazon.rst | 111 +-------- .../docs/sources/installation/windows.rst | 213 +++--------------- 8 files changed, 37 insertions(+), 511 deletions(-) delete mode 100644 components/engine/Vagrantfile diff --git a/components/engine/.gitignore b/components/engine/.gitignore index a40d9e067c..0087b47302 100644 --- a/components/engine/.gitignore +++ b/components/engine/.gitignore @@ -22,3 +22,4 @@ bundles/ .git/ vendor/pkg/ pyenv +Vagrantfile diff --git a/components/engine/MAINTAINERS b/components/engine/MAINTAINERS index 895fba563a..a08f8f3140 100644 --- a/components/engine/MAINTAINERS +++ b/components/engine/MAINTAINERS @@ -6,4 +6,3 @@ Michael Crosby (@crosbymichael) api.go: Victor Vieux (@vieux) Dockerfile: Tianon Gravi (@tianon) Makefile: Tianon Gravi (@tianon) -Vagrantfile: Cristian Staretu (@unclejack) diff --git a/components/engine/Vagrantfile b/components/engine/Vagrantfile deleted file mode 100644 index 23f262020a..0000000000 --- a/components/engine/Vagrantfile +++ /dev/null @@ -1,206 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -BOX_NAME = ENV['BOX_NAME'] || "ubuntu" -BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64.box" -VF_BOX_URI = ENV['BOX_URI'] || "http://files.vagrantup.com/precise64_vmware_fusion.box" -AWS_BOX_URI = ENV['BOX_URI'] || "https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box" -AWS_REGION = ENV['AWS_REGION'] || "us-east-1" -AWS_AMI = ENV['AWS_AMI'] || "ami-69f5a900" -AWS_INSTANCE_TYPE = ENV['AWS_INSTANCE_TYPE'] || 't1.micro' -SSH_PRIVKEY_PATH = ENV['SSH_PRIVKEY_PATH'] -PRIVATE_NETWORK = ENV['PRIVATE_NETWORK'] - -# Boolean that forwards the Docker dynamic ports 49000-49900 -# See http://docs.docker.io/en/latest/use/port_redirection/ for more -# $ FORWARD_DOCKER_PORTS=1 vagrant [up|reload] -FORWARD_DOCKER_PORTS = ENV['FORWARD_DOCKER_PORTS'] -VAGRANT_RAM = ENV['VAGRANT_RAM'] || 512 -VAGRANT_CORES = ENV['VAGRANT_CORES'] || 1 - -# You may also provide a comma-separated list of ports -# for Vagrant to forward. For example: -# $ FORWARD_PORTS=8080,27017 vagrant [up|reload] -FORWARD_PORTS = ENV['FORWARD_PORTS'] - -# A script to upgrade from the 12.04 kernel to the raring backport kernel (3.8) -# and install docker. -$script = <