diff --git a/components/engine/AUTHORS b/components/engine/AUTHORS index 60533c1cc2..f9e9ecdace 100644 --- a/components/engine/AUTHORS +++ b/components/engine/AUTHORS @@ -20,6 +20,7 @@ Antony Messerli Asbjørn Enge Barry Allard Ben Toews +Ben Wiklund Benoit Chesneau Bhiraj Butala Bouke Haarsma @@ -47,6 +48,7 @@ Daniel YC Lin Darren Coxall David Calavera David Sissitka +Dinesh Subhraveti Deni Bertovic Dominik Honnef Don Spaulding @@ -68,6 +70,7 @@ Francisco Souza Frederick F. Kautz IV Gabriel Monroy Gareth Rushgrove +Graydon Hoare Greg Thornton Guillaume J. Charmes Gurjeet Singh @@ -113,6 +116,7 @@ Kyle Conroy Laurie Voss Louis Opter Manuel Meurer +Manuel Woelker Marco Hennings Marcus Farkas Marcus Ramberg diff --git a/components/engine/CHANGELOG.md b/components/engine/CHANGELOG.md index 060a07c21e..55d8d756a8 100644 --- a/components/engine/CHANGELOG.md +++ b/components/engine/CHANGELOG.md @@ -1,5 +1,66 @@ # Changelog +## 0.7.3 (2013-01-02) + +#### Builder + ++ Update ADD to use the image cache, based on a hash of the added content +* Add error message for empty Dockerfile + +#### Documentation + +- Fix outdated link to the "Introduction" on www.docker.io ++ Update the docs to get wider when the screen does +- Add information about needing to install LXC when using raw binaries +* Update Fedora documentation to disentangle the docker and docker.io conflict +* Add a note about using the new `-mtu` flag in several GCE zones ++ Add FrugalWare installation instructions ++ Add a more complete example of `docker run` +- Fix API documentation for creating and starting Privileged containers +- Add missing "name" parameter documentation on "/containers/create" +* Add a mention of `lxc-checkconfig` as a way to check for some of the necessary kernel configuration +- Update the 1.8 API documentation with some additions that were added to the docs for 1.7 + +#### Hack + +- Add missing libdevmapper dependency to the packagers documentation +* Update minimum Go requirement to a hard line at Go 1.2+ +* Many minor improvements to the Vagrantfile ++ Add ability to customize dockerinit search locations when compiling (to be used very sparingly only by packagers of platforms who require a nonstandard location) ++ Add coverprofile generation reporting +- Add `-a` to our Go build flags, removing the need for recompiling the stdlib manually +* Update Dockerfile to be more canonical and have less spurious warnings during build +- Fix some miscellaneous `docker pull` progress bar display issues +* Migrate more miscellaneous packages under the "pkg" folder +* Update TextMate highlighting to automatically be enabled for files named "Dockerfile" +* Reorganize syntax highlighting files under a common "contrib/syntax" directory +* Update install.sh script (https://get.docker.io/) to not fail if busybox fails to download or run at the end of the Ubuntu/Debian installation +* Add support for container names in bash completion + +#### Packaging + ++ Add an official Docker client binary for Darwin (Mac OS X) +* Remove empty "Vendor" string and added "License" on deb package ++ Add a stubbed version of "/etc/default/docker" in the deb package + +#### Runtime + +* Update layer application to extract tars in place, avoiding file churn while handling whiteouts +- Fix permissiveness of mtime comparisons in tar handling (since GNU tar and Go tar do not yet support sub-second mtime precision) +* Reimplement `docker top` in pure Go to work more consistently, and even inside Docker-in-Docker (thus removing the shell injection vulnerability present in some versions of `lxc-ps`) ++ Update `-H unix://` to work similarly to `-H tcp://` by inserting the default values for missing portions +- Fix more edge cases regarding dockerinit and deleted or replaced docker or dockerinit files +* Update container name validation to include '.' +- Fix use of a symlink or non-absolute path as the argument to `-g` to work as expected +* Update to handle external mounts outside of LXC, fixing many small mounting quirks and making future execution backends and other features simpler +* Update to use proper box-drawing characters everywhere in `docker images -tree` +* Move MTU setting from LXC configuration to directly use netlink +* Add `-S` option to external tar invocation for more efficient spare file handling ++ Add arch/os info to User-Agent string, especially for registry requests ++ Add `-mtu` option to Docker daemon for configuring MTU +- Fix `docker build` to exit with a non-zero exit code on error ++ Add `DOCKER_HOST` environment variable to configure the client `-H` flag without specifying it manually for every invocation + ## 0.7.2 (2013-12-16) #### Runtime @@ -15,7 +76,7 @@ - Prevent deletion of image if ANY container is depending on it even if the container is not running * Update docker push to use new progress display * Use os.Lstat to allow mounting unix sockets when inspecting volumes -- Adjusted handling of inactive user login +- Adjust handling of inactive user login - Add missing defines in devicemapper for older kernels - Allow untag operations with no container validation - Add auth config to docker build @@ -110,7 +171,7 @@ #### Runtime -* Improved stability, fixes some race conditons +* Improve stability, fixes some race conditons * Skip the volumes mounted when deleting the volumes of container. * Fix layer size computation: handle hard links correctly * Use the work Path for docker cp CONTAINER:PATH @@ -153,7 +214,7 @@ + Add lock around write operations in graph * Check if port is valid * Fix restart runtime error with ghost container networking -+ Added some more colors and animals to increase the pool of generated names ++ Add some more colors and animals to increase the pool of generated names * Fix issues in docker inspect + Escape apparmor confinement + Set environment variables using a file. @@ -307,7 +368,7 @@ * Improve network performance for VirtualBox * Revamp install.sh to be usable by more people, and to use official install methods whenever possible (apt repo, portage tree, etc.) - Fix contrib/mkimage-debian.sh apt caching prevention -+ Added Dockerfile.tmLanguage to contrib ++ Add Dockerfile.tmLanguage to contrib * Configured FPM to make /etc/init/docker.conf a config file * Enable SSH Agent forwarding in Vagrant VM * Several small tweaks/fixes for contrib/mkimage-debian.sh @@ -421,7 +482,7 @@ * Mount /dev/shm as a tmpfs - Switch from http to https for get.docker.io * Let userland proxy handle container-bound traffic -* Updated the Docker CLI to specify a value for the "Host" header. +* Update the Docker CLI to specify a value for the "Host" header. - Change network range to avoid conflict with EC2 DNS - Reduce connect and read timeout when pinging the registry * Parallel pull @@ -617,7 +678,7 @@ + Builder: 'docker build git://URL' fetches and builds a remote git repository * Runtime: 'docker ps -s' optionally prints container size -* Tests: Improved and simplified +* Tests: improved and simplified - Runtime: fix a regression introduced in 0.4.3 which caused the logs command to fail. - Builder: fix a regression when using ADD with single regular file. @@ -632,7 +693,7 @@ + ADD of a local file will detect tar archives and unpack them * ADD improvements: use tar for copy + automatically unpack local archives * ADD uses tar/untar for copies instead of calling 'cp -ar' -* Fixed the behavior of ADD to be (mostly) reverse-compatible, predictable and well-documented. +* Fix the behavior of ADD to be (mostly) reverse-compatible, predictable and well-documented. - Fix a bug which caused builds to fail if ADD was the first command * Nicer output for 'docker build' @@ -677,7 +738,7 @@ + Detect faulty DNS configuration and replace it with a public default + Allow docker run : + You can now specify public port (ex: -p 80:4500) -* Improved image removal to garbage-collect unreferenced parents +* Improve image removal to garbage-collect unreferenced parents #### Client @@ -731,7 +792,7 @@ #### Documentation -* Improved install instructions. +* Improve install instructions. ## 0.3.3 (2013-05-23) @@ -816,7 +877,7 @@ + Support for data volumes ('docker run -v=PATH') + Share data volumes between containers ('docker run -volumes-from') -+ Improved documentation ++ Improve documentation * Upgrade to Go 1.0.3 * Various upgrades to the dev environment for contributors @@ -872,7 +933,7 @@ - Add debian packaging - Documentation: installing on Arch Linux - Documentation: running Redis on docker -- Fixed lxc 0.9 compatibility +- Fix lxc 0.9 compatibility - Automatically load aufs module - Various bugfixes and stability improvements @@ -907,7 +968,7 @@ - Stabilize process management - Layers can include a commit message - Simplified 'docker attach' -- Fixed support for re-attaching +- Fix support for re-attaching - Various bugfixes and stability improvements - Auto-download at run - Auto-login on push diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index b06c6553e0..9da4e8f039 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -24,40 +24,32 @@ # docker-version 0.6.1 -FROM ubuntu:12.04 -MAINTAINER Solomon Hykes +FROM stackbrew/ubuntu:12.04 +MAINTAINER Tianon Gravi (@tianon) -# Build dependencies -RUN echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list -RUN apt-get update -RUN apt-get install -y -q curl -RUN apt-get install -y -q git -RUN apt-get install -y -q mercurial -RUN apt-get install -y -q build-essential libsqlite3-dev +# Add precise-backports to get s3cmd >= 1.1.0 (so we get ENV variable support in our .s3cfg) +RUN echo 'deb http://archive.ubuntu.com/ubuntu precise-backports main universe' > /etc/apt/sources.list.d/backports.list -# Install Go -RUN curl -s https://go.googlecode.com/files/go1.2.src.tar.gz | tar -v -C /usr/local -xz -ENV PATH /usr/local/go/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin -ENV GOPATH /go:/go/src/github.com/dotcloud/docker/vendor -RUN cd /usr/local/go/src && ./make.bash && go install -ldflags '-w -linkmode external -extldflags "-static -Wl,--unresolved-symbols=ignore-in-shared-libs"' -tags netgo -a std - -# Ubuntu stuff -RUN apt-get install -y -q ruby1.9.3 rubygems libffi-dev -RUN gem install --no-rdoc --no-ri fpm -RUN apt-get install -y -q reprepro dpkg-sig - -RUN apt-get install -y -q python-pip -RUN pip install s3cmd==1.1.0-beta3 -RUN pip install python-magic==0.4.6 -RUN /bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEY\n' > /.s3cfg - -# Runtime dependencies -RUN apt-get install -y -q iptables -RUN apt-get install -y -q lxc -RUN apt-get install -y -q aufs-tools +# Packaged dependencies +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq \ + apt-utils \ + aufs-tools \ + build-essential \ + curl \ + dpkg-sig \ + git \ + iptables \ + libsqlite3-dev \ + lxc \ + mercurial \ + reprepro \ + ruby1.9.1 \ + ruby1.9.1-dev \ + s3cmd=1.1.0* \ + --no-install-recommends # Get lvm2 source for compiling statically -RUN git clone https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /usr/local/lvm2 && git checkout v2_02_103 +RUN git clone https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /usr/local/lvm2 && git checkout -q v2_02_103 # see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags # note: we can't use "git clone -b" above because it requires at least git 1.7.10 to be able to use that on a tag instead of a branch and we only have 1.7.9.5 @@ -65,9 +57,26 @@ RUN git clone https://git.fedorahosted.org/git/lvm2.git /usr/local/lvm2 && cd /u RUN cd /usr/local/lvm2 && ./configure --enable-static_link && make device-mapper && make install_device-mapper # see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL +# Install Go +RUN curl -s https://go.googlecode.com/files/go1.2.src.tar.gz | tar -v -C /usr/local -xz +ENV PATH /usr/local/go/bin:$PATH +ENV GOPATH /go:/go/src/github.com/dotcloud/docker/vendor +RUN cd /usr/local/go/src && ./make.bash --no-clean 2>&1 + +# Compile Go for cross compilation +ENV DOCKER_CROSSPLATFORMS darwin/amd64 darwin/386 +# TODO add linux/386 and linux/arm +RUN cd /usr/local/go/src && bash -xc 'for platform in $DOCKER_CROSSPLATFORMS; do GOOS=${platform%/*} GOARCH=${platform##*/} ./make.bash --no-clean 2>&1; done' + # Grab Go's cover tool for dead-simple code coverage testing RUN go get code.google.com/p/go.tools/cmd/cover +# TODO replace FPM with some very minimal debhelper stuff +RUN gem install --no-rdoc --no-ri fpm --version 1.0.1 + +# Setup s3cmd config +RUN /bin/echo -e '[default]\naccess_key=$AWS_ACCESS_KEY\nsecret_key=$AWS_SECRET_KEY' > /.s3cfg + VOLUME /var/lib/docker WORKDIR /go/src/github.com/dotcloud/docker diff --git a/components/engine/MAINTAINERS b/components/engine/MAINTAINERS index 96469c9814..ef3aeda493 100644 --- a/components/engine/MAINTAINERS +++ b/components/engine/MAINTAINERS @@ -3,4 +3,6 @@ Guillaume Charmes (@creack) Victor Vieux (@vieux) Michael Crosby (@crosbymichael) api.go: Victor Vieux (@vieux) +Dockerfile: Tianon Gravi (@tianon) +Makefile: Tianon Gravi (@tianon) Vagrantfile: Daniel Mizyrycki (@mzdaniel) diff --git a/components/engine/Makefile b/components/engine/Makefile index 21ff0dc57d..86b2a0207f 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -1,4 +1,4 @@ -.PHONY: all binary build default docs shell test +.PHONY: all binary build cross default docs shell test DOCKER_RUN_DOCKER := docker run -rm -i -t -privileged -e TESTFLAGS -v $(CURDIR)/bundles:/go/src/github.com/dotcloud/docker/bundles docker @@ -10,6 +10,9 @@ all: build binary: build $(DOCKER_RUN_DOCKER) hack/make.sh binary +cross: build + $(DOCKER_RUN_DOCKER) hack/make.sh binary cross + docs: docker build -t docker-docs docs && docker run -p 8000:8000 docker-docs diff --git a/components/engine/VERSION b/components/engine/VERSION index 7486fdbc50..f38fc5393f 100644 --- a/components/engine/VERSION +++ b/components/engine/VERSION @@ -1 +1 @@ -0.7.2 +0.7.3 diff --git a/components/engine/Vagrantfile b/components/engine/Vagrantfile index 013027ecde..54fc783c00 100644 --- a/components/engine/Vagrantfile +++ b/components/engine/Vagrantfile @@ -26,7 +26,7 @@ fi # Adding an apt gpg key is idempotent. wget -q -O - https://get.docker.io/gpg | apt-key add - -# Creating the docker.list file is idempotent, but it may overrite desired +# Creating the docker.list file is idempotent, but it may overwrite desired # settings if it already exists. This could be solved with md5sum but it # doesn't seem worth it. echo 'deb http://get.docker.io/ubuntu docker main' > \ @@ -41,7 +41,7 @@ apt-get install -q -y lxc-docker usermod -a -G docker "$user" tmp=`mktemp -q` && { - # Only install the backport kernel, don't bother upgrade if the backport is + # Only install the backport kernel, don't bother upgrading if the backport is # already installed. We want parse the output of apt so we need to save it # with 'tee'. NOTE: The installation of the kernel will trigger dkms to # install vboxguest if needed. diff --git a/components/engine/api.go b/components/engine/api.go index da1fa9edee..1128c7a2b7 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -10,7 +10,7 @@ import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" - "github.com/dotcloud/docker/systemd" + "github.com/dotcloud/docker/pkg/systemd" "github.com/dotcloud/docker/utils" "github.com/gorilla/mux" "io" diff --git a/components/engine/archive/archive.go b/components/engine/archive/archive.go index 838f8f6e7f..f8fcf0b163 100644 --- a/components/engine/archive/archive.go +++ b/components/engine/archive/archive.go @@ -3,6 +3,8 @@ package archive import ( "archive/tar" "bytes" + "compress/gzip" + "compress/bzip2" "fmt" "github.com/dotcloud/docker/utils" "io" @@ -59,6 +61,43 @@ func DetectCompression(source []byte) Compression { return Uncompressed } +func xzDecompress(archive io.Reader) (io.Reader, error) { + args := []string{"xz", "-d", "-c", "-q"} + + return CmdStream(exec.Command(args[0], args[1:]...), archive, nil) +} + +func DecompressStream(archive io.Reader) (io.Reader, error) { + buf := make([]byte, 10) + totalN := 0 + for totalN < 10 { + n, err := archive.Read(buf[totalN:]) + if err != nil { + if err == io.EOF { + return nil, fmt.Errorf("Tarball too short") + } + return nil, err + } + totalN += n + utils.Debugf("[tar autodetect] n: %d", n) + } + compression := DetectCompression(buf) + wrap := io.MultiReader(bytes.NewReader(buf), archive) + + switch compression { + case Uncompressed: + return wrap, nil + case Gzip: + return gzip.NewReader(wrap) + case Bzip2: + return bzip2.NewReader(wrap), nil + case Xz: + return xzDecompress(wrap) + default: + return nil, fmt.Errorf("Unsupported compression format %s", (&compression).Extension()) + } +} + func (compression *Compression) Flag() string { switch *compression { case Bzip2: @@ -110,7 +149,7 @@ func escapeName(name string) string { // Tar creates an archive from the directory at `path`, only including files whose relative // paths are included in `filter`. If `filter` is nil, then all files are included. func TarFilter(path string, options *TarOptions) (io.Reader, error) { - args := []string{"tar", "--numeric-owner", "-f", "-", "-C", path, "-T", "-"} + args := []string{"tar", "-S", "--numeric-owner", "-f", "-", "-C", path, "-T", "-"} if options.Includes == nil { options.Includes = []string{"."} } @@ -155,7 +194,7 @@ func TarFilter(path string, options *TarOptions) (io.Reader, error) { } } - return CmdStream(exec.Command(args[0], args[1:]...), &files, func() { + return CmdStream(exec.Command(args[0], args[1:]...), bytes.NewBufferString(files), func() { if tmpDir != "" { _ = os.RemoveAll(tmpDir) } @@ -189,7 +228,7 @@ func Untar(archive io.Reader, path string, options *TarOptions) error { compression := DetectCompression(buf) utils.Debugf("Archive compression detected: %s", compression.Extension()) - args := []string{"--numeric-owner", "-f", "-", "-C", path, "-x" + compression.Flag()} + args := []string{"-S", "--numeric-owner", "-f", "-", "-C", path, "-x" + compression.Flag()} if options != nil { for _, exclude := range options.Excludes { @@ -301,7 +340,7 @@ func CopyFileWithTar(src, dst string) error { // CmdStream executes a command, and returns its stdout as a stream. // If the command fails to run or doesn't complete successfully, an error // will be returned, including anything written on stderr. -func CmdStream(cmd *exec.Cmd, input *string, atEnd func()) (io.Reader, error) { +func CmdStream(cmd *exec.Cmd, input io.Reader, atEnd func()) (io.Reader, error) { if input != nil { stdin, err := cmd.StdinPipe() if err != nil { @@ -312,7 +351,7 @@ func CmdStream(cmd *exec.Cmd, input *string, atEnd func()) (io.Reader, error) { } // Write stdin if any go func() { - _, _ = stdin.Write([]byte(*input)) + io.Copy(stdin, input) stdin.Close() }() } diff --git a/components/engine/archive/changes.go b/components/engine/archive/changes.go index a4076fc0ad..8fe9ff2233 100644 --- a/components/engine/archive/changes.go +++ b/components/engine/archive/changes.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "syscall" + "time" ) type ChangeType int @@ -34,6 +35,21 @@ func (change *Change) String() string { return fmt.Sprintf("%s %s", kind, change.Path) } +// Gnu tar and the go tar writer don't have sub-second mtime +// precision, which is problematic when we apply changes via tar +// files, we handle this by comparing for exact times, *or* same +// second count and either a or b having exactly 0 nanoseconds +func sameFsTime(a, b time.Time) bool { + return a == b || + (a.Unix() == b.Unix() && + (a.Nanosecond() == 0 || b.Nanosecond() == 0)) +} + +func sameFsTimeSpec(a, b syscall.Timespec) bool { + return a.Sec == b.Sec && + (a.Nsec == b.Nsec || a.Nsec == 0 || b.Nsec == 0) +} + func Changes(layers []string, rw string) ([]Change, error) { var changes []Change err := filepath.Walk(rw, func(path string, f os.FileInfo, err error) error { @@ -85,7 +101,7 @@ func Changes(layers []string, rw string) ([]Change, error) { // However, if it's a directory, maybe it wasn't actually modified. // If you modify /foo/bar/baz, then /foo will be part of the changed files only because it's the parent of bar if stat.IsDir() && f.IsDir() { - if f.Size() == stat.Size() && f.Mode() == stat.Mode() && f.ModTime() == stat.ModTime() { + if f.Size() == stat.Size() && f.Mode() == stat.Mode() && sameFsTime(f.ModTime(), stat.ModTime()) { // Both directories are the same, don't record the change return nil } @@ -181,7 +197,7 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) { oldStat.Rdev != newStat.Rdev || // Don't look at size for dirs, its not a good measure of change (oldStat.Size != newStat.Size && oldStat.Mode&syscall.S_IFDIR != syscall.S_IFDIR) || - getLastModification(oldStat) != getLastModification(newStat) { + !sameFsTimeSpec(getLastModification(oldStat), getLastModification(newStat)) { change := Change{ Path: newChild.path(), Kind: ChangeModify, diff --git a/components/engine/archive/changes_test.go b/components/engine/archive/changes_test.go index 714ab71e2d..1302b76f47 100644 --- a/components/engine/archive/changes_test.go +++ b/components/engine/archive/changes_test.go @@ -258,48 +258,44 @@ func TestChangesDirsMutated(t *testing.T) { } func TestApplyLayer(t *testing.T) { - t.Skip("Skipping TestApplyLayer due to known failures") // Disable this for now as it is broken - return + src, err := ioutil.TempDir("", "docker-changes-test") + if err != nil { + t.Fatal(err) + } + createSampleDir(t, src) + defer os.RemoveAll(src) + dst := src + "-copy" + if err := copyDir(src, dst); err != nil { + t.Fatal(err) + } + mutateSampleDir(t, dst) + defer os.RemoveAll(dst) - // src, err := ioutil.TempDir("", "docker-changes-test") - // if err != nil { - // t.Fatal(err) - // } - // createSampleDir(t, src) - // dst := src + "-copy" - // if err := copyDir(src, dst); err != nil { - // t.Fatal(err) - // } - // mutateSampleDir(t, dst) + changes, err := ChangesDirs(dst, src) + if err != nil { + t.Fatal(err) + } - // changes, err := ChangesDirs(dst, src) - // if err != nil { - // t.Fatal(err) - // } + layer, err := ExportChanges(dst, changes) + if err != nil { + t.Fatal(err) + } - // layer, err := ExportChanges(dst, changes) - // if err != nil { - // t.Fatal(err) - // } + layerCopy, err := NewTempArchive(layer, "") + if err != nil { + t.Fatal(err) + } - // layerCopy, err := NewTempArchive(layer, "") - // if err != nil { - // t.Fatal(err) - // } + if err := ApplyLayer(src, layerCopy); err != nil { + t.Fatal(err) + } - // if err := ApplyLayer(src, layerCopy); err != nil { - // t.Fatal(err) - // } + changes2, err := ChangesDirs(src, dst) + if err != nil { + t.Fatal(err) + } - // changes2, err := ChangesDirs(src, dst) - // if err != nil { - // t.Fatal(err) - // } - - // if len(changes2) != 0 { - // t.Fatalf("Unexpected differences after re applying mutation: %v", changes) - // } - - // os.RemoveAll(src) - // os.RemoveAll(dst) + if len(changes2) != 0 { + t.Fatalf("Unexpected differences after reapplying mutation: %v", changes2) + } } diff --git a/components/engine/archive/diff.go b/components/engine/archive/diff.go index f44991ecb5..464d57a742 100644 --- a/components/engine/archive/diff.go +++ b/components/engine/archive/diff.go @@ -1,6 +1,9 @@ package archive import ( + "archive/tar" + "github.com/dotcloud/docker/utils" + "io" "os" "path/filepath" "strings" @@ -8,87 +11,181 @@ import ( "time" ) +// Linux device nodes are a bit weird due to backwards compat with 16 bit device nodes. +// They are, from low to high: the lower 8 bits of the minor, then 12 bits of the major, +// then the top 12 bits of the minor +func mkdev(major int64, minor int64) uint32 { + return uint32(((minor & 0xfff00) << 12) | ((major & 0xfff) << 8) | (minor & 0xff)) +} +func timeToTimespec(time time.Time) (ts syscall.Timespec) { + if time.IsZero() { + // Return UTIME_OMIT special value + ts.Sec = 0 + ts.Nsec = ((1 << 30) - 2) + return + } + return syscall.NsecToTimespec(time.UnixNano()) +} + // ApplyLayer parses a diff in the standard layer format from `layer`, and // applies it to the directory `dest`. func ApplyLayer(dest string, layer Archive) error { - // Poor man's diff applyer in 2 steps: + // We need to be able to set any perms + oldmask := syscall.Umask(0) + defer syscall.Umask(oldmask) - // Step 1: untar everything in place - if err := Untar(layer, dest, nil); err != nil { + layer, err := DecompressStream(layer) + if err != nil { return err } - modifiedDirs := make(map[string]*syscall.Stat_t) - addDir := func(file string) { - d := filepath.Dir(file) - if _, exists := modifiedDirs[d]; !exists { - if s, err := os.Lstat(d); err == nil { - if sys := s.Sys(); sys != nil { - if stat, ok := sys.(*syscall.Stat_t); ok { - modifiedDirs[d] = stat + tr := tar.NewReader(layer) + + var dirs []*tar.Header + + // Iterate through the files in the archive. + for { + hdr, err := tr.Next() + if err == io.EOF { + // end of tar archive + break + } + if err != nil { + return err + } + + // Normalize name, for safety and for a simple is-root check + hdr.Name = filepath.Clean(hdr.Name) + + if !strings.HasSuffix(hdr.Name, "/") { + // Not the root directory, ensure that the parent directory exists. + // This happened in some tests where an image had a tarfile without any + // parent directories. + parent := filepath.Dir(hdr.Name) + parentPath := filepath.Join(dest, parent) + if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { + err = os.MkdirAll(parentPath, 600) + if err != nil { + return err + } + } + } + + // Skip AUFS metadata dirs + if strings.HasPrefix(hdr.Name, ".wh..wh.") { + continue + } + + path := filepath.Join(dest, hdr.Name) + base := filepath.Base(path) + if strings.HasPrefix(base, ".wh.") { + originalBase := base[len(".wh."):] + originalPath := filepath.Join(filepath.Dir(path), originalBase) + if err := os.RemoveAll(originalPath); err != nil { + return err + } + } else { + // If path exits we almost always just want to remove and replace it. + // The only exception is when it is a directory *and* the file from + // the layer is also a directory. Then we want to merge them (i.e. + // just apply the metadata from the layer). + hasDir := false + if fi, err := os.Lstat(path); err == nil { + if fi.IsDir() && hdr.Typeflag == tar.TypeDir { + hasDir = true + } else { + if err := os.RemoveAll(path); err != nil { + return err + } + } + } + + switch hdr.Typeflag { + case tar.TypeDir: + if !hasDir { + err = os.Mkdir(path, os.FileMode(hdr.Mode)) + if err != nil { + return err + } + } + dirs = append(dirs, hdr) + + case tar.TypeReg, tar.TypeRegA: + // Source is regular file + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(file, tr); err != nil { + file.Close() + return err + } + file.Close() + + case tar.TypeBlock, tar.TypeChar, tar.TypeFifo: + mode := uint32(hdr.Mode & 07777) + switch hdr.Typeflag { + case tar.TypeBlock: + mode |= syscall.S_IFBLK + case tar.TypeChar: + mode |= syscall.S_IFCHR + case tar.TypeFifo: + mode |= syscall.S_IFIFO + } + + if err := syscall.Mknod(path, mode, int(mkdev(hdr.Devmajor, hdr.Devminor))); err != nil { + return err + } + + case tar.TypeLink: + if err := os.Link(filepath.Join(dest, hdr.Linkname), path); err != nil { + return err + } + + case tar.TypeSymlink: + if err := os.Symlink(hdr.Linkname, path); err != nil { + return err + } + + default: + utils.Debugf("unhandled type %d\n", hdr.Typeflag) + } + + if err = syscall.Lchown(path, hdr.Uid, hdr.Gid); err != nil { + return err + } + + // There is no LChmod, so ignore mode for symlink. Also, this + // must happen after chown, as that can modify the file mode + if hdr.Typeflag != tar.TypeSymlink { + err = syscall.Chmod(path, uint32(hdr.Mode&07777)) + if err != nil { + return err + } + } + + // Directories must be handled at the end to avoid further + // file creation in them to modify the mtime + if hdr.Typeflag != tar.TypeDir { + ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} + // syscall.UtimesNano doesn't support a NOFOLLOW flag atm, and + if hdr.Typeflag != tar.TypeSymlink { + if err := syscall.UtimesNano(path, ts); err != nil { + return err + } + } else { + if err := LUtimesNano(path, ts); err != nil { + return err } } } } } - // Step 2: walk for whiteouts and apply them, removing them in the process - err := filepath.Walk(dest, func(fullPath string, f os.FileInfo, err error) error { - if err != nil { - if os.IsNotExist(err) { - // This happens in the case of whiteouts in parent dir removing a directory - // We just ignore it - return filepath.SkipDir - } - return err - } - - // Rebase path - path, err := filepath.Rel(dest, fullPath) - if err != nil { - return err - } - path = filepath.Join("/", path) - - // Skip AUFS metadata - if matched, err := filepath.Match("/.wh..wh.*", path); err != nil { - return err - } else if matched { - addDir(fullPath) - if err := os.RemoveAll(fullPath); err != nil { - return err - } - } - - filename := filepath.Base(path) - if strings.HasPrefix(filename, ".wh.") { - rmTargetName := filename[len(".wh."):] - rmTargetPath := filepath.Join(filepath.Dir(fullPath), rmTargetName) - - // Remove the file targeted by the whiteout - addDir(rmTargetPath) - if err := os.RemoveAll(rmTargetPath); err != nil { - return err - } - // Remove the whiteout itself - addDir(fullPath) - if err := os.RemoveAll(fullPath); err != nil { - return err - } - } - return nil - }) - if err != nil { - return err - } - - for k, v := range modifiedDirs { - lastAccess := getLastAccess(v) - lastModification := getLastModification(v) - aTime := time.Unix(lastAccess.Unix()) - mTime := time.Unix(lastModification.Unix()) - - if err := os.Chtimes(k, aTime, mTime); err != nil { + for _, hdr := range dirs { + path := filepath.Join(dest, hdr.Name) + ts := []syscall.Timespec{timeToTimespec(hdr.AccessTime), timeToTimespec(hdr.ModTime)} + if err := syscall.UtimesNano(path, ts); err != nil { return err } } diff --git a/components/engine/archive/stat_darwin.go b/components/engine/archive/stat_darwin.go index 53ae9dee2f..e041783ec6 100644 --- a/components/engine/archive/stat_darwin.go +++ b/components/engine/archive/stat_darwin.go @@ -9,3 +9,7 @@ func getLastAccess(stat *syscall.Stat_t) syscall.Timespec { func getLastModification(stat *syscall.Stat_t) syscall.Timespec { return stat.Mtimespec } + +func LUtimesNano(path string, ts []syscall.Timespec) error { + return nil +} diff --git a/components/engine/archive/stat_linux.go b/components/engine/archive/stat_linux.go index 50b4627c4a..2f7a520ccd 100644 --- a/components/engine/archive/stat_linux.go +++ b/components/engine/archive/stat_linux.go @@ -1,6 +1,9 @@ package archive -import "syscall" +import ( + "syscall" + "unsafe" +) func getLastAccess(stat *syscall.Stat_t) syscall.Timespec { return stat.Atim @@ -9,3 +12,21 @@ func getLastAccess(stat *syscall.Stat_t) syscall.Timespec { func getLastModification(stat *syscall.Stat_t) syscall.Timespec { return stat.Mtim } + +func LUtimesNano(path string, ts []syscall.Timespec) error { + // These are not currently available in syscall + AT_FDCWD := -100 + AT_SYMLINK_NOFOLLOW := 0x100 + + var _path *byte + _path, err := syscall.BytePtrFromString(path) + if err != nil { + return err + } + + if _, _, err := syscall.Syscall6(syscall.SYS_UTIMENSAT, uintptr(AT_FDCWD), uintptr(unsafe.Pointer(_path)), uintptr(unsafe.Pointer(&ts[0])), uintptr(AT_SYMLINK_NOFOLLOW), 0, 0); err != 0 && err != syscall.ENOSYS { + return err + } + + return nil +} diff --git a/components/engine/auth/auth.go b/components/engine/auth/auth.go index e88fb908f9..770a6a0c0f 100644 --- a/components/engine/auth/auth.go +++ b/components/engine/auth/auth.go @@ -163,7 +163,7 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e loginAgainstOfficialIndex := serverAddress == IndexServerAddress() - // to avoid sending the server address to the server it should be removed before marshalled + // to avoid sending the server address to the server it should be removed before being marshalled authCopy := *authConfig authCopy.ServerAddress = "" @@ -254,11 +254,11 @@ func (config *ConfigFile) ResolveAuthConfig(registry string) AuthConfig { // default to the index server return config.Configs[IndexServerAddress()] } - // if its not the index server there are three cases: + // if it's not the index server there are three cases: // - // 1. this is a full config url -> it should be used as is - // 2. it could be a full url, but with the wrong protocol - // 3. it can be the hostname optionally with a port + // 1. a full config url -> it should be used as is + // 2. a full url, but with the wrong protocol + // 3. a hostname, with an optional port // // as there is only one auth entry which is fully qualified we need to start // parsing and matching diff --git a/components/engine/buildfile.go b/components/engine/buildfile.go index 7d87a17d3a..4f72a73520 100644 --- a/components/engine/buildfile.go +++ b/components/engine/buildfile.go @@ -1,7 +1,10 @@ package docker import ( + "crypto/sha256" + "encoding/hex" "encoding/json" + "errors" "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" @@ -11,11 +14,17 @@ import ( "net/url" "os" "path" + "path/filepath" "reflect" "regexp" + "sort" "strings" ) +var ( + ErrDockerfileEmpty = errors.New("Dockerfile cannot be empty") +) + type BuildFile interface { Build(io.Reader) (string, error) CmdFrom(string) error @@ -26,10 +35,13 @@ type buildFile struct { runtime *Runtime srv *Server - image string - maintainer string - config *Config - context string + image string + maintainer string + config *Config + + contextPath string + context *utils.TarSum + verbose bool utilizeCache bool rm bool @@ -87,6 +99,27 @@ func (b *buildFile) CmdMaintainer(name string) error { return b.commit("", b.config.Cmd, fmt.Sprintf("MAINTAINER %s", name)) } +// probeCache checks to see if image-caching is enabled (`b.utilizeCache`) +// and if so attempts to look up the current `b.image` and `b.config` pair +// in the current server `b.srv`. If an image is found, probeCache returns +// `(true, nil)`. If no image is found, it returns `(false, nil)`. If there +// is any error, it returns `(false, err)`. +func (b *buildFile) probeCache() (bool, error) { + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return false, err + } else if cache != nil { + fmt.Fprintf(b.outStream, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return true, nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } + } + return false, nil +} + func (b *buildFile) CmdRun(args string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to run") @@ -104,17 +137,12 @@ func (b *buildFile) CmdRun(args string) error { utils.Debugf("Command to be executed: %v", b.config.Cmd) - if b.utilizeCache { - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.outStream, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") - } + hit, err := b.probeCache() + if err != nil { + return err + } + if hit { + return nil } cid, err := b.run() @@ -260,44 +288,27 @@ func (b *buildFile) CmdVolume(args string) error { return nil } -func (b *buildFile) addRemote(container *Container, orig, dest string) error { - file, err := utils.Download(orig) +func (b *buildFile) checkPathForAddition(orig string) error { + origPath := path.Join(b.contextPath, orig) + if !strings.HasPrefix(origPath, b.contextPath) { + return fmt.Errorf("Forbidden path outside the build context: %s (%s)", orig, origPath) + } + _, err := os.Stat(origPath) if err != nil { - return err + return fmt.Errorf("%s: no such file or directory", orig) } - defer file.Body.Close() - - // If the destination is a directory, figure out the filename. - if strings.HasSuffix(dest, "/") { - u, err := url.Parse(orig) - if err != nil { - return err - } - path := u.Path - if strings.HasSuffix(path, "/") { - path = path[:len(path)-1] - } - parts := strings.Split(path, "/") - filename := parts[len(parts)-1] - if filename == "" { - return fmt.Errorf("cannot determine filename from url: %s", u) - } - dest = dest + filename - } - - return container.Inject(file.Body, dest) + return nil } func (b *buildFile) addContext(container *Container, orig, dest string) error { - origPath := path.Join(b.context, orig) - destPath := path.Join(container.RootfsPath(), dest) + var ( + origPath = path.Join(b.contextPath, orig) + destPath = path.Join(container.RootfsPath(), dest) + ) // Preserve the trailing '/' if strings.HasSuffix(dest, "/") { destPath = destPath + "/" } - if !strings.HasPrefix(origPath, b.context) { - return fmt.Errorf("Forbidden path outside the build context: %s (%s)", orig, origPath) - } fi, err := os.Stat(origPath) if err != nil { return fmt.Errorf("%s: no such file or directory", orig) @@ -321,7 +332,7 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { } func (b *buildFile) CmdAdd(args string) error { - if b.context == "" { + if b.context == nil { return fmt.Errorf("No context given. Impossible to use ADD") } tmp := strings.SplitN(args, " ", 2) @@ -341,8 +352,90 @@ func (b *buildFile) CmdAdd(args string) error { cmd := b.config.Cmd b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)} - b.config.Image = b.image + + // FIXME: do we really need this? + var ( + origPath = orig + destPath = dest + ) + + if utils.IsURL(orig) { + resp, err := utils.Download(orig) + if err != nil { + return err + } + tmpDirName, err := ioutil.TempDir(b.contextPath, "docker-remote") + if err != nil { + return err + } + tmpFileName := path.Join(tmpDirName, "tmp") + tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer os.RemoveAll(tmpDirName) + if _, err = io.Copy(tmpFile, resp.Body); err != nil { + return err + } + origPath = path.Join(filepath.Base(tmpDirName), filepath.Base(tmpFileName)) + tmpFile.Close() + + // If the destination is a directory, figure out the filename. + if strings.HasSuffix(dest, "/") { + u, err := url.Parse(orig) + if err != nil { + return err + } + path := u.Path + if strings.HasSuffix(path, "/") { + path = path[:len(path)-1] + } + parts := strings.Split(path, "/") + filename := parts[len(parts)-1] + if filename == "" { + return fmt.Errorf("cannot determine filename from url: %s", u) + } + destPath = dest + filename + } + } + + if err := b.checkPathForAddition(origPath); err != nil { + return err + } + + // Hash path and check the cache + if b.utilizeCache { + var ( + hash string + sums = b.context.GetSums() + ) + if fi, err := os.Stat(path.Join(b.contextPath, origPath)); err != nil { + return err + } else if fi.IsDir() { + var subfiles []string + for file, sum := range sums { + if strings.HasPrefix(file, origPath) { + subfiles = append(subfiles, sum) + } + } + sort.Strings(subfiles) + hasher := sha256.New() + hasher.Write([]byte(strings.Join(subfiles, ","))) + hash = "dir:" + hex.EncodeToString(hasher.Sum(nil)) + } else { + hash = "file:" + sums[origPath] + } + b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", hash, dest)} + hit, err := b.probeCache() + if err != nil { + return err + } + if hit { + return nil + } + } + // Create the container and start it container, _, err := b.runtime.Create(b.config, "") if err != nil { @@ -355,14 +448,8 @@ func (b *buildFile) CmdAdd(args string) error { } defer container.Unmount() - if utils.IsURL(orig) { - if err := b.addRemote(container, orig, dest); err != nil { - return err - } - } else { - if err := b.addContext(container, orig, dest); err != nil { - return err - } + if err := b.addContext(container, origPath, destPath); err != nil { + return err } if err := b.commit(container.ID, cmd, fmt.Sprintf("ADD %s in %s", orig, dest)); err != nil { @@ -460,17 +547,12 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { b.config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment} defer func(cmd []string) { b.config.Cmd = cmd }(cmd) - if b.utilizeCache { - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.outStream, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") - } + hit, err := b.probeCache() + if err != nil { + return err + } + if hit { + return nil } container, warnings, err := b.runtime.Create(b.config, "") @@ -511,17 +593,17 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { var lineContinuation = regexp.MustCompile(`\s*\\\s*\n`) func (b *buildFile) Build(context io.Reader) (string, error) { - // FIXME: @creack "name" is a terrible variable name - name, err := ioutil.TempDir("", "docker-build") + tmpdirPath, err := ioutil.TempDir("", "docker-build") if err != nil { return "", err } - if err := archive.Untar(context, name, nil); err != nil { + b.context = &utils.TarSum{Reader: context} + if err := archive.Untar(b.context, tmpdirPath, nil); err != nil { return "", err } - defer os.RemoveAll(name) - b.context = name - filename := path.Join(name, "Dockerfile") + defer os.RemoveAll(tmpdirPath) + b.contextPath = tmpdirPath + filename := path.Join(tmpdirPath, "Dockerfile") if _, err := os.Stat(filename); os.IsNotExist(err) { return "", fmt.Errorf("Can't build a directory with no Dockerfile") } @@ -529,6 +611,9 @@ func (b *buildFile) Build(context io.Reader) (string, error) { if err != nil { return "", err } + if len(fileBytes) == 0 { + return "", ErrDockerfileEmpty + } dockerfile := string(fileBytes) dockerfile = lineContinuation.ReplaceAllString(dockerfile, "") stepN := 0 diff --git a/components/engine/commands.go b/components/engine/commands.go index ef6dce6bd5..97bfda5194 100644 --- a/components/engine/commands.go +++ b/components/engine/commands.go @@ -12,8 +12,8 @@ import ( "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/registry" - "github.com/dotcloud/docker/term" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -238,6 +238,10 @@ func (cli *DockerCli) CmdBuild(args ...string) error { } err = cli.stream("POST", fmt.Sprintf("/build?%s", v.Encode()), body, cli.out, headers) if jerr, ok := err.(*utils.JSONError); ok { + // If no error code is set, default to 1 + if jerr.Code == 0 { + jerr.Code = 1 + } return &utils.StatusError{Status: jerr.Message, StatusCode: jerr.Code} } return err @@ -469,6 +473,13 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "LXC Version: %s\n", remoteInfo.Get("LXCVersion")) fmt.Fprintf(cli.out, "EventsListeners: %d\n", remoteInfo.GetInt("NEventsListener")) fmt.Fprintf(cli.out, "Kernel Version: %s\n", remoteInfo.Get("KernelVersion")) + + if initSha1 := remoteInfo.Get("InitSha1"); initSha1 != "" { + fmt.Fprintf(cli.out, "Init SHA1: %s\n", initSha1) + } + if initPath := remoteInfo.Get("InitPath"); initPath != "" { + fmt.Fprintf(cli.out, "Init Path: %s\n", initPath) + } } if len(remoteInfo.GetList("IndexServerAddress")) != 0 { @@ -1237,9 +1248,9 @@ func (cli *DockerCli) WalkTree(noTrunc bool, images *[]APIImages, byParent map[s cli.WalkTree(noTrunc, &subimages, byParent, prefix+" ", printNode) } } else { - printNode(cli, noTrunc, image, prefix+"|─") + printNode(cli, noTrunc, image, prefix+"├─") if subimages, exists := byParent[image.ID]; exists { - cli.WalkTree(noTrunc, &subimages, byParent, prefix+"| ", printNode) + cli.WalkTree(noTrunc, &subimages, byParent, prefix+"│ ", printNode) } } } @@ -1814,6 +1825,8 @@ func parseRun(cmd *flag.FlagSet, args []string, capabilities *Capabilities) (*Co flVolumes.Set(dstDir) binds = append(binds, bind) flVolumes.Delete(bind) + } else if bind == "/" { + return nil, nil, cmd, fmt.Errorf("Invalid volume: path can't be '/'") } } diff --git a/components/engine/commands_unit_test.go b/components/engine/commands_unit_test.go index 2eac5ce60d..e44d9a1854 100644 --- a/components/engine/commands_unit_test.go +++ b/components/engine/commands_unit_test.go @@ -128,7 +128,9 @@ func TestParseRunVolumes(t *testing.T) { t.Fatalf("Error parsing volume flags, without volume, no volume should be present. Received %v", config.Volumes) } - mustParse(t, "-v /") + if _, _, err := parse(t, "-v /"); err == nil { + t.Fatalf("Expected error, but got none") + } if _, _, err := parse(t, "-v /:/"); err == nil { t.Fatalf("Error parsing volume flags, `-v /:/` should fail but didn't") diff --git a/components/engine/config.go b/components/engine/config.go index 1f743f6290..5a6de7a873 100644 --- a/components/engine/config.go +++ b/components/engine/config.go @@ -18,6 +18,7 @@ type DaemonConfig struct { DefaultIp net.IP InterContainerCommunication bool GraphDriver string + Mtu int } // ConfigFromJob creates and returns a new DaemonConfig object @@ -41,5 +42,10 @@ func ConfigFromJob(job *engine.Job) *DaemonConfig { config.DefaultIp = net.ParseIP(job.Getenv("DefaultIp")) config.InterContainerCommunication = job.GetenvBool("InterContainerCommunication") config.GraphDriver = job.Getenv("GraphDriver") + if mtu := job.GetenvInt("Mtu"); mtu != -1 { + config.Mtu = mtu + } else { + config.Mtu = DefaultNetworkMtu + } return &config } diff --git a/components/engine/container.go b/components/engine/container.go index 206f2dfe63..532a697839 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -7,7 +7,8 @@ import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/graphdriver" - "github.com/dotcloud/docker/term" + "github.com/dotcloud/docker/mount" + "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/utils" "github.com/kr/pty" "io" @@ -48,7 +49,6 @@ type Container struct { network *NetworkInterface NetworkSettings *NetworkSettings - SysInitPath string ResolvConfPath string HostnamePath string HostsPath string @@ -297,7 +297,11 @@ func (container *Container) generateEnvConfig(env []string) error { if err != nil { return err } - ioutil.WriteFile(container.EnvConfigPath(), data, 0600) + p, err := container.EnvConfigPath() + if err != nil { + return err + } + ioutil.WriteFile(p, data, 0600) return nil } @@ -578,6 +582,7 @@ func (container *Container) Start() (err error) { params = append(params, "-g", network.Gateway, "-i", fmt.Sprintf("%s/%d", network.IPAddress, network.IPPrefixLen), + "-mtu", strconv.Itoa(container.runtime.config.Mtu), ) } @@ -681,6 +686,45 @@ func (container *Container) Start() (err error) { } } + root := container.RootfsPath() + envPath, err := container.EnvConfigPath() + if err != nil { + return err + } + + // Mount docker specific files into the containers root fs + if err := mount.Mount(runtime.sysInitPath, path.Join(root, "/.dockerinit"), "none", "bind,ro"); err != nil { + return err + } + if err := mount.Mount(envPath, path.Join(root, "/.dockerenv"), "none", "bind,ro"); err != nil { + return err + } + if err := mount.Mount(container.ResolvConfPath, path.Join(root, "/etc/resolv.conf"), "none", "bind,ro"); err != nil { + return err + } + + if container.HostnamePath != "" && container.HostsPath != "" { + if err := mount.Mount(container.HostnamePath, path.Join(root, "/etc/hostname"), "none", "bind,ro"); err != nil { + return err + } + if err := mount.Mount(container.HostsPath, path.Join(root, "/etc/hosts"), "none", "bind,ro"); err != nil { + return err + } + } + + // Mount user specified volumes + + for r, v := range container.Volumes { + mountAs := "ro" + if container.VolumesRW[v] { + mountAs = "rw" + } + + if err := mount.Mount(v, path.Join(root, r), "none", fmt.Sprintf("bind,%s", mountAs)); err != nil { + return err + } + } + container.cmd = exec.Command(params[0], params[1:]...) // Setup logging of stdout and stderr to disk @@ -836,7 +880,7 @@ func (container *Container) createVolumes() error { volPath = path.Join(container.RootfsPath(), volPath) rootVolPath, err := utils.FollowSymlinkInScope(volPath, container.RootfsPath()) if err != nil { - panic(err) + return err } if _, err := os.Stat(rootVolPath); err != nil { @@ -1358,6 +1402,32 @@ func (container *Container) GetImage() (*Image, error) { } func (container *Container) Unmount() error { + var ( + err error + root = container.RootfsPath() + mounts = []string{ + path.Join(root, "/.dockerinit"), + path.Join(root, "/.dockerenv"), + path.Join(root, "/etc/resolv.conf"), + } + ) + + if container.HostnamePath != "" && container.HostsPath != "" { + mounts = append(mounts, path.Join(root, "/etc/hostname"), path.Join(root, "/etc/hosts")) + } + + for r := range container.Volumes { + mounts = append(mounts, path.Join(root, r)) + } + + for _, m := range mounts { + if lastError := mount.Unmount(m); lastError != nil { + err = lastError + } + } + if err != nil { + return err + } return container.runtime.Unmount(container) } @@ -1377,8 +1447,20 @@ func (container *Container) jsonPath() string { return path.Join(container.root, "config.json") } -func (container *Container) EnvConfigPath() string { - return path.Join(container.root, "config.env") +func (container *Container) EnvConfigPath() (string, error) { + p := path.Join(container.root, "config.env") + if _, err := os.Stat(p); err != nil { + if os.IsNotExist(err) { + f, err := os.Create(p) + if err != nil { + return "", err + } + f.Close() + } else { + return "", err + } + } + return p, nil } func (container *Container) lxcConfigPath() string { diff --git a/components/engine/contrib/completion/bash/docker b/components/engine/contrib/completion/bash/docker index 8e535285e1..f1a515d00a 100755 --- a/components/engine/contrib/completion/bash/docker +++ b/components/engine/contrib/completion/bash/docker @@ -4,7 +4,7 @@ # # This script provides supports completion of: # - commands and their options -# - container ids +# - container ids and names # - image repos and tags # - filepaths # @@ -25,21 +25,24 @@ __docker_containers_all() { local containers containers="$( docker ps -a -q )" - COMPREPLY=( $( compgen -W "$containers" -- "$cur" ) ) + names="$( docker inspect -format '{{.Name}}' $containers | sed 's,^/,,' )" + COMPREPLY=( $( compgen -W "$names $containers" -- "$cur" ) ) } __docker_containers_running() { local containers containers="$( docker ps -q )" - COMPREPLY=( $( compgen -W "$containers" -- "$cur" ) ) + names="$( docker inspect -format '{{.Name}}' $containers | sed 's,^/,,' )" + COMPREPLY=( $( compgen -W "$names $containers" -- "$cur" ) ) } __docker_containers_stopped() { local containers containers="$( comm -13 <(docker ps -q | sort -u) <(docker ps -a -q | sort -u) )" - COMPREPLY=( $( compgen -W "$containers" -- "$cur" ) ) + names="$( docker inspect -format '{{.Name}}' $containers | sed 's,^/,,' )" + COMPREPLY=( $( compgen -W "$names $containers" -- "$cur" ) ) } __docker_image_repos() @@ -70,8 +73,9 @@ __docker_containers_and_images() { local containers images containers="$( docker ps -a -q )" + names="$( docker inspect -format '{{.Name}}' $containers | sed 's,^/,,' )" images="$( docker images | awk 'NR>1{print $1":"$2}' )" - COMPREPLY=( $( compgen -W "$images $containers" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "$images $names $containers" -- "$cur" ) ) __ltrim_colon_completions "$cur" } diff --git a/components/engine/contrib/mkimage-debootstrap.sh b/components/engine/contrib/mkimage-debootstrap.sh index f9992d6e3c..3f268b52da 100755 --- a/components/engine/contrib/mkimage-debootstrap.sh +++ b/components/engine/contrib/mkimage-debootstrap.sh @@ -144,9 +144,9 @@ if [ -z "$strictDebootstrap" ]; then echo 'force-unsafe-io' | sudo tee etc/dpkg/dpkg.cfg.d/02apt-speedup > /dev/null # we want to effectively run "apt-get clean" after every install to keep images small (see output of "apt-get clean -s" for context) { - aptGetClean='rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true' - echo 'DPkg::Post-Invoke { "'$aptGetClean'"; };' - echo 'APT::Update::Post-Invoke { "'$aptGetClean'"; };' + aptGetClean='"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true";' + echo "DPkg::Post-Invoke { ${aptGetClean} };" + echo "APT::Update::Post-Invoke { ${aptGetClean} };" echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' } | sudo tee etc/apt/apt.conf.d/no-cache > /dev/null # and remove the translations, too diff --git a/components/engine/contrib/Dockerfile.tmLanguage/Dockerfile.YAML-tmLanguage b/components/engine/contrib/syntax/textmate/Dockerfile.YAML-tmLanguage similarity index 100% rename from components/engine/contrib/Dockerfile.tmLanguage/Dockerfile.YAML-tmLanguage rename to components/engine/contrib/syntax/textmate/Dockerfile.YAML-tmLanguage diff --git a/components/engine/contrib/Dockerfile.tmLanguage/Dockerfile.tmLanguage b/components/engine/contrib/syntax/textmate/Dockerfile.tmLanguage similarity index 94% rename from components/engine/contrib/Dockerfile.tmLanguage/Dockerfile.tmLanguage rename to components/engine/contrib/syntax/textmate/Dockerfile.tmLanguage index ce5bb5e917..fa8f38992e 100644 --- a/components/engine/contrib/Dockerfile.tmLanguage/Dockerfile.tmLanguage +++ b/components/engine/contrib/syntax/textmate/Dockerfile.tmLanguage @@ -4,6 +4,10 @@ name Dockerfile + fileTypes + + Dockerfile + patterns diff --git a/components/engine/contrib/Dockerfile.tmLanguage/MAINTAINERS b/components/engine/contrib/syntax/textmate/MAINTAINERS similarity index 100% rename from components/engine/contrib/Dockerfile.tmLanguage/MAINTAINERS rename to components/engine/contrib/syntax/textmate/MAINTAINERS diff --git a/components/engine/contrib/Dockerfile.tmLanguage/README.md b/components/engine/contrib/syntax/textmate/README.md similarity index 100% rename from components/engine/contrib/Dockerfile.tmLanguage/README.md rename to components/engine/contrib/syntax/textmate/README.md diff --git a/components/engine/contrib/vim-syntax/LICENSE b/components/engine/contrib/syntax/vim/LICENSE similarity index 100% rename from components/engine/contrib/vim-syntax/LICENSE rename to components/engine/contrib/syntax/vim/LICENSE diff --git a/components/engine/contrib/vim-syntax/README.md b/components/engine/contrib/syntax/vim/README.md similarity index 100% rename from components/engine/contrib/vim-syntax/README.md rename to components/engine/contrib/syntax/vim/README.md diff --git a/components/engine/contrib/vim-syntax/doc/dockerfile.txt b/components/engine/contrib/syntax/vim/doc/dockerfile.txt similarity index 100% rename from components/engine/contrib/vim-syntax/doc/dockerfile.txt rename to components/engine/contrib/syntax/vim/doc/dockerfile.txt diff --git a/components/engine/contrib/vim-syntax/ftdetect/dockerfile.vim b/components/engine/contrib/syntax/vim/ftdetect/dockerfile.vim similarity index 100% rename from components/engine/contrib/vim-syntax/ftdetect/dockerfile.vim rename to components/engine/contrib/syntax/vim/ftdetect/dockerfile.vim diff --git a/components/engine/contrib/vim-syntax/syntax/dockerfile.vim b/components/engine/contrib/syntax/vim/syntax/dockerfile.vim similarity index 100% rename from components/engine/contrib/vim-syntax/syntax/dockerfile.vim rename to components/engine/contrib/syntax/vim/syntax/dockerfile.vim diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 3ee7be9876..2d7e04ce92 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -40,6 +40,7 @@ func main() { flInterContainerComm = flag.Bool("icc", true, "Enable inter-container communication") flGraphDriver = flag.String("s", "", "Force the docker runtime to use a specific storage driver") flHosts = docker.NewListOpts(docker.ValidateHost) + flMtu = flag.Int("mtu", docker.DefaultNetworkMtu, "Set the containers network mtu") ) flag.Var(&flDns, "dns", "Force docker to use specific DNS servers") flag.Var(&flHosts, "H", "Multiple tcp://host:port or unix://path/to/socket to bind in daemon mode, single connection otherwise") @@ -51,8 +52,13 @@ func main() { return } if flHosts.Len() == 0 { - // If we do not have a host, default to unix socket - flHosts.Set(fmt.Sprintf("unix://%s", docker.DEFAULTUNIXSOCKET)) + defaultHost := os.Getenv("DOCKER_HOST") + + if defaultHost == "" || *flDaemon { + // If we do not have a host, default to unix socket + defaultHost = fmt.Sprintf("unix://%s", docker.DEFAULTUNIXSOCKET) + } + flHosts.Set(defaultHost) } if *bridgeName != "" && *bridgeIp != "" { @@ -69,6 +75,7 @@ func main() { flag.Usage() return } + eng, err := engine.New(*flRoot) if err != nil { log.Fatal(err) @@ -86,6 +93,7 @@ func main() { job.Setenv("DefaultIp", *flDefaultIp) job.SetenvBool("InterContainerCommunication", *flInterContainerComm) job.Setenv("GraphDriver", *flGraphDriver) + job.SetenvInt("Mtu", *flMtu) if err := job.Run(); err != nil { log.Fatal(err) } diff --git a/components/engine/docs/README.md b/components/engine/docs/README.md index 768cabdbb1..3fdbad2ead 100644 --- a/components/engine/docs/README.md +++ b/components/engine/docs/README.md @@ -46,7 +46,7 @@ directory: * Linux: `pip install -r docs/requirements.txt` -* Mac OS X: `[sudo] pip-2.7 -r docs/requirements.txt` +* Mac OS X: `[sudo] pip-2.7 install -r docs/requirements.txt` ###Alternative Installation: Docker Container diff --git a/components/engine/docs/sources/api/docker_remote_api.rst b/components/engine/docs/sources/api/docker_remote_api.rst index 7cb7b323c1..b6615ad7d6 100644 --- a/components/engine/docs/sources/api/docker_remote_api.rst +++ b/components/engine/docs/sources/api/docker_remote_api.rst @@ -26,10 +26,10 @@ Docker Remote API 2. Versions =========== -The current version of the API is 1.7 +The current version of the API is 1.8 Calling /images//insert is the same as calling -/v1.7/images//insert +/v1.8/images//insert You can still call an old version of the api using /v1.0/images//insert diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst index b6661bfdb6..ab452798b9 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.3.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.3.rst @@ -1078,7 +1078,7 @@ Monitor Docker's events .. sourcecode:: http - POST /events?since=1374067924 + GET /events?since=1374067924 **Example response**: diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst index 0441e068d0..5c8884b16f 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.4.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.4.rst @@ -1122,7 +1122,7 @@ Monitor Docker's events .. sourcecode:: http - POST /events?since=1374067924 + GET /events?since=1374067924 **Example response**: diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.5.rst b/components/engine/docs/sources/api/docker_remote_api_v1.5.rst index 8cd501e2a1..609fc6b056 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.5.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.5.rst @@ -1093,7 +1093,7 @@ Monitor Docker's events .. sourcecode:: http - POST /events?since=1374067924 + GET /events?since=1374067924 **Example response**: diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.6.rst b/components/engine/docs/sources/api/docker_remote_api_v1.6.rst index 25be478a1b..df53275a4f 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.6.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.6.rst @@ -1228,7 +1228,7 @@ Monitor Docker's events .. sourcecode:: http - POST /events?since=1374067924 + GET /events?since=1374067924 **Example response**: diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.7.rst b/components/engine/docs/sources/api/docker_remote_api_v1.7.rst index 9857948732..d47f672df0 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.7.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.7.rst @@ -122,7 +122,6 @@ Create a container "AttachStdout":true, "AttachStderr":true, "PortSpecs":null, - "Privileged": false, "Tty":false, "OpenStdin":false, "StdinOnce":false, @@ -136,10 +135,12 @@ Create a container "/tmp": {} }, "VolumesFrom":"", - "WorkingDir":"" - + "WorkingDir":"", + "ExposedPorts":{ + "22/tcp": {} + } } - + **Example response**: .. sourcecode:: http @@ -364,10 +365,11 @@ Start a container { "Binds":["/tmp:/tmp"], "LxcConf":{"lxc.utsname":"docker"}, - "PortBindings":null + "PortBindings":{ "22/tcp": [{ "HostPort": "11022" }] }, + "Privileged":false, "PublishAllPorts":false } - + Binds need to reference Volumes that were defined during container creation. **Example response**: @@ -1159,7 +1161,7 @@ Monitor Docker's events .. sourcecode:: http - POST /events?since=1374067924 + GET /events?since=1374067924 **Example response**: diff --git a/components/engine/docs/sources/api/docker_remote_api_v1.8.rst b/components/engine/docs/sources/api/docker_remote_api_v1.8.rst index bc51e209ef..3fe5cd73e0 100644 --- a/components/engine/docs/sources/api/docker_remote_api_v1.8.rst +++ b/components/engine/docs/sources/api/docker_remote_api_v1.8.rst @@ -122,7 +122,6 @@ Create a container "AttachStdout":true, "AttachStderr":true, "PortSpecs":null, - "Privileged": false, "Tty":false, "OpenStdin":false, "StdinOnce":false, @@ -132,12 +131,16 @@ Create a container ], "Dns":null, "Image":"base", - "Volumes":{}, + "Volumes":{ + "/tmp": {} + }, "VolumesFrom":"", - "WorkingDir":"" - + "WorkingDir":"", + "ExposedPorts":{ + "22/tcp": {} + } } - + **Example response**: .. sourcecode:: http @@ -151,6 +154,7 @@ Create a container } :jsonparam config: the container's configuration + :query name: Assign the specified name to the container. Must match ``/?[a-zA-Z0-9_-]+``. :statuscode 201: no error :statuscode 404: no such container :statuscode 406: impossible to attach (container not running) @@ -377,7 +381,10 @@ Start a container { "Binds":["/tmp:/tmp"], - "LxcConf":{"lxc.utsname":"docker"} + "LxcConf":{"lxc.utsname":"docker"}, + "PortBindings":{ "22/tcp": [{ "HostPort": "11022" }] }, + "PublishAllPorts":false, + "Privileged":false } **Example response**: @@ -1173,7 +1180,7 @@ Monitor Docker's events .. sourcecode:: http - POST /events?since=1374067924 + GET /events?since=1374067924 **Example response**: diff --git a/components/engine/docs/sources/api/registry_api.rst b/components/engine/docs/sources/api/registry_api.rst index 8803b70ba8..b5c36cc344 100644 --- a/components/engine/docs/sources/api/registry_api.rst +++ b/components/engine/docs/sources/api/registry_api.rst @@ -19,7 +19,8 @@ Docker Registry API - It doesn’t have a local database - It will be open-sourced at some point -We expect that there will be multiple registries out there. To help to grasp the context, here are some examples of registries: +We expect that there will be multiple registries out there. To help to grasp +the context, here are some examples of registries: - **sponsor registry**: such a registry is provided by a third-party hosting infrastructure as a convenience for their customers and the docker community as a whole. Its costs are supported by the third party, but the management and operation of the registry are supported by dotCloud. It features read/write access, and delegates authentication and authorization to the Index. - **mirror registry**: such a registry is provided by a third-party hosting infrastructure but is targeted at their customers only. Some mechanism (unspecified to date) ensures that public images are pulled from a sponsor registry to the mirror registry, to make sure that the customers of the third-party provider can “docker pull” those images locally. @@ -37,7 +38,10 @@ We expect that there will be multiple registries out there. To help to grasp the - local mount point; - remote docker addressed through SSH. -The latter would only require two new commands in docker, e.g. “registryget” and “registryput”, wrapping access to the local filesystem (and optionally doing consistency checks). Authentication and authorization are then delegated to SSH (e.g. with public keys). +The latter would only require two new commands in docker, e.g. ``registryget`` +and ``registryput``, wrapping access to the local filesystem (and optionally +doing consistency checks). Authentication and authorization are then delegated +to SSH (e.g. with public keys). 2. Endpoints ============ diff --git a/components/engine/docs/sources/api/registry_index_spec.rst b/components/engine/docs/sources/api/registry_index_spec.rst index 449569414e..89f6319f5c 100644 --- a/components/engine/docs/sources/api/registry_index_spec.rst +++ b/components/engine/docs/sources/api/registry_index_spec.rst @@ -15,11 +15,13 @@ Registry & Index Spec --------- The Index is responsible for centralizing information about: + - User accounts - Checksums of the images - Public namespaces The Index has different components: + - Web UI - Meta-data store (comments, stars, list public repositories) - Authentication service @@ -27,7 +29,7 @@ The Index has different components: The index is authoritative for those information. -We expect that there will be only one instance of the index, run and managed by dotCloud. +We expect that there will be only one instance of the index, run and managed by Docker Inc. 1.2 Registry ------------ @@ -53,12 +55,16 @@ We expect that there will be multiple registries out there. To help to grasp the - local mount point; - remote docker addressed through SSH. -The latter would only require two new commands in docker, e.g. “registryget” and “registryput”, wrapping access to the local filesystem (and optionally doing consistency checks). Authentication and authorization are then delegated to SSH (e.g. with public keys). +The latter would only require two new commands in docker, e.g. ``registryget`` +and ``registryput``, wrapping access to the local filesystem (and optionally +doing consistency checks). Authentication and authorization are then delegated +to SSH (e.g. with public keys). 1.3 Docker ---------- On top of being a runtime for LXC, Docker is the Registry client. It supports: + - Push / Pull on the registry - Client authentication on the Index @@ -72,21 +78,33 @@ On top of being a runtime for LXC, Docker is the Registry client. It supports: 1. Contact the Index to know where I should download “samalba/busybox” 2. Index replies: - a. “samalba/busybox” is on Registry A - b. here are the checksums for “samalba/busybox” (for all layers) + a. ``samalba/busybox`` is on Registry A + b. here are the checksums for ``samalba/busybox`` (for all layers) c. token -3. Contact Registry A to receive the layers for “samalba/busybox” (all of them to the base image). Registry A is authoritative for “samalba/busybox” but keeps a copy of all inherited layers and serve them all from the same location. +3. Contact Registry A to receive the layers for ``samalba/busybox`` (all of them to the base image). Registry A is authoritative for “samalba/busybox” but keeps a copy of all inherited layers and serve them all from the same location. 4. registry contacts index to verify if token/user is allowed to download images 5. Index returns true/false lettings registry know if it should proceed or error out 6. Get the payload for all layers -It’s possible to run docker pull \https:///repositories/samalba/busybox. In this case, docker bypasses the Index. However the security is not guaranteed (in case Registry A is corrupted) because there won’t be any checksum checks. +It's possible to run: -Currently registry redirects to s3 urls for downloads, going forward all downloads need to be streamed through the registry. The Registry will then abstract the calls to S3 by a top-level class which implements sub-classes for S3 and local storage. +.. code-block:: bash -Token is only returned when the 'X-Docker-Token' header is sent with request. + docker pull https:///repositories/samalba/busybox -Basic Auth is required to pull private repos. Basic auth isn't required for pulling public repos, but if one is provided, it needs to be valid and for an active account. +In this case, Docker bypasses the Index. However the security is not guaranteed +(in case Registry A is corrupted) because there won’t be any checksum checks. + +Currently registry redirects to s3 urls for downloads, going forward all +downloads need to be streamed through the registry. The Registry will then +abstract the calls to S3 by a top-level class which implements sub-classes for +S3 and local storage. + +Token is only returned when the ``X-Docker-Token`` header is sent with request. + +Basic Auth is required to pull private repos. Basic auth isn't required for +pulling public repos, but if one is provided, it needs to be valid and for an +active account. API (pulling repository foo/bar): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -155,7 +173,9 @@ API (pulling repository foo/bar): **Index can be replaced!** For a private Registry deployed, a custom Index can be used to serve and validate token according to different policies. -Docker computes the checksums and submit them to the Index at the end of the push. When a repository name does not have checksums on the Index, it means that the push is in progress (since checksums are submitted at the end). +Docker computes the checksums and submit them to the Index at the end of the +push. When a repository name does not have checksums on the Index, it means +that the push is in progress (since checksums are submitted at the end). API (pushing repos foo/bar): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -237,10 +257,11 @@ API (pushing repos foo/bar): 2.3 Delete ---------- -If you need to delete something from the index or registry, we need a nice clean way to do that. Here is the workflow. +If you need to delete something from the index or registry, we need a nice +clean way to do that. Here is the workflow. -1. Docker contacts the index to request a delete of a repository “samalba/busybox” (authentication required with user credentials) -2. If authentication works and repository is valid, “samalba/busybox” is marked as deleted and a temporary token is returned +1. Docker contacts the index to request a delete of a repository ``samalba/busybox`` (authentication required with user credentials) +2. If authentication works and repository is valid, ``samalba/busybox`` is marked as deleted and a temporary token is returned 3. Send a delete request to the registry for the repository (along with the token) 4. Registry A contacts the Index to verify the token (token must corresponds to the repository name) 5. Index validates the token. Registry A deletes the repository and everything associated to it. @@ -312,24 +333,40 @@ The Index has two main purposes (along with its fancy social features): 3.1 Without an Index -------------------- -Using the Registry without the Index can be useful to store the images on a private network without having to rely on an external entity controlled by dotCloud. -In this case, the registry will be launched in a special mode (--standalone? --no-index?). In this mode, the only thing which changes is that Registry will never contact the Index to verify a token. It will be the Registry owner responsibility to authenticate the user who pushes (or even pulls) an image using any mechanism (HTTP auth, IP based, etc...). +Using the Registry without the Index can be useful to store the images on a +private network without having to rely on an external entity controlled by +Docker Inc. -In this scenario, the Registry is responsible for the security in case of data corruption since the checksums are not delivered by a trusted entity. +In this case, the registry will be launched in a special mode (--standalone? +--no-index?). In this mode, the only thing which changes is that Registry will +never contact the Index to verify a token. It will be the Registry owner +responsibility to authenticate the user who pushes (or even pulls) an image +using any mechanism (HTTP auth, IP based, etc...). -As hinted previously, a standalone registry can also be implemented by any HTTP server handling GET/PUT requests (or even only GET requests if no write access is necessary). +In this scenario, the Registry is responsible for the security in case of data +corruption since the checksums are not delivered by a trusted entity. + +As hinted previously, a standalone registry can also be implemented by any HTTP +server handling GET/PUT requests (or even only GET requests if no write access +is necessary). 3.2 With an Index ----------------- The Index data needed by the Registry are simple: + - Serve the checksums - Provide and authorize a Token -In the scenario of a Registry running on a private network with the need of centralizing and authorizing, it’s easy to use a custom Index. +In the scenario of a Registry running on a private network with the need of +centralizing and authorizing, it’s easy to use a custom Index. -The only challenge will be to tell Docker to contact (and trust) this custom Index. Docker will be configurable at some point to use a specific Index, it’ll be the private entity responsibility (basically the organization who uses Docker in a private environment) to maintain the Index and the Docker’s configuration among its consumers. +The only challenge will be to tell Docker to contact (and trust) this custom +Index. Docker will be configurable at some point to use a specific Index, it’ll +be the private entity responsibility (basically the organization who uses +Docker in a private environment) to maintain the Index and the Docker’s +configuration among its consumers. 4. The API ========== @@ -339,16 +376,22 @@ The first version of the api is available here: https://github.com/jpetazzo/dock 4.1 Images ---------- -The format returned in the images is not defined here (for layer and json), basically because Registry stores exactly the same kind of information as Docker uses to manage them. +The format returned in the images is not defined here (for layer and JSON), +basically because Registry stores exactly the same kind of information as +Docker uses to manage them. -The format of ancestry is a line-separated list of image ids, in age order. I.e. the image’s parent is on the last line, the parent of the parent on the next-to-last line, etc.; if the image has no parent, the file is empty. +The format of ancestry is a line-separated list of image ids, in age order, +i.e. the image’s parent is on the last line, the parent of the parent on the +next-to-last line, etc.; if the image has no parent, the file is empty. -GET /v1/images//layer -PUT /v1/images//layer -GET /v1/images//json -PUT /v1/images//json -GET /v1/images//ancestry -PUT /v1/images//ancestry +.. code-block:: bash + + GET /v1/images//layer + PUT /v1/images//layer + GET /v1/images//json + PUT /v1/images//json + GET /v1/images//ancestry + PUT /v1/images//ancestry 4.2 Users --------- @@ -393,7 +436,9 @@ PUT /v1/users/ 4.2.3 Login (Index) ^^^^^^^^^^^^^^^^^^^ -Does nothing else but asking for a user authentication. Can be used to validate credentials. HTTP Basic Auth for now, maybe change in future. + +Does nothing else but asking for a user authentication. Can be used to validate +credentials. HTTP Basic Auth for now, maybe change in future. GET /v1/users @@ -405,7 +450,10 @@ GET /v1/users 4.3 Tags (Registry) ------------------- -The Registry does not know anything about users. Even though repositories are under usernames, it’s just a namespace for the registry. Allowing us to implement organizations or different namespaces per user later, without modifying the Registry’s API. +The Registry does not know anything about users. Even though repositories are +under usernames, it’s just a namespace for the registry. Allowing us to +implement organizations or different namespaces per user later, without +modifying the Registry’s API. The following naming restrictions apply: @@ -439,7 +487,10 @@ DELETE /v1/repositories///tags/ 4.4 Images (Index) ------------------ -For the Index to “resolve” the repository name to a Registry location, it uses the X-Docker-Endpoints header. In other terms, this requests always add a “X-Docker-Endpoints” to indicate the location of the registry which hosts this repository. +For the Index to “resolve” the repository name to a Registry location, it uses +the X-Docker-Endpoints header. In other terms, this requests always add a +``X-Docker-Endpoints`` to indicate the location of the registry which hosts this +repository. 4.4.1 Get the images ^^^^^^^^^^^^^^^^^^^^^ @@ -484,17 +535,20 @@ Return 202 OK ====================== It’s possible to chain Registries server for several reasons: + - Load balancing - Delegate the next request to another server -When a Registry is a reference for a repository, it should host the entire images chain in order to avoid breaking the chain during the download. +When a Registry is a reference for a repository, it should host the entire +images chain in order to avoid breaking the chain during the download. The Index and Registry use this mechanism to redirect on one or the other. Example with an image download: -On every request, a special header can be returned: -X-Docker-Endpoints: server1,server2 +On every request, a special header can be returned:: + + X-Docker-Endpoints: server1,server2 On the next request, the client will always pick a server from this list. @@ -504,7 +558,8 @@ On the next request, the client will always pick a server from this list. 6.1 On the Index ----------------- -The Index supports both “Basic” and “Token” challenges. Usually when there is a “401 Unauthorized”, the Index replies this:: +The Index supports both “Basic” and “Token” challenges. Usually when there is a +``401 Unauthorized``, the Index replies this:: 401 Unauthorized WWW-Authenticate: Basic realm="auth required",Token @@ -543,11 +598,13 @@ The Registry only supports the Token challenge:: 401 Unauthorized WWW-Authenticate: Token -The only way is to provide a token on “401 Unauthorized” responses:: +The only way is to provide a token on ``401 Unauthorized`` responses:: - Authorization: Token signature=123abc,repository=”foo/bar”,access=read + Authorization: Token signature=123abc,repository="foo/bar",access=read -Usually, the Registry provides a Cookie when a Token verification succeeded. Every time the Registry passes a Cookie, you have to pass it back the same cookie.:: +Usually, the Registry provides a Cookie when a Token verification succeeded. +Every time the Registry passes a Cookie, you have to pass it back the same +cookie.:: 200 OK Set-Cookie: session="wD/J7LqL5ctqw8haL10vgfhrb2Q=?foo=UydiYXInCnAxCi4=×tamp=RjEzNjYzMTQ5NDcuNDc0NjQzCi4="; Path=/; HttpOnly diff --git a/components/engine/docs/sources/commandline/cli.rst b/components/engine/docs/sources/commandline/cli.rst index 9b1db93556..67c8b06189 100644 --- a/components/engine/docs/sources/commandline/cli.rst +++ b/components/engine/docs/sources/commandline/cli.rst @@ -12,7 +12,7 @@ To list available commands, either run ``docker`` with no parameters or execute $ sudo docker Usage: docker [OPTIONS] COMMAND [arg...] - -H=[unix:///var/run/docker.sock]: tcp://host:port to bind/connect to or unix://path/to/socket to use + -H=[unix:///var/run/docker.sock]: tcp://[host[:port]] to bind/connect to or unix://[/path/to/socket] to use. When host=[0.0.0.0], port=[4243] or path=[/var/run/docker.sock] is omitted, default values are used. A self-sufficient runtime for linux containers. @@ -27,7 +27,7 @@ To list available commands, either run ``docker`` with no parameters or execute Usage of docker: -D=false: Enable debug mode - -H=[unix:///var/run/docker.sock]: Multiple tcp://host:port or unix://path/to/socket to bind in daemon mode, single connection otherwise + -H=[unix:///var/run/docker.sock]: tcp://[host[:port]] to bind or unix://[/path/to/socket] to use. When host=[0.0.0.0], port=[4243] or path=[/var/run/docker.sock] is omitted, default values are used. -api-enable-cors=false: Enable CORS headers in the remote API -b="": Attach containers to a pre-existing network bridge; use 'none' to disable container networking -bip="": Use the provided CIDR notation address for the dynamically created bridge (docker0); Mutually exclusive of -b @@ -37,19 +37,32 @@ To list available commands, either run ``docker`` with no parameters or execute -icc=true: Enable inter-container communication -ip="0.0.0.0": Default IP address to use when binding container ports -iptables=true: Disable docker's addition of iptables rules + -mtu=1500: Set the containers network mtu -p="/var/run/docker.pid": Path to use for daemon PID file -r=true: Restart previously running containers -s="": Force the docker runtime to use a specific storage driver -v=false: Print version information and quit -The docker daemon is the persistent process that manages containers. Docker uses the same binary for both the +The Docker daemon is the persistent process that manages containers. Docker uses the same binary for both the daemon and client. To run the daemon you provide the ``-d`` flag. -To force docker to use devicemapper as the storage driver, use ``docker -d -s devicemapper`` +To force Docker to use devicemapper as the storage driver, use ``docker -d -s devicemapper``. -To set the dns server for all docker containers, use ``docker -d -dns 8.8.8.8`` +To set the DNS server for all Docker containers, use ``docker -d -dns 8.8.8.8``. + +To run the daemon with debug output, use ``docker -d -D``. + +The docker client will also honor the ``DOCKER_HOST`` environment variable to set +the ``-H`` flag for the client. + +:: + + docker -H tcp://0.0.0.0:4243 ps + # or + export DOCKER_HOST="tcp://0.0.0.0:4243" + docker ps + # both are equal -To run the daemon with debug output, use ``docker -d -D`` .. _cli_attach: @@ -68,11 +81,11 @@ To run the daemon with debug output, use ``docker -d -D`` You can detach from the container again (and leave it running) with ``CTRL-c`` (for a quiet exit) or ``CTRL-\`` to get a stacktrace of the Docker client when it quits. When you detach from the container's -process the exit code will be retuned to the client. +process the exit code will be returned to the client. -To stop a container, use ``docker stop`` +To stop a container, use ``docker stop``. -To kill the container, use ``docker kill`` +To kill the container, use ``docker kill``. .. _cli_attach_examples: @@ -128,12 +141,11 @@ Examples: -no-cache: Do not use the cache when building the image. -rm: Remove intermediate containers after a successful build -The files at PATH or URL are called the "context" of the build. The -build process may refer to any of the files in the context, for -example when using an :ref:`ADD ` instruction. When a -single ``Dockerfile`` is given as URL, then no context is set. When a -git repository is set as URL, then the repository is used as the -context +The files at ``PATH`` or ``URL`` are called the "context" of the build. The +build process may refer to any of the files in the context, for example when +using an :ref:`ADD ` instruction. When a single ``Dockerfile`` +is given as ``URL``, then no context is set. When a Git repository is set as +``URL``, then the repository is used as the context .. _cli_build_examples: @@ -168,13 +180,13 @@ Examples: ---> f52f38b7823e Successfully built f52f38b7823e -This example specifies that the PATH is ``.``, and so all the files in -the local directory get tar'd and sent to the Docker daemon. The PATH +This example specifies that the ``PATH`` is ``.``, and so all the files in +the local directory get tar'd and sent to the Docker daemon. The ``PATH`` specifies where to find the files for the "context" of the build on the Docker daemon. Remember that the daemon could be running on a -remote machine and that no parsing of the Dockerfile happens at the +remote machine and that no parsing of the ``Dockerfile`` happens at the client side (where you're running ``docker build``). That means that -*all* the files at PATH get sent, not just the ones listed to +*all* the files at ``PATH`` get sent, not just the ones listed to :ref:`ADD ` in the ``Dockerfile``. The transfer of context from the local machine to the Docker daemon is @@ -197,16 +209,16 @@ tag will be ``2.0`` This will read a ``Dockerfile`` from *stdin* without context. Due to the lack of a context, no contents of any local directory will be sent -to the ``docker`` daemon. Since there is no context, a Dockerfile +to the ``docker`` daemon. Since there is no context, a ``Dockerfile`` ``ADD`` only works if it refers to a remote URL. .. code-block:: bash $ sudo docker build github.com/creack/docker-firefox -This will clone the Github repository and use the cloned repository as +This will clone the GitHub repository and use the cloned repository as context. The ``Dockerfile`` at the root of the repository is used as -``Dockerfile``. Note that you can specify an arbitrary git repository +``Dockerfile``. Note that you can specify an arbitrary Git repository by using the ``git://`` schema. @@ -247,7 +259,7 @@ Change the command that a container runs ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Sometimes you have an application container running just a service and you need -to make a quick change (run bash?) and then change it back. +to make a quick change and then change it back. In this example, we run a container with ``ls`` and then change the image to run ``ls /etc``. @@ -270,9 +282,9 @@ Full -run example The ``-run`` JSON hash changes the ``Config`` section when running ``docker inspect CONTAINERID`` or ``config`` when running ``docker inspect IMAGEID``. -(multiline is ok within a single quote ``'``) +(Multiline is okay within a single quote ``'``) -:: +.. code-block:: bash $ sudo docker commit -run=' { @@ -315,7 +327,7 @@ or ``config`` when running ``docker inspect IMAGEID``. Copy files/folders from the containers filesystem to the host path. Paths are relative to the root of the filesystem. - + .. code-block:: bash $ sudo docker cp 7bb0e258aefe:/etc/debian_version . @@ -329,7 +341,7 @@ or ``config`` when running ``docker inspect IMAGEID``. :: Usage: docker diff CONTAINER - + List the changed files and directories in a container's filesystem There are 3 events that are listed in the 'diff': @@ -338,7 +350,7 @@ There are 3 events that are listed in the 'diff': 2. ```D``` - Delete 3. ```C``` - Change -for example: +For example: .. code-block:: bash @@ -366,7 +378,7 @@ for example: Usage: docker events Get real time events from the server - + -since="": Show previously created events and then stream. (either seconds since epoch, or date string as below) @@ -429,8 +441,8 @@ Show events in the past from a specified time Usage: docker export CONTAINER Export the contents of a filesystem as a tar archive to STDOUT - -for example: + +For example: .. code-block:: bash @@ -450,7 +462,7 @@ for example: -notrunc=false: Don't truncate output -q=false: only show numeric IDs -To see how the docker:latest image was built: +To see how the ``docker:latest`` image was built: .. code-block:: bash @@ -482,7 +494,7 @@ To see how the docker:latest image was built: d5e85dc5b1d8 2 weeks ago /bin/sh -c apt-get update 13e642467c11 2 weeks ago /bin/sh -c echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list ae6dde92a94e 2 weeks ago /bin/sh -c #(nop) MAINTAINER Solomon Hykes - ubuntu:12.04 6 months ago + ubuntu:12.04 6 months ago .. _cli_images: @@ -500,7 +512,7 @@ To see how the docker:latest image was built: -q=false: only show numeric IDs -tree=false: output graph in tree format -viz=false: output graph in graphviz format - + Listing the most recently created images ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -553,15 +565,15 @@ Displaying image hierarchy $ sudo docker images -tree - |─8dbd9e392a96 Size: 131.5 MB (virtual 131.5 MB) Tags: ubuntu:12.04,ubuntu:latest,ubuntu:precise + ├─8dbd9e392a96 Size: 131.5 MB (virtual 131.5 MB) Tags: ubuntu:12.04,ubuntu:latest,ubuntu:precise └─27cf78414709 Size: 180.1 MB (virtual 180.1 MB) └─b750fe79269d Size: 24.65 kB (virtual 180.1 MB) Tags: ubuntu:12.10,ubuntu:quantal - |─f98de3b610d5 Size: 12.29 kB (virtual 180.1 MB) - | └─7da80deb7dbf Size: 16.38 kB (virtual 180.1 MB) - | └─65ed2fee0a34 Size: 20.66 kB (virtual 180.2 MB) - | └─a2b9ea53dddc Size: 819.7 MB (virtual 999.8 MB) - | └─a29b932eaba8 Size: 28.67 kB (virtual 999.9 MB) - | └─e270a44f124d Size: 12.29 kB (virtual 999.9 MB) Tags: progrium/buildstep:latest + ├─f98de3b610d5 Size: 12.29 kB (virtual 180.1 MB) + │ └─7da80deb7dbf Size: 16.38 kB (virtual 180.1 MB) + │ └─65ed2fee0a34 Size: 20.66 kB (virtual 180.2 MB) + │ └─a2b9ea53dddc Size: 819.7 MB (virtual 999.8 MB) + │ └─a29b932eaba8 Size: 28.67 kB (virtual 999.9 MB) + │ └─e270a44f124d Size: 12.29 kB (virtual 999.9 MB) Tags: progrium/buildstep:latest └─17e74ac162d8 Size: 53.93 kB (virtual 180.2 MB) └─339a3f56b760 Size: 24.65 kB (virtual 180.2 MB) └─904fcc40e34d Size: 96.7 MB (virtual 276.9 MB) @@ -588,10 +600,9 @@ Displaying image hierarchy (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) into it, then optionally tag it. At this time, the URL must start with ``http`` and point to a single -file archive (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) containing a +file archive (.tar, .tar.gz, .tgz, .bzip, .tar.xz, or .txz) containing a root filesystem. If you would like to import from a local directory or -archive, you can use the ``-`` parameter to take the data from -standard in. +archive, you can use the ``-`` parameter to take the data from *stdin*. Examples ~~~~~~~~ @@ -601,24 +612,30 @@ Import from a remote location This will create a new untagged image. -``$ sudo docker import http://example.com/exampleimage.tgz`` +.. code-block:: bash + + $ sudo docker import http://example.com/exampleimage.tgz Import from a local file ........................ -Import to docker via pipe and standard in +Import to docker via pipe and *stdin*. -``$ cat exampleimage.tgz | sudo docker import - exampleimagelocal:new`` +.. code-block:: bash + + $ cat exampleimage.tgz | sudo docker import - exampleimagelocal:new Import from a local directory ............................. -``$ sudo tar -c . | docker import - exampleimagedir`` +.. code-block:: bash -Note the ``sudo`` in this example -- you must preserve the ownership -of the files (especially root ownership) during the archiving with -tar. If you are not root (or sudo) when you tar, then the ownerships -might not get preserved. + $ sudo tar -c . | docker import - exampleimagedir + +Note the ``sudo`` in this example -- you must preserve the ownership of the +files (especially root ownership) during the archiving with tar. If you are not +root (or the sudo command) when you tar, then the ownerships might not get +preserved. .. _cli_info: @@ -657,16 +674,16 @@ might not get preserved. Insert a file from URL in the IMAGE at PATH -Use the specified IMAGE as the parent for a new image which adds a -:ref:`layer ` containing the new file. ``insert`` does not modify -the original image, and the new image has the contents of the parent image, -plus the new file. +Use the specified ``IMAGE`` as the parent for a new image which adds a +:ref:`layer ` containing the new file. The ``insert`` command does +not modify the original image, and the new image has the contents of the parent +image, plus the new file. Examples ~~~~~~~~ -Insert file from github +Insert file from GitHub ....................... .. code-block:: bash @@ -681,16 +698,16 @@ Insert file from github :: - Usage: docker inspect [OPTIONS] CONTAINER + Usage: docker inspect CONTAINER|IMAGE [CONTAINER|IMAGE...] - Return low-level information on a container + Return low-level information on a container/image - -format="": template to output results + -format="": Format the output using the given go template. By default, this will render all results in a JSON array. If a format is specified, the given template will be executed for each result. -Go's `text/template ` package +Go's `text/template `_ package describes all the details of the format. Examples @@ -795,14 +812,14 @@ Known Issues (kill) Fetch the logs of a container -``docker logs`` is a convenience which batch-retrieves whatever logs -are present at the time of execution. This does not guarantee -execution order when combined with a ``docker run`` (i.e. your run may -not have generated any logs at the time you execute ``docker logs``). +The ``docker logs`` command is a convenience which batch-retrieves whatever +logs are present at the time of execution. This does not guarantee execution +order when combined with a ``docker run`` (i.e. your run may not have generated +any logs at the time you execute ``docker logs``). -``docker logs -f`` combines ``docker logs`` and ``docker attach``: it -will first return all logs from the beginning and then continue -streaming new output from the container's stdout and stderr. +The ``docker logs -f`` command combines ``docker logs`` and ``docker attach``: +it will first return all logs from the beginning and then continue streaming +new output from the container's stdout and stderr. .. _cli_port: @@ -940,7 +957,7 @@ Removing tagged images ~~~~~~~~~~~~~~~~~~~~~~ Images can be removed either by their short or long ID's, or their image names. -If an image has more than one name, each of them needs to be removed before the +If an image has more than one name, each of them needs to be removed before the image is removed. .. code-block:: bash @@ -952,7 +969,7 @@ image is removed. test2 latest fd484f19954f 23 seconds ago 7 B (virtual 4.964 MB) $ sudo docker rmi fd484f19954f - Error: Conflict, fd484f19954f wasn't deleted + Error: Conflict, cannot delete image fd484f19954f because it is tagged in multiple repositories 2013/12/11 05:47:16 Error: failed to remove one or more images $ sudo docker rmi test1 @@ -1004,13 +1021,14 @@ image is removed. -link="": Add link to another container (name:alias) -name="": Assign the specified name to the container. If no name is specific docker will generate a random name -P=false: Publish all exposed ports to the host interfaces - -``'docker run'`` first ``'creates'`` a writeable container layer over -the specified image, and then ``'starts'`` it using the specified -command. That is, ``'docker run'`` is equivalent to the API -``/containers/create`` then ``/containers/(id)/start``. -``docker run`` can be used in combination with ``docker commit`` to :ref:`change the command that a container runs `. +The ``docker run`` command first ``creates`` a writeable container layer over +the specified image, and then ``starts`` it using the specified command. That +is, ``docker run`` is equivalent to the API ``/containers/create`` then +``/containers/(id)/start``. + +The ``docker run`` command can be used in combination with ``docker commit`` to +:ref:`change the command that a container runs `. Known Issues (run -volumes-from) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1026,10 +1044,10 @@ Examples: $ sudo docker run -cidfile /tmp/docker_test.cid ubuntu echo "test" -This will create a container and print "test" to the console. The -``cidfile`` flag makes docker attempt to create a new file and write the -container ID to it. If the file exists already, docker will return an -error. Docker will close this file when docker run exits. +This will create a container and print ``test`` to the console. The +``cidfile`` flag makes Docker attempt to create a new file and write the +container ID to it. If the file exists already, Docker will return an +error. Docker will close this file when ``docker run`` exits. .. code-block:: bash @@ -1063,7 +1081,7 @@ use-cases, like running Docker within Docker. $ sudo docker run -w /path/to/dir/ -i -t ubuntu pwd The ``-w`` lets the command being executed inside directory given, -here /path/to/dir/. If the path does not exists it is created inside the +here ``/path/to/dir/``. If the path does not exists it is created inside the container. .. code-block:: bash @@ -1080,7 +1098,7 @@ using the container, but inside the current working directory. $ sudo docker run -p 127.0.0.1:80:8080 ubuntu bash -This binds port ``8080`` of the container to port ``80`` on 127.0.0.1 of the +This binds port ``8080`` of the container to port ``80`` on ``127.0.0.1`` of the host machine. :ref:`port_redirection` explains in detail how to manipulate ports in Docker. @@ -1114,11 +1132,31 @@ to the newly created container. $ sudo docker run -volumes-from 777f7dc92da7,ba8c0c54f0f2:ro -i -t ubuntu pwd The ``-volumes-from`` flag mounts all the defined volumes from the -refrence containers. Containers can be specified by a comma seperated +referenced containers. Containers can be specified by a comma seperated list or by repetitions of the ``-volumes-from`` argument. The container -id may be optionally suffixed with ``:ro`` or ``:rw`` to mount the volumes in +ID may be optionally suffixed with ``:ro`` or ``:rw`` to mount the volumes in read-only or read-write mode, respectively. By default, the volumes are mounted -in the same mode (rw or ro) as the reference container. +in the same mode (read write or read only) as the reference container. + +A complete example +.................. + +.. code-block:: bash + + $ sudo docker run -d -name static static-web-files sh + $ sudo docker run -d -expose=8098 -name riak riakserver + $ sudo docker run -d -m 100m -e DEVELOPMENT=1 -e BRANCH=example-code -v $(pwd):/app/bin:ro -name app appserver + $ sudo docker run -d -p 1443:443 -dns=dns.dev.org -v /var/log/httpd -volumes-from static -link riak -link app -h www.sven.dev.org -name web webserver + $ sudo docker run -t -i -rm -volumes-from web -w /var/log/httpd busybox tail -f access.log + +This example shows 5 containers that might be set up to test a web application change: + +1. Start a pre-prepared volume image ``static-web-files`` (in the background) that has CSS, image and static HTML in it, (with a ``VOLUME`` instruction in the ``Dockerfile`` to allow the web server to use those files); +2. Start a pre-prepared ``riakserver`` image, give the container name ``riak`` and expose port ``8098`` to any containers that link to it; +3. Start the ``appserver`` image, restricting its memory usage to 100MB, setting two environment variables ``DEVELOPMENT`` and ``BRANCH`` and bind-mounting the current directory (``$(pwd)``) in the container in read-only mode as ``/app/bin``; +4. Start the ``webserver``, mapping port ``443`` in the container to port ``1443`` on the Docker server, setting the DNS server to ``dns.dev.org``, creating a volume to put the log files into (so we can access it from another container), then importing the files from the volume exposed by the ``static`` container, and linking to all exposed ports from ``riak`` and ``app``. Lastly, we set the hostname to ``web.sven.dev.org`` so its consistent with the pre-generated SSL certificate; +5. Finally, we create a container that runs ``tail -f access.log`` using the logs volume from the ``web`` container, setting the workdir to ``/var/log/httpd``. The ``-rm`` option means that when the container exits, the container's layer is removed. + .. _cli_save: @@ -1205,7 +1243,7 @@ The main process inside the container will receive SIGTERM, and after a grace pe ``version`` ----------- -Show the version of the docker client, daemon, and latest released version. +Show the version of the Docker client, daemon, and latest released version. .. _cli_wait: diff --git a/components/engine/docs/sources/contributing/devenvironment.rst b/components/engine/docs/sources/contributing/devenvironment.rst index cc254bbc68..6675173584 100644 --- a/components/engine/docs/sources/contributing/devenvironment.rst +++ b/components/engine/docs/sources/contributing/devenvironment.rst @@ -136,7 +136,7 @@ You can run an interactive session in the newly built container: Extra Step: Build and view the Documentation -------------------------------------------- +-------------------------------------------- If you want to read the documentation from a local website, or are making changes to it, you can build the documentation and then serve it by: diff --git a/components/engine/docs/sources/examples/running_ssh_service.rst b/components/engine/docs/sources/examples/running_ssh_service.rst index 3d0a782678..a0ce532d8d 100644 --- a/components/engine/docs/sources/examples/running_ssh_service.rst +++ b/components/engine/docs/sources/examples/running_ssh_service.rst @@ -94,5 +94,13 @@ The password is ``screencast``. $ ifconfig $ ssh root@192.168.33.10 -p 49154 # Thanks for watching, Thatcher thatcher@dotcloud.com + +Update: +------- + +For Ubuntu 13.10 using stackbrew/ubuntu, you may need do these additional steps: + +1. change /etc/pam.d/sshd, pam_loginuid line 'required' to 'optional' +2. echo LANG=\"en_US.UTF-8\" > /etc/default/locale diff --git a/components/engine/docs/sources/faq.rst b/components/engine/docs/sources/faq.rst index 23460f4b55..e2e16c362b 100644 --- a/components/engine/docs/sources/faq.rst +++ b/components/engine/docs/sources/faq.rst @@ -111,7 +111,7 @@ What does Docker add to just plain LXC? registry to store and transfer private containers, for internal server deployments for example. - * *Tool ecosystem.* + * *Tool ecosystem.* Docker defines an API for automating and customizing the creation and deployment of containers. There are a huge number of tools integrating with Docker to extend its @@ -122,6 +122,11 @@ What does Docker add to just plain LXC? (Jenkins, Strider, Travis), etc. Docker is rapidly establishing itself as the standard for container-based tooling. +What is different between a Docker container and a VM? +...................................................... + +There's a great StackOverflow answer `showing the differences `_. + Do I lose my data when the container exits? ........................................... @@ -129,6 +134,53 @@ Not at all! Any data that your application writes to disk gets preserved in its container until you explicitly delete the container. The file system for the container persists even after the container halts. +How far do Docker containers scale? +................................... + +Some of the largest server farms in the world today are based on containers. +Large web deployments like Google and Twitter, and platform providers such as +Heroku and dotCloud all run on container technology, at a scale of hundreds of +thousands or even millions of containers running in parallel. + +How do I connect Docker containers? +................................... + +Currently the recommended way to link containers is via the `link` primitive. +You can see details of how to `work with links here +`_. + +Also of useful when enabling more flexible service portability is the +`Ambassador linking pattern +`_. + +How do I run more than one process in a Docker container? +......................................................... + +Any capable process supervisor such as http://supervisord.org/, runit, s6, or +daemontools can do the trick. Docker will start up the process management +daemon which will then fork to run additional processes. As long as the +processor manager daemon continues to run, the container will continue to as +well. You can see a more substantial example `that uses supervisord here +`_. + +What platforms does Docker run on? +.................................. + +Linux: + +- Ubuntu 12.04, 13.04 et al +- Fedora 19/20+ +- RHEL 6.5+ +- Centos 6+ +- Gento +- ArchLinux + +Cloud: + +- Amazon EC2 +- Google Compute Engine +- Rackspace + Can I help by adding some questions and answers? ................................................ diff --git a/components/engine/docs/sources/index.rst b/components/engine/docs/sources/index.rst index 88752ac3bf..1fb82f3bec 100644 --- a/components/engine/docs/sources/index.rst +++ b/components/engine/docs/sources/index.rst @@ -25,7 +25,7 @@ currently in active development, so this documentation will change frequently. For an overview of Docker, please see the `Introduction -`_. When you're ready to start working with +`_. When you're ready to start working with Docker, we have a `quick start `_ and a more in-depth guide to :ref:`ubuntu_linux` and other :ref:`installation_list` paths including prebuilt binaries, diff --git a/components/engine/docs/sources/installation/binaries.rst b/components/engine/docs/sources/installation/binaries.rst index 759f8a7502..f06a8d6c5f 100644 --- a/components/engine/docs/sources/installation/binaries.rst +++ b/components/engine/docs/sources/installation/binaries.rst @@ -21,6 +21,11 @@ Check Your Kernel Your host's Linux kernel must meet the Docker :ref:`kernel` +Check for User Space Tools +-------------------------- + +You must have a working installation of the `lxc `_ utilities and library. + Get the docker binary: ---------------------- diff --git a/components/engine/docs/sources/installation/fedora.rst b/components/engine/docs/sources/installation/fedora.rst index 9525371f03..de296b4df2 100644 --- a/components/engine/docs/sources/installation/fedora.rst +++ b/components/engine/docs/sources/installation/fedora.rst @@ -1,6 +1,6 @@ :title: Requirements and Installation on Fedora :description: Please note this project is currently under heavy development. It should not be used in production. -:keywords: Docker, Docker documentation, fedora, requirements, virtualbox, vagrant, git, ssh, putty, cygwin, linux +:keywords: Docker, Docker documentation, Fedora, requirements, virtualbox, vagrant, git, ssh, putty, cygwin, linux .. _fedora: @@ -18,13 +18,34 @@ architecture. Installation ------------ +The ``docker-io`` package provides Docker on Fedora. + + +If you have the (unrelated) ``docker`` package installed already, it will +conflict with ``docker-io``. There's a `bug report`_ filed for it. +To proceed with ``docker-io`` installation on Fedora 19, please remove +``docker`` first. + +.. code-block:: bash + + sudo yum -y remove docker + +For Fedora 20 and later, the ``wmdocker`` package will provide the same +functionality as ``docker`` and will also not conflict with ``docker-io``. + +.. code-block:: bash + + sudo yum -y install wmdocker + sudo yum -y remove docker + Install the ``docker-io`` package which will install Docker on our host. .. code-block:: bash sudo yum -y install docker-io -To update the ``docker-io`` package + +To update the ``docker-io`` package: .. code-block:: bash @@ -50,3 +71,5 @@ Now let's verify that Docker is working. **Done!**, now continue with the :ref:`hello_world` example. +.. _bug report: https://bugzilla.redhat.com/show_bug.cgi?id=1043676 + diff --git a/components/engine/docs/sources/installation/frugalware.rst b/components/engine/docs/sources/installation/frugalware.rst new file mode 100644 index 0000000000..cda6c4bfc4 --- /dev/null +++ b/components/engine/docs/sources/installation/frugalware.rst @@ -0,0 +1,80 @@ +:title: Installation on FrugalWare +:description: Docker installation on FrugalWare. +:keywords: frugalware linux, virtualization, docker, documentation, installation + +.. _frugalware: + +FrugalWare +========== + +.. include:: install_header.inc + +.. include:: install_unofficial.inc + +Installing on FrugalWare is handled via the official packages: + +* `lxc-docker i686 `_ + +* `lxc-docker x86_64 `_ + +The `lxc-docker` package will install the latest tagged version of Docker. + +Dependencies +------------ + +Docker depends on several packages which are specified as dependencies in +the packages. The core dependencies are: + +* systemd +* lvm2 +* sqlite3 +* libguestfs +* lxc +* iproute2 +* bridge-utils + + +Installation +------------ + +A simple +:: + + pacman -S lxc-docker + +is all that is needed. + + +Starting Docker +--------------- + +There is a systemd service unit created for Docker. To start Docker as service: + +:: + + sudo systemctl start lxc-docker + + +To start on system boot: + +:: + + sudo systemctl enable lxc-docker + +Network Configuration +--------------------- + +IPv4 packet forwarding is disabled by default on FrugalWare, so Internet access from inside +the container may not work. + +To enable packet forwarding, run the following command as the ``root`` user on the host system: + +:: + + sysctl net.ipv4.ip_forward=1 + +And, to make it persistent across reboots, add the following to a file named **/etc/sysctl.d/docker.conf**: + +:: + + net.ipv4.ip_forward=1 diff --git a/components/engine/docs/sources/installation/google.rst b/components/engine/docs/sources/installation/google.rst index e3286d1d90..ff38e1e6e4 100644 --- a/components/engine/docs/sources/installation/google.rst +++ b/components/engine/docs/sources/installation/google.rst @@ -57,9 +57,17 @@ docker-playground:~$ curl get.docker.io | bash docker-playground:~$ sudo update-rc.d docker defaults -7. Start a new container: +7. If running in zones: us-central1-a, europe-west1-1, and europe-west1-b, the docker daemon must be started with the `-mtu` flag. Without the flag, you may experience intermittent network pauses. +`See this issue `_ for more details. + +.. code-block:: bash + + docker -d -mtu 1460 + +8. Start a new container: .. code-block:: bash docker-playground:~$ sudo docker run busybox echo 'docker on GCE \o/' docker on GCE \o/ + diff --git a/components/engine/docs/sources/installation/index.rst b/components/engine/docs/sources/installation/index.rst index b2882a5cb3..9026b1f7f4 100644 --- a/components/engine/docs/sources/installation/index.rst +++ b/components/engine/docs/sources/installation/index.rst @@ -22,6 +22,7 @@ Contents: fedora archlinux gentoolinux + frugalware vagrant windows amazon diff --git a/components/engine/docs/sources/installation/kernel.rst b/components/engine/docs/sources/installation/kernel.rst index b2cf4f1479..8338cfdc88 100644 --- a/components/engine/docs/sources/installation/kernel.rst +++ b/components/engine/docs/sources/installation/kernel.rst @@ -115,6 +115,8 @@ Then run ``update-grub``, and reboot. Details ------- +To automatically check some of the requirements below, you can run `lxc-checkconfig`. + Networking: - CONFIG_BRIDGE diff --git a/components/engine/docs/sources/installation/rhel.rst b/components/engine/docs/sources/installation/rhel.rst index 993d329df7..b928b333f4 100644 --- a/components/engine/docs/sources/installation/rhel.rst +++ b/components/engine/docs/sources/installation/rhel.rst @@ -28,6 +28,15 @@ Installation Firstly, you need to install the EPEL repository. Please follow the `EPEL installation instructions`_. +The ``docker-io`` package provides Docker on EPEL. + + +If you already have the (unrelated) ``docker`` package installed, it will +conflict with ``docker-io``. There's a `bug report`_ filed for it. +To proceed with ``docker-io`` installation, please remove +``docker`` first. + + Next, let's install the ``docker-io`` package which will install Docker on our host. .. code-block:: bash @@ -68,4 +77,5 @@ If you have any issues - please report them directly in the `Red Hat Bugzilla fo .. _Extra Packages for Enterprise Linux (EPEL): https://fedoraproject.org/wiki/EPEL .. _EPEL installation instructions: https://fedoraproject.org/wiki/EPEL#How_can_I_use_these_extra_packages.3F .. _Red Hat Bugzilla for docker-io component : https://bugzilla.redhat.com/enter_bug.cgi?product=Fedora%20EPEL&component=docker-io +.. _bug report: https://bugzilla.redhat.com/show_bug.cgi?id=1043676 diff --git a/components/engine/docs/sources/use/basics.rst b/components/engine/docs/sources/use/basics.rst index 6b2c588817..8c896ae793 100644 --- a/components/engine/docs/sources/use/basics.rst +++ b/components/engine/docs/sources/use/basics.rst @@ -9,7 +9,7 @@ Learn Basic Commands Starting Docker --------------- -If you have used one of the quick install paths', Docker may have been +If you have used one of the quick install paths, Docker may have been installed with upstart, Ubuntu's system for starting processes at boot time. You should be able to run ``sudo docker help`` and get output. @@ -30,8 +30,8 @@ Download a pre-built image # Download an ubuntu image sudo docker pull ubuntu -This will find the ``ubuntu`` image by name in the :ref:`Central Index -` and download it from the top-level Central +This will find the ``ubuntu`` image by name in the :ref:`Central Index +` and download it from the top-level Central Repository to a local image cache. .. NOTE:: When the image has successfully downloaded, you will see a @@ -53,21 +53,23 @@ Running an interactive shell .. _dockergroup: -sudo and the docker Group -------------------------- +The sudo command and the docker Group +------------------------------------- -The ``docker`` daemon always runs as root, and since ``docker`` -version 0.5.2, ``docker`` binds to a Unix socket instead of a TCP -port. By default that Unix socket is owned by the user *root*, and so, -by default, you can access it with ``sudo``. +The ``docker`` daemon always runs as the root user, and since Docker version +0.5.2, the ``docker`` daemon binds to a Unix socket instead of a TCP port. By +default that Unix socket is owned by the user *root*, and so, by default, you +can access it with ``sudo``. Starting in version 0.5.3, if you (or your Docker installer) create a Unix group called *docker* and add users to it, then the ``docker`` daemon will make the ownership of the Unix socket read/writable by the *docker* group when the daemon starts. The ``docker`` daemon must -always run as root, but if you run the ``docker`` client as a user in +always run as the root user, but if you run the ``docker`` client as a user in the *docker* group then you don't need to add ``sudo`` to all the -client commands. Warning: the *docker* group is root-equivalent. +client commands. + +.. warning:: The *docker* group is root-equivalent. **Example:** @@ -97,10 +99,10 @@ Bind Docker to another host/port or a Unix socket `_). Make sure you control access to ``docker``. -With -H it is possible to make the Docker daemon to listen on a -specific ip and port. By default, it will listen on +With ``-H`` it is possible to make the Docker daemon to listen on a +specific IP and port. By default, it will listen on ``unix:///var/run/docker.sock`` to allow only local connections by the -*root* user. You *could* set it to 0.0.0.0:4243 or a specific host ip to +*root* user. You *could* set it to ``0.0.0.0:4243`` or a specific host IP to give access to everybody, but that is **not recommended** because then it is trivial for someone to gain root access to the host where the daemon is running. @@ -179,10 +181,10 @@ Committing (saving) a container state Save your containers state to a container image, so the state can be re-used. -When you commit your container only the differences between the image -the container was created from and the current state of the container -will be stored (as a diff). See which images you already have using -``sudo docker images`` +When you commit your container only the differences between the image the +container was created from and the current state of the container will be +stored (as a diff). See which images you already have using the ``docker +images`` command. .. code-block:: bash @@ -194,7 +196,5 @@ will be stored (as a diff). See which images you already have using You now have a image state from which you can create new instances. - - Read more about :ref:`working_with_the_repository` or continue to the complete :ref:`cli` diff --git a/components/engine/docs/sources/use/builder.rst b/components/engine/docs/sources/use/builder.rst index 10140313a8..81145a6ee8 100644 --- a/components/engine/docs/sources/use/builder.rst +++ b/components/engine/docs/sources/use/builder.rst @@ -251,6 +251,11 @@ All new files and directories are created with mode 0755, uid and gid if you build using STDIN (``docker build - < somefile``), there is no build context, so the Dockerfile can only contain an URL based ADD statement. +.. note:: + if your URL files are protected using authentication, you will need to use + an ``RUN wget`` , ``RUN curl`` or other tool from within the container as + ADD does not support authentication. + The copy obeys the following rules: * The ```` path must be inside the *context* of the build; you cannot diff --git a/components/engine/docs/sources/use/port_redirection.rst b/components/engine/docs/sources/use/port_redirection.rst index b35d27a3db..5cddb238e4 100644 --- a/components/engine/docs/sources/use/port_redirection.rst +++ b/components/engine/docs/sources/use/port_redirection.rst @@ -31,7 +31,7 @@ container, Docker provide ways to bind the container port to an interface of the host system. To simplify communication between containers, Docker provides the linking mechanism. -Binding a port to an host interface +Binding a port to a host interface ----------------------------------- To bind a port of the container to a specific interface of the host diff --git a/components/engine/docs/sources/use/working_with_volumes.rst b/components/engine/docs/sources/use/working_with_volumes.rst index 341bfe8d6a..86576b05e4 100644 --- a/components/engine/docs/sources/use/working_with_volumes.rst +++ b/components/engine/docs/sources/use/working_with_volumes.rst @@ -13,7 +13,7 @@ Share Directories via Volumes A *data volume* is a specially-designated directory within one or more containers that bypasses the :ref:`ufs_def` to provide several useful -features for persistant or shared data: +features for persistent or shared data: * **Data volumes can be shared and reused between containers.** This is the feature that makes data volumes so powerful. You can use it @@ -30,35 +30,58 @@ Each container can have zero or more data volumes. Getting Started ............... -Using data volumes is as simple as adding a new flag: ``-v``. The -parameter ``-v`` can be used more than once in order to create more -volumes within the new container. The example below shows the -instruction to create a container with two new volumes:: +Using data volumes is as simple as adding a ``-v`` parameter to the ``docker run`` +command. The ``-v`` parameter can be used more than once in order to +create more volumes within the new container. To create a new container with +two new volumes:: - docker run -v /var/volume1 -v /var/volume2 shykes/couchdb + $ docker run -v /var/volume1 -v /var/volume2 busybox true -For a Dockerfile, the VOLUME instruction will add one or more new -volumes to any container created from the image:: +This command will create the new container with two new volumes that +exits instantly (``true`` is pretty much the smallest, simplest program +that you can run). Once created you can mount its volumes in any other +container using the ``-volumes-from`` option; irrespecive of whether the +container is running or not. - VOLUME ["/var/volume1", "/var/volume2"] +Or, you can use the VOLUME instruction in a Dockerfile to add one or more new +volumes to any container created from that image:: + # BUILD-USING: docker build -t data . + # RUN-USING: docker run -name DATA data + FROM busybox + VOLUME ["/var/volume1", "/var/volume2"] + CMD ["/usr/bin/true"] -Mount Volumes from an Existing Container: ------------------------------------------ +Creating and mounting a Data Volume Container +--------------------------------------------- -The command below creates a new container which is running as daemon -``-d`` and with one volume ``/var/lib/couchdb``:: +If you have some persistent data that you want to share between containers, +or want to use from non-persistent containers, its best to create a named +Data Volume Container, and then to mount the data from it. - COUCH1=$(sudo docker run -d -v /var/lib/couchdb shykes/couchdb:2013-05-03) +Create a named container with volumes to share (``/var/volume1`` and ``/var/volume2``):: -From the container id of that previous container ``$COUCH1`` it's -possible to create new container sharing the same volume using the -parameter ``-volumes-from container_id``:: + $ docker run -v /var/volume1 -v /var/volume2 -name DATA busybox true - COUCH2=$(sudo docker run -d -volumes-from $COUCH1 shykes/couchdb:2013-05-03) +Then mount those data volumes into your application containers:: -Now, the second container has the all the information from the first volume. + $ docker run -t -i -rm -volumes-from DATA -name client1 ubuntu bash +You can use multiple ``-volumes-from`` parameters to bring together multiple +data volumes from multiple containers. + +Interestingly, you can mount the volumes that came from the ``DATA`` container in +yet another container via the ``client1`` middleman container:: + + $ docker run -t -i -rm -volumes-from client1 ubuntu -name client2 bash + +This allows you to abstract the actual data source from users of that data, +similar to :ref:`ambassador_pattern_linking `. + +If you remove containers that mount volumes, including the initial DATA container, +or the middleman, the volumes will not be deleted until there are no containers still +referencing those volumes. This allows you to upgrade, or effectivly migrate data volumes +between containers. Mount a Host Directory as a Container Volume: --------------------------------------------- @@ -68,13 +91,13 @@ Mount a Host Directory as a Container Volume: -v=[]: Create a bind mount with: [host-dir]:[container-dir]:[rw|ro]. If "host-dir" is missing, then docker creates a new volume. -This is not available for a Dockerfile due the portability and sharing -purpose of it. The [host-dir] volumes is something 100% host dependent -and will break on any other machine. +This is not available from a Dockerfile as it makes the built image less portable +or shareable. [host-dir] volumes are 100% host dependent and will break on any +other machine. For example:: - sudo docker run -v /var/logs:/var/host_logs:ro shykes/couchdb:2013-05-03 + sudo docker run -v /var/logs:/var/host_logs:ro ubuntu bash The command above mounts the host directory ``/var/logs`` into the container with read only permissions as ``/var/host_logs``. @@ -87,3 +110,6 @@ Known Issues * :issue:`2702`: "lxc-start: Permission denied - failed to mount" could indicate a permissions problem with AppArmor. Please see the issue for a workaround. +* :issue:`2528`: the busybox container is used to make the resulting container as small and + simple as possible - whenever you need to interact with the data in the volume + you mount it into another container. diff --git a/components/engine/docs/theme/docker/layout.html b/components/engine/docs/theme/docker/layout.html index 67b3e49745..a966556044 100755 --- a/components/engine/docs/theme/docker/layout.html +++ b/components/engine/docs/theme/docker/layout.html @@ -86,26 +86,26 @@ -
+
-
+
-
+

DOCUMENTATION

{{ toctree(collapse=False, maxdepth=3) }}
- +
-
+
@@ -134,13 +134,22 @@
- Current version: +

Note: You are currently browsing the development documentation. The current release may work differently.

+ + Available versions:
    {% for slug, url in versions %}
  • a.headerlink { .admonition.seealso { border-color: #23cb1f; } - +/* Add styles for other types of comments */ .versionchanged, .versionadded, .versionmodified, @@ -418,15 +463,12 @@ dt:hover > a.headerlink { font-size: larger; font-weight: bold; } - .versionchanged { color: lightseagreen; } - .versionadded { color: mediumblue; } - .deprecated { color: orangered; } diff --git a/components/engine/docs/theme/docker/static/css/main.less b/components/engine/docs/theme/docker/static/css/main.less index ce18a387a6..8c9296d979 100644 --- a/components/engine/docs/theme/docker/static/css/main.less +++ b/components/engine/docs/theme/docker/static/css/main.less @@ -98,7 +98,6 @@ p a { } - .navbar .brand { margin-left: 0px; float: left; @@ -126,9 +125,11 @@ p a { box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065); } -.brand.logo a { +.brand-logo a { color: white; - + img { + width: auto; + } } .logo { @@ -317,10 +318,18 @@ body { margin-top: 40px; } +.sidebar { + width: 215px; + float: left; +} + .main-content { padding: 16px 18px inherit; + margin-left: 230px; /* space for sidebar */ } + + /* ======================= Social footer ======================= */ @@ -330,24 +339,64 @@ body { margin-top: 15px; } -.social .twitter, .social .github, .social .googleplus { - background: url("https://www.docker.io/static/img/footer-links.png") no-repeat transparent; - display: inline-block; - height: 35px; - overflow: hidden; - text-indent: 9999px; - width: 35px; - margin-right: 10px; +.social { + .twitter, .github, .googleplus, .facebook, .slideshare, .linkedin, .flickr, .youtube, .reddit { + background: url("../img/social/docker_social_logos.png") no-repeat transparent; + display: inline-block; + height: 32px; + overflow: hidden; + text-indent: 9999px; + width: 32px; + margin-right: 5px; + } +} + +.social :hover { + -webkit-transform: rotate(-10deg); + -moz-transform: rotate(-10deg); + -o-transform: rotate(-10deg); + -ms-transform: rotate(-10deg); + transform: rotate(-10deg); } .social .twitter { - background-position: 0px 2px; + background-position: -160px 0px; +} + +.social .reddit { + background-position: -256px 0px; } .social .github { - background-position: -59px 2px; + background-position: -64px 0px; } +.social .googleplus { + background-position: -96px 0px; +} + +.social .facebook { + background-position: -0px 0px; +} + +.social .slideshare { + background-position: -128px 0px; +} + +.social .youtube { + background-position: -192px 0px; +} + +.social .flickr { + background-position: -32px 0px; +} + +.social .linkedin { + background-position: -224px 0px; +} + + + // Styles on the forms // ---------------------------------- @@ -528,31 +577,34 @@ div.alert.alert-block { border: 1px solid #88BABC; padding: 5px; font-size: larger; + max-width: 300px; .content { padding-right: 45px; margin-top: 7px; margin-left: 7px; - // display: inline-block; background-image: url('../img/container3.png'); background-position: right center; background-repeat: no-repeat; } .alternative { - visibility: hidden; - display: none; } .active-slug { visibility: visible; display: inline-block; + font-weight: bolder; } &:hover .alternative { animation-duration: 1s; display: inline-block; - visibility: visible; + } + + .version-note { + font-size: 16px; + color: black; } } @@ -612,3 +664,24 @@ dt:hover > a.headerlink { } +/* Add styles for other types of comments */ + +.versionchanged, +.versionadded, +.versionmodified, +.deprecated { + font-size: larger; + font-weight: bold; +} + +.versionchanged { + color: lightseagreen; +} + +.versionadded { + color: mediumblue; +} + +.deprecated { + color: orangered; +} diff --git a/components/engine/docs/theme/docker/static/img/footer-links.png b/components/engine/docs/theme/docker/static/img/footer-links.png deleted file mode 100644 index 4bef964214..0000000000 Binary files a/components/engine/docs/theme/docker/static/img/footer-links.png and /dev/null differ diff --git a/components/engine/docs/theme/docker/static/img/social/docker_social_logos.png b/components/engine/docs/theme/docker/static/img/social/docker_social_logos.png new file mode 100644 index 0000000000..5bde456365 Binary files /dev/null and b/components/engine/docs/theme/docker/static/img/social/docker_social_logos.png differ diff --git a/components/engine/docs/theme/docker/static/js/docs.js b/components/engine/docs/theme/docker/static/js/docs.js index 769922f583..4151ea5a80 100755 --- a/components/engine/docs/theme/docker/static/js/docs.js +++ b/components/engine/docs/theme/docker/static/js/docs.js @@ -53,14 +53,6 @@ $(function(){ } } - if (doc_version == "") { - $('.version-flyer ul').html('
  • Local
    • '); - } - - // mark the active documentation in the version widget - $(".version-flyer a:contains('" + doc_version + "')").parent().addClass('active-slug'); - - // attached handler on click // Do not attach to first element or last (intro, faq) so that // first and last link directly instead of accordian @@ -95,4 +87,18 @@ $(function(){ // add class to all those which have children $('.sidebar > ul > li').not(':last').not(':first').addClass('has-children'); + + if (doc_version == "") { + $('.version-flyer ul').html('
  • Local
    • '); + } + + if (doc_version == "master") { + $('.version-flyer .version-note').hide(); + } + + // mark the active documentation in the version widget + $(".version-flyer a:contains('" + doc_version + "')").parent().addClass('active-slug').setAttribute("title", "Current version"); + + + }); \ No newline at end of file diff --git a/components/engine/engine/engine.go b/components/engine/engine/engine.go index 5da0a97233..ad830ce2e4 100644 --- a/components/engine/engine/engine.go +++ b/components/engine/engine/engine.go @@ -6,6 +6,7 @@ import ( "io" "log" "os" + "path/filepath" "runtime" "strings" ) @@ -79,9 +80,24 @@ func New(root string) (*Engine, error) { } } } + if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) { return nil, err } + + // Docker makes some assumptions about the "absoluteness" of root + // ... so let's make sure it has no symlinks + if p, err := filepath.Abs(root); err != nil { + log.Fatalf("Unable to get absolute root (%s): %s", root, err) + } else { + root = p + } + if p, err := filepath.EvalSymlinks(root); err != nil { + log.Fatalf("Unable to canonicalize root (%s): %s", root, err) + } else { + root = p + } + eng := &Engine{ root: root, handlers: make(map[string]Handler), diff --git a/components/engine/engine/engine_test.go b/components/engine/engine/engine_test.go index 793867f50a..065a19f492 100644 --- a/components/engine/engine/engine_test.go +++ b/components/engine/engine/engine_test.go @@ -18,7 +18,7 @@ func TestRegister(t *testing.T) { eng := newTestEngine(t) - //Should fail because globan handlers are copied + //Should fail because global handlers are copied //at the engine creation if err := eng.Register("dummy1", nil); err == nil { t.Fatalf("Expecting error, got none") diff --git a/components/engine/graph.go b/components/engine/graph.go index da236d3716..058ef7e93d 100644 --- a/components/engine/graph.go +++ b/components/engine/graph.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "syscall" "time" @@ -56,6 +57,7 @@ func (graph *Graph) restore() error { graph.idIndex.Add(id) } } + utils.Debugf("Restored %d elements", len(dir)) return nil } @@ -130,7 +132,8 @@ func (graph *Graph) Create(layerData archive.Archive, container *Container, comm DockerVersion: VERSION, Author: author, Config: config, - Architecture: "x86_64", + Architecture: runtime.GOARCH, + OS: runtime.GOOS, } if container != nil { img.Parent = container.Image diff --git a/components/engine/graphdriver/aufs/aufs.go b/components/engine/graphdriver/aufs/aufs.go index 558c64c5bb..8875cac6d9 100644 --- a/components/engine/graphdriver/aufs/aufs.go +++ b/components/engine/graphdriver/aufs/aufs.go @@ -25,12 +25,12 @@ import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/graphdriver" + mountpk "github.com/dotcloud/docker/mount" "github.com/dotcloud/docker/utils" "os" "os/exec" "path" "strings" - "syscall" ) func init() { @@ -296,7 +296,7 @@ func (a *Driver) unmount(id string) error { func (a *Driver) mounted(id string) (bool, error) { target := path.Join(a.rootPath(), "mnt", id) - return Mounted(target) + return mountpk.Mounted(target) } // During cleanup aufs needs to unmount all mountpoints @@ -327,7 +327,7 @@ func (a *Driver) aufsMount(ro []string, rw, target string) (err error) { for _, layer := range ro { branch := fmt.Sprintf("append:%s=ro+wh", layer) - if err = mount("none", target, "aufs", syscall.MS_REMOUNT, branch); err != nil { + if err = mount("none", target, "aufs", MsRemount, branch); err != nil { return } } diff --git a/components/engine/graphdriver/aufs/mount.go b/components/engine/graphdriver/aufs/mount.go index 6f3476f99c..1f1d98f809 100644 --- a/components/engine/graphdriver/aufs/mount.go +++ b/components/engine/graphdriver/aufs/mount.go @@ -2,9 +2,7 @@ package aufs import ( "github.com/dotcloud/docker/utils" - "os" "os/exec" - "path/filepath" "syscall" ) @@ -17,21 +15,3 @@ func Unmount(target string) error { } return nil } - -func Mounted(mountpoint string) (bool, error) { - mntpoint, err := os.Stat(mountpoint) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - parent, err := os.Stat(filepath.Join(mountpoint, "..")) - if err != nil { - return false, err - } - mntpointSt := mntpoint.Sys().(*syscall.Stat_t) - parentSt := parent.Sys().(*syscall.Stat_t) - - return mntpointSt.Dev != parentSt.Dev, nil -} diff --git a/components/engine/graphdriver/aufs/mount_darwin.go b/components/engine/graphdriver/aufs/mount_darwin.go index ce448036f2..62c84fc7c9 100644 --- a/components/engine/graphdriver/aufs/mount_darwin.go +++ b/components/engine/graphdriver/aufs/mount_darwin.go @@ -2,6 +2,8 @@ package aufs import "errors" +const MsRemount = 0 + func mount(source string, target string, fstype string, flags uintptr, data string) (err error) { return errors.New("mount is not implemented on darwin") } diff --git a/components/engine/graphdriver/aufs/mount_linux.go b/components/engine/graphdriver/aufs/mount_linux.go index 8062bae420..c86f1bbd63 100644 --- a/components/engine/graphdriver/aufs/mount_linux.go +++ b/components/engine/graphdriver/aufs/mount_linux.go @@ -2,6 +2,8 @@ package aufs import "syscall" +const MsRemount = syscall.MS_REMOUNT + func mount(source string, target string, fstype string, flags uintptr, data string) error { return syscall.Mount(source, target, fstype, flags, data) } diff --git a/components/engine/graphdriver/devmapper/deviceset.go b/components/engine/graphdriver/devmapper/deviceset.go index aeda1db24e..7308c0e922 100644 --- a/components/engine/graphdriver/devmapper/deviceset.go +++ b/components/engine/graphdriver/devmapper/deviceset.go @@ -154,7 +154,7 @@ func (devices *DeviceSet) allocateTransactionId() uint64 { func (devices *DeviceSet) saveMetadata() error { jsonData, err := json.Marshal(devices.MetaData) if err != nil { - return fmt.Errorf("Error encoding metaadata to json: %s", err) + return fmt.Errorf("Error encoding metadata to json: %s", err) } tmpFile, err := ioutil.TempFile(filepath.Dir(devices.jsonFile()), ".json") if err != nil { diff --git a/components/engine/hack/PACKAGERS.md b/components/engine/hack/PACKAGERS.md index 90f99a799f..05283b909b 100644 --- a/components/engine/hack/PACKAGERS.md +++ b/components/engine/hack/PACKAGERS.md @@ -36,8 +36,9 @@ To build docker, you will need the following system dependencies * An amd64 machine * A recent version of git and mercurial -* Go version 1.2 or later (see notes below regarding using Go 1.1.2 and dynbinary) +* Go version 1.2 or later * SQLite version 3.7.9 or later +* libdevmapper from lvm2 version 1.02.77 or later (http://www.sourceware.org/lvm2/) * A clean checkout of the source must be added to a valid Go [workspace](http://golang.org/doc/code.html#Workspaces) under the path *src/github.com/dotcloud/docker*. @@ -91,8 +92,7 @@ You would do the users of your distro a disservice and "void the docker warranty A good comparison is Busybox: all distros package it as a statically linked binary, because it just makes sense. Docker is the same way. -If you *must* have a non-static Docker binary, or require Go 1.1.2 (since Go 1.2 is still freshly released -at the time of this writing), please use: +If you *must* have a non-static Docker binary, please use: ```bash ./hack/make.sh dynbinary diff --git a/components/engine/hack/infrastructure/docker-ci/deployment.py b/components/engine/hack/infrastructure/docker-ci/deployment.py index c04219d523..8fb0766bfe 100755 --- a/components/engine/hack/infrastructure/docker-ci/deployment.py +++ b/components/engine/hack/infrastructure/docker-ci/deployment.py @@ -136,7 +136,7 @@ sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n' sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev' ' python-pip supervisor git mercurial linux-image-extra-$(uname -r)' ' aufs-tools make libfontconfig libevent-dev libsqlite3-dev libssl-dev') -sudo('wget -O - https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | ' +sudo('wget -O - https://go.googlecode.com/files/go1.2.linux-amd64.tar.gz | ' 'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go') sudo('GOPATH=/go go get -d github.com/dotcloud/docker') sudo('pip install -r {}/requirements.txt'.format(CFG_PATH)) diff --git a/components/engine/hack/install.sh b/components/engine/hack/install.sh index 34d5b7fbde..a22bb5c48e 100755 --- a/components/engine/hack/install.sh +++ b/components/engine/hack/install.sh @@ -116,7 +116,7 @@ case "$lsb_dist" in ( set -x $sh_c 'docker run busybox echo "Docker has been successfully installed!"' - ) + ) || true fi exit 0 ;; diff --git a/components/engine/hack/make.sh b/components/engine/hack/make.sh index 91f641af89..90683b6ead 100755 --- a/components/engine/hack/make.sh +++ b/components/engine/hack/make.sh @@ -15,8 +15,9 @@ set -e # - The script is intented to be run inside the docker container specified # in the Dockerfile at the root of the source. In other words: # DO NOT CALL THIS SCRIPT DIRECTLY. -# - The right way to call this script is to invoke "docker build ." from -# your checkout of the Docker repository, and then +# - The right way to call this script is to invoke "make" from +# your checkout of the Docker repository. +# the Makefile will so a "docker build -t docker ." and then # "docker run hack/make.sh" in the resulting container image. # @@ -28,7 +29,7 @@ RESOLVCONF=$(readlink --canonicalize /etc/resolv.conf) grep -q "$RESOLVCONF" /proc/mounts || { echo >&2 "# WARNING! I don't seem to be running in a docker container." echo >&2 "# The result of this command might be an incorrect build, and will not be officially supported." - echo >&2 "# Try this: 'docker build -t docker . && docker run docker ./hack/make.sh'" + echo >&2 "# Try this: 'make all'" } # List of bundles to create when no argument is passed @@ -40,6 +41,7 @@ DEFAULT_BUNDLES=( dyntest dyntest-integration cover + cross tgz ubuntu ) @@ -63,10 +65,13 @@ fi # Use these flags when compiling the tests and final binary LDFLAGS='-X main.GITCOMMIT "'$GITCOMMIT'" -X main.VERSION "'$VERSION'" -w' LDFLAGS_STATIC='-X github.com/dotcloud/docker/utils.IAMSTATIC true -linkmode external -extldflags "-lpthread -static -Wl,--unresolved-symbols=ignore-in-object-files"' -BUILDFLAGS='-tags netgo' +BUILDFLAGS='-tags netgo -a' HAVE_GO_TEST_COVER= -if go help testflag | grep -q -- -cover; then +if \ + go help testflag | grep -- -cover > /dev/null \ + && go tool -n cover > /dev/null 2>&1 \ +; then HAVE_GO_TEST_COVER=1 fi @@ -85,10 +90,6 @@ go_test_dir() { coverprofile="$DEST/coverprofiles/${coverprofile//\//-}" testcover=( -cover -coverprofile "$coverprofile" ) fi - ( # we run "go test -i" ouside the "set -x" to provde cleaner output - cd "$dir" - go test -i -ldflags "$LDFLAGS" $BUILDFLAGS - ) ( set -x cd "$dir" diff --git a/components/engine/hack/make/cross b/components/engine/hack/make/cross new file mode 100644 index 0000000000..a67ab6c28a --- /dev/null +++ b/components/engine/hack/make/cross @@ -0,0 +1,23 @@ +#!/bin/bash + +DEST=$1 + +# if we have our linux/amd64 version compiled, let's symlink it in +if [ -x "$DEST/../binary/docker-$VERSION" ]; then + mkdir -p "$DEST/linux/amd64" + ( + cd "$DEST/linux/amd64" + ln -s ../../../binary/* ./ + ) + echo "Created symlinks:" "$DEST/linux/amd64/"* +fi + +for platform in $DOCKER_CROSSPLATFORMS; do + ( + mkdir -p "$DEST/$platform" # bundles/VERSION/cross/GOOS/GOARCH/docker-VERSION + export GOOS=${platform%/*} + export GOARCH=${platform##*/} + export LDFLAGS_STATIC="" # we just need a simple client for these platforms (TODO this might change someday) + source "$(dirname "$BASH_SOURCE")/binary" "$DEST/$platform" + ) +done diff --git a/components/engine/hack/make/dynbinary b/components/engine/hack/make/dynbinary index 100c00eed5..c02094c0c5 100644 --- a/components/engine/hack/make/dynbinary +++ b/components/engine/hack/make/dynbinary @@ -3,7 +3,7 @@ DEST=$1 # dockerinit still needs to be a static binary, even if docker is dynamic -CGO_ENABLED=0 go build -a -o $DEST/dockerinit-$VERSION -ldflags "$LDFLAGS -d" $BUILDFLAGS ./dockerinit +CGO_ENABLED=0 go build -o $DEST/dockerinit-$VERSION -ldflags "$LDFLAGS -d" $BUILDFLAGS ./dockerinit echo "Created binary: $DEST/dockerinit-$VERSION" ln -sf dockerinit-$VERSION $DEST/dockerinit @@ -12,6 +12,6 @@ export DOCKER_INITSHA1="$(sha1sum $DEST/dockerinit-$VERSION | cut -d' ' -f1)" # exported so that "dyntest" can easily access it later without recalculating it ( - export LDFLAGS_STATIC="-X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\"" + export LDFLAGS_STATIC="-X github.com/dotcloud/docker/utils.INITSHA1 \"$DOCKER_INITSHA1\" -X github.com/dotcloud/docker/utils.INITPATH \"$DOCKER_INITPATH\"" source "$(dirname "$BASH_SOURCE")/binary" ) diff --git a/components/engine/hack/make/tgz b/components/engine/hack/make/tgz index 86a7a9827d..5d03306322 100644 --- a/components/engine/hack/make/tgz +++ b/components/engine/hack/make/tgz @@ -1,23 +1,29 @@ #!/bin/bash DEST="$1" -BINARY="$DEST/../binary/docker-$VERSION" -TGZ="$DEST/docker-$VERSION.tgz" +CROSS="$DEST/../cross" set -e -if [ ! -x "$BINARY" ]; then - echo >&2 'error: binary must be run before tgz' +if [ ! -d "$CROSS/linux/amd64" ]; then + echo >&2 'error: binary and cross must be run before tgz' false fi -mkdir -p "$DEST/build" - -mkdir -p "$DEST/build/usr/local/bin" -cp -L "$BINARY" "$DEST/build/usr/local/bin/docker" - -tar --numeric-owner --owner 0 -C "$DEST/build" -czf "$TGZ" usr - -rm -rf "$DEST/build" - -echo "Created tgz: $TGZ" +for d in "$CROSS/"*/*; do + GOARCH="$(basename "$d")" + GOOS="$(basename "$(dirname "$d")")" + mkdir -p "$DEST/$GOOS/$GOARCH" + TGZ="$DEST/$GOOS/$GOARCH/docker-$VERSION.tgz" + + mkdir -p "$DEST/build" + + mkdir -p "$DEST/build/usr/local/bin" + cp -L "$d/docker-$VERSION" "$DEST/build/usr/local/bin/docker" + + tar --numeric-owner --owner 0 -C "$DEST/build" -czf "$TGZ" usr + + rm -rf "$DEST/build" + + echo "Created tgz: $TGZ" +done diff --git a/components/engine/hack/make/ubuntu b/components/engine/hack/make/ubuntu index 578f254558..bcffc6560f 100644 --- a/components/engine/hack/make/ubuntu +++ b/components/engine/hack/make/ubuntu @@ -17,6 +17,7 @@ repeatability across servers. Docker is a great building block for automating distributed systems: large-scale web deployments, database clusters, continuous deployment systems, private PaaS, service-oriented architectures, etc." +PACKAGE_LICENSE="Apache-2.0" # Build docker as an ubuntu package using FPM and REPREPRO (sue me). # bundle_binary must be called first. @@ -30,6 +31,20 @@ bundle_ubuntu() { mkdir -p $DIR/lib/systemd cp -R contrib/init/systemd $DIR/lib/systemd/system + mkdir -p $DIR/etc/default + cat > $DIR/etc/default/docker <<'EOF' +# Docker Upstart and SysVinit configuration file + +# Customize location of Docker binary (especially for development testing). +#DOCKER="/usr/local/bin/docker" + +# Use DOCKER_OPTS to modify the daemon startup options. +#DOCKER_OPTS="-dns 8.8.8.8" + +# If you need Docker to use an HTTP proxy, it can also be specified here. +#export http_proxy=http://127.0.0.1:3128/ +EOF + # Copy the binary # This will fail if the binary bundle hasn't been built mkdir -p $DIR/usr/bin @@ -104,7 +119,7 @@ EOF --replaces lxc-docker \ --replaces lxc-docker-virtual-package \ --url "$PACKAGE_URL" \ - --vendor "$PACKAGE_VENDOR" \ + --license "$PACKAGE_LICENSE" \ --config-files /etc/init/docker.conf \ --config-files /etc/init.d/docker \ --config-files /etc/default/docker \ @@ -118,13 +133,9 @@ EOF --description "$PACKAGE_DESCRIPTION" \ --maintainer "$PACKAGE_MAINTAINER" \ --url "$PACKAGE_URL" \ - --vendor "$PACKAGE_VENDOR" \ - --config-files /etc/init/docker.conf \ - --config-files /etc/init.d/docker \ - --config-files /etc/default/docker \ + --license "$PACKAGE_LICENSE" \ --deb-compression xz \ -t deb . - # note: the --config-files lines have to be duplicated to stop overwrite on package upgrade (since we have to use this funky virtual package) ) } diff --git a/components/engine/hack/release.sh b/components/engine/hack/release.sh index 2ec40677e7..d1278a4c36 100755 --- a/components/engine/hack/release.sh +++ b/components/engine/hack/release.sh @@ -47,6 +47,7 @@ cd /go/src/github.com/dotcloud/docker RELEASE_BUNDLES=( binary + cross tgz ubuntu ) @@ -113,6 +114,77 @@ s3_url() { esac } +release_build() { + GOOS=$1 + GOARCH=$2 + + BINARY=bundles/$VERSION/cross/$GOOS/$GOARCH/docker-$VERSION + TGZ=bundles/$VERSION/tgz/$GOOS/$GOARCH/docker-$VERSION.tgz + + # we need to map our GOOS and GOARCH to uname values + # see https://en.wikipedia.org/wiki/Uname + # ie, GOOS=linux -> "uname -s"=Linux + + S3OS=$GOOS + case "$S3OS" in + darwin) + S3OS=Darwin + ;; + freebsd) + S3OS=FreeBSD + ;; + linux) + S3OS=Linux + ;; + *) + echo >&2 "error: can't convert $S3OS to an appropriate value for 'uname -s'" + exit 1 + ;; + esac + + S3ARCH=$GOARCH + case "$S3ARCH" in + amd64) + S3ARCH=x86_64 + ;; + 386) + S3ARCH=i386 + ;; + arm) + # GOARCH is fine + ;; + *) + echo >&2 "error: can't convert $S3ARCH to an appropriate value for 'uname -m'" + exit 1 + ;; + esac + + S3DIR=s3://$BUCKET/builds/$S3OS/$S3ARCH + + if [ ! -x "$BINARY" ]; then + echo >&2 "error: can't find $BINARY - was it compiled properly?" + exit 1 + fi + if [ ! -f "$TGZ" ]; then + echo >&2 "error: can't find $TGZ - was it packaged properly?" + exit 1 + fi + + echo "Uploading $BINARY to $S3OS/$S3ARCH/docker-$VERSION" + s3cmd --follow-symlinks --preserve --acl-public put $BINARY $S3DIR/docker-$VERSION + + echo "Uploading $TGZ to $S3OS/$S3ARCH/docker-$VERSION.tgz" + s3cmd --follow-symlinks --preserve --acl-public put $TGZ $S3DIR/docker-$VERSION.tgz + + if [ -z "$NOLATEST" ]; then + echo "Copying $S3OS/$S3ARCH/docker-$VERSION to $S3OS/$S3ARCH/docker-latest" + s3cmd --acl-public cp $S3DIR/docker-$VERSION $S3DIR/docker-latest + + echo "Copying $S3OS/$S3ARCH/docker-$VERSION.tgz to $S3OS/$S3ARCH/docker-latest.tgz" + s3cmd --acl-public cp $S3DIR/docker-$VERSION.tgz $S3DIR/docker-latest.tgz + fi +} + # Upload the 'ubuntu' bundle to S3: # 1. A full APT repository is published at $BUCKET/ubuntu/ # 2. Instructions for using the APT repository are uploaded at $BUCKET/ubuntu/index @@ -189,31 +261,21 @@ EOF echo "APT repository uploaded. Instructions available at $(s3_url)/ubuntu" } -# Upload a tgz to S3 -release_tgz() { - [ -e bundles/$VERSION/tgz/docker-$VERSION.tgz ] || { - echo >&2 './hack/make.sh must be run before release_binary' +# Upload binaries and tgz files to S3 +release_binaries() { + [ -e bundles/$VERSION/cross/linux/amd64/docker-$VERSION ] || { + echo >&2 './hack/make.sh must be run before release_binaries' exit 1 } - S3DIR=s3://$BUCKET/builds/Linux/x86_64 - s3cmd --acl-public put bundles/$VERSION/tgz/docker-$VERSION.tgz $S3DIR/docker-$VERSION.tgz + for d in bundles/$VERSION/cross/*/*; do + GOARCH="$(basename "$d")" + GOOS="$(basename "$(dirname "$d")")" + release_build "$GOOS" "$GOARCH" + done - if [ -z "$NOLATEST" ]; then - echo "Copying docker-$VERSION.tgz to docker-latest.tgz" - s3cmd --acl-public cp $S3DIR/docker-$VERSION.tgz $S3DIR/docker-latest.tgz - fi -} + # TODO create redirect from builds/*/i686 to builds/*/i386 -# Upload a static binary to S3 -release_binary() { - [ -e bundles/$VERSION/binary/docker-$VERSION ] || { - echo >&2 './hack/make.sh must be run before release_binary' - exit 1 - } - - S3DIR=s3://$BUCKET/builds/Linux/x86_64 - s3cmd --acl-public put bundles/$VERSION/binary/docker-$VERSION $S3DIR/docker-$VERSION cat < (@crosbymichael) diff --git a/components/engine/pkg/graphdb/conn_darwin.go b/components/engine/pkg/graphdb/conn_darwin.go new file mode 100644 index 0000000000..6e75fd8edb --- /dev/null +++ b/components/engine/pkg/graphdb/conn_darwin.go @@ -0,0 +1,5 @@ +package graphdb + +func NewSqliteConn(root string) (*Database, error) { + panic("Not implemented") +} diff --git a/components/engine/pkg/graphdb/conn_linux.go b/components/engine/pkg/graphdb/conn_linux.go new file mode 100644 index 0000000000..2bd51940ce --- /dev/null +++ b/components/engine/pkg/graphdb/conn_linux.go @@ -0,0 +1,23 @@ +package graphdb + +import ( + _ "code.google.com/p/gosqlite/sqlite3" // registers sqlite + "database/sql" + "os" +) + +func NewSqliteConn(root string) (*Database, error) { + initDatabase := false + if _, err := os.Stat(root); err != nil { + if os.IsNotExist(err) { + initDatabase = true + } else { + return nil, err + } + } + conn, err := sql.Open("sqlite3", root) + if err != nil { + return nil, err + } + return NewDatabase(conn, initDatabase) +} diff --git a/components/engine/graphdb/graphdb.go b/components/engine/pkg/graphdb/graphdb.go similarity index 100% rename from components/engine/graphdb/graphdb.go rename to components/engine/pkg/graphdb/graphdb.go diff --git a/components/engine/graphdb/graphdb_test.go b/components/engine/pkg/graphdb/graphdb_test.go similarity index 100% rename from components/engine/graphdb/graphdb_test.go rename to components/engine/pkg/graphdb/graphdb_test.go diff --git a/components/engine/graphdb/sort.go b/components/engine/pkg/graphdb/sort.go similarity index 100% rename from components/engine/graphdb/sort.go rename to components/engine/pkg/graphdb/sort.go diff --git a/components/engine/graphdb/sort_test.go b/components/engine/pkg/graphdb/sort_test.go similarity index 100% rename from components/engine/graphdb/sort_test.go rename to components/engine/pkg/graphdb/sort_test.go diff --git a/components/engine/graphdb/utils.go b/components/engine/pkg/graphdb/utils.go similarity index 100% rename from components/engine/graphdb/utils.go rename to components/engine/pkg/graphdb/utils.go diff --git a/components/engine/namesgenerator/names-generator.go b/components/engine/pkg/namesgenerator/names-generator.go similarity index 100% rename from components/engine/namesgenerator/names-generator.go rename to components/engine/pkg/namesgenerator/names-generator.go diff --git a/components/engine/namesgenerator/names-generator_test.go b/components/engine/pkg/namesgenerator/names-generator_test.go similarity index 100% rename from components/engine/namesgenerator/names-generator_test.go rename to components/engine/pkg/namesgenerator/names-generator_test.go diff --git a/components/engine/netlink/netlink_darwin.go b/components/engine/pkg/netlink/netlink_darwin.go similarity index 84% rename from components/engine/netlink/netlink_darwin.go rename to components/engine/pkg/netlink/netlink_darwin.go index 9a972fe63f..dcc60b6764 100644 --- a/components/engine/netlink/netlink_darwin.go +++ b/components/engine/pkg/netlink/netlink_darwin.go @@ -25,3 +25,7 @@ func AddDefaultGw(ip net.IP) error { return fmt.Errorf("Not implemented") } + +func NetworkSetMTU(iface *net.Interface, mtu int) error { + return fmt.Errorf("Not implemented") +} diff --git a/components/engine/netlink/netlink_linux.go b/components/engine/pkg/netlink/netlink_linux.go similarity index 94% rename from components/engine/netlink/netlink_linux.go rename to components/engine/pkg/netlink/netlink_linux.go index 239614b5fb..4417f5249e 100644 --- a/components/engine/netlink/netlink_linux.go +++ b/components/engine/pkg/netlink/netlink_linux.go @@ -328,7 +328,6 @@ func AddDefaultGw(ip net.IP) error { } return s.HandleAck(wb.Seq) - } // Bring up a particular network interface @@ -354,6 +353,37 @@ func NetworkLinkUp(iface *net.Interface) error { return s.HandleAck(wb.Seq) } +func NetworkSetMTU(iface *net.Interface, mtu int) error { + s, err := getNetlinkSocket() + if err != nil { + return err + } + defer s.Close() + + wb := newNetlinkRequest(syscall.RTM_SETLINK, syscall.NLM_F_ACK) + + msg := newIfInfomsg(syscall.AF_UNSPEC) + msg.Type = syscall.RTM_SETLINK + msg.Flags = syscall.NLM_F_REQUEST + msg.Index = int32(iface.Index) + msg.Change = 0xFFFFFFFF + wb.AddData(msg) + + var ( + b = make([]byte, 4) + native = nativeEndian() + ) + native.PutUint32(b, uint32(mtu)) + + data := newRtAttr(syscall.IFLA_MTU, b) + wb.AddData(data) + + if err := s.Send(wb); err != nil { + return err + } + return s.HandleAck(wb.Seq) +} + // Add an Ip address to an interface. This is identical to: // ip addr add $ip/$ipNet dev $iface func NetworkLinkAddIp(iface *net.Interface, ip net.IP, ipNet *net.IPNet) error { diff --git a/components/engine/systemd/sd_notify.go b/components/engine/pkg/systemd/sd_notify.go similarity index 100% rename from components/engine/systemd/sd_notify.go rename to components/engine/pkg/systemd/sd_notify.go diff --git a/components/engine/term/MAINTAINERS b/components/engine/pkg/term/MAINTAINERS similarity index 100% rename from components/engine/term/MAINTAINERS rename to components/engine/pkg/term/MAINTAINERS diff --git a/components/engine/term/term.go b/components/engine/pkg/term/term.go similarity index 100% rename from components/engine/term/term.go rename to components/engine/pkg/term/term.go diff --git a/components/engine/term/termios_darwin.go b/components/engine/pkg/term/termios_darwin.go similarity index 100% rename from components/engine/term/termios_darwin.go rename to components/engine/pkg/term/termios_darwin.go diff --git a/components/engine/term/termios_linux.go b/components/engine/pkg/term/termios_linux.go similarity index 100% rename from components/engine/term/termios_linux.go rename to components/engine/pkg/term/termios_linux.go diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 6c9255aa46..3c793cf08c 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -448,6 +448,11 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr if err != nil { return "", fmt.Errorf("Failed to upload layer: %s", err) } + if rc, ok := layer.(io.Closer); ok { + if err := rc.Close(); err != nil { + return "", err + } + } defer res.Body.Close() if res.StatusCode != 200 { diff --git a/components/engine/runtime.go b/components/engine/runtime.go index 3268892d56..7c7f722c5d 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -1,12 +1,10 @@ package docker import ( - _ "code.google.com/p/gosqlite/sqlite3" // registers sqlite "container/list" - "database/sql" "fmt" "github.com/dotcloud/docker/archive" - "github.com/dotcloud/docker/graphdb" + "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/graphdriver/aufs" _ "github.com/dotcloud/docker/graphdriver/devmapper" @@ -31,8 +29,9 @@ import ( const MaxImageDepth = 127 var ( - defaultDns = []string{"8.8.8.8", "8.8.4.4"} - validContainerName = regexp.MustCompile(`^/?[a-zA-Z0-9_-]+$`) + defaultDns = []string{"8.8.8.8", "8.8.4.4"} + validContainerNameChars = `[a-zA-Z0-9_.-]` + validContainerNamePattern = regexp.MustCompile(`^/?` + validContainerNameChars + `+$`) ) type Capabilities struct { @@ -188,7 +187,6 @@ func (runtime *Runtime) Register(container *Container) error { } container.waitLock = make(chan struct{}) - go container.monitor() } } @@ -262,9 +260,8 @@ func (runtime *Runtime) Destroy(container *Container) error { } func (runtime *Runtime) restore() error { - wheel := "-\\|/" if os.Getenv("DEBUG") == "" && os.Getenv("TEST") == "" { - fmt.Printf("Loading containers: ") + fmt.Printf("Loading containers: ") } dir, err := ioutil.ReadDir(runtime.repository) if err != nil { @@ -273,11 +270,11 @@ func (runtime *Runtime) restore() error { containers := make(map[string]*Container) currentDriver := runtime.driver.String() - for i, v := range dir { + for _, v := range dir { id := v.Name() container, err := runtime.load(id) - if i%21 == 0 && os.Getenv("DEBUG") == "" && os.Getenv("TEST") == "" { - fmt.Printf("\b%c", wheel[i%4]) + if os.Getenv("DEBUG") == "" && os.Getenv("TEST") == "" { + fmt.Print(".") } if err != nil { utils.Errorf("Failed to load container %v: %v", id, err) @@ -301,6 +298,9 @@ func (runtime *Runtime) restore() error { if entities := runtime.containerGraph.List("/", -1); entities != nil { for _, p := range entities.Paths() { + if os.Getenv("DEBUG") == "" && os.Getenv("TEST") == "" { + fmt.Print(".") + } e := entities[p] if container, ok := containers[e.ID()]; ok { register(container) @@ -324,7 +324,7 @@ func (runtime *Runtime) restore() error { } if os.Getenv("DEBUG") == "" && os.Getenv("TEST") == "" { - fmt.Printf("\bdone.\n") + fmt.Printf(": done.\n") } return nil @@ -425,8 +425,8 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin name = utils.TruncateID(id) } } else { - if !validContainerName.MatchString(name) { - return nil, nil, fmt.Errorf("Invalid container name (%s), only [a-zA-Z0-9_-] are allowed", name) + if !validContainerNamePattern.MatchString(name) { + return nil, nil, fmt.Errorf("Invalid container name (%s), only %s are allowed", name, validContainerNameChars) } } @@ -485,10 +485,8 @@ func (runtime *Runtime) Create(config *Config, name string) (*Container, []strin hostConfig: &HostConfig{}, Image: img.ID, // Always use the resolved image id NetworkSettings: &NetworkSettings{}, - // FIXME: do we need to store this in the container? - SysInitPath: runtime.sysInitPath, - Name: name, - Driver: runtime.driver.String(), + Name: name, + Driver: runtime.driver.String(), } container.root = runtime.containerRoot(container.ID) // Step 1: create the container directory. @@ -675,14 +673,17 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) { } if ad, ok := driver.(*aufs.Driver); ok { + utils.Debugf("Migrating existing containers") if err := ad.Migrate(config.Root, setupInitLayer); err != nil { return nil, err } } + utils.Debugf("Escaping AppArmor confinement") if err := linkLxcStart(config.Root); err != nil { return nil, err } + utils.Debugf("Creating images graph") g, err := NewGraph(path.Join(config.Root, "graph"), driver) if err != nil { return nil, err @@ -694,10 +695,12 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) { if err != nil { return nil, err } + utils.Debugf("Creating volumes graph") volumes, err := NewGraph(path.Join(config.Root, "volumes"), volumesDriver) if err != nil { return nil, err } + utils.Debugf("Creating repository list") repositories, err := NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g) if err != nil { return nil, fmt.Errorf("Couldn't create Tag store: %s", err) @@ -711,19 +714,7 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) { } graphdbPath := path.Join(config.Root, "linkgraph.db") - initDatabase := false - if _, err := os.Stat(graphdbPath); err != nil { - if os.IsNotExist(err) { - initDatabase = true - } else { - return nil, err - } - } - conn, err := sql.Open("sqlite3", graphdbPath) - if err != nil { - return nil, err - } - graph, err := graphdb.NewDatabase(conn, initDatabase) + graph, err := graphdb.NewSqliteConn(graphdbPath) if err != nil { return nil, err } @@ -734,18 +725,18 @@ func NewRuntimeFromDirectory(config *DaemonConfig) (*Runtime, error) { return nil, fmt.Errorf("Could not locate dockerinit: This usually means docker was built incorrectly. See http://docs.docker.io/en/latest/contributing/devenvironment for official build instructions.") } - if !utils.IAMSTATIC { - if err := os.Mkdir(path.Join(config.Root, fmt.Sprintf("init")), 0700); err != nil && !os.IsExist(err) { + if sysInitPath != localCopy { + // When we find a suitable dockerinit binary (even if it's our local binary), we copy it into config.Root at localCopy for future use (so that the original can go away without that being a problem, for example during a package upgrade). + if err := os.Mkdir(path.Dir(localCopy), 0700); err != nil && !os.IsExist(err) { return nil, err } - if _, err := utils.CopyFile(sysInitPath, localCopy); err != nil { return nil, err } - sysInitPath = localCopy - if err := os.Chmod(sysInitPath, 0700); err != nil { + if err := os.Chmod(localCopy, 0700); err != nil { return nil, err } + sysInitPath = localCopy } runtime := &Runtime{ diff --git a/components/engine/server.go b/components/engine/server.go index 9b492a2927..a49a34d542 100644 --- a/components/engine/server.go +++ b/components/engine/server.go @@ -1,14 +1,13 @@ package docker import ( - "bufio" "encoding/json" "errors" "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/engine" - "github.com/dotcloud/docker/graphdb" + "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/registry" "github.com/dotcloud/docker/utils" "io" @@ -22,6 +21,7 @@ import ( "path" "path/filepath" "runtime" + "sort" "strconv" "strings" "sync" @@ -634,6 +634,13 @@ func (srv *Server) DockerInfo(job *engine.Job) engine.Status { kernelVersion = kv.String() } + // if we still have the original dockerinit binary from before we copied it locally, let's return the path to that, since that's more intuitive (the copied path is trivial to derive by hand given VERSION) + initPath := utils.DockerInitPath("") + if initPath == "" { + // if that fails, we'll just return the path from the runtime + initPath = srv.runtime.sysInitPath + } + v := &engine.Env{} v.SetInt("Containers", len(srv.runtime.List())) v.SetInt("Images", imgcount) @@ -649,6 +656,8 @@ func (srv *Server) DockerInfo(job *engine.Job) engine.Status { v.SetInt("NEventsListener", len(srv.events)) v.Set("KernelVersion", kernelVersion) v.Set("IndexServerAddress", auth.IndexServerAddress()) + v.Set("InitSha1", utils.INITSHA1) + v.Set("InitPath", initPath) if _, err := v.WriteTo(job.Stdout); err != nil { job.Error(err) return engine.StatusErr @@ -690,34 +699,57 @@ func (srv *Server) ImageHistory(name string) ([]APIHistory, error) { func (srv *Server) ContainerTop(name, psArgs string) (*APITop, error) { if container := srv.runtime.Get(name); container != nil { - output, err := exec.Command("lxc-ps", "--name", container.ID, "--", psArgs).CombinedOutput() - if err != nil { - return nil, fmt.Errorf("lxc-ps: %s (%s)", err, output) + if !container.State.IsRunning() { + return nil, fmt.Errorf("Container %s is not running", name) } - procs := APITop{} - for i, line := range strings.Split(string(output), "\n") { + pids, err := utils.GetPidsForContainer(container.ID) + if err != nil { + return nil, err + } + if len(psArgs) == 0 { + psArgs = "-ef" + } + output, err := exec.Command("ps", psArgs).Output() + if err != nil { + return nil, fmt.Errorf("Error running ps: %s", err) + } + + lines := strings.Split(string(output), "\n") + header := strings.Fields(lines[0]) + procs := APITop{ + Titles: header, + } + + pidIndex := -1 + for i, name := range header { + if name == "PID" { + pidIndex = i + } + } + if pidIndex == -1 { + return nil, errors.New("Couldn't find PID field in ps output") + } + + for _, line := range lines[1:] { if len(line) == 0 { continue } - words := []string{} - scanner := bufio.NewScanner(strings.NewReader(line)) - scanner.Split(bufio.ScanWords) - if !scanner.Scan() { - return nil, fmt.Errorf("Wrong output using lxc-ps") + fields := strings.Fields(line) + p, err := strconv.Atoi(fields[pidIndex]) + if err != nil { + return nil, fmt.Errorf("Unexpected pid '%s': %s", fields[pidIndex], err) } - // no scanner.Text because we skip container id - for scanner.Scan() { - if i != 0 && len(words) == len(procs.Titles) { - words[len(words)-1] = fmt.Sprintf("%s %s", words[len(words)-1], scanner.Text()) - } else { - words = append(words, scanner.Text()) + + for _, pid := range pids { + if pid == p { + // Make sure number of fields equals number of header titles + // merging "overhanging" fields + processes := fields[:len(procs.Titles)-1] + processes = append(processes, strings.Join(fields[len(procs.Titles)-1:], " ")) + + procs.Processes = append(procs.Processes, processes) } } - if i == 0 { - procs.Titles = words - } else { - procs.Processes = append(procs.Processes, words) - } } return &procs, nil @@ -1253,6 +1285,7 @@ func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, return "", err } + out.Write(sf.FormatProgress(utils.TruncateID(imgData.ID), "Image successfully pushed", nil)) return imgData.Checksum, nil } @@ -1515,7 +1548,7 @@ func (srv *Server) deleteImageAndChildren(id string, imgs *[]APIRmi, byParents m if err != nil { return err } - if len(byParents[id]) == 0 { + if len(byParents[id]) == 0 && srv.canDeleteImage(id) == nil { if err := srv.runtime.repositories.DeleteAll(id); err != nil { return err } @@ -1567,7 +1600,7 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro } else if repoName != parsedRepo { // the id belongs to multiple repos, like base:latest and user:test, // in that case return conflict - return imgs, nil + return nil, fmt.Errorf("Conflict, cannot delete image %s because it is tagged in multiple repositories", utils.TruncateID(img.ID)) } } } else { @@ -1603,9 +1636,8 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { var ( repository, tag string - validate = true + img, err = srv.runtime.repositories.LookupImage(name) ) - img, err := srv.runtime.repositories.LookupImage(name) if err != nil { return nil, fmt.Errorf("No such image: %s", name) } @@ -1627,31 +1659,34 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { // // i.e. only validate if we are performing an actual delete and not // an untag op - if repository != "" { - validate = len(srv.runtime.repositories.ByID()[img.ID]) == 1 - } - - if validate { + if repository != "" && len(srv.runtime.repositories.ByID()[img.ID]) == 1 { // Prevent deletion if image is used by a container - for _, container := range srv.runtime.List() { - parent, err := srv.runtime.repositories.LookupImage(container.Image) - if err != nil { - return nil, err - } - - if err := parent.WalkHistory(func(p *Image) error { - if img.ID == p.ID { - return fmt.Errorf("Conflict, cannot delete %s because the container %s is using it", name, container.ID) - } - return nil - }); err != nil { - return nil, err - } + if err := srv.canDeleteImage(img.ID); err != nil { + return nil, err } } return srv.deleteImage(img, repository, tag) } +func (srv *Server) canDeleteImage(imgID string) error { + for _, container := range srv.runtime.List() { + parent, err := srv.runtime.repositories.LookupImage(container.Image) + if err != nil { + return err + } + + if err := parent.WalkHistory(func(p *Image) error { + if imgID == p.ID { + return fmt.Errorf("Conflict, cannot delete %s because the container %s is using it", utils.TruncateID(imgID), utils.TruncateID(container.ID)) + } + return nil + }); err != nil { + return err + } + } + return nil +} + func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error) { // Retrieve all images @@ -1661,16 +1696,13 @@ func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error) } // Store the tree in a map of map (map[parentId][childId]) - imageMap := make(map[string]map[string]struct{}) + imageMap := make(map[string][]string) for _, img := range images { - if _, exists := imageMap[img.Parent]; !exists { - imageMap[img.Parent] = make(map[string]struct{}) - } - imageMap[img.Parent][img.ID] = struct{}{} + imageMap[img.Parent] = append(imageMap[img.Parent], img.ID) } - + sort.Strings(imageMap[imgID]) // Loop on the children of the given image and check the config - for elem := range imageMap[imgID] { + for _, elem := range imageMap[imgID] { img, err := srv.runtime.graph.Get(elem) if err != nil { return nil, err @@ -1990,6 +2022,8 @@ func (srv *Server) HTTPRequestFactory(metaHeaders map[string][]string) *utils.HT httpVersion = append(httpVersion, &simpleVersionInfo{"go", v.Get("GoVersion")}) httpVersion = append(httpVersion, &simpleVersionInfo{"git-commit", v.Get("GitCommit")}) httpVersion = append(httpVersion, &simpleVersionInfo{"kernel", v.Get("KernelVersion")}) + httpVersion = append(httpVersion, &simpleVersionInfo{"os", v.Get("Os")}) + httpVersion = append(httpVersion, &simpleVersionInfo{"arch", v.Get("Arch")}) ud := utils.NewHTTPUserAgentDecorator(httpVersion...) md := &utils.HTTPMetaHeadersDecorator{ Headers: metaHeaders, diff --git a/components/engine/sysinit/sysinit.go b/components/engine/sysinit/sysinit.go index 2dc5967dc6..f906b7d2dd 100644 --- a/components/engine/sysinit/sysinit.go +++ b/components/engine/sysinit/sysinit.go @@ -4,7 +4,7 @@ import ( "encoding/json" "flag" "fmt" - "github.com/dotcloud/docker/netlink" + "github.com/dotcloud/docker/pkg/netlink" "github.com/dotcloud/docker/utils" "github.com/syndtr/gocapability/capability" "io/ioutil" @@ -25,6 +25,7 @@ type DockerInitArgs struct { privileged bool env []string args []string + mtu int } func setupHostname(args *DockerInitArgs) error { @@ -32,7 +33,7 @@ func setupHostname(args *DockerInitArgs) error { if hostname == "" { return nil } - return syscall.Sethostname([]byte(hostname)) + return setHostname(hostname) } // Setup networking @@ -50,6 +51,9 @@ func setupNetworking(args *DockerInitArgs) error { if err := netlink.NetworkLinkAddIp(iface, ip, ipNet); err != nil { return fmt.Errorf("Unable to set up networking: %v", err) } + if err := netlink.NetworkSetMTU(iface, args.mtu); err != nil { + return fmt.Errorf("Unable to set MTU: %v", err) + } if err := netlink.NetworkLinkUp(iface); err != nil { return fmt.Errorf("Unable to set up networking: %v", err) } @@ -227,6 +231,7 @@ func SysInit() { ip := flag.String("i", "", "ip address") workDir := flag.String("w", "", "workdir") privileged := flag.Bool("privileged", false, "privileged mode") + mtu := flag.Int("mtu", 1500, "interface mtu") flag.Parse() // Get env @@ -250,6 +255,7 @@ func SysInit() { privileged: *privileged, env: env, args: flag.Args(), + mtu: *mtu, } if err := executeProgram(args); err != nil { diff --git a/components/engine/sysinit/sysinit_darwin.go b/components/engine/sysinit/sysinit_darwin.go new file mode 100644 index 0000000000..64566afb3c --- /dev/null +++ b/components/engine/sysinit/sysinit_darwin.go @@ -0,0 +1,5 @@ +package sysinit + +func setHostname(hostname string) error { + panic("Not supported on darwin") +} diff --git a/components/engine/sysinit/sysinit_linux.go b/components/engine/sysinit/sysinit_linux.go new file mode 100644 index 0000000000..d18d2fab8b --- /dev/null +++ b/components/engine/sysinit/sysinit_linux.go @@ -0,0 +1,9 @@ +package sysinit + +import ( + "syscall" +) + +func setHostname(hostname string) error { + return syscall.Sethostname([]byte(hostname)) +} diff --git a/components/engine/utils.go b/components/engine/utils.go index 96dd9ed442..5bcc678184 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -3,7 +3,7 @@ package docker import ( "fmt" "github.com/dotcloud/docker/archive" - "github.com/dotcloud/docker/namesgenerator" + "github.com/dotcloud/docker/pkg/namesgenerator" "github.com/dotcloud/docker/utils" "io/ioutil" "strconv" diff --git a/components/engine/utils/http.go b/components/engine/utils/http.go index 463d3b4fdd..68e93d8eb9 100644 --- a/components/engine/utils/http.go +++ b/components/engine/utils/http.go @@ -76,9 +76,9 @@ type HTTPUserAgentDecorator struct { } func NewHTTPUserAgentDecorator(versions ...VersionInfo) HTTPRequestDecorator { - ret := new(HTTPUserAgentDecorator) - ret.versions = versions - return ret + return &HTTPUserAgentDecorator{ + versions: versions, + } } func (h *HTTPUserAgentDecorator) ChangeRequest(req *http.Request) (newReq *http.Request, err error) { @@ -108,15 +108,15 @@ func (h *HTTPMetaHeadersDecorator) ChangeRequest(req *http.Request) (newReq *htt } type HTTPAuthDecorator struct { - login string + login string password string } func NewHTTPAuthDecorator(login, password string) HTTPRequestDecorator { - ret := new(HTTPAuthDecorator) - ret.login = login - ret.password = password - return ret + return &HTTPAuthDecorator{ + login: login, + password: password, + } } func (self *HTTPAuthDecorator) ChangeRequest(req *http.Request) (*http.Request, error) { @@ -136,7 +136,7 @@ func NewHTTPRequestFactory(d ...HTTPRequestDecorator) *HTTPRequestFactory { } } -func (self *HTTPRequestFactory) AddDecorator(d... HTTPRequestDecorator) { +func (self *HTTPRequestFactory) AddDecorator(d ...HTTPRequestDecorator) { self.decorators = append(self.decorators, d...) } diff --git a/components/engine/utils/jsonmessage.go b/components/engine/utils/jsonmessage.go index cc467162f9..3e4e0f86ad 100644 --- a/components/engine/utils/jsonmessage.go +++ b/components/engine/utils/jsonmessage.go @@ -3,7 +3,7 @@ package utils import ( "encoding/json" "fmt" - "github.com/dotcloud/docker/term" + "github.com/dotcloud/docker/pkg/term" "io" "strings" "time" @@ -52,7 +52,7 @@ func (p *JSONProgress) String() string { } numbersBox = fmt.Sprintf("%8v/%v", current, total) - if p.Start > 0 { + if p.Start > 0 && percentage < 50 { fromStart := time.Now().UTC().Sub(time.Unix(int64(p.Start), 0)) perEntry := fromStart / time.Duration(p.Current) left := time.Duration(p.Total-p.Current) * perEntry diff --git a/components/engine/utils/progressreader.go b/components/engine/utils/progressreader.go index 81f076682e..a43ee55b0f 100644 --- a/components/engine/utils/progressreader.go +++ b/components/engine/utils/progressreader.go @@ -7,21 +7,18 @@ import ( // Reader with progress bar type progressReader struct { - reader io.ReadCloser // Stream to read from - output io.Writer // Where to send progress bar to - progress JSONProgress - // readTotal int // Expected stream length (bytes) - // readProgress int // How much has been read so far (bytes) + reader io.ReadCloser // Stream to read from + output io.Writer // Where to send progress bar to + progress JSONProgress lastUpdate int // How many bytes read at least update ID string action string - // template string // Template to print. Default "%v/%v (%v)" - sf *StreamFormatter - newLine bool + sf *StreamFormatter + newLine bool } func (r *progressReader) Read(p []byte) (n int, err error) { - read, err := io.ReadCloser(r.reader).Read(p) + read, err := r.reader.Read(p) r.progress.Current += read updateEvery := 1024 * 512 //512kB if r.progress.Total > 0 { @@ -41,7 +38,9 @@ func (r *progressReader) Read(p []byte) (n int, err error) { return read, err } func (r *progressReader) Close() error { - return io.ReadCloser(r.reader).Close() + r.progress.Current = r.progress.Total + r.output.Write(r.sf.FormatProgress(r.ID, r.action, &r.progress)) + return r.reader.Close() } func ProgressReader(r io.ReadCloser, size int, output io.Writer, sf *StreamFormatter, newline bool, ID, action string) *progressReader { return &progressReader{ diff --git a/components/engine/utils/tarsum.go b/components/engine/utils/tarsum.go index 9cba516a34..7de3dbe5be 100644 --- a/components/engine/utils/tarsum.go +++ b/components/engine/utils/tarsum.go @@ -1,38 +1,30 @@ package utils import ( + "archive/tar" "bytes" "compress/gzip" "crypto/sha256" "encoding/hex" - "archive/tar" "hash" "io" "sort" "strconv" + "strings" ) -type verboseHash struct { - hash.Hash -} - -func (h verboseHash) Write(buf []byte) (int, error) { - Debugf("--->%s<---", buf) - return h.Hash.Write(buf) -} - type TarSum struct { io.Reader - tarR *tar.Reader - tarW *tar.Writer - gz *gzip.Writer - bufTar *bytes.Buffer - bufGz *bytes.Buffer - h hash.Hash - h2 verboseHash - sums []string - finished bool - first bool + tarR *tar.Reader + tarW *tar.Writer + gz *gzip.Writer + bufTar *bytes.Buffer + bufGz *bytes.Buffer + h hash.Hash + sums map[string]string + currentFile string + finished bool + first bool } func (ts *TarSum) encodeHeader(h *tar.Header) error { @@ -52,7 +44,6 @@ func (ts *TarSum) encodeHeader(h *tar.Header) error { // {"atime", strconv.Itoa(int(h.AccessTime.UTC().Unix()))}, // {"ctime", strconv.Itoa(int(h.ChangeTime.UTC().Unix()))}, } { - // Debugf("-->%s<-- -->%s<--", elem[0], elem[1]) if _, err := ts.h.Write([]byte(elem[0] + elem[1])); err != nil { return err } @@ -68,9 +59,9 @@ func (ts *TarSum) Read(buf []byte) (int, error) { ts.tarW = tar.NewWriter(ts.bufTar) ts.gz = gzip.NewWriter(ts.bufGz) ts.h = sha256.New() - // ts.h = verboseHash{sha256.New()} ts.h.Reset() ts.first = true + ts.sums = make(map[string]string) } if ts.finished { @@ -85,7 +76,7 @@ func (ts *TarSum) Read(buf []byte) (int, error) { return 0, err } if !ts.first { - ts.sums = append(ts.sums, hex.EncodeToString(ts.h.Sum(nil))) + ts.sums[ts.currentFile] = hex.EncodeToString(ts.h.Sum(nil)) ts.h.Reset() } else { ts.first = false @@ -102,6 +93,7 @@ func (ts *TarSum) Read(buf []byte) (int, error) { } return n, err } + ts.currentFile = strings.TrimSuffix(strings.TrimPrefix(currentHeader.Name, "./"), "/") if err := ts.encodeHeader(currentHeader); err != nil { return 0, err } @@ -143,12 +135,17 @@ func (ts *TarSum) Read(buf []byte) (int, error) { } func (ts *TarSum) Sum(extra []byte) string { - sort.Strings(ts.sums) + var sums []string + + for _, sum := range ts.sums { + sums = append(sums, sum) + } + sort.Strings(sums) h := sha256.New() if extra != nil { h.Write(extra) } - for _, sum := range ts.sums { + for _, sum := range sums { Debugf("-->%s<--", sum) h.Write([]byte(sum)) } @@ -156,3 +153,7 @@ func (ts *TarSum) Sum(extra []byte) string { Debugf("checksum processed: %s", checksum) return checksum } + +func (ts *TarSum) GetSums() map[string]string { + return ts.sums +} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 6864dddf82..ffe90ac734 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -26,6 +26,7 @@ import ( var ( IAMSTATIC bool // whether or not Docker itself was compiled statically via ./hack/make.sh binary INITSHA1 string // sha1sum of separate static dockerinit, if Docker itself was compiled dynamically via ./hack/make.sh dynbinary + INITPATH string // custom location to search for a valid dockerinit binary (available for packagers as a last resort escape hatch) ) // A common interface to access the Fatal method of @@ -162,14 +163,23 @@ func Trunc(s string, maxlen int) string { return s[:maxlen] } -// Figure out the absolute path of our own binary +// Figure out the absolute path of our own binary (if it's still around). func SelfPath() string { path, err := exec.LookPath(os.Args[0]) if err != nil { + if os.IsNotExist(err) { + return "" + } + if execErr, ok := err.(*exec.Error); ok && os.IsNotExist(execErr.Err) { + return "" + } panic(err) } path, err = filepath.Abs(path) if err != nil { + if os.IsNotExist(err) { + return "" + } panic(err) } return path @@ -190,7 +200,13 @@ func dockerInitSha1(target string) string { } func isValidDockerInitPath(target string, selfPath string) bool { // target and selfPath should be absolute (InitPath and SelfPath already do this) + if target == "" { + return false + } if IAMSTATIC { + if selfPath == "" { + return false + } if target == selfPath { return true } @@ -216,6 +232,7 @@ func DockerInitPath(localCopy string) string { } var possibleInits = []string{ localCopy, + INITPATH, filepath.Join(filepath.Dir(selfPath), "dockerinit"), // FHS 3.0 Draft: "/usr/libexec includes internal binaries that are not intended to be executed directly by users or shell scripts. Applications may use a single subdirectory under /usr/libexec." @@ -229,6 +246,9 @@ func DockerInitPath(localCopy string) string { "/usr/local/lib/docker/dockerinit", } for _, dockerInit := range possibleInits { + if dockerInit == "" { + continue + } path, err := exec.LookPath(dockerInit) if err == nil { path, err = filepath.Abs(path) @@ -776,11 +796,21 @@ func GetNameserversAsCIDR(resolvConf []byte) []string { return nameservers } -func ParseHost(host string, port int, addr string) (string, error) { - var proto string +// FIXME: Change this not to receive default value as parameter +func ParseHost(defaultHost string, defaultPort int, defaultUnix, addr string) (string, error) { + var ( + proto string + host string + port int + ) + switch { case strings.HasPrefix(addr, "unix://"): - return addr, nil + proto = "unix" + addr = strings.TrimPrefix(addr, "unix://") + if addr == "" { + addr = defaultUnix + } case strings.HasPrefix(addr, "tcp://"): proto = "tcp" addr = strings.TrimPrefix(addr, "tcp://") @@ -791,19 +821,29 @@ func ParseHost(host string, port int, addr string) (string, error) { proto = "tcp" } - if strings.Contains(addr, ":") { + if proto != "unix" && strings.Contains(addr, ":") { hostParts := strings.Split(addr, ":") if len(hostParts) != 2 { return "", fmt.Errorf("Invalid bind address format: %s", addr) } if hostParts[0] != "" { host = hostParts[0] + } else { + host = defaultHost } - if p, err := strconv.Atoi(hostParts[1]); err == nil { + + if p, err := strconv.Atoi(hostParts[1]); err == nil && p != 0 { port = p + } else { + port = defaultPort } + } else { host = addr + port = defaultPort + } + if proto == "unix" { + return fmt.Sprintf("%s://%s", proto, host), nil } return fmt.Sprintf("%s://%s:%d", proto, host, port), nil } @@ -1117,3 +1157,59 @@ func CopyFile(src, dst string) (int64, error) { defer df.Close() return io.Copy(df, sf) } + +// Returns the relative path to the cgroup docker is running in. +func GetThisCgroup(cgroupType string) (string, error) { + output, err := ioutil.ReadFile("/proc/self/cgroup") + if err != nil { + return "", err + } + for _, line := range strings.Split(string(output), "\n") { + parts := strings.Split(line, ":") + // any type used by docker should work + if parts[1] == cgroupType { + return parts[2], nil + } + } + return "", fmt.Errorf("cgroup '%s' not found in /proc/self/cgroup", cgroupType) +} + +// Returns a list of pids for the given container. +func GetPidsForContainer(id string) ([]int, error) { + pids := []int{} + + // memory is chosen randomly, any cgroup used by docker works + cgroupType := "memory" + + cgroupRoot, err := FindCgroupMountpoint(cgroupType) + if err != nil { + return pids, err + } + + cgroupThis, err := GetThisCgroup(cgroupType) + if err != nil { + return pids, err + } + + filename := filepath.Join(cgroupRoot, cgroupThis, id, "tasks") + if _, err := os.Stat(filename); os.IsNotExist(err) { + // With more recent lxc versions use, cgroup will be in lxc/ + filename = filepath.Join(cgroupRoot, cgroupThis, "lxc", id, "tasks") + } + + output, err := ioutil.ReadFile(filename) + if err != nil { + return pids, err + } + for _, p := range strings.Split(string(output), "\n") { + if len(p) == 0 { + continue + } + pid, err := strconv.Atoi(p) + if err != nil { + return pids, fmt.Errorf("Invalid pid '%s': %s", p, err) + } + pids = append(pids, pid) + } + return pids, nil +} diff --git a/components/engine/utils/utils_test.go b/components/engine/utils/utils_test.go index 350ebb911f..b31938d9b5 100644 --- a/components/engine/utils/utils_test.go +++ b/components/engine/utils/utils_test.go @@ -299,22 +299,33 @@ func assertRAMInBytes(t *testing.T, size string, expectError bool, expectedBytes } func TestParseHost(t *testing.T) { - if addr, err := ParseHost("127.0.0.1", 4243, "0.0.0.0"); err != nil || addr != "tcp://0.0.0.0:4243" { + var ( + defaultHttpHost = "127.0.0.1" + defaultHttpPort = 4243 + defaultUnix = "/var/run/docker.sock" + ) + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "0.0.0.0"); err != nil || addr != "tcp://0.0.0.0:4243" { t.Errorf("0.0.0.0 -> expected tcp://0.0.0.0:4243, got %s", addr) } - if addr, err := ParseHost("127.0.0.1", 4243, "0.0.0.1:5555"); err != nil || addr != "tcp://0.0.0.1:5555" { + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "0.0.0.1:5555"); err != nil || addr != "tcp://0.0.0.1:5555" { t.Errorf("0.0.0.1:5555 -> expected tcp://0.0.0.1:5555, got %s", addr) } - if addr, err := ParseHost("127.0.0.1", 4243, ":6666"); err != nil || addr != "tcp://127.0.0.1:6666" { + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, ":6666"); err != nil || addr != "tcp://127.0.0.1:6666" { t.Errorf(":6666 -> expected tcp://127.0.0.1:6666, got %s", addr) } - if addr, err := ParseHost("127.0.0.1", 4243, "tcp://:7777"); err != nil || addr != "tcp://127.0.0.1:7777" { + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "tcp://:7777"); err != nil || addr != "tcp://127.0.0.1:7777" { t.Errorf("tcp://:7777 -> expected tcp://127.0.0.1:7777, got %s", addr) } - if addr, err := ParseHost("127.0.0.1", 4243, "unix:///var/run/docker.sock"); err != nil || addr != "unix:///var/run/docker.sock" { + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "unix:///var/run/docker.sock"); err != nil || addr != "unix:///var/run/docker.sock" { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) } - if addr, err := ParseHost("127.0.0.1", 4243, "udp://127.0.0.1"); err == nil { + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "unix://"); err != nil || addr != "unix:///var/run/docker.sock" { + t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) + } + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "udp://127.0.0.1"); err == nil { + t.Errorf("udp protocol address expected error return, but err == nil. Got %s", addr) + } + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "udp://127.0.0.1:4243"); err == nil { t.Errorf("udp protocol address expected error return, but err == nil. Got %s", addr) } } diff --git a/components/engine/version.go b/components/engine/version.go index c85e77c59a..a4288245f7 100644 --- a/components/engine/version.go +++ b/components/engine/version.go @@ -25,6 +25,8 @@ func dockerVersion() *engine.Env { v.Set("Version", VERSION) v.Set("GitCommit", GITCOMMIT) v.Set("GoVersion", runtime.Version()) + v.Set("Os", runtime.GOOS) + v.Set("Arch", runtime.GOARCH) // FIXME:utils.GetKernelVersion should only be needed here if kernelVersion, err := utils.GetKernelVersion(); err == nil { v.Set("KernelVersion", kernelVersion.String())