diff --git a/components/engine/CHANGELOG.md b/components/engine/CHANGELOG.md index f7e5d9232e..b81e0d7706 100644 --- a/components/engine/CHANGELOG.md +++ b/components/engine/CHANGELOG.md @@ -1,5 +1,83 @@ # Changelog +## 0.6.4 (2013-10-16) +- Runtime: Add cleanup of container when Start() fails +- Testing: Catch errClosing error when TCP and UDP proxies are terminated +- Testing: Add aggregated docker-ci email report +- Testing: Remove a few errors in tests +* Contrib: Reorganize contributed completion scripts to add zsh completion +* Contrib: Add vim syntax highlighting for Dockerfiles from @honza +* Runtime: Add better comments to utils/stdcopy.go +- Testing: add cleanup to remove leftover containers +* Documentation: Document how to edit and release docs +* Documentation: Add initial draft of the Docker infrastructure doc +* Contrib: Add mkimage-arch.sh +- Builder: Abort build if mergeConfig returns an error and fix duplicate error message +- Runtime: Remove error messages which are not actually errors +* Testing: Only run certain tests with TESTFLAGS='-run TestName' make.sh +* Testing: Prevent docker-ci to test closing PRs +- Documentation: Minor updates to postgresql_service.rst +* Testing: Add nightly release to docker-ci +* Hack: Improve network performance for VirtualBox +* Hack: Add vagrant user to the docker group +* Runtime: Add utils.Errorf for error logging +- Packaging: Remove deprecated packaging directory +* Hack: Revamp install.sh to be usable by more people, and to use official install methods whenever possible (apt repo, portage tree, etc.) +- Hack: Fix contrib/mkimage-debian.sh apt caching prevention +* Documentation: Clarify LGTM process to contributors +- Documentation: Small fixes to parameter names in docs for ADD command +* Runtime: Record termination time in state. +- Registry: Use correct auth config when logging in. +- Documentation: Corrected error in the package name +* Documentation: Document what `vagrant up` is actually doing +- Runtime: Fix `docker rm` with volumes +- Runtime: Use empty string so TempDir uses the OS's temp dir automatically +- Runtime: Make sure to close the network allocators +* Testing: Replace panic by log.Fatal in tests ++ Documentation: improve doc search results +- Runtime: Fix some error cases where a HTTP body might not be closed +* Hack: Add proper bash completion for "docker push" +* Documentation: Add devenvironment link to CONTRIBUTING.md +* Documentation: Cleanup whitespace in API 1.5 docs +* Documentation: use angle brackets in MAINTAINER example email +- Testing: Increase TestRunDetach timeout +* Documentation: Fix help text for -v option ++ Hack: Added Dockerfile.tmLanguage to contrib ++ Runtime: Autorestart containers by default +* Testing: Adding more tests around auth.ResolveAuthConfig +* Hack: Configured FPM to make /etc/init/docker.conf a config file +* Hack: Add xz utils as a runtime dep +* Documentation: Add `apt-get install curl` to Ubuntu docs +* Documentation: Remove Gentoo install notes about #1422 workaround +* Documentation: Fix Ping endpoint documentation +* Runtime: Bump vendor kr/pty to commit 3b1f6487b (syscall.O_NOCTTY) +* Runtime: lxc: Allow set_file_cap capability in container +* Documentation: Update archlinux.rst +- Documentation: Fix ironic typo in changelog +* Documentation: Add explanation for export restrictions +* Hack: Add cleanup/refactor portion of #2010 for hack and Dockerfile updates ++ Documentation: Changes to a new style for the docs. Includes version switcher. +* Documentation: Formatting, add information about multiline json ++ Hack: Add contrib/mkimage-centos.sh back (from #1621), and associated documentation link +- Runtime: Fix panic with wrong dockercfg file +- Runtime: Fix the attach behavior with -i +* Documentation: Add .dockercfg doc +- Runtime: Move run -rm to the cli only +* Hack: Enable SSH Agent forwarding in Vagrant VM ++ Runtime: Add -rm to docker run for removing a container on exit +* Documentation: Improve registry and index REST API documentation +* Runtime: Split stdout stderr +- Documentation: Replace deprecated upgrading reference to docker-latest.tgz, which hasn't been updated since 0.5.3 +* Documentation: Update Gentoo installation documentation now that we're in the portage tree proper +- Registry: Fix the error message so it is the same as the regex +* Runtime: Always create a new session for the container +* Hack: Add several of the small make.sh fixes from #1920, and make the output more consistent and contributor-friendly +* Documentation: Various command fixes in postgres example +* Documentation: Cleanup and reorganize docs and tooling for contributors and maintainers +- Documentation: Minor spelling correction of protocoll -> protocol +* Hack: Several small tweaks/fixes for contrib/mkimage-debian.sh ++ Hack: Add @tianon to hack/MAINTAINERS + ## 0.6.3 (2013-09-23) * Packaging: Update tar vendor dependency - Client: Fix detach issue diff --git a/components/engine/CONTRIBUTING.md b/components/engine/CONTRIBUTING.md index a7d73273ae..43137c67e2 100644 --- a/components/engine/CONTRIBUTING.md +++ b/components/engine/CONTRIBUTING.md @@ -59,8 +59,10 @@ Submit unit tests for your changes. Go has a great test framework built in; use it! Take a look at existing tests for inspiration. Run the full test suite on your branch before submitting a pull request. -Make sure you include relevant updates or additions to documentation when -creating or modifying features. +Update the documentation when creating or modifying features. Test +your documentation changes for clarity, concision, and correctness, as +well as a clean docmuent build. See ``docs/README.md`` for more +information on building the docs and how docs get released. Write clean code. Universally formatted code promotes ease of writing, reading, and maintenance. Always run `go fmt` before committing your changes. Most diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index cf0a04b8ba..e189968eb1 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -12,7 +12,7 @@ # # # # Run the test suite: -# docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker go test -v +# docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker hack/make.sh test # # # Publish a release: # docker run -privileged -lxc-conf=lxc.aa_profile=unconfined \ diff --git a/components/engine/VERSION b/components/engine/VERSION index fe50cf96de..ba6a53c9ac 100644 --- a/components/engine/VERSION +++ b/components/engine/VERSION @@ -1 +1 @@ -0.6.3-dev +0.6.4-dev diff --git a/components/engine/Vagrantfile b/components/engine/Vagrantfile index 6b3ae37c62..93a2219fa0 100644 --- a/components/engine/Vagrantfile +++ b/components/engine/Vagrantfile @@ -80,6 +80,8 @@ Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| config.vm.provider :virtualbox do |vb| config.vm.box = BOX_NAME config.vm.box_url = BOX_URI + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"] end end diff --git a/components/engine/api.go b/components/engine/api.go index 8a958edb1d..fc4a853789 100644 --- a/components/engine/api.go +++ b/components/engine/api.go @@ -349,7 +349,7 @@ func postCommit(srv *Server, version float64, w http.ResponseWriter, r *http.Req return err } config := &Config{} - if err := json.NewDecoder(r.Body).Decode(config); err != nil { + if err := json.NewDecoder(r.Body).Decode(config); err != nil && err != io.EOF { utils.Errorf("%s", err) } repo := r.Form.Get("repo") @@ -909,8 +909,7 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput, !noCache, rm) id, err := b.Build(context) if err != nil { - fmt.Fprintf(w, "Error build: %s\n", err) - return err + return fmt.Errorf("Error build: %s", err) } if repoName != "" { srv.runtime.repositories.Set(repoName, tag, id, false) diff --git a/components/engine/api_test.go b/components/engine/api_test.go index 6ac9421ea9..8bef4e6785 100644 --- a/components/engine/api_test.go +++ b/components/engine/api_test.go @@ -5,6 +5,7 @@ import ( "bufio" "bytes" "encoding/json" + "fmt" "github.com/dotcloud/docker/utils" "io" "net" @@ -12,6 +13,7 @@ import ( "net/http/httptest" "os" "path" + "strings" "testing" "time" ) @@ -40,6 +42,25 @@ func TestGetBoolParam(t *testing.T) { } } +func TesthttpError(t *testing.T) { + r := httptest.NewRecorder() + + httpError(r, fmt.Errorf("No such method")) + if r.Code != http.StatusNotFound { + t.Fatalf("Expected %d, got %d", http.StatusNotFound, r.Code) + } + + httpError(r, fmt.Errorf("This accound hasn't been activated")) + if r.Code != http.StatusForbidden { + t.Fatalf("Expected %d, got %d", http.StatusForbidden, r.Code) + } + + httpError(r, fmt.Errorf("Some error")) + if r.Code != http.StatusInternalServerError { + t.Fatalf("Expected %d, got %d", http.StatusInternalServerError, r.Code) + } +} + func TestGetVersion(t *testing.T) { var err error runtime := mkRuntime(t) @@ -244,7 +265,11 @@ func TestGetImagesJSON(t *testing.T) { t.Fatalf("Error expected, received none") } - httpError(r4, err) + if !strings.HasPrefix(err.Error(), "Bad parameter") { + t.Fatalf("Error should starts with \"Bad parameter\"") + } + http.Error(r4, err.Error(), http.StatusBadRequest) + if r4.Code != http.StatusBadRequest { t.Fatalf("%d Bad Request expected, received %d\n", http.StatusBadRequest, r4.Code) } @@ -784,6 +809,8 @@ func TestPostContainersStart(t *testing.T) { t.Fatal(err) } + req.Header.Set("Content-Type", "application/json") + r := httptest.NewRecorder() if err := postContainersStart(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil { t.Fatal(err) diff --git a/components/engine/commands_test.go b/components/engine/commands_test.go index 2da1f8e0ca..aceb7a6577 100644 --- a/components/engine/commands_test.go +++ b/components/engine/commands_test.go @@ -545,6 +545,7 @@ func TestAttachDisconnect(t *testing.T) { // Expected behaviour: container gets deleted automatically after exit func TestRunAutoRemove(t *testing.T) { + t.Skip("Fixme. Skipping test for now, race condition") stdout, stdoutPipe := io.Pipe() cli := NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) defer cleanup(globalRuntime) diff --git a/components/engine/container.go b/components/engine/container.go index f499559f8e..7d1cb79495 100644 --- a/components/engine/container.go +++ b/components/engine/container.go @@ -396,9 +396,9 @@ func (container *Container) startPty() error { // Copy the PTYs to our broadcasters go func() { defer container.stdout.CloseWriters() - utils.Debugf("[startPty] Begin of stdout pipe") + utils.Debugf("startPty: begin of stdout pipe") io.Copy(container.stdout, ptyMaster) - utils.Debugf("[startPty] End of stdout pipe") + utils.Debugf("startPty: end of stdout pipe") }() // stdin @@ -407,9 +407,9 @@ func (container *Container) startPty() error { container.cmd.SysProcAttr.Setctty = true go func() { defer container.stdin.Close() - utils.Debugf("[startPty] Begin of stdin pipe") + utils.Debugf("startPty: begin of stdin pipe") io.Copy(ptyMaster, container.stdin) - utils.Debugf("[startPty] End of stdin pipe") + utils.Debugf("startPty: end of stdin pipe") }() } if err := container.cmd.Start(); err != nil { @@ -429,9 +429,9 @@ func (container *Container) start() error { } go func() { defer stdin.Close() - utils.Debugf("Begin of stdin pipe [start]") + utils.Debugf("start: begin of stdin pipe") io.Copy(stdin, container.stdin) - utils.Debugf("End of stdin pipe [start]") + utils.Debugf("start: end of stdin pipe") }() } return container.cmd.Start() @@ -448,8 +448,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s errors <- err } else { go func() { - utils.Debugf("[start] attach stdin\n") - defer utils.Debugf("[end] attach stdin\n") + utils.Debugf("attach: stdin: begin") + defer utils.Debugf("attach: stdin: end") // No matter what, when stdin is closed (io.Copy unblock), close stdout and stderr if container.Config.StdinOnce && !container.Config.Tty { defer cStdin.Close() @@ -467,7 +467,7 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s _, err = io.Copy(cStdin, stdin) } if err != nil { - utils.Errorf("[error] attach stdin: %s\n", err) + utils.Errorf("attach: stdin: %s", err) } // Discard error, expecting pipe error errors <- nil @@ -481,8 +481,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s } else { cStdout = p go func() { - utils.Debugf("[start] attach stdout\n") - defer utils.Debugf("[end] attach stdout\n") + utils.Debugf("attach: stdout: begin") + defer utils.Debugf("attach: stdout: end") // If we are in StdinOnce mode, then close stdin if container.Config.StdinOnce && stdin != nil { defer stdin.Close() @@ -491,8 +491,11 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s defer stdinCloser.Close() } _, err := io.Copy(stdout, cStdout) + if err == io.ErrClosedPipe { + err = nil + } if err != nil { - utils.Errorf("[error] attach stdout: %s\n", err) + utils.Errorf("attach: stdout: %s", err) } errors <- err }() @@ -502,9 +505,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s if stdinCloser != nil { defer stdinCloser.Close() } - if cStdout, err := container.StdoutPipe(); err != nil { - utils.Errorf("Error stdout pipe") + utils.Errorf("attach: stdout pipe: %s", err) } else { io.Copy(&utils.NopWriter{}, cStdout) } @@ -517,8 +519,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s } else { cStderr = p go func() { - utils.Debugf("[start] attach stderr\n") - defer utils.Debugf("[end] attach stderr\n") + utils.Debugf("attach: stderr: begin") + defer utils.Debugf("attach: stderr: end") // If we are in StdinOnce mode, then close stdin if container.Config.StdinOnce && stdin != nil { defer stdin.Close() @@ -527,8 +529,11 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s defer stdinCloser.Close() } _, err := io.Copy(stderr, cStderr) + if err == io.ErrClosedPipe { + err = nil + } if err != nil { - utils.Errorf("[error] attach stderr: %s\n", err) + utils.Errorf("attach: stderr: %s", err) } errors <- err }() @@ -540,7 +545,7 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s } if cStderr, err := container.StderrPipe(); err != nil { - utils.Errorf("Error stdout pipe") + utils.Errorf("attach: stdout pipe: %s", err) } else { io.Copy(&utils.NopWriter{}, cStderr) } @@ -554,24 +559,29 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s if cStderr != nil { defer cStderr.Close() } - // FIXME: how do clean up the stdin goroutine without the unwanted side effect + // FIXME: how to clean up the stdin goroutine without the unwanted side effect // of closing the passed stdin? Add an intermediary io.Pipe? for i := 0; i < nJobs; i += 1 { - utils.Debugf("Waiting for job %d/%d\n", i+1, nJobs) + utils.Debugf("attach: waiting for job %d/%d", i+1, nJobs) if err := <-errors; err != nil { - utils.Errorf("Job %d returned error %s. Aborting all jobs\n", i+1, err) + utils.Errorf("attach: job %d returned error %s, aborting all jobs", i+1, err) return err } - utils.Debugf("Job %d completed successfully\n", i+1) + utils.Debugf("attach: job %d completed successfully", i+1) } - utils.Debugf("All jobs completed successfully\n") + utils.Debugf("attach: all jobs completed successfully") return nil }) } -func (container *Container) Start(hostConfig *HostConfig) error { +func (container *Container) Start(hostConfig *HostConfig) (err error) { container.State.Lock() defer container.State.Unlock() + defer func() { + if err != nil { + container.cleanup() + } + }() if hostConfig == nil { // in docker start of docker restart we want to reuse previous HostConfigFile hostConfig, _ = container.ReadHostConfig() @@ -824,7 +834,6 @@ func (container *Container) Start(hostConfig *HostConfig) error { container.cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} - var err error if container.Config.Tty { err = container.startPty() } else { @@ -870,9 +879,14 @@ func (container *Container) Output() (output []byte, err error) { return output, err } -// StdinPipe() returns a pipe connected to the standard input of the container's -// active process. -// +// Container.StdinPipe returns a WriteCloser which can be used to feed data +// to the standard input of the container's active process. +// Container.StdoutPipe and Container.StderrPipe each return a ReadCloser +// which can be used to retrieve the standard output (and error) generated +// by the container's active process. The output (and error) are actually +// copied and delivered to all StdoutPipe and StderrPipe consumers, using +// a kind of "broadcaster". + func (container *Container) StdinPipe() (io.WriteCloser, error) { return container.stdinPipe, nil } @@ -950,7 +964,7 @@ func (container *Container) allocateNetwork() error { } func (container *Container) releaseNetwork() { - if container.Config.NetworkDisabled { + if container.Config.NetworkDisabled || container.network == nil { return } container.network.Release() @@ -974,21 +988,24 @@ func (container *Container) waitLxc() error { func (container *Container) monitor(hostConfig *HostConfig) { // Wait for the program to exit - utils.Debugf("Waiting for process") - // If the command does not exists, try to wait via lxc + // If the command does not exist, try to wait via lxc + // (This probably happens only for ghost containers, i.e. containers that were running when Docker started) if container.cmd == nil { + utils.Debugf("monitor: waiting for container %s using waitLxc", container.ID) if err := container.waitLxc(); err != nil { // Discard the error as any signals or non 0 returns will generate an error - utils.Debugf("%s: Process: %s", container.ShortID(), err) + utils.Debugf("monitor: while waiting for container %s, waitLxc had a problem: %s", container.ShortID(), err) } } else { + utils.Debugf("monitor: waiting for container %s using cmd.Wait", container.ID) if err := container.cmd.Wait(); err != nil { - // Discard the error as any signals or non 0 returns will generate an error - utils.Debugf("%s: Process: %s", container.ShortID(), err) + // Since non-zero exit status and signal terminations will cause err to be non-nil, + // we have to actually discard it. Still, log it anyway, just in case. + utils.Debugf("monitor: cmd.Wait reported exit status %s for container %s", err, container.ID) } } - utils.Debugf("Process finished") + utils.Debugf("monitor: container %s finished", container.ID) exitCode := -1 if container.cmd != nil { @@ -1003,6 +1020,28 @@ func (container *Container) monitor(hostConfig *HostConfig) { } // Cleanup + container.cleanup() + + // Re-create a brand new stdin pipe once the container exited + if container.Config.OpenStdin { + container.stdin, container.stdinPipe = io.Pipe() + } + + // Release the lock + close(container.waitLock) + + if err := container.ToDisk(); err != nil { + // FIXME: there is a race condition here which causes this to fail during the unit tests. + // If another goroutine was waiting for Wait() to return before removing the container's root + // from the filesystem... At this point it may already have done so. + // This is because State.setStopped() has already been called, and has caused Wait() + // to return. + // FIXME: why are we serializing running state to disk in the first place? + //log.Printf("%s: Failed to dump configuration to the disk: %s", container.ID, err) + } +} + +func (container *Container) cleanup() { container.releaseNetwork() if container.Config.OpenStdin { if err := container.stdin.Close(); err != nil { @@ -1025,24 +1064,6 @@ func (container *Container) monitor(hostConfig *HostConfig) { if err := container.Unmount(); err != nil { log.Printf("%v: Failed to umount filesystem: %v", container.ID, err) } - - // Re-create a brand new stdin pipe once the container exited - if container.Config.OpenStdin { - container.stdin, container.stdinPipe = io.Pipe() - } - - // Release the lock - close(container.waitLock) - - if err := container.ToDisk(); err != nil { - // FIXME: there is a race condition here which causes this to fail during the unit tests. - // If another goroutine was waiting for Wait() to return before removing the container's root - // from the filesystem... At this point it may already have done so. - // This is because State.setStopped() has already been called, and has caused Wait() - // to return. - // FIXME: why are we serializing running state to disk in the first place? - //log.Printf("%s: Failed to dump configuration to the disk: %s", container.ID, err) - } } func (container *Container) kill() error { diff --git a/components/engine/contrib/MAINTAINERS b/components/engine/contrib/MAINTAINERS index a710312a05..18e05a3070 100644 --- a/components/engine/contrib/MAINTAINERS +++ b/components/engine/contrib/MAINTAINERS @@ -1,2 +1 @@ Tianon Gravi (@tianon) -Kawsar Saiyeed (@KSid) diff --git a/components/engine/contrib/docker.bash b/components/engine/contrib/completion/bash/docker old mode 100644 new mode 100755 similarity index 100% rename from components/engine/contrib/docker.bash rename to components/engine/contrib/completion/bash/docker diff --git a/components/engine/contrib/completion/zsh/_docker b/components/engine/contrib/completion/zsh/_docker new file mode 100755 index 0000000000..92acdb13dd --- /dev/null +++ b/components/engine/contrib/completion/zsh/_docker @@ -0,0 +1,242 @@ +#compdef docker +# +# zsh completion for docker (http://docker.io) +# +# version: 0.2.2 +# author: Felix Riedel +# license: BSD License +# github: https://github.com/felixr/docker-zsh-completion +# + +__parse_docker_list() { + sed -e '/^ID/d' -e 's/[ ]\{2,\}/|/g' -e 's/ \([hdwm]\)\(inutes\|ays\|ours\|eeks\)/\1/' | awk ' BEGIN {FS="|"} { printf("%s:%7s, %s\n", $1, $4, $2)}' +} + +__docker_stoppedcontainers() { + local expl + declare -a stoppedcontainers + stoppedcontainers=(${(f)"$(docker ps -a | grep --color=never 'Exit' | __parse_docker_list )"}) + _describe -t containers-stopped "Stopped Containers" stoppedcontainers +} + +__docker_runningcontainers() { + local expl + declare -a containers + + containers=(${(f)"$(docker ps | __parse_docker_list)"}) + _describe -t containers-active "Running Containers" containers +} + +__docker_containers () { + __docker_stoppedcontainers + __docker_runningcontainers +} + +__docker_images () { + local expl + declare -a images + images=(${(f)"$(docker images | awk '(NR > 1){printf("%s\\:%s\n", $1,$2)}')"}) + images=($images ${(f)"$(docker images | awk '(NR > 1){printf("%s:%-15s in %s\n", $3,$2,$1)}')"}) + _describe -t docker-images "Images" images +} + +__docker_tags() { + local expl + declare -a tags + tags=(${(f)"$(docker images | awk '(NR>1){print $2}'| sort | uniq)"}) + _describe -t docker-tags "tags" tags +} + +__docker_search() { + # declare -a dockersearch + local cache_policy + zstyle -s ":completion:${curcontext}:" cache-policy cache_policy + if [[ -z "$cache_policy" ]]; then + zstyle ":completion:${curcontext}:" cache-policy __docker_caching_policy + fi + + local searchterm cachename + searchterm="${words[$CURRENT]%/}" + cachename=_docker-search-$searchterm + + local expl + local -a result + if ( [[ ${(P)+cachename} -eq 0 ]] || _cache_invalid ${cachename#_} ) \ + && ! _retrieve_cache ${cachename#_}; then + _message "Searching for ${searchterm}..." + result=(${(f)"$(docker search ${searchterm} | awk '(NR>2){print $1}')"}) + _store_cache ${cachename#_} result + fi + _wanted dockersearch expl 'Available images' compadd -a result +} + +__docker_caching_policy() +{ + # oldp=( "$1"(Nmh+24) ) # 24 hour + oldp=( "$1"(Nmh+1) ) # 24 hour + (( $#oldp )) +} + + +__docker_repositories () { + local expl + declare -a repos + repos=(${(f)"$(docker images | sed -e '1d' -e 's/[ ].*//' | sort | uniq)"}) + _describe -t docker-repos "Repositories" repos +} + +__docker_commands () { + # local -a _docker_subcommands + local cache_policy + + zstyle -s ":completion:${curcontext}:" cache-policy cache_policy + if [[ -z "$cache_policy" ]]; then + zstyle ":completion:${curcontext}:" cache-policy __docker_caching_policy + fi + + if ( [[ ${+_docker_subcommands} -eq 0 ]] || _cache_invalid docker_subcommands) \ + && ! _retrieve_cache docker_subcommands; + then + _docker_subcommands=(${${(f)"$(_call_program commands + docker 2>&1 | sed -e '1,6d' -e '/^[ ]*$/d' -e 's/[ ]*\([^ ]\+\)\s*\([^ ].*\)/\1:\2/' )"}}) + _docker_subcommands=($_docker_subcommands 'help:Show help for a command') + _store_cache docker_subcommands _docker_subcommands + fi + _describe -t docker-commands "docker command" _docker_subcommands +} + +__docker_subcommand () { + local -a _command_args + case "$words[1]" in + (attach|wait) + _arguments ':containers:__docker_runningcontainers' + ;; + (build) + _arguments \ + '-t=-:repository:__docker_repositories' \ + ':path or URL:_directories' + ;; + (commit) + _arguments \ + ':container:__docker_containers' \ + ':repository:__docker_repositories' \ + ':tag: ' + ;; + (diff|export|logs) + _arguments '*:containers:__docker_containers' + ;; + (history) + _arguments '*:images:__docker_images' + ;; + (images) + _arguments \ + '-a[Show all images]' \ + ':repository:__docker_repositories' + ;; + (inspect) + _arguments '*:containers:__docker_containers' + ;; + (history) + _arguments ':images:__docker_images' + ;; + (insert) + _arguments '1:containers:__docker_containers' \ + '2:URL:(http:// file://)' \ + '3:file:_files' + ;; + (kill) + _arguments '*:containers:__docker_runningcontainers' + ;; + (port) + _arguments '1:containers:__docker_runningcontainers' + ;; + (start) + _arguments '*:containers:__docker_stoppedcontainers' + ;; + (rm) + _arguments '-v[Remove the volumes associated to the container]' \ + '*:containers:__docker_stoppedcontainers' + ;; + (rmi) + _arguments '-v[Remove the volumes associated to the container]' \ + '*:images:__docker_images' + ;; + (top) + _arguments '1:containers:__docker_runningcontainers' + ;; + (restart|stop) + _arguments '-t=-[Number of seconds to try to stop for before killing the container]:seconds to before killing:(1 5 10 30 60)' \ + '*:containers:__docker_runningcontainers' + ;; + (top) + _arguments ':containers:__docker_runningcontainers' + ;; + (ps) + _arguments '-a[Show all containers. Only running containers are shown by default]' \ + '-h[Show help]' \ + '-beforeId=-[Show only container created before Id, include non-running one]:containers:__docker_containers' \ + '-n=-[Show n last created containers, include non-running one]:n:(1 5 10 25 50)' + ;; + (tag) + _arguments \ + '-f[force]'\ + ':image:__docker_images'\ + ':repository:__docker_repositories' \ + ':tag:__docker_tags' + ;; + (run) + _arguments \ + '-a=-[Attach to stdin, stdout or stderr]:toggle:(true false)' \ + '-c=-[CPU shares (relative weight)]:CPU shares: ' \ + '-d[Detached mode: leave the container running in the background]' \ + '*-dns=[Set custom dns servers]:dns server: ' \ + '*-e=[Set environment variables]:environment variable: ' \ + '-entrypoint=-[Overwrite the default entrypoint of the image]:entry point: ' \ + '-h=-[Container host name]:hostname:_hosts' \ + '-i[Keep stdin open even if not attached]' \ + '-m=-[Memory limit (in bytes)]:limit: ' \ + '*-p=-[Expose a container''s port to the host]:port:_ports' \ + '-t=-[Allocate a pseudo-tty]:toggle:(true false)' \ + '-u=-[Username or UID]:user:_users' \ + '*-v=-[Bind mount a volume (e.g. from the host: -v /host:/container, from docker: -v /container)]:volume: '\ + '-volumes-from=-[Mount volumes from the specified container]:volume: ' \ + '(-):images:__docker_images' \ + '(-):command: _command_names -e' \ + '*::arguments: _normal' + ;; + (pull|search) + _arguments ':name:__docker_search' + ;; + (help) + _arguments ':subcommand:__docker_commands' + ;; + (*) + _message 'Unknown sub command' + esac + +} + +_docker () { + local curcontext="$curcontext" state line + typeset -A opt_args + + _arguments -C \ + '-H=-[tcp://host:port to bind/connect to]:socket: ' \ + '(-): :->command' \ + '(-)*:: :->option-or-argument' + + if (( CURRENT == 1 )); then + + fi + case $state in + (command) + __docker_commands + ;; + (option-or-argument) + curcontext=${curcontext%:*:*}:docker-$words[1]: + __docker_subcommand + ;; + esac +} + +_docker "$@" diff --git a/components/engine/contrib/mkimage-arch.sh b/components/engine/contrib/mkimage-arch.sh new file mode 100755 index 0000000000..3b86022206 --- /dev/null +++ b/components/engine/contrib/mkimage-arch.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Generate a minimal filesystem for archlinux and load it into the local +# docker as "archlinux" +# requires root +set -e + +PACSTRAP=$(which pacstrap) +[ "$PACSTRAP" ] || { + echo "Could not find pacstrap. Run pacman -S arch-install-scripts" + exit 1 +} +EXPECT=$(which expect) +[ "$EXPECT" ] || { + echo "Could not find expect. Run pacman -S expect" + exit 1 +} + +ROOTFS=~/rootfs-arch-$$-$RANDOM +mkdir $ROOTFS + +#packages to ignore for space savings +PKGIGNORE=linux,jfsutils,lvm2,cryptsetup,groff,man-db,man-pages,mdadm,pciutils,pcmciautils,reiserfsprogs,s-nail,xfsprogs + +expect < $ROOTFS/etc/locale.gen < /etc/pacman.d/mirrorlist' + +# udev doesn't work in containers, rebuild /dev +DEV=${ROOTFS}/dev +mv ${DEV} ${DEV}.old +mkdir -p ${DEV} +mknod -m 666 ${DEV}/null c 1 3 +mknod -m 666 ${DEV}/zero c 1 5 +mknod -m 666 ${DEV}/random c 1 8 +mknod -m 666 ${DEV}/urandom c 1 9 +mkdir -m 755 ${DEV}/pts +mkdir -m 1777 ${DEV}/shm +mknod -m 666 ${DEV}/tty c 5 0 +mknod -m 600 ${DEV}/console c 5 1 +mknod -m 666 ${DEV}/tty0 c 4 0 +mknod -m 666 ${DEV}/full c 1 7 +mknod -m 600 ${DEV}/initctl p +mknod -m 666 ${DEV}/ptmx c 5 2 + +tar -C $ROOTFS -c . | docker import - archlinux +docker run -i -t archlinux echo Success. +rm -rf $ROOTFS diff --git a/components/engine/contrib/vim-syntax/LICENSE b/components/engine/contrib/vim-syntax/LICENSE new file mode 100644 index 0000000000..e67cdabd22 --- /dev/null +++ b/components/engine/contrib/vim-syntax/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2013 Honza Pokorny +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/components/engine/contrib/vim-syntax/README.md b/components/engine/contrib/vim-syntax/README.md new file mode 100644 index 0000000000..b78246617b --- /dev/null +++ b/components/engine/contrib/vim-syntax/README.md @@ -0,0 +1,23 @@ +dockerfile.vim +============== + +Syntax highlighting for Dockerfiles + +Installation +------------ + +Via pathogen, the usual way... + +Features +-------- + +The syntax highlighting includes: + +* The directives (e.g. `FROM`) +* Strings +* Comments + +License +------- + +BSD, short and sweet diff --git a/components/engine/contrib/vim-syntax/doc/dockerfile.txt b/components/engine/contrib/vim-syntax/doc/dockerfile.txt new file mode 100644 index 0000000000..37cc7be915 --- /dev/null +++ b/components/engine/contrib/vim-syntax/doc/dockerfile.txt @@ -0,0 +1,18 @@ +*dockerfile.txt* Syntax highlighting for Dockerfiles + +Author: Honza Pokorny +License: BSD + +INSTALLATION *installation* + +Drop it on your Pathogen path and you're all set. + +FEATURES *features* + +The syntax highlighting includes: + +* The directives (e.g. FROM) +* Strings +* Comments + + vim:tw=78:et:ft=help:norl: diff --git a/components/engine/contrib/vim-syntax/ftdetect/dockerfile.vim b/components/engine/contrib/vim-syntax/ftdetect/dockerfile.vim new file mode 100644 index 0000000000..83281d3326 --- /dev/null +++ b/components/engine/contrib/vim-syntax/ftdetect/dockerfile.vim @@ -0,0 +1 @@ +au BufNewFile,BufRead Dockerfile set filetype=dockerfile diff --git a/components/engine/contrib/vim-syntax/syntax/dockerfile.vim b/components/engine/contrib/vim-syntax/syntax/dockerfile.vim new file mode 100644 index 0000000000..90e0651b66 --- /dev/null +++ b/components/engine/contrib/vim-syntax/syntax/dockerfile.vim @@ -0,0 +1,22 @@ +" dockerfile.vim - Syntax highlighting for Dockerfiles +" Maintainer: Honza Pokorny +" Version: 0.5 + + +if exists("b:current_syntax") + finish +endif + +let b:current_syntax = "dockerfile" + +syntax case ignore + +syntax match dockerfileKeyword /\v^\s*(FROM|MAINTAINER|RUN|CMD|EXPOSE|ENV|ADD)\s/ +syntax match dockerfileKeyword /\v^\s*(ENTRYPOINT|VOLUME|USER|WORKDIR)\s/ +highlight link dockerfileKeyword Keyword + +syntax region dockerfileString start=/\v"/ skip=/\v\\./ end=/\v"/ +highlight link dockerfileString String + +syntax match dockerfileComment "\v^\s*#.*$" +highlight link dockerfileComment Comment diff --git a/components/engine/docs/README.md b/components/engine/docs/README.md index d53b8675b9..02572ee11e 100644 --- a/components/engine/docs/README.md +++ b/components/engine/docs/README.md @@ -1,38 +1,93 @@ Docker Documentation ==================== -Documentation -------------- -This is your definite place to contribute to the docker documentation. After each push to master the documentation -is automatically generated and made available on [docs.docker.io](http://docs.docker.io) +Overview +-------- -Each of the .rst files under sources reflects a page on the documentation. +The source for Docker documentation is here under ``sources/`` in the +form of .rst files. These files use +[reStructuredText](http://docutils.sourceforge.net/rst.html) +formatting with [Sphinx](http://sphinx-doc.org/) extensions for +structure, cross-linking and indexing. -Installation ------------- +The HTML files are built and hosted on +[readthedocs.org](https://readthedocs.org/projects/docker/), appearing +via proxy on https://docs.docker.io. The HTML files update +automatically after each change to the master or release branch of the +[docker files on GitHub](https://github.com/dotcloud/docker) thanks to +post-commit hooks. The "release" branch maps to the "latest" +documentation and the "master" branch maps to the "master" +documentation. + +**Warning**: The "master" documentation may include features not yet +part of any official docker release. "Master" docs should be used only +for understanding bleeding-edge development and "latest" should be +used for the latest official release. + +If you need to manually trigger a build of an existing branch, then +you can do that through the [readthedocs +interface](https://readthedocs.org/builds/docker/). If you would like +to add new build targets, including new branches or tags, then you +must contact one of the existing maintainers and get your +readthedocs.org account added to the maintainers list, or just file an +issue on GitHub describing the branch/tag and why it needs to be added +to the docs, and one of the maintainers will add it for you. + +Getting Started +--------------- + +To edit and test the docs, you'll need to install the Sphinx tool and +its dependencies. There are two main ways to install this tool: + +Native Installation +................... -* Work in your own fork of the code, we accept pull requests. * Install sphinx: `pip install sphinx` - * Mac OS X: `[sudo] pip-2.7 install sphinx`) + * Mac OS X: `[sudo] pip-2.7 install sphinx` * Install sphinx httpdomain contrib package: `pip install sphinxcontrib-httpdomain` * Mac OS X: `[sudo] pip-2.7 install sphinxcontrib-httpdomain` * If pip is not available you can probably install it using your favorite package manager as **python-pip** +Alternative Installation: Docker Container +.......................................... + +If you're running ``docker`` on your development machine then you may +find it easier and cleaner to use the Dockerfile. This installs Sphinx +in a container, adds the local ``docs/`` directory and builds the HTML +docs inside the container, even starting a simple HTTP server on port +8000 so that you can connect and see your changes. Just run ``docker +build .`` and run the resulting image. This is the equivalent to +``make clean server`` since each container starts clean. + Usage ----- -* Change the `.rst` files with your favorite editor to your liking. -* Run `make docs` to clean up old files and generate new ones. -* Your static website can now be found in the `_build` directory. -* To preview what you have generated run `make server` and open http://localhost:8000/ in your favorite browser. +* Follow the contribution guidelines (``../CONTRIBUTING.md``) +* Work in your own fork of the code, we accept pull requests. +* Change the ``.rst`` files with your favorite editor -- try to keep the + lines short and respect RST and Sphinx conventions. +* Run ``make clean docs`` to clean up old files and generate new ones, + or just ``make docs`` to update after small changes. +* Your static website can now be found in the ``_build`` directory. +* To preview what you have generated run ``make server`` and open + http://localhost:8000/ in your favorite browser. + +``make clean docs`` must complete without any warnings or errors. Working using GitHub's file editor ---------------------------------- -Alternatively, for small changes and typo's you might want to use GitHub's built in file editor. It allows -you to preview your changes right online. Just be careful not to create many commits. + +Alternatively, for small changes and typos you might want to use +GitHub's built in file editor. It allows you to preview your changes +right online (though there can be some differences between GitHub +markdown and Sphinx RST). Just be careful not to create many commits. Images ------ -When you need to add images, try to make them as small as possible (e.g. as gif). + +When you need to add images, try to make them as small as possible +(e.g. as gif). Usually images should go in the same directory as the +.rst file which references them, or in a subdirectory if one already +exists. Notes ----- @@ -41,7 +96,7 @@ lessc ``lessc main.less`` or watched using watch-lessc ``watch-lessc -i main.les Guides on using sphinx ---------------------- -* To make links to certain pages create a link target like so: +* To make links to certain sections create a link target like so: ``` .. _hello_world: @@ -52,7 +107,10 @@ Guides on using sphinx This is.. (etc.) ``` - The ``_hello_world:`` will make it possible to link to this position (page and marker) from all other pages. + The ``_hello_world:`` will make it possible to link to this position + (page and section heading) from all other pages. See the [Sphinx + docs](http://sphinx-doc.org/markup/inline.html#role-ref) for more + information and examples. * Notes, warnings and alarms @@ -68,13 +126,17 @@ Guides on using sphinx * Code examples - Start without $, so it's easy to copy and paste. + * Start without $, so it's easy to copy and paste. + * Use "sudo" with docker to ensure that your command is runnable + even if they haven't [used the *docker* + group](http://docs.docker.io/en/latest/use/basics/#why-sudo). Manpages -------- -* To make the manpages, simply run 'make man'. Please note there is a bug in spinx 1.1.3 which makes this fail. -Upgrade to the latest version of sphinx. -* Then preview the manpage by running `man _build/man/docker.1`, where _build/man/docker.1 is the path to the generated -manfile -* The manpages are also autogenerated by our hosted readthedocs here: http://docs-docker.dotcloud.com/projects/docker/downloads/ +* To make the manpages, run ``make man``. Please note there is a bug + in spinx 1.1.3 which makes this fail. Upgrade to the latest version + of Sphinx. +* Then preview the manpage by running ``man _build/man/docker.1``, + where ``_build/man/docker.1`` is the path to the generated manfile + diff --git a/components/engine/docs/sources/api/docker_remote_api.rst b/components/engine/docs/sources/api/docker_remote_api.rst index f410183c58..86cacd17d9 100644 --- a/components/engine/docs/sources/api/docker_remote_api.rst +++ b/components/engine/docs/sources/api/docker_remote_api.rst @@ -22,7 +22,6 @@ Docker Remote API - Since API version 1.2, the auth configuration is now handled client side, so the client has to send the authConfig as POST in /images/(name)/push -- Known client libraries may be found in :ref:`remote_api_client_libs` 2. Versions =========== diff --git a/components/engine/docs/sources/api/remote_api_client_libraries.rst b/components/engine/docs/sources/api/remote_api_client_libraries.rst index f391613ec4..bd8610eaf1 100644 --- a/components/engine/docs/sources/api/remote_api_client_libraries.rst +++ b/components/engine/docs/sources/api/remote_api_client_libraries.rst @@ -3,18 +3,14 @@ :keywords: API, Docker, index, registry, REST, documentation, clients, Python, Ruby, Javascript, Erlang, Go -.. _remote_api_client_libs: - ================================== Docker Remote API Client Libraries ================================== These libraries have not been tested by the Docker Maintainers for -compatibility with the :doc:`docker_remote_api`. Please file issues -with the library owners. If you find more library implementations, -please list them in `Docker doc issues -`_ -or make a pull request, and we will add the libraries here. +compatibility. Please file issues with the library owners. If you +find more library implementations, please list them in Docker doc bugs +and we will add the libraries here. +----------------------+----------------+--------------------------------------------+ | Language/Framework | Name | Repository | diff --git a/components/engine/docs/sources/commandline/cli.rst b/components/engine/docs/sources/commandline/cli.rst index 62dd3707a1..71a902d46f 100644 --- a/components/engine/docs/sources/commandline/cli.rst +++ b/components/engine/docs/sources/commandline/cli.rst @@ -4,11 +4,8 @@ .. _cli: -Overview -====================== - -Docker Usage -~~~~~~~~~~~~~~~~~~ +Command Line Help +----------------- To list available commands, either run ``docker`` with no parameters or execute ``docker help``:: @@ -21,71 +18,666 @@ To list available commands, either run ``docker`` with no parameters or execute ... +.. _cli_attach: + +``attach`` +---------- + +:: + + Usage: docker attach CONTAINER + + Attach to a running container. + +You can detach from the container again (and leave it running) with +``CTRL-c`` (for a quiet exit) or ``CTRL-\`` to get a stacktrace of +the Docker client when it quits. + +To stop a container, use ``docker stop`` + +To kill the container, use ``docker kill`` + +.. _cli_attach_examples: + +Examples: +~~~~~~~~~ + +.. code-block:: bash + + $ ID=$(sudo docker run -d ubuntu /usr/bin/top -b) + $ sudo docker attach $ID + top - 02:05:52 up 3:05, 0 users, load average: 0.01, 0.02, 0.05 + Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie + Cpu(s): 0.1%us, 0.2%sy, 0.0%ni, 99.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st + Mem: 373572k total, 355560k used, 18012k free, 27872k buffers + Swap: 786428k total, 0k used, 786428k free, 221740k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND + 1 root 20 0 17200 1116 912 R 0 0.3 0:00.03 top + + top - 02:05:55 up 3:05, 0 users, load average: 0.01, 0.02, 0.05 + Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie + Cpu(s): 0.0%us, 0.2%sy, 0.0%ni, 99.8%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st + Mem: 373572k total, 355244k used, 18328k free, 27872k buffers + Swap: 786428k total, 0k used, 786428k free, 221776k cached + + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND + 1 root 20 0 17208 1144 932 R 0 0.3 0:00.03 top -Available Commands -~~~~~~~~~~~~~~~~~~ + top - 02:05:58 up 3:06, 0 users, load average: 0.01, 0.02, 0.05 + Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie + Cpu(s): 0.2%us, 0.3%sy, 0.0%ni, 99.5%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st + Mem: 373572k total, 355780k used, 17792k free, 27880k buffers + Swap: 786428k total, 0k used, 786428k free, 221776k cached -.. include:: command/attach.rst + PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND + 1 root 20 0 17208 1144 932 R 0 0.3 0:00.03 top + ^C$ + $ sudo docker stop $ID -.. include:: command/build.rst +.. _cli_build: -.. include:: command/commit.rst +``build`` +--------- -.. include:: command/cp.rst +:: -.. include:: command/diff.rst + Usage: docker build [OPTIONS] PATH | URL | - + Build a new container image from the source code at PATH + -t="": Repository name (and optionally a tag) to be applied to the resulting image in case of success. + -q=false: Suppress verbose build output. + -no-cache: Do not use the cache when building the image. + -rm: Remove intermediate containers after a successful build + When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context -.. include:: command/events.rst +.. _cli_build_examples: -.. include:: command/export.rst +Examples +~~~~~~~~ -.. include:: command/history.rst +.. code-block:: bash -.. include:: command/images.rst + sudo docker build . -.. include:: command/import.rst +This will read the ``Dockerfile`` from the current directory. It will +also send any other files and directories found in the current +directory to the ``docker`` daemon. -.. include:: command/info.rst +The contents of this directory would be used by ``ADD`` commands found +within the ``Dockerfile``. This will send a lot of data to the +``docker`` daemon if the current directory contains a lot of data. If +the absolute path is provided instead of ``.`` then only the files and +directories required by the ADD commands from the ``Dockerfile`` will be +added to the context and transferred to the ``docker`` daemon. -.. include:: command/insert.rst +.. code-block:: bash -.. include:: command/inspect.rst + sudo docker build -t vieux/apache:2.0 . + +This will build like the previous example, but it will then tag the +resulting image. The repository name will be ``vieux/apache`` and the +tag will be ``2.0`` + + +.. code-block:: bash + + sudo docker build - < Dockerfile + +This will read a ``Dockerfile`` from *stdin* without context. Due to +the lack of a context, no contents of any local directory will be sent +to the ``docker`` daemon. ``ADD`` doesn't work when running in this +mode because the absence of the context provides no source files to +copy to the container. + + +.. code-block:: bash + + sudo docker build github.com/creack/docker-firefox + +This will clone the Github repository and use it as context. The +``Dockerfile`` at the root of the repository is used as +``Dockerfile``. Note that you can specify an arbitrary git repository +by using the ``git://`` schema. + + +.. _cli_commit: + +``commit`` +---------- + +:: + + Usage: docker commit [OPTIONS] CONTAINER [REPOSITORY [TAG]] + + Create a new image from a container's changes + + -m="": Commit message + -author="": Author (eg. "John Hannibal Smith " + -run="": Configuration to be applied when the image is launched with `docker run`. + (ex: '{"Cmd": ["cat", "/world"], "PortSpecs": ["22"]}') + +Full -run example (multiline is ok within a single quote ``'``) + +:: + + $ sudo docker commit -run=' + { + "Entrypoint" : null, + "Privileged" : false, + "User" : "", + "VolumesFrom" : "", + "Cmd" : ["cat", "-e", "/etc/resolv.conf"], + "Dns" : ["8.8.8.8", "8.8.4.4"], + "MemorySwap" : 0, + "AttachStdin" : false, + "AttachStderr" : false, + "CpuShares" : 0, + "OpenStdin" : false, + "Volumes" : null, + "Hostname" : "122612f45831", + "PortSpecs" : ["22", "80", "443"], + "Image" : "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "Tty" : false, + "Env" : [ + "HOME=/", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ], + "StdinOnce" : false, + "Domainname" : "", + "WorkingDir" : "/", + "NetworkDisabled" : false, + "Memory" : 0, + "AttachStdout" : false + }' $CONTAINER_ID + +.. _cli_cp: + +``cp`` +------ + +:: + + Usage: docker cp CONTAINER:RESOURCE HOSTPATH + + Copy files/folders from the containers filesystem to the host + path. Paths are relative to the root of the filesystem. + +.. _cli_diff: + +``diff`` +-------- + +:: + + Usage: docker diff CONTAINER [OPTIONS] + + Inspect changes on a container's filesystem + +.. _cli_events: + +``events`` +---------- + +:: + + Usage: docker events + + Get real time events from the server + +.. _cli_events_example: + +Examples +~~~~~~~~ + +You'll need two shells for this example. + +Shell 1: Listening for events +............................. + +.. code-block:: bash + + $ sudo docker events + +Shell 2: Start and Stop a Container +................................... + +.. code-block:: bash + + $ sudo docker start 4386fb97867d + $ sudo docker stop 4386fb97867d + +Shell 1: (Again .. now showing events) +...................................... + +.. code-block:: bash + + [2013-09-03 15:49:26 +0200 CEST] 4386fb97867d: (from 12de384bfb10) start + [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die + [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop + + +.. _cli_export: + +``export`` +---------- + +:: + + Usage: docker export CONTAINER + + Export the contents of a filesystem as a tar archive + +.. _cli_history: + +``history`` +----------- + +:: + + Usage: docker history [OPTIONS] IMAGE + + Show the history of an image + +.. _cli_images: + +``images`` +---------- + +:: + + Usage: docker images [OPTIONS] [NAME] + + List images + + -a=false: show all images + -q=false: only show numeric IDs + -viz=false: output in graphviz format + +Displaying images visually +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + sudo docker images -viz | dot -Tpng -o docker.png + +.. image:: docker_images.gif + :alt: Example inheritance graph of Docker images. + +.. _cli_import: + +``import`` +---------- + +:: + + Usage: docker import URL|- [REPOSITORY [TAG]] + + Create a new filesystem image from the contents of a tarball + +At this time, the URL must start with ``http`` and point to a single +file archive (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) containing a +root filesystem. If you would like to import from a local directory or +archive, you can use the ``-`` parameter to take the data from +standard in. + +Examples +~~~~~~~~ + +Import from a remote location +............................. + +``$ sudo docker import http://example.com/exampleimage.tgz exampleimagerepo`` + +Import from a local file +........................ + +Import to docker via pipe and standard in + +``$ cat exampleimage.tgz | sudo docker import - exampleimagelocal`` + +Import from a local directory +............................. + +``$ sudo tar -c . | docker import - exampleimagedir`` + +Note the ``sudo`` in this example -- you must preserve the ownership +of the files (especially root ownership) during the archiving with +tar. If you are not root (or sudo) when you tar, then the ownerships +might not get preserved. + +.. _cli_info: + +``info`` +-------- + +:: + + Usage: docker info + + Display system-wide information. + +.. _cli_insert: + +``insert`` +---------- + +:: + + Usage: docker insert IMAGE URL PATH + + Insert a file from URL in the IMAGE at PATH + +Examples +~~~~~~~~ + +Insert file from github +....................... + +.. code-block:: bash + + $ sudo docker insert 8283e18b24bc https://raw.github.com/metalivedev/django/master/postinstall /tmp/postinstall.sh + +.. _cli_inspect: + +``inspect`` +----------- + +:: + + Usage: docker inspect [OPTIONS] CONTAINER + + Return low-level information on a container + +.. _cli_kill: + +``kill`` +-------- + +:: + + Usage: docker kill [OPTIONS] CONTAINER [CONTAINER...] + + Kill a running container + +.. _cli_login: + +``login`` +--------- + +:: + + Usage: docker login [OPTIONS] [SERVER] + + Register or Login to the docker registry server + + -e="": email + -p="": password + -u="": username + + If you want to login to a private registry you can + specify this by adding the server name. + + example: + docker login localhost:8080 + + +.. _cli_logs: + +``logs`` +-------- + + +:: + + Usage: docker logs [OPTIONS] CONTAINER + + Fetch the logs of a container + + +.. _cli_port: + +``port`` +-------- + +:: + + Usage: docker port [OPTIONS] CONTAINER PRIVATE_PORT + + Lookup the public-facing port which is NAT-ed to PRIVATE_PORT + + +.. _cli_ps: + +``ps`` +------ + +:: + + Usage: docker ps [OPTIONS] + + List containers + + -a=false: Show all containers. Only running containers are shown by default. + -notrunc=false: Don't truncate output + -q=false: Only display numeric IDs + +.. _cli_pull: + +``pull`` +-------- + +:: + + Usage: docker pull NAME + + Pull an image or a repository from the registry + + +.. _cli_push: + +``push`` +-------- + +:: + + Usage: docker push NAME + + Push an image or a repository to the registry + + +.. _cli_restart: + +``restart`` +----------- + +:: + + Usage: docker restart [OPTIONS] NAME + + Restart a running container + +.. _cli_rm: + +``rm`` +------ + +:: + + Usage: docker rm [OPTIONS] CONTAINER + + Remove one or more containers + +.. _cli_rmi: + +``rmi`` +------- + +:: + + Usage: docker rmi IMAGE [IMAGE...] + + Remove one or more images + +.. _cli_run: + +``run`` +------- + +:: + + Usage: docker run [OPTIONS] IMAGE[:TAG] [COMMAND] [ARG...] + + Run a command in a new container + + -a=map[]: Attach to stdin, stdout or stderr. + -c=0: CPU shares (relative weight) + -cidfile="": Write the container ID to the file + -d=false: Detached mode: Run container in the background, print new container id + -e=[]: Set environment variables + -h="": Container host name + -i=false: Keep stdin open even if not attached + -privileged=false: Give extended privileges to this container + -m=0: Memory limit (in bytes) + -n=true: Enable networking for this container + -p=[]: Map a network port to the container + -rm=false: Automatically remove the container when it exits (incompatible with -d) + -t=false: Allocate a pseudo-tty + -u="": Username or UID + -dns=[]: Set custom dns servers for the container + -v=[]: Create a bind mount with: [host-dir]:[container-dir]:[rw|ro]. If "container-dir" is missing, then docker creates a new volume. + -volumes-from="": Mount all volumes from the given container. + -entrypoint="": Overwrite the default entrypoint set by the image. + -w="": Working directory inside the container + -lxc-conf=[]: Add custom lxc options -lxc-conf="lxc.cgroup.cpuset.cpus = 0,1" + +Examples +~~~~~~~~ + +.. code-block:: bash + + sudo docker run -cidfile /tmp/docker_test.cid ubuntu echo "test" + +This will create a container and print "test" to the console. The +``cidfile`` flag makes docker attempt to create a new file and write the +container ID to it. If the file exists already, docker will return an +error. Docker will close this file when docker run exits. + +.. code-block:: bash + + docker run mount -t tmpfs none /var/spool/squid + +This will *not* work, because by default, most potentially dangerous +kernel capabilities are dropped; including ``cap_sys_admin`` (which is +required to mount filesystems). However, the ``-privileged`` flag will +allow it to run: + +.. code-block:: bash + + docker run -privileged mount -t tmpfs none /var/spool/squid + +The ``-privileged`` flag gives *all* capabilities to the container, +and it also lifts all the limitations enforced by the ``device`` +cgroup controller. In other words, the container can then do almost +everything that the host can do. This flag exists to allow special +use-cases, like running Docker within Docker. + +.. code-block:: bash + + docker run -w /path/to/dir/ -i -t ubuntu pwd + +The ``-w`` lets the command being executed inside directory given, +here /path/to/dir/. If the path does not exists it is created inside the +container. + +.. code-block:: bash + + docker run -v `pwd`:`pwd` -w `pwd` -i -t ubuntu pwd + +The ``-v`` flag mounts the current working directory into the container. +The ``-w`` lets the command being executed inside the current +working directory, by changing into the directory to the value +returned by ``pwd``. So this combination executes the command +using the container, but inside the current working directory. + +.. _cli_search: + +``search`` +---------- + +:: + + Usage: docker search TERM + + Searches for the TERM parameter on the Docker index and prints out + a list of repositories that match. + +.. _cli_start: + +``start`` +--------- + +:: + + Usage: docker start [OPTIONS] NAME + + Start a stopped container + +.. _cli_stop: + +``stop`` +-------- + +:: + + Usage: docker stop [OPTIONS] CONTAINER [CONTAINER...] + + Stop a running container + + -t=10: Number of seconds to wait for the container to stop before killing it. + +.. _cli_tag: + +``tag`` +------- + +:: + + Usage: docker tag [OPTIONS] IMAGE REPOSITORY [TAG] + + Tag an image into a repository + + -f=false: Force + +.. _cli_top: + +``top`` +------- + +:: + + Usage: docker top CONTAINER + + Lookup the running processes of a container + +.. _cli_version: + +``version`` +----------- + +Show the version of the docker client, daemon, and latest released version. + + +.. _cli_wait: + +``wait`` +-------- + +:: + + Usage: docker wait [OPTIONS] NAME + + Block until a container stops, then print its exit code. -.. include:: command/kill.rst - -.. include:: command/login.rst - -.. include:: command/logs.rst - -.. include:: command/port.rst - -.. include:: command/ps.rst - -.. include:: command/pull.rst - -.. include:: command/push.rst - -.. include:: command/restart.rst - -.. include:: command/rm.rst - -.. include:: command/rmi.rst - -.. include:: command/run.rst - -.. include:: command/search.rst - -.. include:: command/start.rst - -.. include:: command/stop.rst - -.. include:: command/tag.rst - -.. include:: command/top.rst - -.. include:: command/version.rst - -.. include:: command/wait.rst diff --git a/components/engine/docs/sources/commandline/command/attach.rst b/components/engine/docs/sources/commandline/command/attach.rst deleted file mode 100644 index 12ed802a02..0000000000 --- a/components/engine/docs/sources/commandline/command/attach.rst +++ /dev/null @@ -1,59 +0,0 @@ -:title: Attach Command -:description: Attach to a running container -:keywords: attach, container, docker, documentation - -=========================================== -``attach`` -- Attach to a running container -=========================================== - -:: - - Usage: docker attach CONTAINER - - Attach to a running container. - -You can detach from the container again (and leave it running) with -``CTRL-c`` (for a quiet exit) or ``CTRL-\`` to get a stacktrace of -the Docker client when it quits. - -To stop a container, use ``docker stop`` - -To kill the container, use ``docker kill`` - -Examples: ---------- - -.. code-block:: bash - - $ ID=$(sudo docker run -d ubuntu /usr/bin/top -b) - $ sudo docker attach $ID - top - 02:05:52 up 3:05, 0 users, load average: 0.01, 0.02, 0.05 - Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie - Cpu(s): 0.1%us, 0.2%sy, 0.0%ni, 99.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st - Mem: 373572k total, 355560k used, 18012k free, 27872k buffers - Swap: 786428k total, 0k used, 786428k free, 221740k cached - - PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND - 1 root 20 0 17200 1116 912 R 0 0.3 0:00.03 top - - top - 02:05:55 up 3:05, 0 users, load average: 0.01, 0.02, 0.05 - Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie - Cpu(s): 0.0%us, 0.2%sy, 0.0%ni, 99.8%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st - Mem: 373572k total, 355244k used, 18328k free, 27872k buffers - Swap: 786428k total, 0k used, 786428k free, 221776k cached - - PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND - 1 root 20 0 17208 1144 932 R 0 0.3 0:00.03 top - - - top - 02:05:58 up 3:06, 0 users, load average: 0.01, 0.02, 0.05 - Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie - Cpu(s): 0.2%us, 0.3%sy, 0.0%ni, 99.5%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st - Mem: 373572k total, 355780k used, 17792k free, 27880k buffers - Swap: 786428k total, 0k used, 786428k free, 221776k cached - - PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND - 1 root 20 0 17208 1144 932 R 0 0.3 0:00.03 top - ^C$ - $ sudo docker stop $ID - diff --git a/components/engine/docs/sources/commandline/command/build.rst b/components/engine/docs/sources/commandline/command/build.rst deleted file mode 100644 index 5ce70d85d9..0000000000 --- a/components/engine/docs/sources/commandline/command/build.rst +++ /dev/null @@ -1,65 +0,0 @@ -:title: Build Command -:description: Build a new image from the Dockerfile passed via stdin -:keywords: build, docker, container, documentation - -================================================ -``build`` -- Build a container from a Dockerfile -================================================ - -:: - - Usage: docker build [OPTIONS] PATH | URL | - - Build a new container image from the source code at PATH - -t="": Repository name (and optionally a tag) to be applied to the resulting image in case of success. - -q=false: Suppress verbose build output. - -no-cache: Do not use the cache when building the image. - -rm: Remove intermediate containers after a successful build - When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context - - -Examples --------- - -.. code-block:: bash - - sudo docker build . - -This will read the ``Dockerfile`` from the current directory. It will -also send any other files and directories found in the current -directory to the ``docker`` daemon. - -The contents of this directory would be used by ``ADD`` commands found -within the ``Dockerfile``. This will send a lot of data to the -``docker`` daemon if the current directory contains a lot of data. If -the absolute path is provided instead of ``.`` then only the files and -directories required by the ADD commands from the ``Dockerfile`` will be -added to the context and transferred to the ``docker`` daemon. - -.. code-block:: bash - - sudo docker build -t vieux/apache:2.0 . - -This will build like the previous example, but it will then tag the -resulting image. The repository name will be ``vieux/apache`` and the -tag will be ``2.0`` - - -.. code-block:: bash - - sudo docker build - < Dockerfile - -This will read a ``Dockerfile`` from *stdin* without context. Due to -the lack of a context, no contents of any local directory will be sent -to the ``docker`` daemon. ``ADD`` doesn't work when running in this -mode because the absence of the context provides no source files to -copy to the container. - - -.. code-block:: bash - - sudo docker build github.com/creack/docker-firefox - -This will clone the Github repository and use it as context. The -``Dockerfile`` at the root of the repository is used as -``Dockerfile``. Note that you can specify an arbitrary git repository -by using the ``git://`` schema. diff --git a/components/engine/docs/sources/commandline/command/commit.rst b/components/engine/docs/sources/commandline/command/commit.rst deleted file mode 100644 index ef0d863688..0000000000 --- a/components/engine/docs/sources/commandline/command/commit.rst +++ /dev/null @@ -1,52 +0,0 @@ -:title: Commit Command -:description: Create a new image from a container's changes -:keywords: commit, docker, container, documentation - -=========================================================== -``commit`` -- Create a new image from a container's changes -=========================================================== - -:: - - Usage: docker commit [OPTIONS] CONTAINER [REPOSITORY [TAG]] - - Create a new image from a container's changes - - -m="": Commit message - -author="": Author (eg. "John Hannibal Smith " - -run="": Configuration to be applied when the image is launched with `docker run`. - (ex: '{"Cmd": ["cat", "/world"], "PortSpecs": ["22"]}') - -Full -run example (multiline is ok within a single quote ``'``) - -:: - - $ sudo docker commit -run=' - { - "Entrypoint" : null, - "Privileged" : false, - "User" : "", - "VolumesFrom" : "", - "Cmd" : ["cat", "-e", "/etc/resolv.conf"], - "Dns" : ["8.8.8.8", "8.8.4.4"], - "MemorySwap" : 0, - "AttachStdin" : false, - "AttachStderr" : false, - "CpuShares" : 0, - "OpenStdin" : false, - "Volumes" : null, - "Hostname" : "122612f45831", - "PortSpecs" : ["22", "80", "443"], - "Image" : "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", - "Tty" : false, - "Env" : [ - "HOME=/", - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - ], - "StdinOnce" : false, - "Domainname" : "", - "WorkingDir" : "/", - "NetworkDisabled" : false, - "Memory" : 0, - "AttachStdout" : false - }' $CONTAINER_ID diff --git a/components/engine/docs/sources/commandline/command/cp.rst b/components/engine/docs/sources/commandline/command/cp.rst deleted file mode 100644 index ea84fa1f90..0000000000 --- a/components/engine/docs/sources/commandline/command/cp.rst +++ /dev/null @@ -1,14 +0,0 @@ -:title: Cp Command -:description: Copy files/folders from the containers filesystem to the host path -:keywords: cp, docker, container, documentation, copy - -============================================================================ -``cp`` -- Copy files/folders from the containers filesystem to the host path -============================================================================ - -:: - - Usage: docker cp CONTAINER:RESOURCE HOSTPATH - - Copy files/folders from the containers filesystem to the host - path. Paths are relative to the root of the filesystem. diff --git a/components/engine/docs/sources/commandline/command/diff.rst b/components/engine/docs/sources/commandline/command/diff.rst deleted file mode 100644 index 2901a7f215..0000000000 --- a/components/engine/docs/sources/commandline/command/diff.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Diff Command -:description: Inspect changes on a container's filesystem -:keywords: diff, docker, container, documentation - -======================================================= -``diff`` -- Inspect changes on a container's filesystem -======================================================= - -:: - - Usage: docker diff CONTAINER [OPTIONS] - - Inspect changes on a container's filesystem diff --git a/components/engine/docs/sources/commandline/command/events.rst b/components/engine/docs/sources/commandline/command/events.rst deleted file mode 100644 index b8dd591fb1..0000000000 --- a/components/engine/docs/sources/commandline/command/events.rst +++ /dev/null @@ -1,34 +0,0 @@ -:title: Events Command -:description: Get real time events from the server -:keywords: events, docker, documentation - -================================================================= -``events`` -- Get real time events from the server -================================================================= - -:: - - Usage: docker events - - Get real time events from the server - -Examples --------- - -Starting and stopping a container -................................. - -.. code-block:: bash - - $ sudo docker start 4386fb97867d - $ sudo docker stop 4386fb97867d - -In another shell - -.. code-block:: bash - - $ sudo docker events - [2013-09-03 15:49:26 +0200 CEST] 4386fb97867d: (from 12de384bfb10) start - [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) die - [2013-09-03 15:49:29 +0200 CEST] 4386fb97867d: (from 12de384bfb10) stop - diff --git a/components/engine/docs/sources/commandline/command/export.rst b/components/engine/docs/sources/commandline/command/export.rst deleted file mode 100644 index 9d7e6b369c..0000000000 --- a/components/engine/docs/sources/commandline/command/export.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Export Command -:description: Export the contents of a filesystem as a tar archive -:keywords: export, docker, container, documentation - -================================================================= -``export`` -- Stream the contents of a container as a tar archive -================================================================= - -:: - - Usage: docker export CONTAINER - - Export the contents of a filesystem as a tar archive diff --git a/components/engine/docs/sources/commandline/command/history.rst b/components/engine/docs/sources/commandline/command/history.rst deleted file mode 100644 index 2f9d3f2814..0000000000 --- a/components/engine/docs/sources/commandline/command/history.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: History Command -:description: Show the history of an image -:keywords: history, docker, container, documentation - -=========================================== -``history`` -- Show the history of an image -=========================================== - -:: - - Usage: docker history [OPTIONS] IMAGE - - Show the history of an image diff --git a/components/engine/docs/sources/commandline/command/images.rst b/components/engine/docs/sources/commandline/command/images.rst deleted file mode 100644 index 279bf240c9..0000000000 --- a/components/engine/docs/sources/commandline/command/images.rst +++ /dev/null @@ -1,26 +0,0 @@ -:title: Images Command -:description: List images -:keywords: images, docker, container, documentation - -========================= -``images`` -- List images -========================= - -:: - - Usage: docker images [OPTIONS] [NAME] - - List images - - -a=false: show all images - -q=false: only show numeric IDs - -viz=false: output in graphviz format - -Displaying images visually --------------------------- - -:: - - sudo docker images -viz | dot -Tpng -o docker.png - -.. image:: https://docs.docker.io/en/latest/_static/docker_images.gif diff --git a/components/engine/docs/sources/commandline/command/import.rst b/components/engine/docs/sources/commandline/command/import.rst deleted file mode 100644 index e236c5bc2b..0000000000 --- a/components/engine/docs/sources/commandline/command/import.rst +++ /dev/null @@ -1,44 +0,0 @@ -:title: Import Command -:description: Create a new filesystem image from the contents of a tarball -:keywords: import, tarball, docker, url, documentation - -========================================================================== -``import`` -- Create a new filesystem image from the contents of a tarball -========================================================================== - -:: - - Usage: docker import URL|- [REPOSITORY [TAG]] - - Create a new filesystem image from the contents of a tarball - -At this time, the URL must start with ``http`` and point to a single -file archive (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) containing a -root filesystem. If you would like to import from a local directory or -archive, you can use the ``-`` parameter to take the data from -standard in. - -Examples --------- - -Import from a remote location -............................. - -``$ sudo docker import http://example.com/exampleimage.tgz exampleimagerepo`` - -Import from a local file -........................ - -Import to docker via pipe and standard in - -``$ cat exampleimage.tgz | sudo docker import - exampleimagelocal`` - -Import from a local directory -............................. - -``$ sudo tar -c . | docker import - exampleimagedir`` - -Note the ``sudo`` in this example -- you must preserve the ownership -of the files (especially root ownership) during the archiving with -tar. If you are not root (or sudo) when you tar, then the ownerships -might not get preserved. diff --git a/components/engine/docs/sources/commandline/command/info.rst b/components/engine/docs/sources/commandline/command/info.rst deleted file mode 100644 index 6df3486c53..0000000000 --- a/components/engine/docs/sources/commandline/command/info.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Info Command -:description: Display system-wide information. -:keywords: info, docker, information, documentation - -=========================================== -``info`` -- Display system-wide information -=========================================== - -:: - - Usage: docker info - - Display system-wide information. diff --git a/components/engine/docs/sources/commandline/command/insert.rst b/components/engine/docs/sources/commandline/command/insert.rst deleted file mode 100644 index 0f2612c9af..0000000000 --- a/components/engine/docs/sources/commandline/command/insert.rst +++ /dev/null @@ -1,23 +0,0 @@ -:title: Insert Command -:description: Insert a file in an image -:keywords: insert, image, docker, documentation - -========================================================================== -``insert`` -- Insert a file in an image -========================================================================== - -:: - - Usage: docker insert IMAGE URL PATH - - Insert a file from URL in the IMAGE at PATH - -Examples --------- - -Insert file from github -....................... - -.. code-block:: bash - - $ sudo docker insert 8283e18b24bc https://raw.github.com/metalivedev/django/master/postinstall /tmp/postinstall.sh diff --git a/components/engine/docs/sources/commandline/command/inspect.rst b/components/engine/docs/sources/commandline/command/inspect.rst deleted file mode 100644 index 90dbe959eb..0000000000 --- a/components/engine/docs/sources/commandline/command/inspect.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Inspect Command -:description: Return low-level information on a container -:keywords: inspect, container, docker, documentation - -========================================================== -``inspect`` -- Return low-level information on a container -========================================================== - -:: - - Usage: docker inspect [OPTIONS] CONTAINER - - Return low-level information on a container diff --git a/components/engine/docs/sources/commandline/command/kill.rst b/components/engine/docs/sources/commandline/command/kill.rst deleted file mode 100644 index f53d3883b0..0000000000 --- a/components/engine/docs/sources/commandline/command/kill.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Kill Command -:description: Kill a running container -:keywords: kill, container, docker, documentation - -==================================== -``kill`` -- Kill a running container -==================================== - -:: - - Usage: docker kill [OPTIONS] CONTAINER [CONTAINER...] - - Kill a running container diff --git a/components/engine/docs/sources/commandline/command/login.rst b/components/engine/docs/sources/commandline/command/login.rst deleted file mode 100644 index 46f354d6be..0000000000 --- a/components/engine/docs/sources/commandline/command/login.rst +++ /dev/null @@ -1,24 +0,0 @@ -:title: Login Command -:description: Register or Login to the docker registry server -:keywords: login, docker, documentation - -============================================================ -``login`` -- Register or Login to the docker registry server -============================================================ - -:: - - Usage: docker login [OPTIONS] [SERVER] - - Register or Login to the docker registry server - - -e="": email - -p="": password - -u="": username - - If you want to login to a private registry you can - specify this by adding the server name. - - example: - docker login localhost:8080 - diff --git a/components/engine/docs/sources/commandline/command/logs.rst b/components/engine/docs/sources/commandline/command/logs.rst deleted file mode 100644 index a3423f6e0c..0000000000 --- a/components/engine/docs/sources/commandline/command/logs.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Logs Command -:description: Fetch the logs of a container -:keywords: logs, container, docker, documentation - -========================================= -``logs`` -- Fetch the logs of a container -========================================= - -:: - - Usage: docker logs [OPTIONS] CONTAINER - - Fetch the logs of a container diff --git a/components/engine/docs/sources/commandline/command/port.rst b/components/engine/docs/sources/commandline/command/port.rst deleted file mode 100644 index 8d59fedab6..0000000000 --- a/components/engine/docs/sources/commandline/command/port.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Port Command -:description: Lookup the public-facing port which is NAT-ed to PRIVATE_PORT -:keywords: port, docker, container, documentation - -========================================================================= -``port`` -- Lookup the public-facing port which is NAT-ed to PRIVATE_PORT -========================================================================= - -:: - - Usage: docker port [OPTIONS] CONTAINER PRIVATE_PORT - - Lookup the public-facing port which is NAT-ed to PRIVATE_PORT diff --git a/components/engine/docs/sources/commandline/command/ps.rst b/components/engine/docs/sources/commandline/command/ps.rst deleted file mode 100644 index 597dbd9ae7..0000000000 --- a/components/engine/docs/sources/commandline/command/ps.rst +++ /dev/null @@ -1,17 +0,0 @@ -:title: Ps Command -:description: List containers -:keywords: ps, docker, documentation, container - -========================= -``ps`` -- List containers -========================= - -:: - - Usage: docker ps [OPTIONS] - - List containers - - -a=false: Show all containers. Only running containers are shown by default. - -notrunc=false: Don't truncate output - -q=false: Only display numeric IDs diff --git a/components/engine/docs/sources/commandline/command/pull.rst b/components/engine/docs/sources/commandline/command/pull.rst deleted file mode 100644 index 4348f28d0f..0000000000 --- a/components/engine/docs/sources/commandline/command/pull.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Pull Command -:description: Pull an image or a repository from the registry -:keywords: pull, image, repo, repository, documentation, docker - -========================================================================= -``pull`` -- Pull an image or a repository from the docker registry server -========================================================================= - -:: - - Usage: docker pull NAME - - Pull an image or a repository from the registry diff --git a/components/engine/docs/sources/commandline/command/push.rst b/components/engine/docs/sources/commandline/command/push.rst deleted file mode 100644 index 9304f9acc5..0000000000 --- a/components/engine/docs/sources/commandline/command/push.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Push Command -:description: Push an image or a repository to the registry -:keywords: push, docker, image, repository, documentation, repo - -======================================================================= -``push`` -- Push an image or a repository to the docker registry server -======================================================================= - -:: - - Usage: docker push NAME - - Push an image or a repository to the registry diff --git a/components/engine/docs/sources/commandline/command/restart.rst b/components/engine/docs/sources/commandline/command/restart.rst deleted file mode 100644 index dfc0dfea6e..0000000000 --- a/components/engine/docs/sources/commandline/command/restart.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Restart Command -:description: Restart a running container -:keywords: restart, container, docker, documentation - -========================================== -``restart`` -- Restart a running container -========================================== - -:: - - Usage: docker restart [OPTIONS] NAME - - Restart a running container diff --git a/components/engine/docs/sources/commandline/command/rm.rst b/components/engine/docs/sources/commandline/command/rm.rst deleted file mode 100644 index 8a2309ce79..0000000000 --- a/components/engine/docs/sources/commandline/command/rm.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Rm Command -:description: Remove a container -:keywords: remove, container, docker, documentation, rm - -============================ -``rm`` -- Remove a container -============================ - -:: - - Usage: docker rm [OPTIONS] CONTAINER - - Remove one or more containers diff --git a/components/engine/docs/sources/commandline/command/rmi.rst b/components/engine/docs/sources/commandline/command/rmi.rst deleted file mode 100644 index 954e5222c6..0000000000 --- a/components/engine/docs/sources/commandline/command/rmi.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Rmi Command -:description: Remove an image -:keywords: rmi, remove, image, docker, documentation - -========================== -``rmi`` -- Remove an image -========================== - -:: - - Usage: docker rmi IMAGE [IMAGE...] - - Remove one or more images diff --git a/components/engine/docs/sources/commandline/command/run.rst b/components/engine/docs/sources/commandline/command/run.rst deleted file mode 100644 index b68e08303f..0000000000 --- a/components/engine/docs/sources/commandline/command/run.rst +++ /dev/null @@ -1,85 +0,0 @@ -:title: Run Command -:description: Run a command in a new container -:keywords: run, container, docker, documentation - -=========================================== -``run`` -- Run a command in a new container -=========================================== - -:: - - Usage: docker run [OPTIONS] IMAGE[:TAG] [COMMAND] [ARG...] - - Run a command in a new container - - -a=map[]: Attach to stdin, stdout or stderr. - -c=0: CPU shares (relative weight) - -cidfile="": Write the container ID to the file - -d=false: Detached mode: Run container in the background, print new container id - -e=[]: Set environment variables - -h="": Container host name - -i=false: Keep stdin open even if not attached - -privileged=false: Give extended privileges to this container - -m=0: Memory limit (in bytes) - -n=true: Enable networking for this container - -p=[]: Map a network port to the container - -rm=false: Automatically remove the container when it exits (incompatible with -d) - -t=false: Allocate a pseudo-tty - -u="": Username or UID - -dns=[]: Set custom dns servers for the container - -v=[]: Create a bind mount with: [host-dir]:[container-dir]:[rw|ro]. If "container-dir" is missing, then docker creates a new volume. - -volumes-from="": Mount all volumes from the given container. - -entrypoint="": Overwrite the default entrypoint set by the image. - -w="": Working directory inside the container - -lxc-conf=[]: Add custom lxc options -lxc-conf="lxc.cgroup.cpuset.cpus = 0,1" - -Examples --------- - -.. code-block:: bash - - sudo docker run -cidfile /tmp/docker_test.cid ubuntu echo "test" - -This will create a container and print "test" to the console. The -``cidfile`` flag makes docker attempt to create a new file and write the -container ID to it. If the file exists already, docker will return an -error. Docker will close this file when docker run exits. - -.. code-block:: bash - - docker run mount -t tmpfs none /var/spool/squid - -This will *not* work, because by default, most potentially dangerous -kernel capabilities are dropped; including ``cap_sys_admin`` (which is -required to mount filesystems). However, the ``-privileged`` flag will -allow it to run: - -.. code-block:: bash - - docker run -privileged mount -t tmpfs none /var/spool/squid - -The ``-privileged`` flag gives *all* capabilities to the container, -and it also lifts all the limitations enforced by the ``device`` -cgroup controller. In other words, the container can then do almost -everything that the host can do. This flag exists to allow special -use-cases, like running Docker within Docker. - -.. code-block:: bash - - docker run -w /path/to/dir/ -i -t ubuntu pwd - -The ``-w`` lets the command being executed inside directory given, -here /path/to/dir/. If the path does not exists it is created inside the -container. - -.. code-block:: bash - - docker run -v `pwd`:`pwd` -w `pwd` -i -t ubuntu pwd - -The ``-v`` flag mounts the current working directory into the container. -The ``-w`` lets the command being executed inside the current -working directory, by changing into the directory to the value -returned by ``pwd``. So this combination executes the command -using the container, but inside the current working directory. - - diff --git a/components/engine/docs/sources/commandline/command/search.rst b/components/engine/docs/sources/commandline/command/search.rst deleted file mode 100644 index 8107016a41..0000000000 --- a/components/engine/docs/sources/commandline/command/search.rst +++ /dev/null @@ -1,14 +0,0 @@ -:title: Search Command -:description: Searches for the TERM parameter on the Docker index and prints out a list of repositories that match. -:keywords: search, docker, image, documentation - -=================================================================== -``search`` -- Search for an image in the docker index -=================================================================== - -:: - - Usage: docker search TERM - - Searches for the TERM parameter on the Docker index and prints out - a list of repositories that match. diff --git a/components/engine/docs/sources/commandline/command/start.rst b/components/engine/docs/sources/commandline/command/start.rst deleted file mode 100644 index b70ad21cfd..0000000000 --- a/components/engine/docs/sources/commandline/command/start.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Start Command -:description: Start a stopped container -:keywords: start, docker, container, documentation - -====================================== -``start`` -- Start a stopped container -====================================== - -:: - - Usage: docker start [OPTIONS] NAME - - Start a stopped container diff --git a/components/engine/docs/sources/commandline/command/stop.rst b/components/engine/docs/sources/commandline/command/stop.rst deleted file mode 100644 index 6a64908eae..0000000000 --- a/components/engine/docs/sources/commandline/command/stop.rst +++ /dev/null @@ -1,15 +0,0 @@ -:title: Stop Command -:description: Stop a running container -:keywords: stop, container, docker, documentation - -==================================== -``stop`` -- Stop a running container -==================================== - -:: - - Usage: docker stop [OPTIONS] CONTAINER [CONTAINER...] - - Stop a running container - - -t=10: Number of seconds to wait for the container to stop before killing it. diff --git a/components/engine/docs/sources/commandline/command/tag.rst b/components/engine/docs/sources/commandline/command/tag.rst deleted file mode 100644 index a9e831aae9..0000000000 --- a/components/engine/docs/sources/commandline/command/tag.rst +++ /dev/null @@ -1,15 +0,0 @@ -:title: Tag Command -:description: Tag an image into a repository -:keywords: tag, docker, image, repository, documentation, repo - -========================================= -``tag`` -- Tag an image into a repository -========================================= - -:: - - Usage: docker tag [OPTIONS] IMAGE REPOSITORY [TAG] - - Tag an image into a repository - - -f=false: Force diff --git a/components/engine/docs/sources/commandline/command/top.rst b/components/engine/docs/sources/commandline/command/top.rst deleted file mode 100644 index bdd35adcfa..0000000000 --- a/components/engine/docs/sources/commandline/command/top.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Top Command -:description: Lookup the running processes of a container -:keywords: top, docker, container, documentation - -======================================================= -``top`` -- Lookup the running processes of a container -======================================================= - -:: - - Usage: docker top CONTAINER - - Lookup the running processes of a container diff --git a/components/engine/docs/sources/commandline/command/version.rst b/components/engine/docs/sources/commandline/command/version.rst deleted file mode 100644 index fb3d3b450e..0000000000 --- a/components/engine/docs/sources/commandline/command/version.rst +++ /dev/null @@ -1,7 +0,0 @@ -:title: Version Command -:description: -:keywords: version, docker, documentation - -================================================== -``version`` -- Show the docker version information -================================================== diff --git a/components/engine/docs/sources/commandline/command/wait.rst b/components/engine/docs/sources/commandline/command/wait.rst deleted file mode 100644 index 23bd54513c..0000000000 --- a/components/engine/docs/sources/commandline/command/wait.rst +++ /dev/null @@ -1,13 +0,0 @@ -:title: Wait Command -:description: Block until a container stops, then print its exit code. -:keywords: wait, docker, container, documentation - -=================================================================== -``wait`` -- Block until a container stops, then print its exit code -=================================================================== - -:: - - Usage: docker wait [OPTIONS] NAME - - Block until a container stops, then print its exit code. diff --git a/components/engine/docs/sources/static_files/docker_images.gif b/components/engine/docs/sources/commandline/docker_images.gif similarity index 100% rename from components/engine/docs/sources/static_files/docker_images.gif rename to components/engine/docs/sources/commandline/docker_images.gif diff --git a/components/engine/docs/sources/commandline/index.rst b/components/engine/docs/sources/commandline/index.rst index 0e7c8738b3..5536e1012e 100644 --- a/components/engine/docs/sources/commandline/index.rst +++ b/components/engine/docs/sources/commandline/index.rst @@ -1,6 +1,6 @@ :title: Commands -:description: -- todo: change me -:keywords: todo, commands, command line, help, docker, documentation +:description: docker command line interface +:keywords: commands, command line, help, docker Commands @@ -12,34 +12,3 @@ Contents: :maxdepth: 1 cli - attach - build - commit - cp - diff - events - export - history - images - import - info - insert - inspect - kill - login - logs - port - ps - pull - push - restart - rm - rmi - run - search - start - stop - tag - top - version - wait diff --git a/components/engine/docs/sources/contributing/devenvironment.rst b/components/engine/docs/sources/contributing/devenvironment.rst index 8d95b7ca1c..59f16646cd 100644 --- a/components/engine/docs/sources/contributing/devenvironment.rst +++ b/components/engine/docs/sources/contributing/devenvironment.rst @@ -124,7 +124,7 @@ You can run an interactive session in the newly built container: -.. note:: The binary is availalbe outside the container in the directory ``./bundles/-dev/binary/``. +.. note:: The binary is available outside the container in the directory ``./bundles/-dev/binary/``. You can swap your host docker executable with this binary for live testing - for example, on ubuntu: ``sudo service docker stop ; sudo cp $(which docker) $(which docker)_ ; sudo cp ./bundles/-dev/binary/docker--dev $(which docker);sudo service docker start``. **Need More Help?** diff --git a/components/engine/docs/sources/examples/postgresql_service.rst b/components/engine/docs/sources/examples/postgresql_service.rst index ba6bb4fe4e..35bcc01b95 100644 --- a/components/engine/docs/sources/examples/postgresql_service.rst +++ b/components/engine/docs/sources/examples/postgresql_service.rst @@ -43,8 +43,8 @@ Install ``python-software-properties``. .. code-block:: bash - apt-get install python-software-properties - apt-get install software-properties-common + apt-get -y install python-software-properties + apt-get -y install software-properties-common Add Pitti's PostgreSQL repository. It contains the most recent stable release of PostgreSQL i.e. ``9.2``. @@ -77,7 +77,8 @@ role. Adjust PostgreSQL configuration so that remote connections to the database are possible. Make sure that inside -``/etc/postgresql/9.2/main/pg_hba.conf`` you have following line: +``/etc/postgresql/9.2/main/pg_hba.conf`` you have following line (you will need +to install an editor, e.g. ``apt-get install vim``): .. code-block:: bash @@ -90,9 +91,17 @@ uncomment ``listen_addresses`` so it is as follows: listen_addresses='*' -*Note:* this PostgreSQL setup is for development only purposes. Refer -to PostgreSQL documentation how to fine-tune these settings so that it -is enough secure. +.. note:: + + This PostgreSQL setup is for development only purposes. Refer + to PostgreSQL documentation how to fine-tune these settings so that it + is enough secure. + +Exit. + +.. code-block:: bash + + exit Create an image and assign it a name. ```` is in the Bash prompt; you can also locate it using ``docker ps -a``. @@ -111,7 +120,9 @@ Finally, run PostgreSQL server via ``docker``. -D /var/lib/postgresql/9.2/main \ -c config_file=/etc/postgresql/9.2/main/postgresql.conf') -Connect the PostgreSQL server using ``psql``. +Connect the PostgreSQL server using ``psql`` (You will need postgres installed +on the machine. For ubuntu, use something like +``sudo apt-get install postgresql``). .. code-block:: bash @@ -128,7 +139,7 @@ As before, create roles or databases if needed. docker=# CREATE DATABASE foo OWNER=docker; CREATE DATABASE -Additionally, publish there your newly created image on Docker Index. +Additionally, publish your newly created image on Docker Index. .. code-block:: bash @@ -149,10 +160,11 @@ container starts. .. code-block:: bash - sudo docker commit postgresql -run='{"Cmd": \ + sudo docker commit -run='{"Cmd": \ ["/bin/su", "postgres", "-c", "/usr/lib/postgresql/9.2/bin/postgres -D \ /var/lib/postgresql/9.2/main -c \ - config_file=/etc/postgresql/9.2/main/postgresql.conf"], "PortSpecs": ["5432"]}' + config_file=/etc/postgresql/9.2/main/postgresql.conf"], "PortSpecs": ["5432"]}' \ + /postgresql From now on, just type ``docker run /postgresql`` and PostgreSQL should automatically start. diff --git a/components/engine/docs/sources/index.rst b/components/engine/docs/sources/index.rst index 99d2159aa7..88752ac3bf 100644 --- a/components/engine/docs/sources/index.rst +++ b/components/engine/docs/sources/index.rst @@ -2,8 +2,6 @@ :description: An overview of the Docker Documentation :keywords: containers, lxc, concepts, explanation -.. image:: https://www.docker.io/static/img/linked/dockerlogo-horizontal.png - Introduction ------------ diff --git a/components/engine/docs/sources/installation/archlinux.rst b/components/engine/docs/sources/installation/archlinux.rst index a593b28f50..a4de066f76 100644 --- a/components/engine/docs/sources/installation/archlinux.rst +++ b/components/engine/docs/sources/installation/archlinux.rst @@ -19,10 +19,6 @@ The lxc-docker-git package will build from the current master branch. Dependencies ------------ -.. versionchanged:: v0.7 - This section may need to be updated since Docker no longer depends - on AUFS. Please see :ref:`kernel`. - Docker depends on several packages which are specified as dependencies in either AUR package. diff --git a/components/engine/docs/sources/installation/gentoolinux.rst b/components/engine/docs/sources/installation/gentoolinux.rst index 374301e648..c4f2d220f3 100644 --- a/components/engine/docs/sources/installation/gentoolinux.rst +++ b/components/engine/docs/sources/installation/gentoolinux.rst @@ -25,11 +25,6 @@ properly installing and using the overlay can be found in `the overlay README Installation ^^^^^^^^^^^^ -.. versionchanged:: v0.7 - This section may need to be updated since Docker no longer depends - on AUFS. Please see :ref:`kernel`. - - The package should properly pull in all the necessary dependencies and prompt for all necessary kernel options. For the most straightforward installation experience, use ``sys-kernel/aufs-sources`` as your kernel sources. If you diff --git a/components/engine/docs/sources/installation/kernel.rst b/components/engine/docs/sources/installation/kernel.rst index 6e22677e03..2959fa4fcd 100644 --- a/components/engine/docs/sources/installation/kernel.rst +++ b/components/engine/docs/sources/installation/kernel.rst @@ -11,7 +11,7 @@ In short, Docker has the following kernel requirements: - Linux version 3.8 or above. -- `Device Mapper support `_. +- `AUFS support `_. - Cgroups and namespaces must be enabled. @@ -48,17 +48,15 @@ detects something older than 3.8. See issue `#407 `_ for details. -Device Mapper support ---------------------- +AUFS support +------------ -The `Device Mapper `_ replaces the -previous Docker dependency on AUFS and has been in the kernel since -2.6.9, so the device-mapper module is more broadly-supported across -Linux distributions. Docker uses `thin-provisioning -`_ -to provide a :ref:`unioning file system `. If you'd like to -check for the presence of the device-mapper module, please see the -`LVM-HOWTO. `_ +Docker currently relies on AUFS, an unioning filesystem. +While AUFS is included in the kernels built by the Debian and Ubuntu +distributions, is not part of the standard kernel. This means that if +you decide to roll your own kernel, you will have to patch your +kernel tree to add AUFS. The process is documented on +`AUFS webpage `_. Cgroups and namespaces @@ -71,7 +69,7 @@ to run LXC containers. Note that 2.6.32 has some documented issues regarding network namespace setup and teardown; those issues are not a risk if you run containers in a private environment, but can lead to denial-of-service attacks if you want to run untrusted code in your containers. For more details, -see `LP#720095 `_. +see `[LP#720095 `_. Kernels 2.6.38, and every version since 3.2, have been deployed successfully to run containerized production workloads. Feature-wise, there is no huge diff --git a/components/engine/docs/sources/installation/ubuntulinux.rst b/components/engine/docs/sources/installation/ubuntulinux.rst index 39a7b4f6bf..e653f704a8 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.rst +++ b/components/engine/docs/sources/installation/ubuntulinux.rst @@ -7,6 +7,11 @@ Ubuntu Linux ============ +.. warning:: + + These instructions have changed for 0.6. If you are upgrading from + an earlier version, you will need to follow them again. + .. include:: install_header.inc Right now, the officially supported distribution are: @@ -14,10 +19,10 @@ Right now, the officially supported distribution are: - :ref:`ubuntu_precise` - :ref:`ubuntu_raring` -Docker has the following dependencies (read more in :ref:`kernel`): +Docker has the following dependencies -* Linux kernel 3.8 -* Device-mapper module +* Linux kernel 3.8 (read more about :ref:`kernel`) +* AUFS file system support (we are working on BTRFS support as an alternative) Please read :ref:`ufw`, if you plan to use `UFW (Uncomplicated Firewall) `_ @@ -37,12 +42,12 @@ Dependencies Due to a bug in LXC, docker works best on the 3.8 kernel. Precise comes with a 3.2 kernel, so we need to upgrade it. The kernel you'll -install when following these steps comes with device-mapper built -in. We also include the generic headers to enable packages that depend -on them, like ZFS and the VirtualBox guest additions. If you didn't -install the headers for your "precise" kernel, then you can skip these -headers for the "raring" kernel. But it is safer to include them if -you're not sure. +install when following these steps comes with AUFS built in. We also +include the generic headers to enable packages that depend on them, +like ZFS and the VirtualBox guest additions. If you didn't install the +headers for your "precise" kernel, then you can skip these headers for +the "raring" kernel. But it is safer to include them if you're not +sure. .. code-block:: bash @@ -58,7 +63,8 @@ you're not sure. Installation ------------ -.. versionchanged:: v0.6 +.. warning:: + These instructions have changed for 0.6. If you are upgrading from an earlier version, you will need to follow them again. @@ -100,19 +106,13 @@ Ubuntu Raring 13.04 (64 bit) Dependencies ------------ -.. versionchanged:: v0.7 - Starting in 0.7 you no longer need to add support for AUFS. - We now use the device-mapper module instead, and this module - is included with kernels since kernel version 2.6 +**AUFS filesystem support** -Ubuntu Raring already comes with the 3.8 kernel, so we don't need to -install it. However, not all systems have AUFS filesystem support -enabled, so if you're on a Docker version before 0.7, then we need to -install it. +Ubuntu Raring already comes with the 3.8 kernel, so we don't need to install it. However, not all systems +have AUFS filesystem support enabled, so we need to install it. .. code-block:: bash - # Only required for versions before v0.7 sudo apt-get update sudo apt-get install linux-image-extra-`uname -r` @@ -122,9 +122,8 @@ Installation Docker is available as a Debian package, which makes installation easy. -*Please note that these instructions have changed for 0.6. If you are -upgrading from an earlier version, you will need to follow them -again.* +*Please note that these instructions have changed for 0.6. If you are upgrading from an earlier version, you will need +to follow them again.* .. code-block:: bash @@ -161,8 +160,8 @@ Verify it worked Docker and UFW ^^^^^^^^^^^^^^ -Docker uses a bridge to manage container networking, and by default -UFW drops all `forwarding`. A first step is to enable forwarding: +Docker uses a bridge to manage containers networking, by default UFW +drop all `forwarding`, a first step is to enable forwarding: .. code-block:: bash @@ -180,3 +179,11 @@ Then reload UFW: sudo ufw reload +UFW's default set of rules denied all `incoming`, so if you want to be +able to reach your containers from another host, you should allow +incoming connections on the docker port (default 4243): + +.. code-block:: bash + + sudo ufw allow 4243/tcp + diff --git a/components/engine/docs/sources/terms/images/docker-filesystems-busyboxrw.png b/components/engine/docs/sources/terms/images/docker-filesystems-busyboxrw.png index d672083b0d..ad41c940e4 100644 Binary files a/components/engine/docs/sources/terms/images/docker-filesystems-busyboxrw.png and b/components/engine/docs/sources/terms/images/docker-filesystems-busyboxrw.png differ diff --git a/components/engine/docs/sources/terms/images/docker-filesystems-debian.png b/components/engine/docs/sources/terms/images/docker-filesystems-debian.png index 70badfaa40..823a215d3e 100644 Binary files a/components/engine/docs/sources/terms/images/docker-filesystems-debian.png and b/components/engine/docs/sources/terms/images/docker-filesystems-debian.png differ diff --git a/components/engine/docs/sources/terms/images/docker-filesystems-debianrw.png b/components/engine/docs/sources/terms/images/docker-filesystems-debianrw.png index c39cbd9d61..97c69a9944 100644 Binary files a/components/engine/docs/sources/terms/images/docker-filesystems-debianrw.png and b/components/engine/docs/sources/terms/images/docker-filesystems-debianrw.png differ diff --git a/components/engine/docs/sources/terms/images/docker-filesystems-generic.png b/components/engine/docs/sources/terms/images/docker-filesystems-generic.png index a9f04e9129..fb734b75c6 100644 Binary files a/components/engine/docs/sources/terms/images/docker-filesystems-generic.png and b/components/engine/docs/sources/terms/images/docker-filesystems-generic.png differ diff --git a/components/engine/docs/sources/terms/images/docker-filesystems-multilayer.png b/components/engine/docs/sources/terms/images/docker-filesystems-multilayer.png index fc6fc88597..0b3ae19c2c 100644 Binary files a/components/engine/docs/sources/terms/images/docker-filesystems-multilayer.png and b/components/engine/docs/sources/terms/images/docker-filesystems-multilayer.png differ diff --git a/components/engine/docs/sources/terms/images/docker-filesystems-multiroot.png b/components/engine/docs/sources/terms/images/docker-filesystems-multiroot.png index f68bd1be7d..5e864273f3 100644 Binary files a/components/engine/docs/sources/terms/images/docker-filesystems-multiroot.png and b/components/engine/docs/sources/terms/images/docker-filesystems-multiroot.png differ diff --git a/components/engine/docs/sources/terms/images/docker-filesystems.svg b/components/engine/docs/sources/terms/images/docker-filesystems.svg index 06d5ae8b0c..d41aff2522 100644 --- a/components/engine/docs/sources/terms/images/docker-filesystems.svg +++ b/components/engine/docs/sources/terms/images/docker-filesystems.svg @@ -9,15 +9,15 @@ xmlns="http://www.w3.org/2000/svg" xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" - inkscape:version="0.48.2 r9819" - version="1.1" - id="svg2" - height="600" - width="800" - sodipodi:docname="docker-filesystems.svg" - inkscape:export-filename="/Users/arothfusz/src/metalivedev/docker/docs/sources/terms/images/docker-filesystems-debianrw.png" + inkscape:export-ydpi="90" inkscape:export-xdpi="90" - inkscape:export-ydpi="90"> + inkscape:export-filename="/Users/arothfusz/src/metalivedev/docker/docs/sources/terms/images/docker-filesystems-multiroot.png" + sodipodi:docname="docker-filesystems.svg" + width="800" + height="600" + id="svg2" + version="1.1" + inkscape:version="0.48.2 r9819"> - - - - - - image/svg+xml - + @@ -321,146 +295,69 @@ inkscape:connector-curvature="0" /> + id="text3655" + style="font-size:40px;font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;line-height:125%;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;font-family:Arial;-inkscape-font-specification:Arial" + transform="matrix(0.67123869,0,0,0.67123869,53.68199,126.56876)"> - - - - - - - - - - - - - - - - - - - @@ -962,37 +859,6 @@ id="path6350" /> - diff --git a/components/engine/docs/sources/use/builder.rst b/components/engine/docs/sources/use/builder.rst index a0548126d8..7a356f8450 100644 --- a/components/engine/docs/sources/use/builder.rst +++ b/components/engine/docs/sources/use/builder.rst @@ -54,8 +54,14 @@ Docker evaluates the instructions in a Dockerfile in order. **The first instruction must be `FROM`** in order to specify the :ref:`base_image_def` from which you are building. -Docker will ignore **comment lines** *beginning* with ``#``. A comment -marker anywhere in the rest of the line will be treated as an argument. +Docker will treat lines that *begin* with ``#`` as a comment. A ``#`` +marker anywhere else in the line will be treated as an argument. This +allows statements like: + +:: + + # Comment + RUN echo 'we are running some # of cool things' 3. Instructions =============== diff --git a/components/engine/hack/RELEASE-CHECKLIST.md b/components/engine/hack/RELEASE-CHECKLIST.md index 0adbdba6dd..5f873c6c5c 100644 --- a/components/engine/hack/RELEASE-CHECKLIST.md +++ b/components/engine/hack/RELEASE-CHECKLIST.md @@ -57,7 +57,13 @@ EXAMPLES: FIXME -### 5. Commit and create a pull request to the "release" branch +### 5. Test the docs + +Make sure that your tree includes documentation for any modified or +new features, syntax or semantic changes. Instructions for building +the docs are in ``docs/README.md`` + +### 6. Commit and create a pull request to the "release" branch ```bash git add CHANGELOG.md @@ -65,9 +71,9 @@ git commit -m "Bump version to $VERSION" git push origin bump_$VERSION ``` -### 6. Get 2 other maintainers to validate the pull request +### 7. Get 2 other maintainers to validate the pull request -### 7. Merge the pull request and apply tags +### 8. Merge the pull request and apply tags ```bash git checkout release @@ -78,7 +84,13 @@ git push git push --tags ``` -### 8. Publish binaries +Merging the pull request to the release branch will automatically +update the documentation on the "latest" revision of the docs. You +should see the updated docs 5-10 minutes after the merge. The docs +will appear on http://docs.docker.io/. For more information about +documentation releases, see ``docs/README.md`` + +### 9. Publish binaries To run this you will need access to the release credentials. Get them from [the infrastructure maintainers]( @@ -100,6 +112,6 @@ use get-nightly.docker.io for general testing, and once everything is fine, switch to get.docker.io). -### 9. Rejoice! +### 10. Rejoice! Congratulations! You're done. diff --git a/components/engine/hack/infrastructure/README.md b/components/engine/hack/infrastructure/README.md index 126792a6b0..a74f1f4095 100644 --- a/components/engine/hack/infrastructure/README.md +++ b/components/engine/hack/infrastructure/README.md @@ -1,5 +1,83 @@ # Docker project infrastructure -This directory holds all information about the technical infrastructure of the docker project; servers, dns, email, and all the corresponding tools and configuration. +This is an overview of the Docker infrastructure. -Obviously credentials should not be stored in this repo, but how to obtain and use them should be documented here. +**Note: obviously, credentials should not be stored in this repository.** +However, when there are credentials, we should list how to obtain them +(e.g. who has them). + + +## Providers + +This should be the list of all the entities providing some kind of +infrastructure service to the Docker project (either for free, +or paid by dotCloud). + + +Provider | Service +--------------|------------------------------------------------- +AWS | packages (S3 bucket), dotCloud PAAS, dev-env, ci +CloudFlare | cdn +Digital Ocean | ci +dotCloud PAAS | website, index, registry, ssl, blog +DynECT | dns (docker.io) +GitHub | repository +Linode | stackbrew +Mailgun | outgoing e-mail +ReadTheDocs | docs + +*Ordered-by: lexicographic* + + +## URLs + +This should be the list of all the infrastructure-related URLs +and which service is handling them. + +URL | Service +---------------------------------------------|--------------------------------- + http://blog.docker.io/ | blog +*http://cdn-registry-1.docker.io/ | registry (pull) + http://debug.docker.io/ | debug tool + http://docs.docker.io/ | docsproxy (proxy to readthedocs) + http://docker-ci.dotcloud.com/ | ci + http://docker.io/ | redirect to www.docker.io (dynect) + http://docker.readthedocs.org/ | docs +*http://get.docker.io/ | packages + https://github.com/dotcloud/docker | repository +*https://index.docker.io/ | index + http://registry-1.docker.io/ | registry (push) + http://staging-docker-ci.dotcloud.com/ | ci +*http://test.docker.io/ | packages +*http://www.docker.io/ | website + http://? (internal URL, not for public use) | stackbrew + +*Ordered-by: lexicographic* + +**Note:** an asterisk in front of the URL means that it is cached by CloudFlare. + + +## Services + +This should be the list of all services referenced above. + +Service | Maintainer(s) | How to update | Source +--------------------|--------------------|------------------|------- +blog | @jbarbier | dotcloud push | https://github.com/dotcloud/blog.docker.io +cdn | @jpetazzo @samalba | cloudflare panel | N/A +ci | @mzdaniel | See [docker-ci] | See [docker-ci] +docs | @metalivedev | github webhook | docker repo +docsproxy | @dhrp | dotcloud push | https://github.com/dotcloud/docker-docs-dotcloud-proxy +index | @kencochrane | dotcloud push | private +packages | @jpetazzo | hack/release | docker repo +registry | @samalba | dotcloud push | https://github.com/dotcloud/docker-registry +repository (github) | N/A | N/A | N/A +ssl (dotcloud) | @jpetazzo | dotcloud ops | N/A +ssl (cloudflare) | @jpetazzo | cloudflare panel | N/A +stackbrew | @shin- | manual | https://github.com/dotcloud/stackbrew/stackbrew +website | @dhrp | dotcloud push | https://github.com/dotcloud/www.docker.io + +*Ordered-by: lexicographic* + + +[docker-ci]: docker-ci.rst diff --git a/components/engine/hack/infrastructure/docker-ci.rst b/components/engine/hack/infrastructure/docker-ci.rst index abb8492cf0..0be530d302 100644 --- a/components/engine/hack/infrastructure/docker-ci.rst +++ b/components/engine/hack/infrastructure/docker-ci.rst @@ -1,5 +1,38 @@ -docker-ci github pull request -============================= +docker-ci +========= + +docker-ci is our buildbot continuous integration server, +building and testing docker, hosted on EC2 and reachable at +http://docker-ci.dotcloud.com + + +Deployment +========== + +# Load AWS credentials +export AWS_ACCESS_KEY_ID='' +export AWS_SECRET_ACCESS_KEY='' +export AWS_KEYPAIR_NAME='' +export AWS_SSH_PRIVKEY='' + +# Load buildbot credentials and config +export BUILDBOT_PWD='' +export IRC_PWD='' +export IRC_CHANNEL='docker-dev' +export SMTP_USER='' +export SMTP_PWD='' +export EMAIL_RCP='' + +# Load registry test credentials +export REGISTRY_USER='' +export REGISTRY_PWD='' + +cd docker/testing +vagrant up --provider=aws + + +github pull request +=================== The entire docker pull request test workflow is event driven by github. Its usage is fully automatic and the results are logged in docker-ci.dotcloud.com @@ -13,3 +46,11 @@ buildbot (0.8.7p1) was patched using ./testing/buildbot/github.py, so it can understand the PR data github sends to it. Originally PR #1603 (ee64e099e0) implemented this capability. Also we added a new scheduler to exclusively filter PRs. and the 'pullrequest' builder to rebase the PR on top of master and test it. + + +nighthly release +================ + +The nightly release process is done by buildbot, running a DinD container that downloads +the docker repository and builds the release container. The resulting docker +binary is then tested, and if everything is fine, the release is done. diff --git a/components/engine/hack/infrastructure/docker-ci/Dockerfile b/components/engine/hack/infrastructure/docker-ci/Dockerfile new file mode 100644 index 0000000000..3ac8d90d24 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/Dockerfile @@ -0,0 +1,43 @@ +# VERSION: 0.22 +# DOCKER-VERSION 0.6.3 +# AUTHOR: Daniel Mizyrycki +# DESCRIPTION: Deploy docker-ci on Amazon EC2 +# COMMENTS: +# CONFIG_JSON is an environment variable json string loaded as: +# +# export CONFIG_JSON=' +# { "AWS_TAG": "EC2_instance_name", +# "AWS_ACCESS_KEY": "EC2_access_key", +# "AWS_SECRET_KEY": "EC2_secret_key", +# "DOCKER_CI_PUB": "$(cat docker-ci_ssh_public_key.pub)", +# "DOCKER_CI_KEY": "$(cat docker-ci_ssh_private_key.key)", +# "BUILDBOT_PWD": "Buildbot_server_password", +# "IRC_PWD": "Buildbot_IRC_password", +# "SMTP_USER": "SMTP_server_user", +# "SMTP_PWD": "SMTP_server_password", +# "PKG_ACCESS_KEY": "Docker_release_S3_bucket_access_key", +# "PKG_SECRET_KEY": "Docker_release_S3_bucket_secret_key", +# "PKG_GPG_PASSPHRASE": "Docker_release_gpg_passphrase", +# "INDEX_AUTH": "Index_encripted_user_password", +# "REGISTRY_USER": "Registry_test_user", +# "REGISTRY_PWD": "Registry_test_password", +# "REGISTRY_BUCKET": "Registry_S3_bucket_name", +# "REGISTRY_ACCESS_KEY": "Registry_S3_bucket_access_key", +# "REGISTRY_SECRET_KEY": "Registry_S3_bucket_secret_key", +# "IRC_CHANNEL": "Buildbot_IRC_channel", +# "EMAIL_RCP": "Buildbot_mailing_receipient" }' +# +# +# TO_BUILD: docker build -t docker-ci . +# TO_DEPLOY: docker run -e CONFIG_JSON="${CONFIG_JSON}" docker-ci + +from ubuntu:12.04 + +run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list +run apt-get update; apt-get install -y python2.7 python-dev python-pip ssh rsync less vim +run pip install boto fabric + +# Add deployment code and set default container command +add . /docker-ci +cmd "/docker-ci/deployment.py" + diff --git a/components/engine/testing/MAINTAINERS b/components/engine/hack/infrastructure/docker-ci/MAINTAINERS similarity index 100% rename from components/engine/testing/MAINTAINERS rename to components/engine/hack/infrastructure/docker-ci/MAINTAINERS diff --git a/components/engine/hack/infrastructure/docker-ci/README.rst b/components/engine/hack/infrastructure/docker-ci/README.rst new file mode 100644 index 0000000000..33a14359bf --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/README.rst @@ -0,0 +1,26 @@ +======= +testing +======= + +This directory contains docker-ci testing related files. + + +Buildbot +======== + +Buildbot is a continuous integration system designed to automate the +build/test cycle. By automatically rebuilding and testing the tree each time +something has changed, build problems are pinpointed quickly, before other +developers are inconvenienced by the failure. + +We are running buildbot in Amazon's EC2 to verify docker passes all +tests when commits get pushed to the master branch and building +nightly releases using Docker in Docker awesome implementation made +by Jerome Petazzoni. + +https://github.com/jpetazzo/dind + +Docker's buildbot instance is at http://docker-ci.dotcloud.com/waterfall + +For deployment instructions, please take a look at +hack/infrastructure/docker-ci/Dockerfile diff --git a/components/engine/hack/infrastructure/docker-ci/buildbot/README.rst b/components/engine/hack/infrastructure/docker-ci/buildbot/README.rst new file mode 100644 index 0000000000..6cbcb8d93a --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/buildbot/README.rst @@ -0,0 +1 @@ +Buildbot configuration and setup files diff --git a/components/engine/testing/buildbot/buildbot.conf b/components/engine/hack/infrastructure/docker-ci/buildbot/buildbot.conf similarity index 100% rename from components/engine/testing/buildbot/buildbot.conf rename to components/engine/hack/infrastructure/docker-ci/buildbot/buildbot.conf diff --git a/components/engine/testing/buildbot/github.py b/components/engine/hack/infrastructure/docker-ci/buildbot/github.py similarity index 85% rename from components/engine/testing/buildbot/github.py rename to components/engine/hack/infrastructure/docker-ci/buildbot/github.py index b0fe98a135..ff6b6c62dd 100644 --- a/components/engine/testing/buildbot/github.py +++ b/components/engine/hack/infrastructure/docker-ci/buildbot/github.py @@ -86,12 +86,16 @@ def getChanges(request, options = None): the http request object """ payload = json.loads(request.args['payload'][0]) - if 'pull_request' in payload: - user = payload['repository']['owner']['login'] - repo = payload['repository']['name'] - repo_url = payload['repository']['html_url'] - else: - user = payload['repository']['owner']['name'] + import urllib,datetime + fname = str(datetime.datetime.now()).replace(' ','_').replace(':','-')[:19] + open('github_{0}.json'.format(fname),'w').write(json.dumps(json.loads(urllib.unquote(request.args['payload'][0])), sort_keys = True, indent = 2)) + + if 'pull_request' in payload: + user = payload['pull_request']['user']['login'] + repo = payload['pull_request']['head']['repo']['name'] + repo_url = payload['pull_request']['head']['repo']['html_url'] + else: + user = payload['repository']['owner']['name'] repo = payload['repository']['name'] repo_url = payload['repository']['url'] project = request.args.get('project', None) @@ -115,7 +119,7 @@ def process_change(payload, user, repo, repo_url, project): Hook. """ changes = [] - + newrev = payload['after'] if 'after' in payload else payload['pull_request']['head']['sha'] refname = payload['ref'] if 'ref' in payload else payload['pull_request']['head']['ref'] @@ -130,10 +134,13 @@ def process_change(payload, user, repo, repo_url, project): log.msg("Branch `%s' deleted, ignoring" % branch) return [] else: - if 'pull_request' in payload: - changes = [{ - 'category' : 'github_pullrequest', - 'who' : user, + if 'pull_request' in payload: + if payload['action'] == 'closed': + log.msg("PR#{} closed, ignoring".format(payload['number'])) + return [] + changes = [{ + 'category' : 'github_pullrequest', + 'who' : '{0} - PR#{1}'.format(user,payload['number']), 'files' : [], 'comments' : payload['pull_request']['title'], 'revision' : newrev, @@ -142,7 +149,7 @@ def process_change(payload, user, repo, repo_url, project): 'revlink' : '{0}/commit/{1}'.format(repo_url,newrev), 'repository' : repo_url, 'project' : project }] - return changes + return changes for commit in payload['commits']: files = [] if 'added' in commit: @@ -166,4 +173,3 @@ def process_change(payload, user, repo, repo_url, project): project = project) changes.append(chdict) return changes - diff --git a/components/engine/testing/buildbot/master.cfg b/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg similarity index 70% rename from components/engine/testing/buildbot/master.cfg rename to components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg index 7962fe9e9d..78a9f71cf2 100644 --- a/components/engine/testing/buildbot/master.cfg +++ b/components/engine/hack/infrastructure/docker-ci/buildbot/master.cfg @@ -20,7 +20,8 @@ TEST_PWD = 'docker' # Credential to authenticate build triggers BUILDER_NAME = 'docker' GITHUB_DOCKER = 'github.com/dotcloud/docker' BUILDBOT_PATH = '/data/buildbot' -DOCKER_PATH = '/data/docker' +DOCKER_PATH = '/go/src/github.com/dotcloud/docker' +DOCKER_CI_PATH = '/docker-ci' BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME) PULL_REQUEST_PATH = '/data/buildbot/slave/pullrequest/build' @@ -45,49 +46,41 @@ c['slavePortnum'] = PORT_MASTER # Schedulers c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME, - 'index','registry','coverage'])] + 'index','registry','coverage','nightlyrelease'])] c['schedulers'] += [SingleBranchScheduler(name="all", change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=[BUILDER_NAME])] c['schedulers'] += [SingleBranchScheduler(name='pullrequest', change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None, builderNames=['pullrequest'])] -c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage'], - hour=0, minute=30)] +c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease'], + hour=7, minute=00)] c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'], hour=range(0,24,4), minute=15)] # Builders # Docker commit test factory = BuildFactory() -factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, - command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; " - "cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; " - "git reset --hard %(src::revision)s; go test -v".format( - BUILDER_PATH, BUILDER_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))])) -c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], +factory.addStep(ShellCommand(description='Docker', logEnviron=False, + usePTY=True, command=['sh', '-c', Interpolate( + '{0}/docker-test/test_docker.sh %(src::revision)s'.format(DOCKER_CI_PATH))])) +c['builders'] = [BuilderConfig(name='docker',slavenames=['buildworker'], factory=factory)] # Docker pull request test factory = BuildFactory() -factory.addStep(ShellCommand(description='pull_request',logEnviron=False,usePTY=True, - command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; " - "cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; " - "git fetch %(src::repository)s %(src::branch)s:PR-%(src::branch)s; " - "git checkout %(src::revision)s; git rebase master; go test -v".format( - PULL_REQUEST_PATH, PULL_REQUEST_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))])) +factory.addStep(ShellCommand(description='pull_request', logEnviron=False, + usePTY=True, command=['sh', '-c', Interpolate( + '{0}/docker-test/test_docker.sh %(src::revision)s %(src::repository)s' + ' %(src::branch)s'.format(DOCKER_CI_PATH))])) c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'], factory=factory)] # Docker coverage test -coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n' - 'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n' - 'sudo -E GOPATH=`pwd` ./bin/gocov test -deps -exclude-goroot -v' - ' -exclude github.com/gorilla/context,github.com/gorilla/mux,github.com/kr/pty,' - 'code.google.com/p/go.net/websocket github.com/dotcloud/docker | ./bin/gocov report') factory = BuildFactory() -factory.addStep(ShellCommand(description='Coverage',logEnviron=False,usePTY=True, - command=coverage_cmd)) +factory.addStep(ShellCommand(description='Coverage', logEnviron=False, + usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format( + DOCKER_CI_PATH))) c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], factory=factory)] @@ -95,8 +88,8 @@ c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], factory = BuildFactory() factory.addStep(ShellCommand(description='registry', logEnviron=False, command='. {0}/master/credentials.cfg; ' - '{1}/testing/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, - DOCKER_PATH), usePTY=True)) + '/docker-ci/functionaltests/test_registry.sh'.format(BUILDBOT_PATH), + usePTY=True)) c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], factory=factory)] @@ -109,6 +102,14 @@ factory.addStep(ShellCommand(description='index', logEnviron=False, c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'], factory=factory)] +# Docker nightly release +nightlyrelease_cmd = ('docker run -i -t -privileged -lxc-conf=lxc.aa_profile=unconfined' + ' -e AWS_S3_BUCKET=test.docker.io dockerbuilder') +factory = BuildFactory() +factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False,usePTY=True, + command=nightlyrelease_cmd)) +c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'], + factory=factory)] # Status authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]), diff --git a/components/engine/testing/buildbot/requirements.txt b/components/engine/hack/infrastructure/docker-ci/buildbot/requirements.txt similarity index 100% rename from components/engine/testing/buildbot/requirements.txt rename to components/engine/hack/infrastructure/docker-ci/buildbot/requirements.txt diff --git a/components/engine/testing/buildbot/setup.sh b/components/engine/hack/infrastructure/docker-ci/buildbot/setup.sh similarity index 72% rename from components/engine/testing/buildbot/setup.sh rename to components/engine/hack/infrastructure/docker-ci/buildbot/setup.sh index 99e4f7f104..c7e89c44b2 100755 --- a/components/engine/testing/buildbot/setup.sh +++ b/components/engine/hack/infrastructure/docker-ci/buildbot/setup.sh @@ -6,16 +6,22 @@ USER=$1 CFG_PATH=$2 -BUILDBOT_PWD=$3 -IRC_PWD=$4 -IRC_CHANNEL=$5 -SMTP_USER=$6 -SMTP_PWD=$7 -EMAIL_RCP=$8 +DOCKER_PATH=$3 +BUILDBOT_PWD=$4 +IRC_PWD=$5 +IRC_CHANNEL=$6 +SMTP_USER=$7 +SMTP_PWD=$8 +EMAIL_RCP=$9 +REGISTRY_USER=${10} +REGISTRY_PWD=${11} +REGISTRY_BUCKET=${12} +REGISTRY_ACCESS_KEY=${13} +REGISTRY_SECRET_KEY=${14} BUILDBOT_PATH="/data/buildbot" -DOCKER_PATH="/data/docker" SLAVE_NAME="buildworker" SLAVE_SOCKET="localhost:9989" + export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" function run { su $USER -c "$1"; } @@ -35,6 +41,10 @@ run "sed -i -E 's#(SMTP_USER = ).+#\1\"$SMTP_USER\"#' master/master.cfg" run "sed -i -E 's#(SMTP_PWD = ).+#\1\"$SMTP_PWD\"#' master/master.cfg" run "sed -i -E 's#(EMAIL_RCP = ).+#\1\"$EMAIL_RCP\"#' master/master.cfg" run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD" +run "echo 'export DOCKER_CREDS=\"$REGISTRY_USER:$REGISTRY_PWD\"' > $BUILDBOT_PATH/master/credentials.cfg" +run "echo 'export S3_BUCKET=\"$REGISTRY_BUCKET\"' >> $BUILDBOT_PATH/master/credentials.cfg" +run "echo 'export S3_ACCESS_KEY=\"$REGISTRY_ACCESS_KEY\"' >> $BUILDBOT_PATH/master/credentials.cfg" +run "echo 'export S3_SECRET_KEY=\"$REGISTRY_SECRET_KEY\"' >> $BUILDBOT_PATH/master/credentials.cfg" # Patch github webstatus to capture pull requests cp $CFG_PATH/github.py /usr/local/lib/python2.7/dist-packages/buildbot/status/web/hooks diff --git a/components/engine/hack/infrastructure/docker-ci/deployment.py b/components/engine/hack/infrastructure/docker-ci/deployment.py new file mode 100755 index 0000000000..0ff21ceda1 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/deployment.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python + +import os, sys, re, json, base64 +from boto.ec2.connection import EC2Connection +from subprocess import call +from fabric import api +from fabric.api import cd, run, put, sudo +from os import environ as env +from time import sleep + +# Remove SSH private key as it needs more processing +CONFIG = json.loads(re.sub(r'("DOCKER_CI_KEY".+?"(.+?)",)','', + env['CONFIG_JSON'], flags=re.DOTALL)) + +# Populate environment variables +for key in CONFIG: + env[key] = CONFIG[key] + +# Load SSH private key +env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1', + env['CONFIG_JSON'],flags=re.DOTALL) + + +AWS_TAG = env.get('AWS_TAG','docker-ci') +AWS_KEY_NAME = 'dotcloud-dev' # Same as CONFIG_JSON['DOCKER_CI_PUB'] +AWS_AMI = 'ami-d582d6bc' # Ubuntu 13.04 +AWS_REGION = 'us-east-1' +AWS_TYPE = 'm1.small' +AWS_SEC_GROUPS = 'gateway' +AWS_IMAGE_USER = 'ubuntu' +DOCKER_PATH = '/go/src/github.com/dotcloud/docker' +DOCKER_CI_PATH = '/docker-ci' +CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH) + + +class AWS_EC2: + '''Amazon EC2''' + def __init__(self, access_key, secret_key): + '''Set default API parameters''' + self.handler = EC2Connection(access_key, secret_key) + def create_instance(self, tag, instance_type): + reservation = self.handler.run_instances(**instance_type) + instance = reservation.instances[0] + sleep(10) + while instance.state != 'running': + sleep(5) + instance.update() + print "Instance state: %s" % (instance.state) + instance.add_tag("Name",tag) + print "instance %s done!" % (instance.id) + return instance.ip_address + def get_instances(self): + return self.handler.get_all_instances() + def get_tags(self): + return dict([(i.instances[0].id, i.instances[0].tags['Name']) + for i in self.handler.get_all_instances() if i.instances[0].tags]) + def del_instance(self, instance_id): + self.handler.terminate_instances(instance_ids=[instance_id]) + + +def json_fmt(data): + '''Format json output''' + return json.dumps(data, sort_keys = True, indent = 2) + + +# Create EC2 API handler +ec2 = AWS_EC2(env['AWS_ACCESS_KEY'], env['AWS_SECRET_KEY']) + +# Stop processing if AWS_TAG exists on EC2 +if AWS_TAG in ec2.get_tags().values(): + print ('Instance: {} already deployed. Not further processing.' + .format(AWS_TAG)) + exit(1) + +ip = ec2.create_instance(AWS_TAG, {'image_id':AWS_AMI, 'instance_type':AWS_TYPE, + 'security_groups':[AWS_SEC_GROUPS], 'key_name':AWS_KEY_NAME}) + +# Wait 30 seconds for the machine to boot +sleep(30) + +# Create docker-ci ssh private key so docker-ci docker container can communicate +# with its EC2 instance +os.makedirs('/root/.ssh') +open('/root/.ssh/id_rsa','w').write(env['DOCKER_CI_KEY']) +os.chmod('/root/.ssh/id_rsa',0600) +open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n') + +api.env.host_string = ip +api.env.user = AWS_IMAGE_USER +api.env.key_filename = '/root/.ssh/id_rsa' + +# Correct timezone +sudo('echo "America/Los_Angeles" >/etc/timezone') +sudo('dpkg-reconfigure --frontend noninteractive tzdata') + +# Load public docker-ci key +sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB'])) + +# Create docker nightly release credentials file +credentials = { + 'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'], + 'AWS_SECRET_KEY': env['PKG_SECRET_KEY'], + 'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE'], + 'INDEX_AUTH': env['INDEX_AUTH']} +open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write( + base64.b64encode(json.dumps(credentials))) + +# Transfer docker +sudo('mkdir -p ' + DOCKER_CI_PATH) +sudo('chown {}.{} {}'.format(AWS_IMAGE_USER, AWS_IMAGE_USER, DOCKER_CI_PATH)) +call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, AWS_IMAGE_USER, ip, + os.path.dirname(DOCKER_CI_PATH)), shell=True) + +# Install Docker and Buildbot dependencies +sudo('addgroup docker') +sudo('usermod -a -G docker ubuntu') +sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker') +sudo('wget -q -O - https://get.docker.io/gpg | apt-key add -') +sudo('echo deb https://get.docker.io/ubuntu docker main >' + ' /etc/apt/sources.list.d/docker.list') +sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n' + 'deb http://us.archive.ubuntu.com/ubuntu/ raring-security main universe\n"' + ' > /etc/apt/sources.list; apt-get update') +sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev' + ' python-pip supervisor git mercurial linux-image-extra-$(uname -r)' + ' aufs-tools make libfontconfig libevent-dev') +sudo('wget -O - https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | ' + 'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go') +sudo('GOPATH=/go go get -d github.com/dotcloud/docker') +sudo('pip install -r {}/requirements.txt'.format(CFG_PATH)) + +# Install docker and testing dependencies +sudo('apt-get install -y -q lxc-docker') +sudo('curl -s https://phantomjs.googlecode.com/files/' + 'phantomjs-1.9.1-linux-x86_64.tar.bz2 | tar jx -C /usr/bin' + ' --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs') + +#### FIXME. Temporarily install docker with proper apparmor handling +sudo('stop docker') +sudo('wget -q -O /usr/bin/docker http://test.docker.io/test/docker') +sudo('start docker') + +# Build docker-ci containers +sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH)) +sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format( + DOCKER_CI_PATH)) + +# Setup buildbot +sudo('mkdir /data') +sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}' + ' {11} {12}'.format(CFG_PATH, DOCKER_PATH, env['BUILDBOT_PWD'], + env['IRC_PWD'], env['IRC_CHANNEL'], env['SMTP_USER'], + env['SMTP_PWD'], env['EMAIL_RCP'], env['REGISTRY_USER'], + env['REGISTRY_PWD'], env['REGISTRY_BUCKET'], env['REGISTRY_ACCESS_KEY'], + env['REGISTRY_SECRET_KEY'])) diff --git a/components/engine/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh b/components/engine/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh new file mode 100755 index 0000000000..2ca3b6e801 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -x +# Generate a random string of $1 characters +function random { + cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1 +} + +# Compute test paths +BASE_PATH=`pwd`/test_docker_$(random 12) +DOCKER_PATH=$BASE_PATH/go/src/github.com/dotcloud/docker +export GOPATH=$BASE_PATH/go:$DOCKER_PATH/vendor + +# Fetch latest master +mkdir -p $DOCKER_PATH +cd $DOCKER_PATH +git init . +git fetch -q http://github.com/dotcloud/docker master +git reset --hard FETCH_HEAD + +# Fetch go coverage +cd $BASE_PATH/go +GOPATH=$BASE_PATH/go go get github.com/axw/gocov/gocov +sudo -E GOPATH=$GOPATH ./bin/gocov test -deps -exclude-goroot -v\ + -exclude github.com/gorilla/context,github.com/gorilla/mux,github.com/kr/pty,\ +code.google.com/p/go.net/websocket,github.com/dotcloud/tar\ + github.com/dotcloud/docker | ./bin/gocov report; exit_status=$? + +# Cleanup testing directory +rm -rf $BASE_PATH + +exit $exit_status diff --git a/components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh b/components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh new file mode 100755 index 0000000000..46075b76e0 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/docker-test/test_docker.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -x +COMMIT=${1-HEAD} +REPO=${2-http://github.com/dotcloud/docker} +BRANCH=${3-master} + +# Generate a random string of $1 characters +function random { + cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1 +} + +# Compute test paths +BASE_PATH=`pwd`/test_docker_$(random 12) +DOCKER_PATH=$BASE_PATH/go/src/github.com/dotcloud/docker +export GOPATH=$BASE_PATH/go:$DOCKER_PATH/vendor + +# Fetch latest master +mkdir -p $DOCKER_PATH +cd $DOCKER_PATH +git init . +git fetch -q http://github.com/dotcloud/docker master +git reset --hard FETCH_HEAD + +# Merge commit +git fetch -q "$REPO" "$BRANCH" +git merge --no-edit $COMMIT || exit 1 + +# Test commit +go test -v; exit_status=$? + +# Cleanup testing directory +rm -rf $BASE_PATH + +exit $exit_status diff --git a/components/engine/testing/functionaltests/test_index.py b/components/engine/hack/infrastructure/docker-ci/functionaltests/test_index.py similarity index 100% rename from components/engine/testing/functionaltests/test_index.py rename to components/engine/hack/infrastructure/docker-ci/functionaltests/test_index.py diff --git a/components/engine/hack/infrastructure/docker-ci/functionaltests/test_registry.sh b/components/engine/hack/infrastructure/docker-ci/functionaltests/test_registry.sh new file mode 100755 index 0000000000..8bcd355c7c --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/functionaltests/test_registry.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +set -x + +# Cleanup +rm -rf docker-registry + +# Setup the environment +export SETTINGS_FLAVOR=test +export DOCKER_REGISTRY_CONFIG=config_test.yml + +# Get latest docker registry +git clone -q https://github.com/dotcloud/docker-registry.git +cd docker-registry + +# Get dependencies +pip install -q -r requirements.txt +pip install -q -r test-requirements.txt +pip install -q tox + +# Run registry tests +tox || exit 1 +export PYTHONPATH=$(pwd)/docker-registry +python -m unittest discover -p s3.py -s test || exit 1 +python -m unittest discover -p workflow.py -s test + diff --git a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile new file mode 100644 index 0000000000..8eb0b72118 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile @@ -0,0 +1,37 @@ +# VERSION: 1.2 +# DOCKER-VERSION 0.6.3 +# AUTHOR: Daniel Mizyrycki +# DESCRIPTION: Build docker nightly release using Docker in Docker. +# REFERENCES: This code reuses the excellent implementation of docker in docker +# made by Jerome Petazzoni. https://github.com/jpetazzo/dind +# COMMENTS: +# release_credentials.json is a base64 json encoded file containing: +# { "AWS_ACCESS_KEY": "Test_docker_AWS_S3_bucket_id", +# "AWS_SECRET_KEY='Test_docker_AWS_S3_bucket_key' +# "GPG_PASSPHRASE='Test_docker_GPG_passphrase_signature' +# "INDEX_AUTH='Encripted_index_authentication' } +# TO_BUILD: docker build -t dockerbuilder . +# TO_RELEASE: docker run -i -t -privileged -lxc-conf="lxc.aa_profile = unconfined" -e AWS_S3_BUCKET="test.docker.io" dockerbuilder + +from docker +maintainer Daniel Mizyrycki + +# Add docker dependencies and downloading packages +run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list +run apt-get update; apt-get install -y -q wget python2.7 + +# Add production docker binary +run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker + +#### FIXME. Temporarily install docker with proper apparmor handling +run wget -q -O /usr/bin/docker http://test.docker.io/test/docker; chmod +x /usr/bin/docker + +# Add proto docker builder +add ./dockerbuild /usr/bin/dockerbuild +run chmod +x /usr/bin/dockerbuild + +# Add release credentials +add ./release_credentials.json /root/release_credentials.json + +# Launch build process in a container +cmd dockerbuild diff --git a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild new file mode 100644 index 0000000000..857e0f8848 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild @@ -0,0 +1,45 @@ +#!/bin/bash + +# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY, PG_PASSPHRASE and INDEX_AUTH +# are decoded from /root/release_credentials.json +# Variable AWS_S3_BUCKET is passed to the environment from docker run -e + +# Enable debugging +set -x + +# Fetch docker master branch +rm -rf /go/src/github.com/dotcloud/docker +cd / +git clone -q http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker +cd /go/src/github.com/dotcloud/docker + +echo FIXME. Temporarily add Jerome changeset with proper apparmor handling +git fetch http://github.com/jpetazzo/docker escape-apparmor-confinement:escape-apparmor-confinement +git rebase --onto master master escape-apparmor-confinement + +# Launch docker daemon using dind inside the container +./hack/dind /usr/bin/docker -d & +sleep 5 + +# Add an uncommitted change to generate a timestamped release +date > timestamp + +# Build the docker package using /Dockerfile +docker build -t docker . + +# Run Docker unittests binary and Ubuntu package +docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker hack/make.sh || exit 1 + +# Turn debug off to load credentials from the environment +set +x +eval $(cat /root/release_credentials.json | python -c ' +import sys,json,base64; +d=json.loads(base64.b64decode(sys.stdin.read())); +exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")') +echo '{"https://index.docker.io/v1/":{"auth":"'$INDEX_AUTH'","email":"engineering@dotcloud.com"}}' > /.dockercfg +set -x + +# Push docker nightly +echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX release hack/release.sh +set +x +docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE release hack/release.sh diff --git a/components/engine/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json new file mode 100644 index 0000000000..ed6d53ecd1 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json @@ -0,0 +1 @@ +eyAiQVdTX0FDQ0VTU19LRVkiOiAiIiwKICAiQVdTX1NFQ1JFVF9LRVkiOiAiIiwKICAiR1BHX1BBU1NQSFJBU0UiOiAiIiwKICAiSU5ERVhfQVVUSCI6ICIiIH0= diff --git a/components/engine/hack/infrastructure/docker-ci/report/Dockerfile b/components/engine/hack/infrastructure/docker-ci/report/Dockerfile new file mode 100644 index 0000000000..32600c4c58 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/report/Dockerfile @@ -0,0 +1,28 @@ +# VERSION: 0.22 +# DOCKER-VERSION 0.6.3 +# AUTHOR: Daniel Mizyrycki +# DESCRIPTION: Generate docker-ci daily report +# COMMENTS: The build process is initiated by deployment.py + Report configuration is passed through ./credentials.json at +# deployment time. +# TO_BUILD: docker build -t report . +# TO_DEPLOY: docker run report + +from ubuntu:12.04 +maintainer Daniel Mizyrycki + +env PYTHONPATH /report + + +# Add report dependencies +run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > \ + /etc/apt/sources.list +run apt-get update; apt-get install -y python2.7 python-pip ssh rsync + +# Set San Francisco timezone +run echo "America/Los_Angeles" >/etc/timezone +run dpkg-reconfigure --frontend noninteractive tzdata + +# Add report code and set default container command +add . /report +cmd "/report/report.py" diff --git a/components/engine/hack/infrastructure/docker-ci/report/deployment.py b/components/engine/hack/infrastructure/docker-ci/report/deployment.py new file mode 100755 index 0000000000..d5efb4a960 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/report/deployment.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python + +'''Deploy docker-ci report container on Digital Ocean. +Usage: + export CONFIG_JSON=' + { "DROPLET_NAME": "Digital_Ocean_dropplet_name", + "DO_CLIENT_ID": "Digital_Ocean_client_id", + "DO_API_KEY": "Digital_Ocean_api_key", + "DOCKER_KEY_ID": "Digital_Ocean_ssh_key_id", + "DOCKER_CI_KEY_PATH": "docker-ci_private_key_path", + "DOCKER_CI_PUB": "$(cat docker-ci_ssh_public_key.pub)", + "DOCKER_CI_ADDRESS" "user@docker-ci_fqdn_server", + "SMTP_USER": "SMTP_server_user", + "SMTP_PWD": "SMTP_server_password", + "EMAIL_SENDER": "Buildbot_mailing_sender", + "EMAIL_RCP": "Buildbot_mailing_receipient" }' + python deployment.py +''' + +import re, json, requests, base64 +from fabric import api +from fabric.api import cd, run, put, sudo +from os import environ as env +from time import sleep +from datetime import datetime + +# Populate environment variables +CONFIG = json.loads(env['CONFIG_JSON']) +for key in CONFIG: + env[key] = CONFIG[key] + +# Load DOCKER_CI_KEY +env['DOCKER_CI_KEY'] = open(env['DOCKER_CI_KEY_PATH']).read() + +DROPLET_NAME = env.get('DROPLET_NAME','report') +TIMEOUT = 120 # Seconds before timeout droplet creation +IMAGE_ID = 894856 # Docker on Ubuntu 13.04 +REGION_ID = 4 # New York 2 +SIZE_ID = 66 # memory 512MB +DO_IMAGE_USER = 'root' # Image user on Digital Ocean +API_URL = 'https://api.digitalocean.com/' + + +class digital_ocean(): + + def __init__(self, key, client): + '''Set default API parameters''' + self.key = key + self.client = client + self.api_url = API_URL + + def api(self, cmd_path, api_arg={}): + '''Make api call''' + api_arg.update({'api_key':self.key, 'client_id':self.client}) + resp = requests.get(self.api_url + cmd_path, params=api_arg).text + resp = json.loads(resp) + if resp['status'] != 'OK': + raise Exception(resp['error_message']) + return resp + + def droplet_data(self, name): + '''Get droplet data''' + data = self.api('droplets') + data = [droplet for droplet in data['droplets'] + if droplet['name'] == name] + return data[0] if data else {} + +def json_fmt(data): + '''Format json output''' + return json.dumps(data, sort_keys = True, indent = 2) + + +do = digital_ocean(env['DO_API_KEY'], env['DO_CLIENT_ID']) + +# Get DROPLET_NAME data +data = do.droplet_data(DROPLET_NAME) + +# Stop processing if DROPLET_NAME exists on Digital Ocean +if data: + print ('Droplet: {} already deployed. Not further processing.' + .format(DROPLET_NAME)) + exit(1) + +# Create droplet +do.api('droplets/new', {'name':DROPLET_NAME, 'region_id':REGION_ID, + 'image_id':IMAGE_ID, 'size_id':SIZE_ID, + 'ssh_key_ids':[env['DOCKER_KEY_ID']]}) + +# Wait for droplet to be created. +start_time = datetime.now() +while (data.get('status','') != 'active' and ( + datetime.now()-start_time).seconds < TIMEOUT): + data = do.droplet_data(DROPLET_NAME) + print data['status'] + sleep(3) + +# Wait for the machine to boot +sleep(15) + +# Get droplet IP +ip = str(data['ip_address']) +print 'droplet: {} ip: {}'.format(DROPLET_NAME, ip) + +api.env.host_string = ip +api.env.user = DO_IMAGE_USER +api.env.key_filename = env['DOCKER_CI_KEY_PATH'] + +# Correct timezone +sudo('echo "America/Los_Angeles" >/etc/timezone') +sudo('dpkg-reconfigure --frontend noninteractive tzdata') + +# Load JSON_CONFIG environment for Dockerfile +CONFIG_JSON= base64.b64encode( + '{{"DOCKER_CI_PUB": "{DOCKER_CI_PUB}",' + ' "DOCKER_CI_KEY": "{DOCKER_CI_KEY}",' + ' "DOCKER_CI_ADDRESS": "{DOCKER_CI_ADDRESS}",' + ' "SMTP_USER": "{SMTP_USER}",' + ' "SMTP_PWD": "{SMTP_PWD}",' + ' "EMAIL_SENDER": "{EMAIL_SENDER}",' + ' "EMAIL_RCP": "{EMAIL_RCP}"}}'.format(**env)) + +run('mkdir -p /data/report') +put('./', '/data/report') +with cd('/data/report'): + run('chmod 700 report.py') + run('echo "{}" > credentials.json'.format(CONFIG_JSON)) + run('docker build -t report .') + run('rm credentials.json') + run("echo -e '30 09 * * * /usr/bin/docker run report\n' |" + " /usr/bin/crontab -") diff --git a/components/engine/hack/infrastructure/docker-ci/report/report.py b/components/engine/hack/infrastructure/docker-ci/report/report.py new file mode 100755 index 0000000000..7018cabc27 --- /dev/null +++ b/components/engine/hack/infrastructure/docker-ci/report/report.py @@ -0,0 +1,145 @@ +#!/usr/bin/python + +'''CONFIG_JSON is a json encoded string base64 environment variable. It is used +to clone docker-ci database, generate docker-ci report and submit it by email. +CONFIG_JSON data comes from the file /report/credentials.json inserted in this +container by deployment.py: + +{ "DOCKER_CI_PUB": "$(cat docker-ci_ssh_public_key.pub)", + "DOCKER_CI_KEY": "$(cat docker-ci_ssh_private_key.key)", + "DOCKER_CI_ADDRESS": "user@docker-ci_fqdn_server", + "SMTP_USER": "SMTP_server_user", + "SMTP_PWD": "SMTP_server_password", + "EMAIL_SENDER": "Buildbot_mailing_sender", + "EMAIL_RCP": "Buildbot_mailing_receipient" } ''' + +import os, re, json, sqlite3, datetime, base64 +import smtplib +from datetime import timedelta +from subprocess import call +from os import environ as env + +TODAY = datetime.date.today() + +# Load credentials to the environment +env['CONFIG_JSON'] = base64.b64decode(open('/report/credentials.json').read()) + +# Remove SSH private key as it needs more processing +CONFIG = json.loads(re.sub(r'("DOCKER_CI_KEY".+?"(.+?)",)','', + env['CONFIG_JSON'], flags=re.DOTALL)) + +# Populate environment variables +for key in CONFIG: + env[key] = CONFIG[key] + +# Load SSH private key +env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1', + env['CONFIG_JSON'],flags=re.DOTALL) + +# Prevent rsync to validate host on first connection to docker-ci +os.makedirs('/root/.ssh') +open('/root/.ssh/id_rsa','w').write(env['DOCKER_CI_KEY']) +os.chmod('/root/.ssh/id_rsa',0600) +open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n') + + +# Sync buildbot database from docker-ci +call('rsync {}:/data/buildbot/master/state.sqlite .'.format( + env['DOCKER_CI_ADDRESS']), shell=True) + +class SQL: + def __init__(self, database_name): + sql = sqlite3.connect(database_name) + # Use column names as keys for fetchall rows + sql.row_factory = sqlite3.Row + sql = sql.cursor() + self.sql = sql + + def query(self,query_statement): + return self.sql.execute(query_statement).fetchall() + +sql = SQL("state.sqlite") + + +class Report(): + + def __init__(self,period='',date=''): + self.data = [] + self.period = 'date' if not period else period + self.date = str(TODAY) if not date else date + self.compute() + + def compute(self): + '''Compute report''' + if self.period == 'week': + self.week_report(self.date) + else: + self.date_report(self.date) + + + def date_report(self,date): + '''Create a date test report''' + builds = [] + # Get a queryset with all builds from date + rows = sql.query('SELECT * FROM builds JOIN buildrequests' + ' WHERE builds.brid=buildrequests.id and' + ' date(start_time, "unixepoch", "localtime") = "{0}"' + ' GROUP BY number'.format(date)) + build_names = sorted(set([row['buildername'] for row in rows])) + # Create a report build line for a given build + for build_name in build_names: + tried = len([row['buildername'] + for row in rows if row['buildername'] == build_name]) + fail_tests = [row['buildername'] for row in rows if ( + row['buildername'] == build_name and row['results'] != 0)] + fail = len(fail_tests) + fail_details = '' + fail_pct = int(100.0*fail/tried) if tried != 0 else 100 + builds.append({'name': build_name, 'tried': tried, 'fail': fail, + 'fail_pct': fail_pct, 'fail_details':fail_details}) + if builds: + self.data.append({'date': date, 'builds': builds}) + + + def week_report(self,date): + '''Add the week's date test reports to report.data''' + date = datetime.datetime.strptime(date,'%Y-%m-%d').date() + last_monday = date - datetime.timedelta(days=date.weekday()) + week_dates = [last_monday + timedelta(days=x) for x in range(7,-1,-1)] + for date in week_dates: + self.date_report(str(date)) + + def render_text(self): + '''Return rendered report in text format''' + retval = '' + fail_tests = {} + for builds in self.data: + retval += 'Test date: {0}\n'.format(builds['date'],retval) + table = '' + for build in builds['builds']: + table += ('Build {name:15} Tried: {tried:4} ' + ' Failures: {fail:4} ({fail_pct}%)\n'.format(**build)) + if build['name'] in fail_tests: + fail_tests[build['name']] += build['fail_details'] + else: + fail_tests[build['name']] = build['fail_details'] + retval += '{0}\n'.format(table) + retval += '\n Builds failing' + for fail_name in fail_tests: + retval += '\n' + fail_name + '\n' + for (fail_id,fail_url,rn_tests,nr_errors,log_errors, + tracelog_errors) in fail_tests[fail_name]: + retval += fail_url + '\n' + retval += '\n\n' + return retval + + +# Send email +smtp_from = env['EMAIL_SENDER'] +subject = '[docker-ci] Daily report for {}'.format(str(TODAY)) +msg = "From: {}\r\nTo: {}\r\nSubject: {}\r\n\r\n".format( + smtp_from, env['EMAIL_RCP'], subject) +msg = msg + Report('week').render_text() +server = smtplib.SMTP_SSL('smtp.mailgun.org') +server.login(env['SMTP_USER'], env['SMTP_PWD']) +server.sendmail(smtp_from, env['EMAIL_RCP'], msg) diff --git a/components/engine/network_proxy.go b/components/engine/network_proxy.go index 1bb849538b..86545801b5 100644 --- a/components/engine/network_proxy.go +++ b/components/engine/network_proxy.go @@ -103,7 +103,11 @@ func (proxy *TCPProxy) Run() { for { client, err := proxy.listener.Accept() if err != nil { - utils.Errorf("Stopping proxy on tcp/%v for tcp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) + if utils.IsClosedError(err) { + utils.Debugf("Stopping proxy on tcp/%v for tcp/%v (socket was closed)", proxy.frontendAddr, proxy.backendAddr) + } else { + utils.Errorf("Stopping proxy on tcp/%v for tcp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) + } return } go proxy.clientLoop(client.(*net.TCPConn), quit) @@ -205,7 +209,11 @@ func (proxy *UDPProxy) Run() { // NOTE: Apparently ReadFrom doesn't return // ECONNREFUSED like Read do (see comment in // UDPProxy.replyLoop) - utils.Errorf("Stopping proxy on udp/%v for udp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) + if utils.IsClosedError(err) { + utils.Debugf("Stopping proxy on udp/%v for udp/%v (socket was closed)", proxy.frontendAddr, proxy.backendAddr) + } else { + utils.Errorf("Stopping proxy on udp/%v for udp/%v (%v)", proxy.frontendAddr, proxy.backendAddr, err.Error()) + } break } diff --git a/components/engine/runtime.go b/components/engine/runtime.go index d2705ee3ef..07708f4798 100644 --- a/components/engine/runtime.go +++ b/components/engine/runtime.go @@ -421,7 +421,9 @@ func (runtime *Runtime) Create(config *Config) (*Container, error) { } if img.Config != nil { - MergeConfig(config, img.Config) + if err := MergeConfig(config, img.Config); err != nil { + return nil, err + } } if len(config.Entrypoint) != 0 && config.Cmd == nil { diff --git a/components/engine/runtime_test.go b/components/engine/runtime_test.go index cc1b22ba59..6576134f87 100644 --- a/components/engine/runtime_test.go +++ b/components/engine/runtime_test.go @@ -313,6 +313,29 @@ func TestRuntimeCreate(t *testing.T) { if err == nil { t.Fatal("Builder.Create should throw an error when Cmd is empty") } + + config := &Config{ + Image: GetTestImage(runtime).ID, + Cmd: []string{"/bin/ls"}, + PortSpecs: []string{"80"}, + } + container, err = runtime.Create(config) + + image, err := runtime.Commit(container, "testrepo", "testtag", "", "", config) + if err != nil { + t.Error(err) + } + + _, err = runtime.Create( + &Config{ + Image: image.ID, + PortSpecs: []string{"80000:80"}, + }, + ) + if err == nil { + t.Fatal("Builder.Create should throw an error when PortSpecs is invalid") + } + } func TestDestroy(t *testing.T) { diff --git a/components/engine/term/term.go b/components/engine/term/term.go index 5929c2caa1..8c53a20ca6 100644 --- a/components/engine/term/term.go +++ b/components/engine/term/term.go @@ -21,11 +21,19 @@ type Winsize struct { func GetWinsize(fd uintptr) (*Winsize, error) { ws := &Winsize{} _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TIOCGWINSZ), uintptr(unsafe.Pointer(ws))) + // Skipp errno = 0 + if err == 0 { + return ws, nil + } return ws, err } func SetWinsize(fd uintptr, ws *Winsize) error { _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TIOCSWINSZ), uintptr(unsafe.Pointer(ws))) + // Skipp errno = 0 + if err == 0 { + return nil + } return err } diff --git a/components/engine/testing/README.rst b/components/engine/testing/README.rst deleted file mode 100644 index ce5aa837a4..0000000000 --- a/components/engine/testing/README.rst +++ /dev/null @@ -1,58 +0,0 @@ -======= -testing -======= - -This directory contains testing related files. - - -Buildbot -======== - -Buildbot is a continuous integration system designed to automate the -build/test cycle. By automatically rebuilding and testing the tree each time -something has changed, build problems are pinpointed quickly, before other -developers are inconvenienced by the failure. - -We are running buildbot in an AWS instance to verify docker passes all tests -when commits get pushed to the master branch. - -You can check docker's buildbot instance at http://docker-ci.dotcloud.com/waterfall - - -Deployment -~~~~~~~~~~ - -:: - - # Define AWS credential environment variables - export AWS_ACCESS_KEY_ID=xxxxxxxxxxxx - export AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxx - export AWS_KEYPAIR_NAME=xxxxxxxxxxxx - export AWS_SSH_PRIVKEY=xxxxxxxxxxxx - - # Define email recipient and IRC channel - export EMAIL_RCP=xxxxxx@domain.com - export IRC_CHANNEL=docker - - # Define buildbot credentials - export BUILDBOT_PWD=xxxxxxxxxxxx - export IRC_PWD=xxxxxxxxxxxx - export SMTP_USER=xxxxxxxxxxxx - export SMTP_PWD=xxxxxxxxxxxx - - # Define docker registry functional test credentials - export REGISTRY_USER=xxxxxxxxxxxx - export REGISTRY_PWD=xxxxxxxxxxxx - - # Checkout docker - git clone git://github.com/dotcloud/docker.git - - # Deploy docker on AWS - cd docker/testing - vagrant up --provider=aws - - -Buildbot AWS dependencies -------------------------- - -vagrant, virtualbox packages and vagrant aws plugin diff --git a/components/engine/testing/Vagrantfile b/components/engine/testing/Vagrantfile deleted file mode 100644 index 1d15709a36..0000000000 --- a/components/engine/testing/Vagrantfile +++ /dev/null @@ -1,74 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -BOX_NAME = "docker-ci" -BOX_URI = "http://cloud-images.ubuntu.com/vagrant/raring/current/raring-server-cloudimg-amd64-vagrant-disk1.box" -AWS_AMI = "ami-10314d79" -DOCKER_PATH = "/data/docker" -CFG_PATH = "#{DOCKER_PATH}/testing/buildbot" -on_vbox = File.file?("#{File.dirname(__FILE__)}/.vagrant/machines/default/virtualbox/id") | \ - Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? & \ - (on_vbox=true; ARGV.each do |arg| on_vbox &&= !arg.downcase.start_with?("--provider") end; on_vbox) -USER = on_vbox ? "vagrant": "ubuntu" - -Vagrant::Config.run do |config| - # Setup virtual machine box. This VM configuration code is always executed. - config.vm.box = BOX_NAME - config.vm.box_url = BOX_URI - config.vm.forward_port 8010, 8010 - config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.." - - - # Deploy buildbot and its dependencies if it was not done - if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? - # Add memory limitation capabilities - pkg_cmd = 'sed -Ei \'s/^(GRUB_CMDLINE_LINUX_DEFAULT)=.+/\\1="cgroup_enable=memory swapaccount=1 quiet"/\' /etc/default/grub; ' - # Adjust kernel - pkg_cmd << "apt-get update -qq; " - if on_vbox - pkg_cmd << "apt-get install -q -y linux-image-extra-`uname -r`; " - else - pkg_cmd << "apt-get install -q -y linux-image-generic; " - end - - # Deploy buildbot CI - pkg_cmd << "apt-get install -q -y python-dev python-pip supervisor; " \ - "pip install -r #{CFG_PATH}/requirements.txt; " \ - "chown #{USER}.#{USER} /data; cd /data; " \ - "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \ - "#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \ - "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " \ - "#{CFG_PATH}/setup_credentials.sh #{USER} " \ - "#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; " - # Install docker and testing dependencies - pkg_cmd << "curl -s https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | " \ - " tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go; " \ - "curl -s https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2 | " \ - " tar jx -C /usr/bin --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs; " \ - "DEBIAN_FRONTEND=noninteractive apt-get install -q -y lxc git mercurial aufs-tools make libfontconfig; " \ - "export GOPATH=/data/docker-dependencies; go get -d github.com/dotcloud/docker; " \ - "rm -rf ${GOPATH}/src/github.com/dotcloud/docker; " - # Activate new kernel options - pkg_cmd << "shutdown -r +1; " - config.vm.provision :shell, :inline => pkg_cmd - end -end - -# Providers were added on Vagrant >= 1.1.0 -Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| - config.vm.provider :aws do |aws, override| - aws.tags = { 'Name' => 'docker-ci' } - aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"] - aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"] - aws.keypair_name = ENV["AWS_KEYPAIR_NAME"] - override.ssh.private_key_path = ENV["AWS_SSH_PRIVKEY"] - override.ssh.username = USER - aws.ami = AWS_AMI - aws.region = "us-east-1" - aws.instance_type = "m1.small" - aws.security_groups = "gateway" - end - - config.vm.provider :virtualbox do |vb| - end -end diff --git a/components/engine/testing/buildbot/README.rst b/components/engine/testing/buildbot/README.rst deleted file mode 100644 index 7edbf32d0a..0000000000 --- a/components/engine/testing/buildbot/README.rst +++ /dev/null @@ -1 +0,0 @@ -Buildbot configuration and setup files (except Vagrantfile located on ..) diff --git a/components/engine/testing/buildbot/credentials.cfg b/components/engine/testing/buildbot/credentials.cfg deleted file mode 100644 index fbdd35d578..0000000000 --- a/components/engine/testing/buildbot/credentials.cfg +++ /dev/null @@ -1,5 +0,0 @@ -# Credentials for tests. Buildbot source this file on tests -# when needed. - -# Docker registry credentials. Format: 'username:password' -export DOCKER_CREDS='' diff --git a/components/engine/testing/buildbot/setup_credentials.sh b/components/engine/testing/buildbot/setup_credentials.sh deleted file mode 100755 index f093815d60..0000000000 --- a/components/engine/testing/buildbot/setup_credentials.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Setup of test credentials. Called by Vagrantfile -export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" - -USER=$1 -REGISTRY_USER=$2 -REGISTRY_PWD=$3 - -BUILDBOT_PATH="/data/buildbot" -DOCKER_PATH="/data/docker" - -function run { su $USER -c "$1"; } - -run "cp $DOCKER_PATH/testing/buildbot/credentials.cfg $BUILDBOT_PATH/master" -cd $BUILDBOT_PATH/master -run "sed -i -E 's#(export DOCKER_CREDS=).+#\1\"$REGISTRY_USER:$REGISTRY_PWD\"#' credentials.cfg" diff --git a/components/engine/testing/functionaltests/test_registry.sh b/components/engine/testing/functionaltests/test_registry.sh deleted file mode 100755 index 095a731631..0000000000 --- a/components/engine/testing/functionaltests/test_registry.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -# Cleanup -rm -rf docker-registry - -# Get latest docker registry -git clone https://github.com/dotcloud/docker-registry.git - -# Configure and run registry tests -cd docker-registry; cp config_sample.yml config.yml -cd test; python -m unittest workflow diff --git a/components/engine/utils.go b/components/engine/utils.go index 99600d1882..0e357051f2 100644 --- a/components/engine/utils.go +++ b/components/engine/utils.go @@ -90,7 +90,7 @@ func CompareConfig(a, b *Config) bool { return true } -func MergeConfig(userConf, imageConf *Config) { +func MergeConfig(userConf, imageConf *Config) error { if userConf.User == "" { userConf.User = imageConf.User } @@ -108,9 +108,15 @@ func MergeConfig(userConf, imageConf *Config) { } else { for _, imagePortSpec := range imageConf.PortSpecs { found := false - imageNat, _ := parseNat(imagePortSpec) + imageNat, err := parseNat(imagePortSpec) + if err != nil { + return err + } for _, userPortSpec := range userConf.PortSpecs { - userNat, _ := parseNat(userPortSpec) + userNat, err := parseNat(userPortSpec) + if err != nil { + return err + } if imageNat.Proto == userNat.Proto && imageNat.Backend == userNat.Backend { found = true } @@ -171,6 +177,7 @@ func MergeConfig(userConf, imageConf *Config) { userConf.Volumes[k] = v } } + return nil } func parseLxcConfOpts(opts ListOpts) ([]KeyValuePair, error) { diff --git a/components/engine/utils/stdcopy.go b/components/engine/utils/stdcopy.go index 42dad738e4..3cb8ab02b3 100644 --- a/components/engine/utils/stdcopy.go +++ b/components/engine/utils/stdcopy.go @@ -37,8 +37,12 @@ func (w *StdWriter) Write(buf []byte) (n int, err error) { return n - StdWriterPrefixLen, err } -// NewStdWriter instanciate a new Writer based on the given type `t`. -// the utils package contains the valid parametres for `t`: +// NewStdWriter instanciates a new Writer. +// Everything written to it will be encapsulated using a custom format, +// and written to the underlying `w` stream. +// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection. +// `t` indicates the id of the stream to encapsulate. +// It can be utils.Stdin, utils.Stdout, utils.Stderr. func NewStdWriter(w io.Writer, t StdType) *StdWriter { if len(t) != StdWriterPrefixLen { return nil @@ -55,16 +59,14 @@ var ErrInvalidStdHeader = errors.New("Unrecognized input header") // StdCopy is a modified version of io.Copy. // -// StdCopy copies from src to dstout or dsterr until either EOF is reached -// on src or an error occurs. It returns the number of bytes -// copied and the first error encountered while copying, if any. +// StdCopy will demultiplex `src`, assuming that it contains two streams, +// previously multiplexed together using a StdWriter instance. +// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`. // -// A successful Copy returns err == nil, not err == EOF. -// Because Copy is defined to read from src until EOF, it does -// not treat an EOF from Read as an error to be reported. +// StdCopy will read until it hits EOF on `src`. It will then return a nil error. +// In other words: if `err` is non nil, it indicates a real underlying error. // -// The source needs to be writter via StdWriter, dstout or dsterr is selected -// based on the prefix added by StdWriter +// `written` will hold the total number of bytes written to `dstout` and `dsterr`. func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) { var ( buf = make([]byte, 32*1024+StdWriterPrefixLen+1) diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 4187da1aec..cb06937bc2 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -1063,3 +1063,13 @@ func ShellQuoteArguments(args []string) string { } return buf.String() } + +func IsClosedError(err error) bool { + /* This comparison is ugly, but unfortunately, net.go doesn't export errClosing. + * See: + * http://golang.org/src/pkg/net/net.go + * https://code.google.com/p/go/issues/detail?id=4337 + * https://groups.google.com/forum/#!msg/golang-nuts/0_aaCvBmOcM/SptmDyX1XJMJ + */ + return strings.HasSuffix(err.Error(), "use of closed network connection") +}