From 54030128e3e10d7dd9261810d3f3513f6af1e917 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Tue, 3 Jun 2014 15:46:01 +0800 Subject: [PATCH 001/516] mount of /.dockerinit is not needed for native driver, so move it into lxc driver Docker-DCO-1.1-Signed-off-by: Albert Zhang (github: zhgwenming) Upstream-commit: b611198d286d2f4ebd7526c623dff8e523691698 Component: engine --- .../engine/daemon/execdriver/execdrivers/execdrivers.go | 2 +- components/engine/daemon/execdriver/lxc/driver.go | 5 ++++- components/engine/daemon/volumes.go | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/components/engine/daemon/execdriver/execdrivers/execdrivers.go b/components/engine/daemon/execdriver/execdrivers/execdrivers.go index 2e18454a09..f43514408b 100644 --- a/components/engine/daemon/execdriver/execdrivers/execdrivers.go +++ b/components/engine/daemon/execdriver/execdrivers/execdrivers.go @@ -15,7 +15,7 @@ func NewDriver(name, root, initPath string, sysInfo *sysinfo.SysInfo) (execdrive // we want to give the lxc driver the full docker root because it needs // to access and write config and template files in /var/lib/docker/containers/* // to be backwards compatible - return lxc.NewDriver(root, sysInfo.AppArmor) + return lxc.NewDriver(root, initPath, sysInfo.AppArmor) case "native": return native.NewDriver(path.Join(root, "execdriver", "native"), initPath) } diff --git a/components/engine/daemon/execdriver/lxc/driver.go b/components/engine/daemon/execdriver/lxc/driver.go index 59daf1afe1..d4a00dcca6 100644 --- a/components/engine/daemon/execdriver/lxc/driver.go +++ b/components/engine/daemon/execdriver/lxc/driver.go @@ -54,11 +54,12 @@ func init() { type driver struct { root string // root path for the driver to use + initPath string apparmor bool sharedRoot bool } -func NewDriver(root string, apparmor bool) (*driver, error) { +func NewDriver(root, initPath string, apparmor bool) (*driver, error) { // setup unconfined symlink if err := linkLxcStart(root); err != nil { return nil, err @@ -66,6 +67,7 @@ func NewDriver(root string, apparmor bool) (*driver, error) { return &driver{ apparmor: apparmor, root: root, + initPath: initPath, sharedRoot: rootIsShared(), }, nil } @@ -79,6 +81,7 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba if err := execdriver.SetTerminal(c, pipes); err != nil { return -1, err } + c.Mounts = append(c.Mounts, execdriver.Mount{d.initPath, c.InitPath, false, true}) if err := d.generateEnvConfig(c); err != nil { return -1, err } diff --git a/components/engine/daemon/volumes.go b/components/engine/daemon/volumes.go index f4b3921c9a..1de8ff98ac 100644 --- a/components/engine/daemon/volumes.go +++ b/components/engine/daemon/volumes.go @@ -36,7 +36,6 @@ func prepareVolumesForContainer(container *Container) error { func setupMountsForContainer(container *Container) error { mounts := []execdriver.Mount{ - {container.daemon.sysInitPath, "/.dockerinit", false, true}, {container.ResolvConfPath, "/etc/resolv.conf", false, true}, } From 363db75bc019c24a8cbf53f262d95a6de9507b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Petazzoni?= Date: Wed, 11 Jun 2014 17:09:19 -0700 Subject: [PATCH 002/516] Select masquerade by outgoing interface rather than by destination subnet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker-DCO-1.1-Signed-off-by: Jérôme Petazzoni (github: jpetazzo) Upstream-commit: a084f4bc61a81eb9076246d85d6cc5168b32a949 Component: engine --- components/engine/daemon/networkdriver/bridge/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/daemon/networkdriver/bridge/driver.go b/components/engine/daemon/networkdriver/bridge/driver.go index 8c5db9f843..1cde7c9be0 100644 --- a/components/engine/daemon/networkdriver/bridge/driver.go +++ b/components/engine/daemon/networkdriver/bridge/driver.go @@ -175,7 +175,7 @@ func InitDriver(job *engine.Job) engine.Status { func setupIPTables(addr net.Addr, icc bool) error { // Enable NAT - natArgs := []string{"POSTROUTING", "-t", "nat", "-s", addr.String(), "!", "-d", addr.String(), "-j", "MASQUERADE"} + natArgs := []string{"POSTROUTING", "-t", "nat", "-s", addr.String(), "!", "-o", bridgeIface, "-j", "MASQUERADE"} if !iptables.Exists(natArgs...) { if output, err := iptables.Raw(append([]string{"-I"}, natArgs...)...); err != nil { From 946f83d348147b619ea578ceeb242cd7e2bce6be Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 4 Dec 2013 15:03:51 +0100 Subject: [PATCH 003/516] Add support for client certificates for registries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This lets you specify custom client TLS certificates and CA root for a specific registry hostname. Docker will then verify the registry against the CA and present the client cert when talking to that registry. This allows the registry to verify that the client has a proper key, indicating that the client is allowed to access the images. A custom cert is configured by creating a directory in /etc/docker/certs.d with the same name as the registry hostname. Inside this directory all *.crt files are added as CA Roots (if none exists, the system default is used) and pair of files .key and .cert indicate a custom certificate to present to the registry. If there are multiple certificates each one will be tried in alphabetical order, proceeding to the next if we get a 403 of 5xx response. So, an example setup would be: /etc/docker/certs.d/ └── localhost ├── client.cert ├── client.key └── localhost.crt A simple way to test this setup is to use an apache server to host a registry. Just copy a registry tree into the apache root, here is an example one containing the busybox image: http://people.gnome.org/~alexl/v1.tar.gz Then add this conf file as /etc/httpd/conf.d/registry.conf: # This must be in the root context, otherwise it causes a re-negotiation # which is not supported by the tls implementation in go SSLVerifyClient optional_no_ca Action cert-protected /cgi-bin/cert.cgi SetHandler cert-protected Header set x-docker-registry-version "0.6.2" SetEnvIf Host (.*) custom_host=$1 Header set X-Docker-Endpoints "%{custom_host}e" And this as /var/www/cgi-bin/cert.cgi #!/bin/bash if [ "$HTTPS" != "on" ]; then echo "Status: 403 Not using SSL" echo "x-docker-registry-version: 0.6.2" echo exit 0 fi if [ "$SSL_CLIENT_VERIFY" == "NONE" ]; then echo "Status: 403 Client certificate invalid" echo "x-docker-registry-version: 0.6.2" echo exit 0 fi echo "Content-length: $(stat --printf='%s' $PATH_TRANSLATED)" echo "x-docker-registry-version: 0.6.2" echo "X-Docker-Endpoints: $SERVER_NAME" echo "X-Docker-Size: 0" echo cat $PATH_TRANSLATED This will return 403 for all accessed to /v1 unless *any* client cert is presented. Obviously a real implementation would verify more details about the certificate. Example client certs can be generated with: openssl genrsa -out client.key 1024 openssl req -new -x509 -text -key client.key -out client.cert Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 05243104fc0a0ef9537766cf5bd920824665eb78 Component: engine --- components/engine/docs/mkdocs.yml | 1 + .../docs/sources/articles/certificates.md | 83 +++++++ components/engine/docs/sources/use.md | 14 ++ components/engine/registry/registry.go | 227 ++++++++++++++---- 4 files changed, 272 insertions(+), 53 deletions(-) create mode 100644 components/engine/docs/sources/articles/certificates.md create mode 100644 components/engine/docs/sources/use.md diff --git a/components/engine/docs/mkdocs.yml b/components/engine/docs/mkdocs.yml index 5c3147a285..00810e2cb0 100755 --- a/components/engine/docs/mkdocs.yml +++ b/components/engine/docs/mkdocs.yml @@ -83,6 +83,7 @@ pages: - ['articles/security.md', 'Articles', 'Security'] - ['articles/https.md', 'Articles', 'Running Docker with HTTPS'] - ['articles/host_integration.md', 'Articles', 'Automatically starting Containers'] +- ['articles/certificates.md', 'Articles', 'Using certificates for repository client verification'] - ['articles/using_supervisord.md', 'Articles', 'Using Supervisor'] - ['articles/cfengine_process_management.md', 'Articles', 'Process management with CFEngine'] - ['articles/puppet.md', 'Articles', 'Using Puppet'] diff --git a/components/engine/docs/sources/articles/certificates.md b/components/engine/docs/sources/articles/certificates.md new file mode 100644 index 0000000000..3772a7bc57 --- /dev/null +++ b/components/engine/docs/sources/articles/certificates.md @@ -0,0 +1,83 @@ +page_title: Using certificates for repository client verification +page_description: How to set up per-repository client certificates +page_keywords: Usage, repository, certificate, root, docker, documentation, examples + +# Using certificates for repository client verification + +This lets you specify custom client TLS certificates and CA root for a +specific registry hostname. Docker will then verify the registry +against the CA and present the client cert when talking to that +registry. This allows the registry to verify that the client has a +proper key, indicating that the client is allowed to access the +images. + +A custom cert is configured by creating a directory in +`/etc/docker/certs.d` with the same name as the registry hostname. Inside +this directory all .crt files are added as CA Roots (if none exists, +the system default is used) and pair of files `$filename.key` and +`$filename.cert` indicate a custom certificate to present to the +registry. + +If there are multiple certificates each one will be tried in +alphabetical order, proceeding to the next if we get a 403 of 5xx +response. + +So, an example setup would be:: + + /etc/docker/certs.d/ + └── localhost + ├── client.cert + ├── client.key + └── localhost.crt + +A simple way to test this setup is to use an apache server to host a +registry. Just copy a registry tree into the apache root, +[here](http://people.gnome.org/~alexl/v1.tar.gz) is an example one +containing the busybox image. + +Then add this conf file as `/etc/httpd/conf.d/registry.conf`: + + # This must be in the root context, otherwise it causes a re-negotiation + # which is not supported by the tls implementation in go + SSLVerifyClient optional_no_ca + + + Action cert-protected /cgi-bin/cert.cgi + SetHandler cert-protected + + Header set x-docker-registry-version "0.6.2" + SetEnvIf Host (.*) custom_host=$1 + Header set X-Docker-Endpoints "%{custom_host}e" + + +And this as `/var/www/cgi-bin/cert.cgi`: + + #!/bin/bash + if [ "$HTTPS" != "on" ]; then + echo "Status: 403 Not using SSL" + echo "x-docker-registry-version: 0.6.2" + echo + exit 0 + fi + if [ "$SSL_CLIENT_VERIFY" == "NONE" ]; then + echo "Status: 403 Client certificate invalid" + echo "x-docker-registry-version: 0.6.2" + echo + exit 0 + fi + echo "Content-length: $(stat --printf='%s' $PATH_TRANSLATED)" + echo "x-docker-registry-version: 0.6.2" + echo "X-Docker-Endpoints: $SERVER_NAME" + echo "X-Docker-Size: 0" + echo + + cat $PATH_TRANSLATED + +This will return 403 for all accessed to `/v1` unless any client cert is +presented. Obviously a real implementation would verify more details +about the certificate. + +Example client certs can be generated with:: + + openssl genrsa -out client.key 1024 + openssl req -new -x509 -text -key client.key -out client.cert diff --git a/components/engine/docs/sources/use.md b/components/engine/docs/sources/use.md new file mode 100644 index 0000000000..20c5eb2c50 --- /dev/null +++ b/components/engine/docs/sources/use.md @@ -0,0 +1,14 @@ +# Use + +## Contents: + + - [First steps with Docker](basics/) + - [Share Images via Repositories](workingwithrepository/) + - [Redirect Ports](port_redirection/) + - [Configure Networking](networking/) + - [Automatically Start Containers](host_integration/) + - [Share Directories via Volumes](working_with_volumes/) + - [Link Containers](working_with_links_names/) + - [Link via an Ambassador Container](ambassador_pattern_linking/) + - [Using Puppet](puppet/) + - [Using certificates for repository client verification](certificates/) diff --git a/components/engine/registry/registry.go b/components/engine/registry/registry.go index 24c55125ce..748636dca8 100644 --- a/components/engine/registry/registry.go +++ b/components/engine/registry/registry.go @@ -4,6 +4,8 @@ import ( "bytes" "crypto/sha256" _ "crypto/sha512" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -13,6 +15,8 @@ import ( "net/http" "net/http/cookiejar" "net/url" + "os" + "path" "regexp" "runtime" "strconv" @@ -29,31 +33,155 @@ var ( errLoginRequired = errors.New("Authentication is required.") ) +type TimeoutType uint32 + +const ( + NoTimeout TimeoutType = iota + ReceiveTimeout + ConnectTimeout +) + +func newClient(jar http.CookieJar, roots *x509.CertPool, cert *tls.Certificate, timeout TimeoutType) *http.Client { + tlsConfig := tls.Config{RootCAs: roots} + + if cert != nil { + tlsConfig.Certificates = append(tlsConfig.Certificates, *cert) + } + + httpTransport := &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tlsConfig, + } + + switch timeout { + case ConnectTimeout: + httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { + // Set the connect timeout to 5 seconds + conn, err := net.DialTimeout(proto, addr, 5*time.Second) + if err != nil { + return nil, err + } + // Set the recv timeout to 10 seconds + conn.SetDeadline(time.Now().Add(10 * time.Second)) + return conn, nil + } + case ReceiveTimeout: + httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { + conn, err := net.Dial(proto, addr) + if err != nil { + return nil, err + } + conn = utils.NewTimeoutConn(conn, 1*time.Minute) + return conn, nil + } + } + + return &http.Client{ + Transport: httpTransport, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + Jar: jar, + } +} + +func doRequest(req *http.Request, jar http.CookieJar, timeout TimeoutType) (*http.Response, *http.Client, error) { + hasFile := func(files []os.FileInfo, name string) bool { + for _, f := range files { + if f.Name() == name { + return true + } + } + return false + } + + hostDir := path.Join("/etc/docker/certs.d", req.URL.Host) + fs, err := ioutil.ReadDir(hostDir) + if err != nil && !os.IsNotExist(err) { + return nil, nil, err + } + + var ( + pool *x509.CertPool + certs []*tls.Certificate + ) + + for _, f := range fs { + if strings.HasSuffix(f.Name(), ".crt") { + if pool == nil { + pool = x509.NewCertPool() + } + data, err := ioutil.ReadFile(path.Join(hostDir, f.Name())) + if err != nil { + return nil, nil, err + } else { + pool.AppendCertsFromPEM(data) + } + } + if strings.HasSuffix(f.Name(), ".cert") { + certName := f.Name() + keyName := certName[:len(certName)-5] + ".key" + if !hasFile(fs, keyName) { + return nil, nil, fmt.Errorf("Missing key %s for certificate %s", keyName, certName) + } else { + cert, err := tls.LoadX509KeyPair(path.Join(hostDir, certName), path.Join(hostDir, keyName)) + if err != nil { + return nil, nil, err + } + certs = append(certs, &cert) + } + } + if strings.HasSuffix(f.Name(), ".key") { + keyName := f.Name() + certName := keyName[:len(keyName)-4] + ".cert" + if !hasFile(fs, certName) { + return nil, nil, fmt.Errorf("Missing certificate %s for key %s", certName, keyName) + } + } + } + + if len(certs) == 0 { + client := newClient(jar, pool, nil, timeout) + res, err := client.Do(req) + if err != nil { + return nil, nil, err + } + return res, client, nil + } else { + for i, cert := range certs { + client := newClient(jar, pool, cert, timeout) + res, err := client.Do(req) + if i == len(certs)-1 { + // If this is the last cert, always return the result + return res, client, err + } else { + // Otherwise, continue to next cert if 403 or 5xx + if err == nil && res.StatusCode != 403 && !(res.StatusCode >= 500 && res.StatusCode < 600) { + return res, client, err + } + } + } + } + + return nil, nil, nil +} + func pingRegistryEndpoint(endpoint string) (RegistryInfo, error) { if endpoint == IndexServerAddress() { // Skip the check, we now this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil } - httpDial := func(proto string, addr string) (net.Conn, error) { - // Set the connect timeout to 5 seconds - conn, err := net.DialTimeout(proto, addr, 5*time.Second) - if err != nil { - return nil, err - } - // Set the recv timeout to 10 seconds - conn.SetDeadline(time.Now().Add(10 * time.Second)) - return conn, nil - } - httpTransport := &http.Transport{ - Dial: httpDial, - Proxy: http.ProxyFromEnvironment, - } - client := &http.Client{Transport: httpTransport} - resp, err := client.Get(endpoint + "_ping") + + req, err := http.NewRequest("GET", endpoint+"_ping", nil) if err != nil { return RegistryInfo{Standalone: false}, err } + + resp, _, err := doRequest(req, nil, ConnectTimeout) + if err != nil { + return RegistryInfo{Standalone: false}, err + } + defer resp.Body.Close() jsonString, err := ioutil.ReadAll(resp.Body) @@ -171,6 +299,10 @@ func setTokenAuth(req *http.Request, token []string) { } } +func (r *Registry) doRequest(req *http.Request) (*http.Response, *http.Client, error) { + return doRequest(req, r.jar, r.timeout) +} + // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { @@ -179,7 +311,7 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s return nil, err } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -214,7 +346,7 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo return false } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { utils.Errorf("Error in LookupRemoteImage %s", err) return false @@ -231,7 +363,7 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ return nil, -1, fmt.Errorf("Failed to download json: %s", err) } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } @@ -260,6 +392,7 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i var ( retries = 5 headRes *http.Response + client *http.Client hasResume bool = false imageURL = fmt.Sprintf("%simages/%s/layer", registry, imgID) ) @@ -267,9 +400,10 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } + setTokenAuth(headReq, token) for i := 1; i <= retries; i++ { - headRes, err = r.client.Do(headReq) + headRes, client, err = r.doRequest(headReq) if err != nil && i == retries { return nil, fmt.Errorf("Eror while making head request: %s\n", err) } else if err != nil { @@ -290,10 +424,10 @@ func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string, i setTokenAuth(req, token) if hasResume { utils.Debugf("server supports resume") - return utils.ResumableRequestReader(r.client, req, 5, imgSize), nil + return utils.ResumableRequestReader(client, req, 5, imgSize), nil } utils.Debugf("server doesn't support resume") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -319,7 +453,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -380,7 +514,7 @@ func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { } req.Header.Set("X-Docker-Token", "true") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -448,13 +582,13 @@ func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, req.Header.Set("X-Docker-Checksum", imgData.Checksum) req.Header.Set("X-Docker-Checksum-Payload", imgData.ChecksumPayload) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } defer res.Body.Close() if len(res.Cookies()) > 0 { - r.client.Jar.SetCookies(req.URL, res.Cookies()) + r.jar.SetCookies(req.URL, res.Cookies()) } if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) @@ -484,7 +618,7 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) } @@ -525,7 +659,7 @@ func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registr req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} setTokenAuth(req, token) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return "", "", fmt.Errorf("Failed to upload layer: %s", err) } @@ -562,7 +696,7 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token req.Header.Add("Content-type", "application/json") setTokenAuth(req, token) req.ContentLength = int64(len(revision)) - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return err } @@ -610,7 +744,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat req.Header["X-Docker-Endpoints"] = regs } - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -629,7 +763,7 @@ func (r *Registry) PushImageJSONIndex(remote string, imgList []*ImgData, validat if validate { req.Header["X-Docker-Endpoints"] = regs } - res, err = r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -688,7 +822,7 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") - res, err := r.client.Do(req) + res, _, err := r.doRequest(req) if err != nil { return nil, err } @@ -750,10 +884,11 @@ type RegistryInfo struct { } type Registry struct { - client *http.Client authConfig *AuthConfig reqFactory *utils.HTTPRequestFactory indexEndpoint string + jar *cookiejar.Jar + timeout TimeoutType } func trustedLocation(req *http.Request) bool { @@ -791,30 +926,16 @@ func AddRequiredHeadersToRedirectedRequests(req *http.Request, via []*http.Reque } func NewRegistry(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, indexEndpoint string, timeout bool) (r *Registry, err error) { - httpTransport := &http.Transport{ - DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - } - if timeout { - httpTransport.Dial = func(proto string, addr string) (net.Conn, error) { - conn, err := net.Dial(proto, addr) - if err != nil { - return nil, err - } - conn = utils.NewTimeoutConn(conn, 1*time.Minute) - return conn, nil - } - } r = &Registry{ - authConfig: authConfig, - client: &http.Client{ - Transport: httpTransport, - CheckRedirect: AddRequiredHeadersToRedirectedRequests, - }, + authConfig: authConfig, indexEndpoint: indexEndpoint, } - r.client.Jar, err = cookiejar.New(nil) + if timeout { + r.timeout = ReceiveTimeout + } + + r.jar, err = cookiejar.New(nil) if err != nil { return nil, err } From 7d1a57a5bef46fe21d36dc7eb3326f7cbd4687e2 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Wed, 18 Jun 2014 10:51:44 -0400 Subject: [PATCH 004/516] Increase size of buffer for signals The chan struct used to forward signals to containers was one element only, which caused some signals to be dropped when many were being received. Increasing the size of the chan buffer makes this much less likely to happen. Docker-DCO-1.1-Signed-off-by: Matt Heon (github: mheon) Upstream-commit: 06cd125e64c30886f66e6490a1cd4e7a7ba23655 Component: engine --- components/engine/api/client/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/api/client/commands.go b/components/engine/api/client/commands.go index a6a2e35539..0a30bc3d5b 100644 --- a/components/engine/api/client/commands.go +++ b/components/engine/api/client/commands.go @@ -535,7 +535,7 @@ func (cli *DockerCli) CmdRestart(args ...string) error { } func (cli *DockerCli) forwardAllSignals(cid string) chan os.Signal { - sigc := make(chan os.Signal, 1) + sigc := make(chan os.Signal, 128) signal.CatchAll(sigc) go func() { for s := range sigc { From 2ad31b8e2f8a323694f5fa971afcb954bb0ccba7 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Fri, 20 Jun 2014 09:40:29 -0400 Subject: [PATCH 005/516] Add cpuguy83 as volumes maintainer Docker-DCO-1.1-Signed-off-by: Brian Goff (github: cpuguy83) Upstream-commit: ff1ea0064b49bd1d72cafe5d02be9eab4d54b683 Component: engine --- components/engine/daemon/MAINTAINERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 components/engine/daemon/MAINTAINERS diff --git a/components/engine/daemon/MAINTAINERS b/components/engine/daemon/MAINTAINERS new file mode 100644 index 0000000000..7ab013cd82 --- /dev/null +++ b/components/engine/daemon/MAINTAINERS @@ -0,0 +1 @@ +volumes.go: Brian Goff (@cpuguy83) From 3ee2658a4c6bfdc6713ecd64b278067f7e11bfee Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Fri, 20 Jun 2014 12:22:31 -0400 Subject: [PATCH 006/516] Escape control and nonprintable characters in docker ps The docker ps command displays the user-entered command running in a container. If that command contained \n, \t, or other control characters, they were interpreted literally, and newlines and tabs would be printed in the output. Escape the command string to make things more readable. Docker-DCO-1.1-Signed-off-by: Matt Heon (github: mheon) Upstream-commit: f55fa8211b6faf08091dd99ed8e5e3f08ab9cf1f Component: engine --- components/engine/api/client/commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/api/client/commands.go b/components/engine/api/client/commands.go index 0cdf3f1acb..8f76b6e26a 100644 --- a/components/engine/api/client/commands.go +++ b/components/engine/api/client/commands.go @@ -1476,6 +1476,7 @@ func (cli *DockerCli) CmdPs(args ...string) error { outCommand = out.Get("Command") ports = engine.NewTable("", 0) ) + outCommand = strconv.Quote(outCommand) if !*noTrunc { outCommand = utils.Trunc(outCommand, 20) } From 417ba2a8cb60717b11b7b1081baad2e417eb97a7 Mon Sep 17 00:00:00 2001 From: Senthil Kumar Selvaraj Date: Tue, 24 Jun 2014 11:57:24 +0530 Subject: [PATCH 007/516] Fix typo in README.md Upstream-commit: 21e258d7328e56fd451e6ed0324cff094ec70798 Component: engine --- components/engine/daemon/graphdriver/devmapper/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/daemon/graphdriver/devmapper/README.md b/components/engine/daemon/graphdriver/devmapper/README.md index c8ab1d1ee1..d97baaeab2 100644 --- a/components/engine/daemon/graphdriver/devmapper/README.md +++ b/components/engine/daemon/graphdriver/devmapper/README.md @@ -17,7 +17,7 @@ stored in the `$graph/devicemapper/json` file (encoded as Json). In order to support multiple devicemapper graphs on a system the thin pool will be named something like: `docker-0:33-19478248-pool`, where -the `0:30` part is the minor/major device nr and `19478248` is the +the `0:33` part is the minor/major device nr and `19478248` is the inode number of the $graph directory. On the thin pool docker automatically creates a base thin device, From 28d12ecd022e4d11502cc1591396f12700810f32 Mon Sep 17 00:00:00 2001 From: Jonathan Camp Date: Mon, 26 May 2014 09:40:24 +0200 Subject: [PATCH 008/516] Replace dashes in link name with underscores Docker-DCO-1.1-Signed-off-by: Jonathan Camp (github: kung-foo) Upstream-commit: 6e74754a504e80378ed56ec765d62b762f80fcb4 Component: engine --- components/engine/links/links.go | 2 +- components/engine/links/links_test.go | 30 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/components/engine/links/links.go b/components/engine/links/links.go index 7665a06a11..45c1935c20 100644 --- a/components/engine/links/links.go +++ b/components/engine/links/links.go @@ -49,7 +49,7 @@ func (l *Link) Alias() string { func (l *Link) ToEnv() []string { env := []string{} - alias := strings.ToUpper(l.Alias()) + alias := strings.Replace(strings.ToUpper(l.Alias()), "-", "_", -1) if p := l.getDefaultPort(); p != nil { env = append(env, fmt.Sprintf("%s_PORT=%s://%s:%s", alias, p.Proto(), l.ChildIP, p.Port())) diff --git a/components/engine/links/links_test.go b/components/engine/links/links_test.go index e66f9bfb78..e79ef1a136 100644 --- a/components/engine/links/links_test.go +++ b/components/engine/links/links_test.go @@ -6,6 +6,36 @@ import ( "testing" ) +func TestLinkNaming(t *testing.T) { + ports := make(nat.PortSet) + ports[nat.Port("6379/tcp")] = struct{}{} + + link, err := NewLink("172.0.17.3", "172.0.17.2", "/db/docker-1", nil, ports, nil) + if err != nil { + t.Fatal(err) + } + + rawEnv := link.ToEnv() + env := make(map[string]string, len(rawEnv)) + for _, e := range rawEnv { + parts := strings.Split(e, "=") + if len(parts) != 2 { + t.FailNow() + } + env[parts[0]] = parts[1] + } + + value, ok := env["DOCKER_1_PORT"] + + if !ok { + t.Fatalf("DOCKER_1_PORT not found in env") + } + + if value != "tcp://172.0.17.2:6379" { + t.Fatalf("Expected 172.0.17.2:6379, got %s", env["DOCKER_1_PORT"]) + } +} + func TestLinkNew(t *testing.T) { ports := make(nat.PortSet) ports[nat.Port("6379/tcp")] = struct{}{} From 157a125d05675ad67d6dec42cb0b7219f1e5b752 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 20 Jun 2014 11:45:42 -0400 Subject: [PATCH 009/516] devmapper: fix reloadPool() to also specify '1 skip_block_zeroing' createPool() and reloadPool() should be consistent with the thin-pool table params they use. Since createPool() specifies '1 skip_block_zeroing' reloadPool() should too. Otherwise, if the pool is reloaded (as is done when resizing loopback devices) block zeroing will be enabled after the reload completes. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) Upstream-commit: d420134fa2ea8ceedf2a9a64263d09b308263c2c Component: engine --- components/engine/daemon/graphdriver/devmapper/devmapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/daemon/graphdriver/devmapper/devmapper.go b/components/engine/daemon/graphdriver/devmapper/devmapper.go index a6602c276e..2590ec0fc1 100644 --- a/components/engine/daemon/graphdriver/devmapper/devmapper.go +++ b/components/engine/daemon/graphdriver/devmapper/devmapper.go @@ -369,7 +369,7 @@ func reloadPool(poolName string, dataFile, metadataFile *os.File) error { return fmt.Errorf("Can't get data size %s", err) } - params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768" + params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768 1 skip_block_zeroing" if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil { return fmt.Errorf("Can't add target %s", err) } From d61150b2d43fa003d2200ffb1348cecff9fd559b Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Mon, 23 Jun 2014 15:36:08 -0400 Subject: [PATCH 010/516] devmapper: use RAMInBytes() rather than FromHumanSize() Device Mapper needs device sizes in binary (1024) multiples. Otherwise kernel checks can find that the specified thin-pool device sizes aren't a multiple of the specified thin-pool blocksize. The name for "RAMInBytes" is likely too narrow given the new consumers but... Also add "tebibyte" support to RAMInBytes. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) Upstream-commit: 2470a5ed999d9dc8d4b9bd250727ac13964db5b3 Component: engine --- .../engine/daemon/graphdriver/devmapper/deviceset.go | 6 +++--- components/engine/pkg/units/size.go | 8 +++++--- components/engine/pkg/units/size_test.go | 3 +++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go index 31c3f391e4..b0e0819ba8 100644 --- a/components/engine/daemon/graphdriver/devmapper/deviceset.go +++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go @@ -1170,19 +1170,19 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error key = strings.ToLower(key) switch key { case "dm.basesize": - size, err := units.FromHumanSize(val) + size, err := units.RAMInBytes(val) if err != nil { return nil, err } devices.baseFsSize = uint64(size) case "dm.loopdatasize": - size, err := units.FromHumanSize(val) + size, err := units.RAMInBytes(val) if err != nil { return nil, err } devices.dataLoopbackSize = size case "dm.loopmetadatasize": - size, err := units.FromHumanSize(val) + size, err := units.RAMInBytes(val) if err != nil { return nil, err } diff --git a/components/engine/pkg/units/size.go b/components/engine/pkg/units/size.go index 480ec2f141..e0eec3ec7d 100644 --- a/components/engine/pkg/units/size.go +++ b/components/engine/pkg/units/size.go @@ -58,11 +58,11 @@ func FromHumanSize(size string) (int64, error) { } // Parses a human-readable string representing an amount of RAM -// in bytes, kibibytes, mebibytes or gibibytes, and returns the -// number of bytes, or -1 if the string is unparseable. +// in bytes, kibibytes, mebibytes, gibibytes, or tebibytes and +// returns the number of bytes, or -1 if the string is unparseable. // Units are case-insensitive, and the 'b' suffix is optional. func RAMInBytes(size string) (bytes int64, err error) { - re, error := regexp.Compile("^(\\d+)([kKmMgG])?[bB]?$") + re, error := regexp.Compile("^(\\d+)([kKmMgGtT])?[bB]?$") if error != nil { return -1, error } @@ -86,6 +86,8 @@ func RAMInBytes(size string) (bytes int64, err error) { memLimit *= 1024 * 1024 } else if unit == "g" { memLimit *= 1024 * 1024 * 1024 + } else if unit == "t" { + memLimit *= 1024 * 1024 * 1024 * 1024 } return memLimit, nil diff --git a/components/engine/pkg/units/size_test.go b/components/engine/pkg/units/size_test.go index 5240bbd9f0..2c4982b16f 100644 --- a/components/engine/pkg/units/size_test.go +++ b/components/engine/pkg/units/size_test.go @@ -64,7 +64,10 @@ func TestRAMInBytes(t *testing.T) { assertRAMInBytes(t, "32kb", false, 32*1024) assertRAMInBytes(t, "32Kb", false, 32*1024) assertRAMInBytes(t, "32Mb", false, 32*1024*1024) + assertRAMInBytes(t, "32MB", false, 32*1024*1024) assertRAMInBytes(t, "32Gb", false, 32*1024*1024*1024) + assertRAMInBytes(t, "32G", false, 32*1024*1024*1024) + assertRAMInBytes(t, "32Tb", false, 32*1024*1024*1024*1024) assertRAMInBytes(t, "", true, -1) assertRAMInBytes(t, "hello", true, -1) From e00d89f257dce9ec1c666b084646041e2d9f1ed7 Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Fri, 20 Jun 2014 11:53:18 -0400 Subject: [PATCH 011/516] devmapper: Add option for specifying the thin pool blocksize Add dm.blocksize option that you can use with --storage-opt to set a specific blocksize for the thin provisioning pool. Also change the default dm-thin-pool blocksize from 64K to 512K. This strikes a balance between the desire to have smaller blocksize given docker's use of snapshots versus the desire to have more performance that comes with using a larger blocksize. But if very small files will be used on average the user is encouraged to override this default. Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) Upstream-commit: 09ee269d998ad04733ef577739fa051df9d3f12e Component: engine --- .../engine/daemon/graphdriver/devmapper/README.md | 8 ++++++++ .../daemon/graphdriver/devmapper/deviceset.go | 14 ++++++++++++-- .../daemon/graphdriver/devmapper/devmapper.go | 8 ++++---- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/components/engine/daemon/graphdriver/devmapper/README.md b/components/engine/daemon/graphdriver/devmapper/README.md index c8ab1d1ee1..849cfce64c 100644 --- a/components/engine/daemon/graphdriver/devmapper/README.md +++ b/components/engine/daemon/graphdriver/devmapper/README.md @@ -126,6 +126,14 @@ Here is the list of supported options: ``docker -d --storage-opt dm.datadev=/dev/sdb1 --storage-opt dm.metadatadev=/dev/sdc1`` + * `dm.blocksize` + + Specifies a custom blocksize to use for the thin pool. + + Example use: + + ``docker -d --storage-opt dm.blocksize=64K`` + * `dm.blkdiscard` Enables or disables the use of blkdiscard when removing diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go index b0e0819ba8..c42d9c5a72 100644 --- a/components/engine/daemon/graphdriver/devmapper/deviceset.go +++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go @@ -28,6 +28,7 @@ var ( DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 + DefaultThinpBlockSize uint32 = 1024 // 512K = 1024 512b sectors ) type DevInfo struct { @@ -78,6 +79,7 @@ type DeviceSet struct { dataDevice string metadataDevice string doBlkDiscard bool + thinpBlockSize uint32 } type DiskUsage struct { @@ -510,7 +512,7 @@ func (devices *DeviceSet) ResizePool(size int64) error { } // Reload with the new block sizes - if err := reloadPool(devices.getPoolName(), dataloopback, metadataloopback); err != nil { + if err := reloadPool(devices.getPoolName(), dataloopback, metadataloopback, devices.thinpBlockSize); err != nil { return fmt.Errorf("Unable to reload pool: %s", err) } @@ -640,7 +642,7 @@ func (devices *DeviceSet) initDevmapper(doInit bool) error { } defer metadataFile.Close() - if err := createPool(devices.getPoolName(), dataFile, metadataFile); err != nil { + if err := createPool(devices.getPoolName(), dataFile, metadataFile, devices.thinpBlockSize); err != nil { return err } } @@ -1159,6 +1161,7 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error baseFsSize: DefaultBaseFsSize, filesystem: "ext4", doBlkDiscard: true, + thinpBlockSize: DefaultThinpBlockSize, } foundBlkDiscard := false @@ -1206,6 +1209,13 @@ func NewDeviceSet(root string, doInit bool, options []string) (*DeviceSet, error if err != nil { return nil, err } + case "dm.blocksize": + size, err := units.RAMInBytes(val) + if err != nil { + return nil, err + } + // convert to 512b sectors + devices.thinpBlockSize = uint32(size) >> 9 default: return nil, fmt.Errorf("Unknown option %s\n", key) } diff --git a/components/engine/daemon/graphdriver/devmapper/devmapper.go b/components/engine/daemon/graphdriver/devmapper/devmapper.go index 2590ec0fc1..ee4f2a8159 100644 --- a/components/engine/daemon/graphdriver/devmapper/devmapper.go +++ b/components/engine/daemon/graphdriver/devmapper/devmapper.go @@ -328,7 +328,7 @@ func BlockDeviceDiscard(path string) error { } // This is the programmatic example of "dmsetup create" -func createPool(poolName string, dataFile, metadataFile *os.File) error { +func createPool(poolName string, dataFile, metadataFile *os.File, poolBlockSize uint32) error { task, err := createTask(DeviceCreate, poolName) if task == nil { return err @@ -339,7 +339,7 @@ func createPool(poolName string, dataFile, metadataFile *os.File) error { return fmt.Errorf("Can't get data size %s", err) } - params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768 1 skip_block_zeroing" + params := fmt.Sprintf("%s %s %d 32768 1 skip_block_zeroing", metadataFile.Name(), dataFile.Name(), poolBlockSize) if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil { return fmt.Errorf("Can't add target %s", err) } @@ -358,7 +358,7 @@ func createPool(poolName string, dataFile, metadataFile *os.File) error { return nil } -func reloadPool(poolName string, dataFile, metadataFile *os.File) error { +func reloadPool(poolName string, dataFile, metadataFile *os.File, poolBlockSize uint32) error { task, err := createTask(DeviceReload, poolName) if task == nil { return err @@ -369,7 +369,7 @@ func reloadPool(poolName string, dataFile, metadataFile *os.File) error { return fmt.Errorf("Can't get data size %s", err) } - params := metadataFile.Name() + " " + dataFile.Name() + " 128 32768 1 skip_block_zeroing" + params := fmt.Sprintf("%s %s %d 32768 1 skip_block_zeroing", metadataFile.Name(), dataFile.Name(), poolBlockSize) if err := task.AddTarget(0, size/512, "thin-pool", params); err != nil { return fmt.Errorf("Can't add target %s", err) } From bda9d074fe27d5dc88986ca42fb015df2c92513c Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Tue, 24 Jun 2014 12:43:45 -0400 Subject: [PATCH 012/516] devmapper: remove extra space in DefaultThinpBlockSize assignment Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) Upstream-commit: f9c078ef38106d00de7774374f3ac71bb0d562d3 Component: engine --- components/engine/daemon/graphdriver/devmapper/deviceset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/daemon/graphdriver/devmapper/deviceset.go b/components/engine/daemon/graphdriver/devmapper/deviceset.go index c42d9c5a72..61ba81852d 100644 --- a/components/engine/daemon/graphdriver/devmapper/deviceset.go +++ b/components/engine/daemon/graphdriver/devmapper/deviceset.go @@ -28,7 +28,7 @@ var ( DefaultDataLoopbackSize int64 = 100 * 1024 * 1024 * 1024 DefaultMetaDataLoopbackSize int64 = 2 * 1024 * 1024 * 1024 DefaultBaseFsSize uint64 = 10 * 1024 * 1024 * 1024 - DefaultThinpBlockSize uint32 = 1024 // 512K = 1024 512b sectors + DefaultThinpBlockSize uint32 = 1024 // 512K = 1024 512b sectors ) type DevInfo struct { From fbe2366f74a7668540bccf2f4638e109f57b78ac Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 26 Jun 2014 12:06:41 -0400 Subject: [PATCH 013/516] devmapper: document the default DM thin pool blocksize Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) Upstream-commit: 79f217e3501a43326bd532e8eb03182f7488f61c Component: engine --- components/engine/daemon/graphdriver/devmapper/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/engine/daemon/graphdriver/devmapper/README.md b/components/engine/daemon/graphdriver/devmapper/README.md index 849cfce64c..ed99d85a93 100644 --- a/components/engine/daemon/graphdriver/devmapper/README.md +++ b/components/engine/daemon/graphdriver/devmapper/README.md @@ -128,7 +128,8 @@ Here is the list of supported options: * `dm.blocksize` - Specifies a custom blocksize to use for the thin pool. + Specifies a custom blocksize to use for the thin pool. The default + blocksize is 512K. Example use: From 9c28be75423453c86db11f288fb397e1c867623b Mon Sep 17 00:00:00 2001 From: Mike Snitzer Date: Thu, 26 Jun 2014 12:39:16 -0400 Subject: [PATCH 014/516] devmapper: add thin-pool blocksize to the 'docker info' output Docker-DCO-1.1-Signed-off-by: Mike Snitzer (github: snitm) Upstream-commit: a2f3ce2294bbe998df24edd50f2b571d0b21bac8 Component: engine --- components/engine/daemon/graphdriver/devmapper/driver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/daemon/graphdriver/devmapper/driver.go b/components/engine/daemon/graphdriver/devmapper/driver.go index cf82ad62ed..29335377f6 100644 --- a/components/engine/daemon/graphdriver/devmapper/driver.go +++ b/components/engine/daemon/graphdriver/devmapper/driver.go @@ -54,6 +54,7 @@ func (d *Driver) Status() [][2]string { status := [][2]string{ {"Pool Name", s.PoolName}, + {"Pool Blocksize", fmt.Sprintf("%d Kb", s.SectorSize/1024)}, {"Data file", s.DataLoopback}, {"Metadata file", s.MetadataLoopback}, {"Data Space Used", fmt.Sprintf("%.1f Mb", float64(s.Data.Used)/(1024*1024))}, From 018ff8d66c3ce332faf41daec585cbcd8d8ddacd Mon Sep 17 00:00:00 2001 From: Jason Giedymin Date: Fri, 20 Jun 2014 07:39:28 -0400 Subject: [PATCH 015/516] Add H3 menu entries for leftnav: - Add H3 formatting and entries to leftnav, to docs css. Docker-DCO-1.1-Signed-off-by: Jason Giedymin (github: JasonGiedymin) Upstream-commit: e4742dbb942b9db5f91dfbfb829525bdcac57ae1 Component: engine --- components/engine/docs/theme/mkdocs/css/docs.css | 12 ++++++++++++ components/engine/docs/theme/mkdocs/css/main.css | 2 +- components/engine/docs/theme/mkdocs/toc.html | 9 ++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/components/engine/docs/theme/mkdocs/css/docs.css b/components/engine/docs/theme/mkdocs/css/docs.css index 0f42e22ef7..52f170c141 100644 --- a/components/engine/docs/theme/mkdocs/css/docs.css +++ b/components/engine/docs/theme/mkdocs/css/docs.css @@ -24,6 +24,18 @@ height: 100%; } +#leftnav h3 { + font-size: 10px; + font-weight: 700; + color: #394d54; + line-height: 1; + margin: 0px 0 10px 0; + padding-left: 20px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + .content-body { padding: 0px 0px 0px 20px; } diff --git a/components/engine/docs/theme/mkdocs/css/main.css b/components/engine/docs/theme/mkdocs/css/main.css index 42a7a18a56..b8fc3b48a5 100644 --- a/components/engine/docs/theme/mkdocs/css/main.css +++ b/components/engine/docs/theme/mkdocs/css/main.css @@ -902,7 +902,7 @@ div + .form-inline { margin-bottom: 22px; } #leftnav .nav { - margin: 0; + margin: 0, 0, 20px, 0; } #leftnav .nav > li > a { line-height: 22px; diff --git a/components/engine/docs/theme/mkdocs/toc.html b/components/engine/docs/theme/mkdocs/toc.html index 96d15c265c..1de2a4209f 100644 --- a/components/engine/docs/theme/mkdocs/toc.html +++ b/components/engine/docs/theme/mkdocs/toc.html @@ -1,5 +1,8 @@ - {% for toc_item in toc %} - {% for toc_item in toc_item.children %} -
  • {{ toc_item.title }}
  • +{% for toc_item in toc %} + {% for toc_h2_item in toc_item.children %} +
  • {{ toc_h2_item.title }}
  • + {% for toc_h3_item in toc_h2_item.children %} +

    {{ toc_h3_item.title }}

    + {% endfor %} {% endfor %} {% endfor %} From fd631db79c1675c06fa3c3add2284ba8615e1de4 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Sat, 28 Jun 2014 23:09:23 -0600 Subject: [PATCH 016/516] Update bash completion for "docker run --link" and "docker run -v" to complete on running containers and the local filesystem respectively Also, add completion for "docker run -a" ("stdin", "stdout", and "stderr"), "docker run --env-file" (local filesystem), and some other minor code style tweaks. Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 86f06b6deb6e5b9ddb30f781e81cc5b6241e20ae Component: engine --- .../engine/contrib/completion/bash/docker | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/components/engine/contrib/completion/bash/docker b/components/engine/contrib/completion/bash/docker index 89395560f9..532c2538b9 100755 --- a/components/engine/contrib/completion/bash/docker +++ b/components/engine/contrib/completion/bash/docker @@ -485,21 +485,52 @@ _docker_rmi() _docker_run() { case "$prev" in - --cidfile) + -a|--attach) + COMPREPLY=( $( compgen -W 'stdin stdout stderr' -- "$cur" ) ) + return + ;; + --cidfile|--env-file) _filedir + return ;; --volumes-from) __docker_containers_all + return ;; -v|--volume) - # TODO something magical with colons and _filedir ? + case "$cur" in + *:*) + # TODO somehow do _filedir for stuff inside the image, if it's already specified (which is also somewhat difficult to determine) + ;; + '') + COMPREPLY=( $( compgen -W '/' -- "$cur" ) ) + compopt -o nospace + ;; + /*) + _filedir + compopt -o nospace + ;; + esac return ;; -e|--env) COMPREPLY=( $( compgen -e -- "$cur" ) ) + compopt -o nospace return ;; - --entrypoint|-h|--hostname|-m|--memory|-u|--user|-w|--workdir|-c|--cpu-shares|-n|--name|-a|--attach|--link|-p|--publish|--expose|--dns|--lxc-conf) + --link) + case "$cur" in + *:*) + ;; + *) + __docker_containers_running + COMPREPLY=( $( compgen -W "${COMPREPLY[*]}" -S ':' ) ) + compopt -o nospace + ;; + esac + return + ;; + --entrypoint|-h|--hostname|-m|--memory|-u|--user|-w|--workdir|-c|--cpu-shares|-n|--name|-p|--publish|--expose|--dns|--lxc-conf) return ;; *) From a5827383a93066f83d4d3cede73db1b481f23561 Mon Sep 17 00:00:00 2001 From: Kato Kazuyoshi Date: Sun, 29 Jun 2014 16:50:53 +0900 Subject: [PATCH 017/516] Skip lxc_template_unit_test.go on non-Linux platforms It doesn't work without lxc-start. Docker-DCO-1.1-Signed-off-by: Kato Kazuyoshi (github: kzys) Upstream-commit: dda0ce645995d6dd2b6eb5848c30d305a3d220d2 Component: engine --- .../engine/daemon/execdriver/lxc/lxc_template_unit_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/engine/daemon/execdriver/lxc/lxc_template_unit_test.go b/components/engine/daemon/execdriver/lxc/lxc_template_unit_test.go index a9a67c421c..9e4d33f9e6 100644 --- a/components/engine/daemon/execdriver/lxc/lxc_template_unit_test.go +++ b/components/engine/daemon/execdriver/lxc/lxc_template_unit_test.go @@ -1,3 +1,5 @@ +// +build linux + package lxc import ( From f17116cbd31c254c7b0fae1c0933cb7bda5aadef Mon Sep 17 00:00:00 2001 From: Alexandr Morozov Date: Mon, 30 Jun 2014 10:52:11 +0400 Subject: [PATCH 018/516] Add synchronization in server.Containers This fixes races between ps and other operations Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) Upstream-commit: da853e98c94c9ce8001d3d1516ba0fd2614db3b0 Component: engine --- components/engine/server/server.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/components/engine/server/server.go b/components/engine/server/server.go index 0bbb9f31ca..8be4dcd7b3 100644 --- a/components/engine/server/server.go +++ b/components/engine/server/server.go @@ -23,6 +23,7 @@ package server import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -955,22 +956,25 @@ func (srv *Server) Containers(job *engine.Job) engine.Status { } } - for _, container := range srv.daemon.List() { + errLast := errors.New("last container") + writeCont := func(container *daemon.Container) error { + container.Lock() + defer container.Unlock() if !container.State.IsRunning() && !all && n <= 0 && since == "" && before == "" { - continue + return nil } if before != "" && !foundBefore { if container.ID == beforeCont.ID { foundBefore = true } - continue + return nil } if n > 0 && displayed == n { - break + return errLast } if since != "" { if container.ID == sinceCont.ID { - break + return errLast } } displayed++ @@ -997,7 +1001,7 @@ func (srv *Server) Containers(job *engine.Job) engine.Status { out.Set("Status", container.State.String()) str, err := container.NetworkSettings.PortMappingAPI().ToListString() if err != nil { - return job.Error(err) + return err } out.Set("Ports", str) if size { @@ -1006,6 +1010,16 @@ func (srv *Server) Containers(job *engine.Job) engine.Status { out.SetInt64("SizeRootFs", sizeRootFs) } outs.Add(out) + return nil + } + + for _, container := range srv.daemon.List() { + if err := writeCont(container); err != nil { + if err != errLast { + return job.Error(err) + } + break + } } outs.ReverseSort() if _, err := outs.WriteListTo(job.Stdout); err != nil { From e3c0c2772d7ee8ba16ddecc67f027a53998f4734 Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Tue, 1 Jul 2014 21:59:11 +0400 Subject: [PATCH 019/516] Refactoring portallocator Faster, more documented, less code. Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) Upstream-commit: f387cc1205b79ffbe30b28f5df13da034d8221fd Component: engine --- .../portallocator/portallocator.go | 138 ++++++++---------- 1 file changed, 63 insertions(+), 75 deletions(-) diff --git a/components/engine/daemon/networkdriver/portallocator/portallocator.go b/components/engine/daemon/networkdriver/portallocator/portallocator.go index c722ba98ba..d4fcc6e725 100644 --- a/components/engine/daemon/networkdriver/portallocator/portallocator.go +++ b/components/engine/daemon/networkdriver/portallocator/portallocator.go @@ -12,10 +12,22 @@ type portMap struct { last int } -type ( - protocolMap map[string]*portMap - ipMapping map[string]protocolMap -) +func newPortMap() *portMap { + return &portMap{ + p: map[int]struct{}{}, + } +} + +type protoMap map[string]*portMap + +func newProtoMap() protoMap { + return protoMap{ + "tcp": newPortMap(), + "udp": newPortMap(), + } +} + +type ipMapping map[string]protoMap const ( BeginPortRange = 49153 @@ -62,107 +74,83 @@ func (e ErrPortAlreadyAllocated) Error() string { return fmt.Sprintf("Bind for %s:%d failed: port is already allocated", e.ip, e.port) } +// RequestPort requests new port from global ports pool for specified ip and proto. +// If port is 0 it returns first free port. Otherwise it cheks port availability +// in pool and return that port or error if port is already busy. func RequestPort(ip net.IP, proto string, port int) (int, error) { mutex.Lock() defer mutex.Unlock() - if err := validateProto(proto); err != nil { + if proto != "tcp" && proto != "udp" { + return 0, ErrUnknownProtocol + } + + if ip == nil { + ip = defaultIP + } + ipstr := ip.String() + protomap, ok := globalMap[ipstr] + if !ok { + protomap = newProtoMap() + globalMap[ipstr] = protomap + } + mapping := protomap[proto] + if port > 0 { + if _, ok := mapping.p[port]; !ok { + mapping.p[port] = struct{}{} + return port, nil + } + return 0, NewErrPortAlreadyAllocated(ipstr, port) + } + + port, err := mapping.findPort() + if err != nil { return 0, err } - - ip = getDefault(ip) - - mapping := getOrCreate(ip) - - if port > 0 { - if _, ok := mapping[proto].p[port]; !ok { - mapping[proto].p[port] = struct{}{} - return port, nil - } else { - return 0, NewErrPortAlreadyAllocated(ip.String(), port) - } - } else { - port, err := findPort(ip, proto) - - if err != nil { - return 0, err - } - - return port, nil - } + return port, nil } +// ReleasePort releases port from global ports pool for specified ip and proto. func ReleasePort(ip net.IP, proto string, port int) error { mutex.Lock() defer mutex.Unlock() - ip = getDefault(ip) - - mapping := getOrCreate(ip)[proto] - delete(mapping.p, port) - + if ip == nil { + ip = defaultIP + } + protomap, ok := globalMap[ip.String()] + if !ok { + return nil + } + delete(protomap[proto].p, port) return nil } +// ReleaseAll releases all ports for all ips. func ReleaseAll() error { mutex.Lock() - defer mutex.Unlock() - globalMap = ipMapping{} - + mutex.Unlock() return nil } -func getOrCreate(ip net.IP) protocolMap { - ipstr := ip.String() - - if _, ok := globalMap[ipstr]; !ok { - globalMap[ipstr] = protocolMap{ - "tcp": &portMap{p: map[int]struct{}{}, last: 0}, - "udp": &portMap{p: map[int]struct{}{}, last: 0}, - } - } - - return globalMap[ipstr] -} - -func findPort(ip net.IP, proto string) (int, error) { - mapping := getOrCreate(ip)[proto] - - if mapping.last == 0 { - mapping.p[BeginPortRange] = struct{}{} - mapping.last = BeginPortRange +func (pm *portMap) findPort() (int, error) { + if pm.last == 0 { + pm.p[BeginPortRange] = struct{}{} + pm.last = BeginPortRange return BeginPortRange, nil } - for port := mapping.last + 1; port != mapping.last; port++ { + for port := pm.last + 1; port != pm.last; port++ { if port > EndPortRange { port = BeginPortRange } - if _, ok := mapping.p[port]; !ok { - mapping.p[port] = struct{}{} - mapping.last = port + if _, ok := pm.p[port]; !ok { + pm.p[port] = struct{}{} + pm.last = port return port, nil } - } - return 0, ErrAllPortsAllocated } - -func getDefault(ip net.IP) net.IP { - if ip == nil { - return defaultIP - } - - return ip -} - -func validateProto(proto string) error { - if proto != "tcp" && proto != "udp" { - return ErrUnknownProtocol - } - - return nil -} From 4d29e9b718567a90349ed9bdeb0ce15702a9fb17 Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Wed, 2 Jul 2014 18:48:37 +0400 Subject: [PATCH 020/516] Move WriteBroadcaster to separate package as BroadcastWriter Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) Upstream-commit: 9d4e80222144c0aef3e78202ecb4904b3ce5ed9b Component: engine --- components/engine/daemon/container.go | 9 +- components/engine/daemon/daemon.go | 7 +- .../utils/broadcastwriter/broadcastwriter.go | 92 +++++++++++++++ .../broadcastwriter/broadcastwriter_test.go | 108 ++++++++++++++++++ components/engine/utils/utils.go | 86 -------------- components/engine/utils/utils_test.go | 101 ---------------- 6 files changed, 209 insertions(+), 194 deletions(-) create mode 100644 components/engine/utils/broadcastwriter/broadcastwriter.go create mode 100644 components/engine/utils/broadcastwriter/broadcastwriter_test.go diff --git a/components/engine/daemon/container.go b/components/engine/daemon/container.go index 5b41438680..c7a6774601 100644 --- a/components/engine/daemon/container.go +++ b/components/engine/daemon/container.go @@ -29,6 +29,7 @@ import ( "github.com/dotcloud/docker/pkg/symlink" "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" + "github.com/dotcloud/docker/utils/broadcastwriter" ) const DefaultPathEnv = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" @@ -66,8 +67,8 @@ type Container struct { ExecDriver string command *execdriver.Command - stdout *utils.WriteBroadcaster - stderr *utils.WriteBroadcaster + stdout *broadcastwriter.BroadcastWriter + stderr *broadcastwriter.BroadcastWriter stdin io.ReadCloser stdinPipe io.WriteCloser @@ -502,10 +503,10 @@ func (container *Container) cleanup() { utils.Errorf("%s: Error close stdin: %s", container.ID, err) } } - if err := container.stdout.CloseWriters(); err != nil { + if err := container.stdout.Close(); err != nil { utils.Errorf("%s: Error close stdout: %s", container.ID, err) } - if err := container.stderr.CloseWriters(); err != nil { + if err := container.stderr.Close(); err != nil { utils.Errorf("%s: Error close stderr: %s", container.ID, err) } if container.command != nil && container.command.Terminal != nil { diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go index 9500526f9e..0b51715ecd 100644 --- a/components/engine/daemon/daemon.go +++ b/components/engine/daemon/daemon.go @@ -34,6 +34,7 @@ import ( "github.com/dotcloud/docker/pkg/truncindex" "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" + "github.com/dotcloud/docker/utils/broadcastwriter" ) // Set the max depth to the aufs default that most @@ -169,8 +170,8 @@ func (daemon *Daemon) register(container *Container, updateSuffixarray bool, con container.daemon = daemon // Attach to stdout and stderr - container.stderr = utils.NewWriteBroadcaster() - container.stdout = utils.NewWriteBroadcaster() + container.stderr = broadcastwriter.New() + container.stdout = broadcastwriter.New() // Attach to stdin if container.Config.OpenStdin { container.stdin, container.stdinPipe = io.Pipe() @@ -255,7 +256,7 @@ func (daemon *Daemon) ensureName(container *Container) error { return nil } -func (daemon *Daemon) LogToDisk(src *utils.WriteBroadcaster, dst, stream string) error { +func (daemon *Daemon) LogToDisk(src *broadcastwriter.BroadcastWriter, dst, stream string) error { log, err := os.OpenFile(dst, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600) if err != nil { return err diff --git a/components/engine/utils/broadcastwriter/broadcastwriter.go b/components/engine/utils/broadcastwriter/broadcastwriter.go new file mode 100644 index 0000000000..81ca9e5bdd --- /dev/null +++ b/components/engine/utils/broadcastwriter/broadcastwriter.go @@ -0,0 +1,92 @@ +package broadcastwriter + +import ( + "bytes" + "encoding/json" + "io" + "sync" + "time" + + "github.com/dotcloud/docker/utils" +) + +type BroadcastWriter struct { + sync.Mutex + buf *bytes.Buffer + streams map[string](map[io.WriteCloser]struct{}) +} + +func (w *BroadcastWriter) AddWriter(writer io.WriteCloser, stream string) { + w.Lock() + if _, ok := w.streams[stream]; !ok { + w.streams[stream] = make(map[io.WriteCloser]struct{}) + } + w.streams[stream][writer] = struct{}{} + w.Unlock() +} + +func (w *BroadcastWriter) Write(p []byte) (n int, err error) { + created := time.Now().UTC() + w.Lock() + defer w.Unlock() + if writers, ok := w.streams[""]; ok { + for sw := range writers { + if n, err := sw.Write(p); err != nil || n != len(p) { + // On error, evict the writer + delete(writers, sw) + } + } + } + w.buf.Write(p) + lines := []string{} + for { + line, err := w.buf.ReadString('\n') + if err != nil { + w.buf.Write([]byte(line)) + break + } + lines = append(lines, line) + } + + if len(lines) != 0 { + for stream, writers := range w.streams { + if stream == "" { + continue + } + var lp []byte + for _, line := range lines { + b, err := json.Marshal(&utils.JSONLog{Log: line, Stream: stream, Created: created}) + if err != nil { + utils.Errorf("Error making JSON log line: %s", err) + } + lp = append(lp, b...) + lp = append(lp, '\n') + } + for sw := range writers { + if _, err := sw.Write(lp); err != nil { + delete(writers, sw) + } + } + } + } + return len(p), nil +} + +func (w *BroadcastWriter) Close() error { + w.Lock() + defer w.Unlock() + for _, writers := range w.streams { + for w := range writers { + w.Close() + } + } + w.streams = make(map[string](map[io.WriteCloser]struct{})) + return nil +} + +func New() *BroadcastWriter { + return &BroadcastWriter{ + streams: make(map[string](map[io.WriteCloser]struct{})), + buf: bytes.NewBuffer(nil), + } +} diff --git a/components/engine/utils/broadcastwriter/broadcastwriter_test.go b/components/engine/utils/broadcastwriter/broadcastwriter_test.go new file mode 100644 index 0000000000..8d946e2f45 --- /dev/null +++ b/components/engine/utils/broadcastwriter/broadcastwriter_test.go @@ -0,0 +1,108 @@ +package broadcastwriter + +import ( + "bytes" + "errors" + + "testing" +) + +type dummyWriter struct { + buffer bytes.Buffer + failOnWrite bool +} + +func (dw *dummyWriter) Write(p []byte) (n int, err error) { + if dw.failOnWrite { + return 0, errors.New("Fake fail") + } + return dw.buffer.Write(p) +} + +func (dw *dummyWriter) String() string { + return dw.buffer.String() +} + +func (dw *dummyWriter) Close() error { + return nil +} + +func TestBroadcastWriter(t *testing.T) { + writer := New() + + // Test 1: Both bufferA and bufferB should contain "foo" + bufferA := &dummyWriter{} + writer.AddWriter(bufferA, "") + bufferB := &dummyWriter{} + writer.AddWriter(bufferB, "") + writer.Write([]byte("foo")) + + if bufferA.String() != "foo" { + t.Errorf("Buffer contains %v", bufferA.String()) + } + + if bufferB.String() != "foo" { + t.Errorf("Buffer contains %v", bufferB.String()) + } + + // Test2: bufferA and bufferB should contain "foobar", + // while bufferC should only contain "bar" + bufferC := &dummyWriter{} + writer.AddWriter(bufferC, "") + writer.Write([]byte("bar")) + + if bufferA.String() != "foobar" { + t.Errorf("Buffer contains %v", bufferA.String()) + } + + if bufferB.String() != "foobar" { + t.Errorf("Buffer contains %v", bufferB.String()) + } + + if bufferC.String() != "bar" { + t.Errorf("Buffer contains %v", bufferC.String()) + } + + // Test3: Test eviction on failure + bufferA.failOnWrite = true + writer.Write([]byte("fail")) + if bufferA.String() != "foobar" { + t.Errorf("Buffer contains %v", bufferA.String()) + } + if bufferC.String() != "barfail" { + t.Errorf("Buffer contains %v", bufferC.String()) + } + // Even though we reset the flag, no more writes should go in there + bufferA.failOnWrite = false + writer.Write([]byte("test")) + if bufferA.String() != "foobar" { + t.Errorf("Buffer contains %v", bufferA.String()) + } + if bufferC.String() != "barfailtest" { + t.Errorf("Buffer contains %v", bufferC.String()) + } + + writer.Close() +} + +type devNullCloser int + +func (d devNullCloser) Close() error { + return nil +} + +func (d devNullCloser) Write(buf []byte) (int, error) { + return len(buf), nil +} + +// This test checks for races. It is only useful when run with the race detector. +func TestRaceBroadcastWriter(t *testing.T) { + writer := New() + c := make(chan bool) + go func() { + writer.AddWriter(devNullCloser(0), "") + c <- true + }() + writer.Write([]byte("hello")) + <-c +} diff --git a/components/engine/utils/utils.go b/components/engine/utils/utils.go index 333468d361..ef28aceca7 100644 --- a/components/engine/utils/utils.go +++ b/components/engine/utils/utils.go @@ -265,21 +265,6 @@ func (r *bufReader) Close() error { return closer.Close() } -type WriteBroadcaster struct { - sync.Mutex - buf *bytes.Buffer - streams map[string](map[io.WriteCloser]struct{}) -} - -func (w *WriteBroadcaster) AddWriter(writer io.WriteCloser, stream string) { - w.Lock() - if _, ok := w.streams[stream]; !ok { - w.streams[stream] = make(map[io.WriteCloser]struct{}) - } - w.streams[stream][writer] = struct{}{} - w.Unlock() -} - type JSONLog struct { Log string `json:"log,omitempty"` Stream string `json:"stream,omitempty"` @@ -316,77 +301,6 @@ func WriteLog(src io.Reader, dst io.WriteCloser, format string) error { } } -type LogFormatter struct { - wc io.WriteCloser - timeFormat string -} - -func (w *WriteBroadcaster) Write(p []byte) (n int, err error) { - created := time.Now().UTC() - w.Lock() - defer w.Unlock() - if writers, ok := w.streams[""]; ok { - for sw := range writers { - if n, err := sw.Write(p); err != nil || n != len(p) { - // On error, evict the writer - delete(writers, sw) - } - } - } - w.buf.Write(p) - lines := []string{} - for { - line, err := w.buf.ReadString('\n') - if err != nil { - w.buf.Write([]byte(line)) - break - } - lines = append(lines, line) - } - - if len(lines) != 0 { - for stream, writers := range w.streams { - if stream == "" { - continue - } - var lp []byte - for _, line := range lines { - b, err := json.Marshal(&JSONLog{Log: line, Stream: stream, Created: created}) - if err != nil { - Errorf("Error making JSON log line: %s", err) - } - lp = append(lp, b...) - lp = append(lp, '\n') - } - for sw := range writers { - if _, err := sw.Write(lp); err != nil { - delete(writers, sw) - } - } - } - } - return len(p), nil -} - -func (w *WriteBroadcaster) CloseWriters() error { - w.Lock() - defer w.Unlock() - for _, writers := range w.streams { - for w := range writers { - w.Close() - } - } - w.streams = make(map[string](map[io.WriteCloser]struct{})) - return nil -} - -func NewWriteBroadcaster() *WriteBroadcaster { - return &WriteBroadcaster{ - streams: make(map[string](map[io.WriteCloser]struct{})), - buf: bytes.NewBuffer(nil), - } -} - func GetTotalUsedFds() int { if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil { Errorf("Error opening /proc/%d/fd: %s", os.Getpid(), err) diff --git a/components/engine/utils/utils_test.go b/components/engine/utils/utils_test.go index 049c0e30a2..6fd09b97db 100644 --- a/components/engine/utils/utils_test.go +++ b/components/engine/utils/utils_test.go @@ -2,7 +2,6 @@ package utils import ( "bytes" - "errors" "io" "io/ioutil" "os" @@ -35,106 +34,6 @@ func TestBufReader(t *testing.T) { } } -type dummyWriter struct { - buffer bytes.Buffer - failOnWrite bool -} - -func (dw *dummyWriter) Write(p []byte) (n int, err error) { - if dw.failOnWrite { - return 0, errors.New("Fake fail") - } - return dw.buffer.Write(p) -} - -func (dw *dummyWriter) String() string { - return dw.buffer.String() -} - -func (dw *dummyWriter) Close() error { - return nil -} - -func TestWriteBroadcaster(t *testing.T) { - writer := NewWriteBroadcaster() - - // Test 1: Both bufferA and bufferB should contain "foo" - bufferA := &dummyWriter{} - writer.AddWriter(bufferA, "") - bufferB := &dummyWriter{} - writer.AddWriter(bufferB, "") - writer.Write([]byte("foo")) - - if bufferA.String() != "foo" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - - if bufferB.String() != "foo" { - t.Errorf("Buffer contains %v", bufferB.String()) - } - - // Test2: bufferA and bufferB should contain "foobar", - // while bufferC should only contain "bar" - bufferC := &dummyWriter{} - writer.AddWriter(bufferC, "") - writer.Write([]byte("bar")) - - if bufferA.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - - if bufferB.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferB.String()) - } - - if bufferC.String() != "bar" { - t.Errorf("Buffer contains %v", bufferC.String()) - } - - // Test3: Test eviction on failure - bufferA.failOnWrite = true - writer.Write([]byte("fail")) - if bufferA.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - if bufferC.String() != "barfail" { - t.Errorf("Buffer contains %v", bufferC.String()) - } - // Even though we reset the flag, no more writes should go in there - bufferA.failOnWrite = false - writer.Write([]byte("test")) - if bufferA.String() != "foobar" { - t.Errorf("Buffer contains %v", bufferA.String()) - } - if bufferC.String() != "barfailtest" { - t.Errorf("Buffer contains %v", bufferC.String()) - } - - writer.CloseWriters() -} - -type devNullCloser int - -func (d devNullCloser) Close() error { - return nil -} - -func (d devNullCloser) Write(buf []byte) (int, error) { - return len(buf), nil -} - -// This test checks for races. It is only useful when run with the race detector. -func TestRaceWriteBroadcaster(t *testing.T) { - writer := NewWriteBroadcaster() - c := make(chan bool) - go func() { - writer.AddWriter(devNullCloser(0), "") - c <- true - }() - writer.Write([]byte("hello")) - <-c -} - func assertKernelVersion(t *testing.T, a, b *KernelVersionInfo, result int) { if r := CompareKernelVersion(a, b); r != result { t.Fatalf("Unexpected kernel version comparison result. Found %d, expected %d", r, result) From 18c97f39c341d46c02d7851f482888dc0a7e8278 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Mon, 17 Mar 2014 17:03:22 +0100 Subject: [PATCH 021/516] server: Break out setHostConfig() from ContainerStart This will be reused for ContainerCreate Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: a4a80b64a7f6366f7fb94cb8cca042a6324a0e99 Component: engine --- components/engine/server/server.go | 46 ++++++++++++++++++------------ 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/components/engine/server/server.go b/components/engine/server/server.go index 3e6de00c1e..b443eba321 100644 --- a/components/engine/server/server.go +++ b/components/engine/server/server.go @@ -2035,6 +2035,32 @@ func (srv *Server) ImageGetCached(imgID string, config *runconfig.Config) (*imag return match, nil } +func (srv *Server) setHostConfig(container *daemon.Container, hostConfig *runconfig.HostConfig) error { + // Validate the HostConfig binds. Make sure that: + // the source exists + for _, bind := range hostConfig.Binds { + splitBind := strings.Split(bind, ":") + source := splitBind[0] + + // ensure the source exists on the host + _, err := os.Stat(source) + if err != nil && os.IsNotExist(err) { + err = os.MkdirAll(source, 0755) + if err != nil { + return fmt.Errorf("Could not create local directory '%s' for bind mount: %s!", source, err.Error()) + } + } + } + // Register any links from the host config before starting the container + if err := srv.daemon.RegisterLinks(container, hostConfig); err != nil { + return err + } + container.SetHostConfig(hostConfig) + container.ToDisk() + + return nil +} + func (srv *Server) ContainerStart(job *engine.Job) engine.Status { if len(job.Args) < 1 { return job.Errorf("Usage: %s container_id", job.Name) @@ -2056,27 +2082,9 @@ func (srv *Server) ContainerStart(job *engine.Job) engine.Status { // If no environment was set, then no hostconfig was passed. if len(job.Environ()) > 0 { hostConfig := runconfig.ContainerHostConfigFromJob(job) - // Validate the HostConfig binds. Make sure that: - // the source exists - for _, bind := range hostConfig.Binds { - splitBind := strings.Split(bind, ":") - source := splitBind[0] - - // ensure the source exists on the host - _, err := os.Stat(source) - if err != nil && os.IsNotExist(err) { - err = os.MkdirAll(source, 0755) - if err != nil { - return job.Errorf("Could not create local directory '%s' for bind mount: %s!", source, err.Error()) - } - } - } - // Register any links from the host config before starting the container - if err := srv.daemon.RegisterLinks(container, hostConfig); err != nil { + if err := srv.setHostConfig(container, hostConfig); err != nil { return job.Error(err) } - container.SetHostConfig(hostConfig) - container.ToDisk() } if err := container.Start(); err != nil { return job.Errorf("Cannot start container %s: %s", name, err) From c4b0fae9528e13fba0f152960d50f745ba86e79a Mon Sep 17 00:00:00 2001 From: Mrunal Patel Date: Wed, 2 Jul 2014 16:33:14 -0400 Subject: [PATCH 022/516] api.DockerCli: Extract pullImage into separate function This lets us reuse this code later. Docker-DCO-1.1-Signed-off-by: Alexander Larsson (github: alexlarsson) Upstream-commit: 2b3959c414ffe9403e53617ed1a160c1daabeeaa Component: engine --- components/engine/api/client/commands.go | 66 +++++++++++++----------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/components/engine/api/client/commands.go b/components/engine/api/client/commands.go index 2d9c74f432..93470c2918 100644 --- a/components/engine/api/client/commands.go +++ b/components/engine/api/client/commands.go @@ -1894,6 +1894,41 @@ func (cli *DockerCli) CmdTag(args ...string) error { return nil } +func (cli *DockerCli) pullImage(image string) error { + v := url.Values{} + repos, tag := utils.ParseRepositoryTag(image) + // pull only the image tagged 'latest' if no tag was specified + if tag == "" { + tag = "latest" + } + v.Set("fromImage", repos) + v.Set("tag", tag) + + // Resolve the Repository name from fqn to hostname + name + hostname, _, err := registry.ResolveRepositoryName(repos) + if err != nil { + return err + } + + // Load the auth config file, to be able to pull the image + cli.LoadConfigFile() + + // Resolve the Auth config relevant for this server + authConfig := cli.configFile.ResolveAuthConfig(hostname) + buf, err := json.Marshal(authConfig) + if err != nil { + return err + } + + registryAuthHeader := []string{ + base64.URLEncoding.EncodeToString(buf), + } + if err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err, map[string][]string{"X-Registry-Auth": registryAuthHeader}); err != nil { + return err + } + return nil +} + func (cli *DockerCli) CmdRun(args ...string) error { // FIXME: just use runconfig.Parse already config, hostConfig, cmd, err := runconfig.ParseSubcommand(cli.Subcmd("run", "[OPTIONS] IMAGE [COMMAND] [ARG...]", "Run a command in a new container"), args, nil) @@ -1955,37 +1990,10 @@ func (cli *DockerCli) CmdRun(args ...string) error { if statusCode == 404 { fmt.Fprintf(cli.err, "Unable to find image '%s' locally\n", config.Image) - v := url.Values{} - repos, tag := utils.ParseRepositoryTag(config.Image) - // pull only the image tagged 'latest' if no tag was specified - if tag == "" { - tag = "latest" - } - v.Set("fromImage", repos) - v.Set("tag", tag) - - // Resolve the Repository name from fqn to hostname + name - hostname, _, err := registry.ResolveRepositoryName(repos) - if err != nil { - return err - } - - // Load the auth config file, to be able to pull the image - cli.LoadConfigFile() - - // Resolve the Auth config relevant for this server - authConfig := cli.configFile.ResolveAuthConfig(hostname) - buf, err := json.Marshal(authConfig) - if err != nil { - return err - } - - registryAuthHeader := []string{ - base64.URLEncoding.EncodeToString(buf), - } - if err = cli.stream("POST", "/images/create?"+v.Encode(), nil, cli.err, map[string][]string{"X-Registry-Auth": registryAuthHeader}); err != nil { + if err = cli.pullImage(config.Image); err != nil { return err } + // Retry if stream, _, err = cli.call("POST", "/containers/create?"+containerValues.Encode(), config, false); err != nil { return err } From 8ae73ea011d8784482d3f9388b2df60329d7b913 Mon Sep 17 00:00:00 2001 From: Fabio Falci Date: Fri, 27 Jun 2014 16:47:32 +0100 Subject: [PATCH 023/516] Move `docker restart` tests to integration cli Docker-DCO-1.1-Signed-off-by: Fabio Falci (github: fabiofalci) Upstream-commit: 9da6c80533f80060575840e1e2fd80cc11826f7b Component: engine --- .../docker_cli_restart_test.go | 122 ++++++++++++++++++ .../engine/integration/container_test.go | 75 ----------- 2 files changed, 122 insertions(+), 75 deletions(-) create mode 100644 components/engine/integration-cli/docker_cli_restart_test.go diff --git a/components/engine/integration-cli/docker_cli_restart_test.go b/components/engine/integration-cli/docker_cli_restart_test.go new file mode 100644 index 0000000000..c62f108cc7 --- /dev/null +++ b/components/engine/integration-cli/docker_cli_restart_test.go @@ -0,0 +1,122 @@ +package main + +import ( + "os/exec" + "strings" + "testing" +) + +func TestDockerRestartStoppedContainer(t *testing.T) { + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "echo", "foobar") + out, _, err := runCommandWithOutput(runCmd) + errorOut(err, t, out) + + cleanedContainerID := stripTrailingCharacters(out) + + runCmd = exec.Command(dockerBinary, "wait", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + runCmd = exec.Command(dockerBinary, "logs", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out != "foobar\n" { + t.Errorf("container should've printed 'foobar'") + } + + runCmd = exec.Command(dockerBinary, "restart", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + runCmd = exec.Command(dockerBinary, "logs", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out != "foobar\nfoobar\n" { + t.Errorf("container should've printed 'foobar' twice") + } + + deleteAllContainers() + + logDone("restart - echo foobar for stopped container") +} + +func TestDockerRestartRunningContainer(t *testing.T) { + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", "echo foobar && sleep 30 && echo 'should not print this'") + out, _, err := runCommandWithOutput(runCmd) + errorOut(err, t, out) + + cleanedContainerID := stripTrailingCharacters(out) + + runCmd = exec.Command(dockerBinary, "logs", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out != "foobar\n" { + t.Errorf("container should've printed 'foobar'") + } + + runCmd = exec.Command(dockerBinary, "restart", "-t", "1", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + runCmd = exec.Command(dockerBinary, "logs", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out != "foobar\nfoobar\n" { + t.Errorf("container should've printed 'foobar' twice") + } + + deleteAllContainers() + + logDone("restart - echo foobar for running container") +} + +// Test that restarting a container with a volume does not create a new volume on restart. Regression test for #819. +func TestDockerRestartWithVolumes(t *testing.T) { + runCmd := exec.Command(dockerBinary, "run", "-d", "-v", "/test", "busybox", "top") + out, _, err := runCommandWithOutput(runCmd) + errorOut(err, t, out) + + cleanedContainerID := stripTrailingCharacters(out) + + runCmd = exec.Command(dockerBinary, "inspect", "--format", "{{ len .Volumes }}", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out = strings.Trim(out, " \n\r"); out != "1" { + t.Errorf("expect 1 volume received %s", out) + } + + runCmd = exec.Command(dockerBinary, "inspect", "--format", "{{ .Volumes }}", cleanedContainerID) + volumes, _, err := runCommandWithOutput(runCmd) + errorOut(err, t, volumes) + + runCmd = exec.Command(dockerBinary, "restart", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + runCmd = exec.Command(dockerBinary, "inspect", "--format", "{{ len .Volumes }}", cleanedContainerID) + out, _, err = runCommandWithOutput(runCmd) + errorOut(err, t, out) + + if out = strings.Trim(out, " \n\r"); out != "1" { + t.Errorf("expect 1 volume after restart received %s", out) + } + + runCmd = exec.Command(dockerBinary, "inspect", "--format", "{{ .Volumes }}", cleanedContainerID) + volumesAfterRestart, _, err := runCommandWithOutput(runCmd) + errorOut(err, t, volumesAfterRestart) + + if volumes != volumesAfterRestart { + volumes = strings.Trim(volumes, " \n\r") + volumesAfterRestart = strings.Trim(volumesAfterRestart, " \n\r") + t.Errorf("expected volume path: %s Actual path: %s", volumes, volumesAfterRestart) + } + + deleteAllContainers() + + logDone("restart - does not create a new volume on restart") +} diff --git a/components/engine/integration/container_test.go b/components/engine/integration/container_test.go index 48b3321a50..e70da5f14d 100644 --- a/components/engine/integration/container_test.go +++ b/components/engine/integration/container_test.go @@ -71,37 +71,6 @@ func TestKillDifferentUser(t *testing.T) { } } -func TestRestart(t *testing.T) { - daemon := mkDaemon(t) - defer nuke(daemon) - container, _, err := daemon.Create(&runconfig.Config{ - Image: GetTestImage(daemon).ID, - Cmd: []string{"echo", "-n", "foobar"}, - }, - "", - ) - if err != nil { - t.Fatal(err) - } - defer daemon.Destroy(container) - output, err := container.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "foobar" { - t.Error(string(output)) - } - - // Run the container again and check the output - output, err = container.Output() - if err != nil { - t.Fatal(err) - } - if string(output) != "foobar" { - t.Error(string(output)) - } -} - func TestRestartStdin(t *testing.T) { daemon := mkDaemon(t) defer nuke(daemon) @@ -526,47 +495,3 @@ func TestBindMounts(t *testing.T) { t.Fatal("Container failed to write to bind mount file") } } - -// Test that restarting a container with a volume does not create a new volume on restart. Regression test for #819. -func TestRestartWithVolumes(t *testing.T) { - daemon := mkDaemon(t) - defer nuke(daemon) - - container, _, err := daemon.Create(&runconfig.Config{ - Image: GetTestImage(daemon).ID, - Cmd: []string{"echo", "-n", "foobar"}, - Volumes: map[string]struct{}{"/test": {}}, - }, - "", - ) - if err != nil { - t.Fatal(err) - } - defer daemon.Destroy(container) - - for key := range container.Config.Volumes { - if key != "/test" { - t.Fail() - } - } - - _, err = container.Output() - if err != nil { - t.Fatal(err) - } - - expected := container.Volumes["/test"] - if expected == "" { - t.Fail() - } - // Run the container again to verify the volume path persists - _, err = container.Output() - if err != nil { - t.Fatal(err) - } - - actual := container.Volumes["/test"] - if expected != actual { - t.Fatalf("Expected volume path: %s Actual path: %s", expected, actual) - } -} From 6b22b2d896a41fa37bb360253fc8ec3cb893dea6 Mon Sep 17 00:00:00 2001 From: Adrien Folie Date: Wed, 2 Jul 2014 03:52:59 +0200 Subject: [PATCH 024/516] Move TestGetImagesByName Docker-DCO-1.1-Signed-off-by: Adrien Folie (github: folieadrien) Upstream-commit: f337a52a6ec01684c326a827adbe3d0b8b5f1f53 Component: engine --- .../engine/api/server/server_unit_test.go | 37 +++++++++++++++++++ .../docker_cli_inspect_test.go | 22 +++++++++++ components/engine/integration/api_test.go | 25 ------------- 3 files changed, 59 insertions(+), 25 deletions(-) create mode 100644 components/engine/integration-cli/docker_cli_inspect_test.go diff --git a/components/engine/api/server/server_unit_test.go b/components/engine/api/server/server_unit_test.go index 2d14f89551..f8068a29c5 100644 --- a/components/engine/api/server/server_unit_test.go +++ b/components/engine/api/server/server_unit_test.go @@ -319,6 +319,43 @@ func TestGetImagesHistory(t *testing.T) { } } +func TestGetImagesByName(t *testing.T) { + eng := engine.New() + name := "image_name" + var called bool + eng.Register("image_inspect", func(job *engine.Job) engine.Status { + called = true + if job.Args[0] != name { + t.Fatalf("name != '%s': %#v", name, job.Args[0]) + } + if api.APIVERSION.LessThan("1.12") && !job.GetenvBool("dirty") { + t.Fatal("dirty env variable not set") + } else if api.APIVERSION.GreaterThanOrEqualTo("1.12") && job.GetenvBool("dirty") { + t.Fatal("dirty env variable set when it shouldn't") + } + v := &engine.Env{} + v.SetBool("dirty", true) + if _, err := v.WriteTo(job.Stdout); err != nil { + return job.Error(err) + } + return engine.StatusOK + }) + r := serveRequest("GET", "/images/"+name+"/json", nil, eng, t) + if !called { + t.Fatal("handler was not called") + } + if r.HeaderMap.Get("Content-Type") != "application/json" { + t.Fatalf("%#v\n", r) + } + var stdoutJson interface{} + if err := json.Unmarshal(r.Body.Bytes(), &stdoutJson); err != nil { + t.Fatalf("%#v", err) + } + if stdoutJson.(map[string]interface{})["dirty"].(float64) != 1 { + t.Fatalf("%#v", stdoutJson) + } +} + func serveRequest(method, target string, body io.Reader, eng *engine.Engine, t *testing.T) *httptest.ResponseRecorder { r := httptest.NewRecorder() req, err := http.NewRequest(method, target, body) diff --git a/components/engine/integration-cli/docker_cli_inspect_test.go b/components/engine/integration-cli/docker_cli_inspect_test.go new file mode 100644 index 0000000000..7e5dd69ae6 --- /dev/null +++ b/components/engine/integration-cli/docker_cli_inspect_test.go @@ -0,0 +1,22 @@ +package main + +import ( + "os/exec" + "strings" + "testing" +) + +func TestInspectImage(t *testing.T) { + imageTest := "scratch" + imageTestId := "511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158" + imagesCmd := exec.Command(dockerBinary, "inspect", "--format='{{.Id}}'", imageTest) + + out, exitCode, err := runCommandWithOutput(imagesCmd) + if exitCode != 0 || err != nil { + t.Fatalf("failed to inspect image") + } + if id := strings.TrimSuffix(out, "\n"); id != imageTestId { + t.Fatalf("Expected id: %s for image: %s but received id: %s", imageTestId, imageTest, id) + } + logDone("inspect - inspect an image") +} diff --git a/components/engine/integration/api_test.go b/components/engine/integration/api_test.go index b2e44717a5..f5e5599e3f 100644 --- a/components/engine/integration/api_test.go +++ b/components/engine/integration/api_test.go @@ -16,7 +16,6 @@ import ( "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/api/server" "github.com/dotcloud/docker/engine" - "github.com/dotcloud/docker/image" "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" ) @@ -124,30 +123,6 @@ func TestGetImagesJSON(t *testing.T) { } } -func TestGetImagesByName(t *testing.T) { - eng := NewTestEngine(t) - defer mkDaemonFromEngine(eng, t).Nuke() - - req, err := http.NewRequest("GET", "/images/"+unitTestImageName+"/json", nil) - if err != nil { - t.Fatal(err) - } - - r := httptest.NewRecorder() - if err := server.ServeRequest(eng, api.APIVERSION, r, req); err != nil { - t.Fatal(err) - } - assertHttpNotError(r, t) - - img := &image.Image{} - if err := json.Unmarshal(r.Body.Bytes(), img); err != nil { - t.Fatal(err) - } - if img.ID != unitTestImageID { - t.Errorf("Error inspecting image") - } -} - func TestGetContainersJSON(t *testing.T) { eng := NewTestEngine(t) defer mkDaemonFromEngine(eng, t).Nuke() From 1a0c71351cff843770d2da7844c0b3dcd2b16b41 Mon Sep 17 00:00:00 2001 From: James Turnbull Date: Sun, 29 Jun 2014 18:01:11 -0400 Subject: [PATCH 025/516] Minor updates and fixes to the CONTRIBUTING doc * Fixed some uses of docker v. Docker * Formatting and line wrapping. * Spelling errors and grammar fixes. Docker-DCO-1.1-Signed-off-by: James Turnbull (github: jamtur01) Upstream-commit: d013b76e10262624c9f8ee05b9877e26eb58b516 Component: engine --- components/engine/CONTRIBUTING.md | 72 +++++++++++++++++-------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/components/engine/CONTRIBUTING.md b/components/engine/CONTRIBUTING.md index d07b972eb7..fd15f956a7 100644 --- a/components/engine/CONTRIBUTING.md +++ b/components/engine/CONTRIBUTING.md @@ -6,12 +6,17 @@ feels wrong or incomplete. ## Reporting Issues -When reporting [issues](https://github.com/dotcloud/docker/issues) -on GitHub please include your host OS (Ubuntu 12.04, Fedora 19, etc), -the output of `uname -a` and the output of `docker version` along with -the output of `docker -D info`. Please include the steps required to reproduce -the problem if possible and applicable. -This information will help us review and fix your issue faster. +When reporting [issues](https://github.com/dotcloud/docker/issues) on +GitHub please include your host OS (Ubuntu 12.04, Fedora 19, etc). +Please include: + +* The output of `uname -a`. +* The output of `docker version`. +* The output of `docker -D info`. + +Please also include the steps required to reproduce the problem if +possible and applicable. This information will help us review and fix +your issue faster. ## Build Environment @@ -34,7 +39,7 @@ received feedback on what to improve. We're trying very hard to keep Docker lean and focused. We don't want it to do everything for everybody. This means that we might decide against incorporating a new feature. However, there might be a way to implement -that feature *on top of* docker. +that feature *on top of* Docker. ### Discuss your design on the mailing list @@ -60,12 +65,12 @@ help prioritize the most common problems and requests. ### Conventions -Fork the repo and make changes on your fork in a feature branch: +Fork the repository and make changes on your fork in a feature branch: -- If it's a bugfix branch, name it XXX-something where XXX is the number of the - issue +- If it's a bug fix branch, name it XXXX-something where XXXX is the number of the + issue. - If it's a feature branch, create an enhancement issue to announce your - intentions, and name it XXX-something where XXX is the number of the issue. + intentions, and name it XXXX-something where XXXX is the number of the issue. Submit unit tests for your changes. Go has a great test framework built in; use it! Take a look at existing tests for inspiration. Run the full test suite on @@ -73,12 +78,12 @@ your branch before submitting a pull request. Update the documentation when creating or modifying features. Test your documentation changes for clarity, concision, and correctness, as -well as a clean documentation build. See ``docs/README.md`` for more -information on building the docs and how docs get released. +well as a clean documentation build. See `docs/README.md` for more +information on building the docs and how they get released. Write clean code. Universally formatted code promotes ease of writing, reading, and maintenance. Always run `gofmt -s -w file.go` on each changed file before -committing your changes. Most editors have plugins that do this automatically. +committing your changes. Most editors have plug-ins that do this automatically. Pull requests descriptions should be as clear as possible and include a reference to all the issues that they address. @@ -100,21 +105,22 @@ logical units of work using `git rebase -i` and `git push -f`. After every commit the test suite should be passing. Include documentation changes in the same commit so that a revert would remove all traces of the feature or fix. -Commits that fix or close an issue should include a reference like `Closes #XXX` -or `Fixes #XXX`, which will automatically close the issue when merged. +Commits that fix or close an issue should include a reference like +`Closes #XXXX` or `Fixes #XXXX`, which will automatically close the +issue when merged. -Please do not add yourself to the AUTHORS file, as it is regenerated +Please do not add yourself to the `AUTHORS` file, as it is regenerated regularly from the Git history. ### Merge approval -Docker maintainers use LGTM (looks good to me) in comments on the code review +Docker maintainers use LGTM (Looks Good To Me) in comments on the code review to indicate acceptance. A change requires LGTMs from an absolute majority of the maintainers of each -component affected. For example, if a change affects docs/ and registry/, it -needs an absolute majority from the maintainers of docs/ AND, separately, an -absolute majority of the maintainers of registry. +component affected. For example, if a change affects `docs/` and `registry/`, it +needs an absolute majority from the maintainers of `docs/` AND, separately, an +absolute majority of the maintainers of `registry/`. For more details see [MAINTAINERS.md](hack/MAINTAINERS.md) @@ -137,7 +143,6 @@ San Francisco, CA 94110 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. - Developer's Certificate of Origin 1.1 By making a contribution to this project, I certify that: @@ -165,20 +170,20 @@ By making a contribution to this project, I certify that: this project or the open source license(s) involved. ``` -then you just add a line to every git commit message: +Then you just add a line to every git commit message: Docker-DCO-1.1-Signed-off-by: Joe Smith (github: github_handle) -using your real name (sorry, no pseudonyms or anonymous contributions.) +Using your real name (sorry, no pseudonyms or anonymous contributions.) -One way to automate this, is customise your get ``commit.template`` by adding -a ``prepare-commit-msg`` hook to your docker checkout: +One way to automate this, is customize your git `commit.template` by adding +a `prepare-commit-msg` hook to your Docker repository: ``` curl -o .git/hooks/prepare-commit-msg https://raw.githubusercontent.com/dotcloud/docker/master/contrib/prepare-commit-msg.hook && chmod +x .git/hooks/prepare-commit-msg ``` -* Note: the above script expects to find your GitHub user name in ``git config --get github.user`` +* Note: the above script expects to find your GitHub user name in `git config --get github.user` #### Small patch exception @@ -194,11 +199,12 @@ If you have any questions, please refer to the FAQ in the [docs](http://docs.doc ### How can I become a maintainer? -* Step 1: learn the component inside out -* Step 2: make yourself useful by contributing code, bugfixes, support etc. -* Step 3: volunteer on the irc channel (#docker@freenode) -* Step 4: propose yourself at a scheduled docker meeting in #docker-dev +* Step 1: Learn the component inside out +* Step 2: Make yourself useful by contributing code, bug fixes, support etc. +* Step 3: Volunteer on the IRC channel (#docker at Freenode) +* Step 4: Propose yourself at a scheduled docker meeting in #docker-dev -Don't forget: being a maintainer is a time investment. Make sure you will have time to make yourself available. -You don't have to be a maintainer to make a difference on the project! +Don't forget: being a maintainer is a time investment. Make sure you +will have time to make yourself available. You don't have to be a +maintainer to make a difference on the project! From 6e61659fda04f1aa094a4492553dc4875b6e8874 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 2 Jul 2014 21:19:48 -0700 Subject: [PATCH 026/516] Fix 2 docker.com theme regressions on docs.docker.com - Hover colours of doc nav wrong - Primary Docs Nav not active when user is Logged in. Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: 873241790f802351d36c490046435215b33e4551 Component: engine --- components/engine/docs/theme/mkdocs/css/docs.css | 7 ++----- components/engine/docs/theme/mkdocs/header.html | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/components/engine/docs/theme/mkdocs/css/docs.css b/components/engine/docs/theme/mkdocs/css/docs.css index 0f42e22ef7..8d7aca2f0c 100644 --- a/components/engine/docs/theme/mkdocs/css/docs.css +++ b/components/engine/docs/theme/mkdocs/css/docs.css @@ -56,12 +56,9 @@ ol li { #nav_menu > #docsnav > #main-nav > li > a { color: #253237; } -#nav_menu > #docsnav > #main-nav > li.dd_on_hover > a { - color: #5992a3; -} #nav_menu > #docsnav > #main-nav > li.dd_on_hover { - background: #b1d5df; - color: #5992a3; + background: #d3f1fb; + color: #253237; } #nav_menu > #docsnav > #main-nav > li > span > b { border-top-color: #b1d5df !important; diff --git a/components/engine/docs/theme/mkdocs/header.html b/components/engine/docs/theme/mkdocs/header.html index 3560929cac..a3b1d9bd78 100644 --- a/components/engine/docs/theme/mkdocs/header.html +++ b/components/engine/docs/theme/mkdocs/header.html @@ -24,7 +24,7 @@ From ee3e18ed8e5dc364d87a6b1d182a12887b465249 Mon Sep 17 00:00:00 2001 From: unclejack Date: Thu, 3 Jul 2014 08:28:39 +0300 Subject: [PATCH 027/516] update vendored archive/tar to lower allocations Docker-DCO-1.1-Signed-off-by: Cristian Staretu (github: unclejack) Upstream-commit: b885f61a61df1e8d8f71914e17c2a2861167f96e Component: engine --- components/engine/hack/vendor.sh | 6 +++--- .../p/go/src/pkg/archive/tar/reader.go | 13 ++++++++----- .../p/go/src/pkg/archive/tar/writer.go | 8 +++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/components/engine/hack/vendor.sh b/components/engine/hack/vendor.sh index 2ee530a736..b0a89db68d 100755 --- a/components/engine/hack/vendor.sh +++ b/components/engine/hack/vendor.sh @@ -53,9 +53,9 @@ clone hg code.google.com/p/go.net 84a4013f96e0 clone hg code.google.com/p/gosqlite 74691fb6f837 -# get Go tip's archive/tar, for xattr support -# TODO after Go 1.3 drops, bump our minimum supported version and drop this vendored dep -clone hg code.google.com/p/go 3458ba248590 +# get Go tip's archive/tar, for xattr support and improved performance +# TODO after Go 1.4 drops, bump our minimum supported version and drop this vendored dep +clone hg code.google.com/p/go 17404efd6b02 mv src/code.google.com/p/go/src/pkg/archive/tar tmp-tar rm -rf src/code.google.com/p/go mkdir -p src/code.google.com/p/go/src/pkg/archive diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go index 920a9b08f9..a27559d0f0 100644 --- a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/reader.go @@ -29,10 +29,11 @@ const maxNanoSecondIntSize = 9 // The Next method advances to the next file in the archive (including the first), // and then it can be treated as an io.Reader to access the file's data. type Reader struct { - r io.Reader - err error - pad int64 // amount of padding (ignored) after current file entry - curr numBytesReader // reader for current file entry + r io.Reader + err error + pad int64 // amount of padding (ignored) after current file entry + curr numBytesReader // reader for current file entry + hdrBuff [blockSize]byte // buffer to use in readHeader } // A numBytesReader is an io.Reader with a numBytes method, returning the number @@ -426,7 +427,9 @@ func (tr *Reader) verifyChecksum(header []byte) bool { } func (tr *Reader) readHeader() *Header { - header := make([]byte, blockSize) + header := tr.hdrBuff[:] + copy(header, zeroBlock) + if _, tr.err = io.ReadFull(tr.r, header); tr.err != nil { return nil } diff --git a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go index 6eff6f6f84..d107dbbb51 100644 --- a/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go +++ b/components/engine/vendor/src/code.google.com/p/go/src/pkg/archive/tar/writer.go @@ -37,8 +37,9 @@ type Writer struct { nb int64 // number of unwritten bytes for current file entry pad int64 // amount of padding to write after current file entry closed bool - usedBinary bool // whether the binary numeric field extension was used - preferPax bool // use pax header instead of binary numeric header + usedBinary bool // whether the binary numeric field extension was used + preferPax bool // use pax header instead of binary numeric header + hdrBuff [blockSize]byte // buffer to use in writeHeader } // NewWriter creates a new Writer writing to w. @@ -160,7 +161,8 @@ func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { // subsecond time resolution, but for now let's just capture // too long fields or non ascii characters - header := make([]byte, blockSize) + header := tw.hdrBuff[:] + copy(header, zeroBlock) s := slicer(header) // keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax From eec6ca2f45ca7fec8f094323ca9585a731fe4acc Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Wed, 2 Jul 2014 19:15:56 +0400 Subject: [PATCH 028/516] Benchmark for BroadcastWriter Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) Upstream-commit: a8d95b7c573dedad26395372dd0d4ff8882f6cb6 Component: engine --- .../broadcastwriter/broadcastwriter_test.go | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/components/engine/utils/broadcastwriter/broadcastwriter_test.go b/components/engine/utils/broadcastwriter/broadcastwriter_test.go index 8d946e2f45..d5c9152467 100644 --- a/components/engine/utils/broadcastwriter/broadcastwriter_test.go +++ b/components/engine/utils/broadcastwriter/broadcastwriter_test.go @@ -106,3 +106,39 @@ func TestRaceBroadcastWriter(t *testing.T) { writer.Write([]byte("hello")) <-c } + +func BenchmarkBroadcastWriter(b *testing.B) { + writer := New() + setUpWriter := func() { + for i := 0; i < 100; i++ { + writer.AddWriter(devNullCloser(0), "stdout") + writer.AddWriter(devNullCloser(0), "stderr") + writer.AddWriter(devNullCloser(0), "") + } + } + testLine := "Line that thinks that it is log line from docker" + var buf bytes.Buffer + for i := 0; i < 100; i++ { + buf.Write([]byte(testLine + "\n")) + } + // line without eol + buf.Write([]byte(testLine)) + testText := buf.Bytes() + b.SetBytes(int64(5 * len(testText))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + setUpWriter() + b.StartTimer() + + for j := 0; j < 5; j++ { + if _, err := writer.Write(testText); err != nil { + b.Fatal(err) + } + } + + b.StopTimer() + writer.Close() + b.StartTimer() + } +} From 31988404b579703d98f0ec0e3ed3928c34fd43ca Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Wed, 2 Jul 2014 23:28:08 +0400 Subject: [PATCH 029/516] BroadcastWriter refactoring It became slightly faster and lighter possibly fixes #5923 problems Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) Upstream-commit: 7bdd23bfeec1968a3061e63e24af049837baf4c4 Component: engine --- components/engine/daemon/container.go | 4 +-- .../utils/broadcastwriter/broadcastwriter.go | 34 +++++++++---------- .../broadcastwriter/broadcastwriter_test.go | 4 +-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/components/engine/daemon/container.go b/components/engine/daemon/container.go index c7a6774601..419314b8ba 100644 --- a/components/engine/daemon/container.go +++ b/components/engine/daemon/container.go @@ -503,10 +503,10 @@ func (container *Container) cleanup() { utils.Errorf("%s: Error close stdin: %s", container.ID, err) } } - if err := container.stdout.Close(); err != nil { + if err := container.stdout.Clean(); err != nil { utils.Errorf("%s: Error close stdout: %s", container.ID, err) } - if err := container.stderr.Close(); err != nil { + if err := container.stderr.Clean(); err != nil { utils.Errorf("%s: Error close stderr: %s", container.ID, err) } if container.command != nil && container.command.Terminal != nil { diff --git a/components/engine/utils/broadcastwriter/broadcastwriter.go b/components/engine/utils/broadcastwriter/broadcastwriter.go index 81ca9e5bdd..9bd2c1471e 100644 --- a/components/engine/utils/broadcastwriter/broadcastwriter.go +++ b/components/engine/utils/broadcastwriter/broadcastwriter.go @@ -10,12 +10,16 @@ import ( "github.com/dotcloud/docker/utils" ) +// BroadcastWriter accumulate multiple io.WriteCloser by stream. type BroadcastWriter struct { sync.Mutex buf *bytes.Buffer streams map[string](map[io.WriteCloser]struct{}) } +// AddWriter adds new io.WriteCloser for stream. +// If stream is "", then all writes proceed as is. Otherwise every line from +// input will be packed to serialized utils.JSONLog. func (w *BroadcastWriter) AddWriter(writer io.WriteCloser, stream string) { w.Lock() if _, ok := w.streams[stream]; !ok { @@ -25,10 +29,11 @@ func (w *BroadcastWriter) AddWriter(writer io.WriteCloser, stream string) { w.Unlock() } +// Write writes bytes to all writers. Failed writers will be evicted during +// this call. func (w *BroadcastWriter) Write(p []byte) (n int, err error) { created := time.Now().UTC() w.Lock() - defer w.Unlock() if writers, ok := w.streams[""]; ok { for sw := range writers { if n, err := sw.Write(p); err != nil || n != len(p) { @@ -38,49 +43,44 @@ func (w *BroadcastWriter) Write(p []byte) (n int, err error) { } } w.buf.Write(p) - lines := []string{} for { line, err := w.buf.ReadString('\n') if err != nil { w.buf.Write([]byte(line)) break } - lines = append(lines, line) - } - - if len(lines) != 0 { for stream, writers := range w.streams { if stream == "" { continue } - var lp []byte - for _, line := range lines { - b, err := json.Marshal(&utils.JSONLog{Log: line, Stream: stream, Created: created}) - if err != nil { - utils.Errorf("Error making JSON log line: %s", err) - } - lp = append(lp, b...) - lp = append(lp, '\n') + b, err := json.Marshal(utils.JSONLog{Log: line, Stream: stream, Created: created}) + if err != nil { + utils.Errorf("Error making JSON log line: %s", err) + continue } + b = append(b, '\n') for sw := range writers { - if _, err := sw.Write(lp); err != nil { + if _, err := sw.Write(b); err != nil { delete(writers, sw) } } } } + w.Unlock() return len(p), nil } -func (w *BroadcastWriter) Close() error { +// Clean closes and removes all writers. Last non-eol-terminated part of data +// will be saved. +func (w *BroadcastWriter) Clean() error { w.Lock() - defer w.Unlock() for _, writers := range w.streams { for w := range writers { w.Close() } } w.streams = make(map[string](map[io.WriteCloser]struct{})) + w.Unlock() return nil } diff --git a/components/engine/utils/broadcastwriter/broadcastwriter_test.go b/components/engine/utils/broadcastwriter/broadcastwriter_test.go index d5c9152467..62ca12659a 100644 --- a/components/engine/utils/broadcastwriter/broadcastwriter_test.go +++ b/components/engine/utils/broadcastwriter/broadcastwriter_test.go @@ -82,7 +82,7 @@ func TestBroadcastWriter(t *testing.T) { t.Errorf("Buffer contains %v", bufferC.String()) } - writer.Close() + writer.Clean() } type devNullCloser int @@ -138,7 +138,7 @@ func BenchmarkBroadcastWriter(b *testing.B) { } b.StopTimer() - writer.Close() + writer.Clean() b.StartTimer() } } From 9b1af46554e73a7ba4d742d28302c7f8d009d5ef Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Wed, 4 Jun 2014 16:38:06 -0400 Subject: [PATCH 030/516] Error if Docker daemon starts with BTRFS graph driver and SELinux enabled The Docker btrfs graph driver does not interact well with SELinux at present. If btrfs mounts the same file in several locations, the same SELinux label will be applied to all mountpoints. In the context of the graph driver, things such as shared libraries become inaccessible to containers due to SELInux, causing all dynamically linked applications to fail when run in a container. Consequently, error when we detect the daemon is being run with SELinux enabled and the btrfs driver. Documentation has been added for this behavior. Docker-DCO-1.1-Signed-off-by: Matthew Heon (github: mheon) Upstream-commit: 4318802f645cdd4fa63a894160f153a69a97af59 Component: engine --- components/engine/daemon/daemon.go | 5 +++++ components/engine/docker/docker.go | 2 +- components/engine/docs/man/docker.1.md | 2 +- components/engine/docs/sources/reference/commandline/cli.md | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/components/engine/daemon/daemon.go b/components/engine/daemon/daemon.go index 23402d9518..8863407979 100644 --- a/components/engine/daemon/daemon.go +++ b/components/engine/daemon/daemon.go @@ -778,6 +778,11 @@ func NewDaemonFromDirectory(config *daemonconfig.Config, eng *engine.Engine) (*D } utils.Debugf("Using graph driver %s", driver) + // As Docker on btrfs and SELinux are incompatible at present, error on both being enabled + if config.EnableSelinuxSupport && driver.String() == "btrfs" { + return nil, fmt.Errorf("SELinux is not supported with the BTRFS graph driver!") + } + daemonRepo := path.Join(config.Root, "containers") if err := os.MkdirAll(daemonRepo, 0700); err != nil && !os.IsExist(err) { diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 30d43bc6a8..5367e759af 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -66,7 +66,7 @@ func main() { flCa = flag.String([]string{"-tlscacert"}, dockerConfDir+defaultCaFile, "Trust only remotes providing a certificate signed by the CA given here") flCert = flag.String([]string{"-tlscert"}, dockerConfDir+defaultCertFile, "Path to TLS certificate file") flKey = flag.String([]string{"-tlskey"}, dockerConfDir+defaultKeyFile, "Path to TLS key file") - flSelinuxEnabled = flag.Bool([]string{"-selinux-enabled"}, false, "Enable selinux support") + flSelinuxEnabled = flag.Bool([]string{"-selinux-enabled"}, false, "Enable selinux support. SELinux does not presently support the BTRFS storage driver") ) flag.Var(&flDns, []string{"#dns", "-dns"}, "Force Docker to use specific DNS servers") flag.Var(&flDnsSearch, []string{"-dns-search"}, "Force Docker to use specific DNS search domains") diff --git a/components/engine/docs/man/docker.1.md b/components/engine/docs/man/docker.1.md index a7a826ed9f..602c6e2ace 100644 --- a/components/engine/docs/man/docker.1.md +++ b/components/engine/docs/man/docker.1.md @@ -74,7 +74,7 @@ unix://[/path/to/socket] to use. Print version information and quit. Default is false. **--selinux-enabled**=*true*|*false* - Enable selinux support. Default is false. + Enable selinux support. Default is false. SELinux does not presently support the BTRFS storage driver. # COMMANDS **docker-attach(1)** diff --git a/components/engine/docs/sources/reference/commandline/cli.md b/components/engine/docs/sources/reference/commandline/cli.md index 301593f2f1..9a6d27f0eb 100644 --- a/components/engine/docs/sources/reference/commandline/cli.md +++ b/components/engine/docs/sources/reference/commandline/cli.md @@ -73,7 +73,7 @@ expect an integer, and they can only be specified once. -p, --pidfile="/var/run/docker.pid" Path to use for daemon PID file -r, --restart=true Restart previously running containers -s, --storage-driver="" Force the Docker runtime to use a specific storage driver - --selinux-enabled=false Enable selinux support + --selinux-enabled=false Enable selinux support. SELinux does not presently support the BTRFS storage driver --storage-opt=[] Set storage driver options --tls=false Use TLS; implied by tls-verify flags --tlscacert="/home/sven/.docker/ca.pem" Trust only remotes providing a certificate signed by the CA given here From a796dc81bfe74774cf05658cb15212794f0dad48 Mon Sep 17 00:00:00 2001 From: Victor Vieux Date: Thu, 3 Jul 2014 18:01:36 +0000 Subject: [PATCH 031/516] Change version to 1.1.0-dev Docker-DCO-1.1-Signed-off-by: Victor Vieux (github: vieux) Upstream-commit: d55cafb72e25e5ef9b400c1c010e9832ec8b5561 Component: engine --- components/engine/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/VERSION b/components/engine/VERSION index 9084fa2f71..336c367755 100644 --- a/components/engine/VERSION +++ b/components/engine/VERSION @@ -1 +1 @@ -1.1.0 +1.1.0-dev From 984e0d89c3cee9c9a1f54e493d06087f57ab96c5 Mon Sep 17 00:00:00 2001 From: Tobias Gesellchen Date: Thu, 3 Jul 2014 22:24:24 +0200 Subject: [PATCH 032/516] docs: add Groovy Docker-Client link Docker-DCO-1.1-Signed-off-by: Tobias Gesellchen (github: gesellix) Upstream-commit: 5c246c931eec7ec31a90fc4ae1577eb54f938427 Component: engine --- .../sources/reference/api/remote_api_client_libraries.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/engine/docs/sources/reference/api/remote_api_client_libraries.md b/components/engine/docs/sources/reference/api/remote_api_client_libraries.md index d1d26a1ddf..3dcebeedaa 100644 --- a/components/engine/docs/sources/reference/api/remote_api_client_libraries.md +++ b/components/engine/docs/sources/reference/api/remote_api_client_libraries.md @@ -140,5 +140,11 @@ will add the libraries here. https://github.com/spotify/docker-client Active + + Groovy + docker-client + https://github.com/gesellix-docker/docker-client + Active + From f9a8a473054044f1738db53ce34d79afe3235986 Mon Sep 17 00:00:00 2001 From: Naoki Orii Date: Thu, 3 Jul 2014 14:02:40 -0700 Subject: [PATCH 033/516] doc fix: remove trailing * The trailing `*` makes it seem like there is some kind of annotation Upstream-commit: 6e4b37fffdfc324141aae9f86268db4e8ed7aeb0 Component: engine --- .../engine/docs/sources/introduction/understanding-docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/introduction/understanding-docker.md b/components/engine/docs/sources/introduction/understanding-docker.md index c79573a635..18b8d5926a 100644 --- a/components/engine/docs/sources/introduction/understanding-docker.md +++ b/components/engine/docs/sources/introduction/understanding-docker.md @@ -55,7 +55,7 @@ Docker's portability and lightweight nature also make dynamically managing workloads easy. You can use Docker to quickly scale up or tear down applications and services. Docker's speed means that scaling can be near real time. -*Achieving higher density and running more workloads** +*Achieving higher density and running more workloads* Docker is lightweight and fast. It provides a viable, cost-effective alternative to hypervisor-based virtual machines. This is especially useful in high density From 0c1cce45f2d601767042933a62faa60bb6060463 Mon Sep 17 00:00:00 2001 From: Brian Goff Date: Tue, 1 Jul 2014 16:51:30 -0400 Subject: [PATCH 034/516] Remove dup tests integration/server_test.go/TestCreateNumberUsername == integration-cli/docker_cli_run_test.go/TestUserByID integration/server_test.go/TestRmi == integration-cli/docker_cli_images_test.go/TestCLIImageTagRemove Docker-DCO-1.1-Signed-off-by: Brian Goff (github: cpuguy83) Upstream-commit: 0a3abe33f0bd9b7dfc36021b6f10d0857a432d40 Component: engine --- components/engine/integration/server_test.go | 99 -------------------- 1 file changed, 99 deletions(-) diff --git a/components/engine/integration/server_test.go b/components/engine/integration/server_test.go index 151490cdc6..f70bbb0b77 100644 --- a/components/engine/integration/server_test.go +++ b/components/engine/integration/server_test.go @@ -2,7 +2,6 @@ package docker import ( "bytes" - "strings" "testing" "time" @@ -23,18 +22,6 @@ func TestCreateNumberHostname(t *testing.T) { createTestContainer(eng, config, t) } -func TestCreateNumberUsername(t *testing.T) { - eng := NewTestEngine(t) - defer mkDaemonFromEngine(eng, t).Nuke() - - config, _, _, err := runconfig.Parse([]string{"-u", "1002", unitTestImageID, "echo test"}, nil) - if err != nil { - t.Fatal(err) - } - - createTestContainer(eng, config, t) -} - func TestCommit(t *testing.T) { eng := NewTestEngine(t) defer mkDaemonFromEngine(eng, t).Nuke() @@ -271,92 +258,6 @@ func TestRunWithTooLowMemoryLimit(t *testing.T) { } } -func TestRmi(t *testing.T) { - eng := NewTestEngine(t) - srv := mkServerFromEngine(eng, t) - defer mkDaemonFromEngine(eng, t).Nuke() - - initialImages := getAllImages(eng, t) - - config, hostConfig, _, err := runconfig.Parse([]string{unitTestImageID, "echo", "test"}, nil) - if err != nil { - t.Fatal(err) - } - - containerID := createTestContainer(eng, config, t) - - //To remove - job := eng.Job("start", containerID) - if err := job.ImportEnv(hostConfig); err != nil { - t.Fatal(err) - } - if err := job.Run(); err != nil { - t.Fatal(err) - } - - if err := eng.Job("wait", containerID).Run(); err != nil { - t.Fatal(err) - } - - job = eng.Job("commit", containerID) - job.Setenv("repo", "test") - var outputBuffer = bytes.NewBuffer(nil) - job.Stdout.Add(outputBuffer) - if err := job.Run(); err != nil { - t.Fatal(err) - } - - if err := eng.Job("tag", engine.Tail(outputBuffer, 1), "test", "0.1").Run(); err != nil { - t.Fatal(err) - } - - containerID = createTestContainer(eng, config, t) - - //To remove - job = eng.Job("start", containerID) - if err := job.ImportEnv(hostConfig); err != nil { - t.Fatal(err) - } - if err := job.Run(); err != nil { - t.Fatal(err) - } - - if err := eng.Job("wait", containerID).Run(); err != nil { - t.Fatal(err) - } - - job = eng.Job("commit", containerID) - job.Setenv("repo", "test") - if err := job.Run(); err != nil { - t.Fatal(err) - } - - images := getAllImages(eng, t) - - if images.Len()-initialImages.Len() != 2 { - t.Fatalf("Expected 2 new images, found %d.", images.Len()-initialImages.Len()) - } - - if err = srv.DeleteImage(engine.Tail(outputBuffer, 1), engine.NewTable("", 0), true, false, false); err != nil { - t.Fatal(err) - } - - images = getAllImages(eng, t) - - if images.Len()-initialImages.Len() != 1 { - t.Fatalf("Expected 1 new image, found %d.", images.Len()-initialImages.Len()) - } - - for _, image := range images.Data { - if strings.Contains(unitTestImageID, image.Get("Id")) { - continue - } - if image.GetList("RepoTags")[0] == ":" { - t.Fatalf("Expected tagged image, got untagged one.") - } - } -} - func TestImagesFilter(t *testing.T) { eng := NewTestEngine(t) defer nuke(mkDaemonFromEngine(eng, t)) From 525592aa62347b1545226ef342aa09fc9d007a9e Mon Sep 17 00:00:00 2001 From: Fabio Falci Date: Thu, 26 Jun 2014 12:03:23 +0100 Subject: [PATCH 035/516] Relax dns search to accept empty domain In that case /etc/resolv.conf will be generated with no search option. Usage: --dns-search=. Docker-DCO-1.1-Signed-off-by: Fabio Falci (github: fabiofalci) Upstream-commit: 804b00cd7d1f084a872211e5043d255c454c8e51 Component: engine --- components/engine/docker/docker.go | 2 +- .../integration-cli/docker_cli_run_test.go | 109 ++++++++++++++++++ components/engine/opts/opts.go | 11 +- components/engine/opts/opts_test.go | 14 ++- .../pkg/networkfs/resolvconf/resolvconf.go | 6 +- .../networkfs/resolvconf/resolvconf_test.go | 25 ++++ components/engine/runconfig/parse.go | 2 +- 7 files changed, 158 insertions(+), 11 deletions(-) diff --git a/components/engine/docker/docker.go b/components/engine/docker/docker.go index 30d43bc6a8..147c91d21d 100644 --- a/components/engine/docker/docker.go +++ b/components/engine/docker/docker.go @@ -52,7 +52,7 @@ func main() { flSocketGroup = flag.String([]string{"G", "-group"}, "docker", "Group to assign the unix socket specified by -H when running in daemon mode\nuse '' (the empty string) to disable setting of a group") flEnableCors = flag.Bool([]string{"#api-enable-cors", "-api-enable-cors"}, false, "Enable CORS headers in the remote API") flDns = opts.NewListOpts(opts.ValidateIp4Address) - flDnsSearch = opts.NewListOpts(opts.ValidateDomain) + flDnsSearch = opts.NewListOpts(opts.ValidateDnsSearch) flEnableIptables = flag.Bool([]string{"#iptables", "-iptables"}, true, "Enable Docker's addition of iptables rules") flEnableIpForward = flag.Bool([]string{"#ip-forward", "-ip-forward"}, true, "Enable net.ipv4.ip_forward") flDefaultIp = flag.String([]string{"#ip", "-ip"}, "0.0.0.0", "Default IP address to use when binding container ports") diff --git a/components/engine/integration-cli/docker_cli_run_test.go b/components/engine/integration-cli/docker_cli_run_test.go index eda1b3f0a5..9f52d58d12 100644 --- a/components/engine/integration-cli/docker_cli_run_test.go +++ b/components/engine/integration-cli/docker_cli_run_test.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "io/ioutil" "os" "os/exec" "reflect" @@ -10,6 +11,8 @@ import ( "strings" "sync" "testing" + + "github.com/dotcloud/docker/pkg/networkfs/resolvconf" ) // "test123" should be printed by docker run @@ -983,3 +986,109 @@ func TestDisallowBindMountingRootToRoot(t *testing.T) { logDone("run - bind mount /:/ as volume should fail") } + +func TestDnsDefaultOptions(t *testing.T) { + cmd := exec.Command(dockerBinary, "run", "busybox", "cat", "/etc/resolv.conf") + + actual, _, err := runCommandWithOutput(cmd) + if err != nil { + t.Fatal(err, actual) + } + + resolvConf, err := ioutil.ReadFile("/etc/resolv.conf") + if os.IsNotExist(err) { + t.Fatalf("/etc/resolv.conf does not exist") + } + + if actual != string(resolvConf) { + t.Fatalf("expected resolv.conf is not the same of actual") + } + + deleteAllContainers() + + logDone("run - dns default options") +} + +func TestDnsOptions(t *testing.T) { + cmd := exec.Command(dockerBinary, "run", "--dns=127.0.0.1", "--dns-search=mydomain", "busybox", "cat", "/etc/resolv.conf") + + out, _, err := runCommandWithOutput(cmd) + if err != nil { + t.Fatal(err, out) + } + + actual := strings.Replace(strings.Trim(out, "\r\n"), "\n", " ", -1) + if actual != "nameserver 127.0.0.1 search mydomain" { + t.Fatalf("expected 'nameserver 127.0.0.1 search mydomain', but says: '%s'", actual) + } + + cmd = exec.Command(dockerBinary, "run", "--dns=127.0.0.1", "--dns-search=.", "busybox", "cat", "/etc/resolv.conf") + + out, _, err = runCommandWithOutput(cmd) + if err != nil { + t.Fatal(err, out) + } + + actual = strings.Replace(strings.Trim(strings.Trim(out, "\r\n"), " "), "\n", " ", -1) + if actual != "nameserver 127.0.0.1" { + t.Fatalf("expected 'nameserver 127.0.0.1', but says: '%s'", actual) + } + + logDone("run - dns options") +} + +func TestDnsOptionsBasedOnHostResolvConf(t *testing.T) { + resolvConf, err := ioutil.ReadFile("/etc/resolv.conf") + if os.IsNotExist(err) { + t.Fatalf("/etc/resolv.conf does not exist") + } + + hostNamservers := resolvconf.GetNameservers(resolvConf) + hostSearch := resolvconf.GetSearchDomains(resolvConf) + + cmd := exec.Command(dockerBinary, "run", "--dns=127.0.0.1", "busybox", "cat", "/etc/resolv.conf") + + out, _, err := runCommandWithOutput(cmd) + if err != nil { + t.Fatal(err, out) + } + + if actualNameservers := resolvconf.GetNameservers([]byte(out)); string(actualNameservers[0]) != "127.0.0.1" { + t.Fatalf("expected '127.0.0.1', but says: '%s'", string(actualNameservers[0])) + } + + actualSearch := resolvconf.GetSearchDomains([]byte(out)) + if len(actualSearch) != len(hostSearch) { + t.Fatalf("expected '%s' search domain(s), but it has: '%s'", len(hostSearch), len(actualSearch)) + } + for i := range actualSearch { + if actualSearch[i] != hostSearch[i] { + t.Fatalf("expected '%s' domain, but says: '%s'", actualSearch[i], hostSearch[i]) + } + } + + cmd = exec.Command(dockerBinary, "run", "--dns-search=mydomain", "busybox", "cat", "/etc/resolv.conf") + + out, _, err = runCommandWithOutput(cmd) + if err != nil { + t.Fatal(err, out) + } + + actualNameservers := resolvconf.GetNameservers([]byte(out)) + if len(actualNameservers) != len(hostNamservers) { + t.Fatalf("expected '%s' nameserver(s), but it has: '%s'", len(hostNamservers), len(actualNameservers)) + } + for i := range actualNameservers { + if actualNameservers[i] != hostNamservers[i] { + t.Fatalf("expected '%s' nameserver, but says: '%s'", actualNameservers[i], hostNamservers[i]) + } + } + + if actualSearch = resolvconf.GetSearchDomains([]byte(out)); string(actualSearch[0]) != "mydomain" { + t.Fatalf("expected 'mydomain', but says: '%s'", string(actualSearch[0])) + } + + deleteAllContainers() + + logDone("run - dns options based on host resolv.conf") +} diff --git a/components/engine/opts/opts.go b/components/engine/opts/opts.go index 67f1c8fd48..d17b57e07c 100644 --- a/components/engine/opts/opts.go +++ b/components/engine/opts/opts.go @@ -137,7 +137,16 @@ func ValidateIp4Address(val string) (string, error) { return "", fmt.Errorf("%s is not an ip4 address", val) } -func ValidateDomain(val string) (string, error) { +// Validates domain for resolvconf search configuration. +// A zero length domain is represented by . +func ValidateDnsSearch(val string) (string, error) { + if val = strings.Trim(val, " "); val == "." { + return val, nil + } + return validateDomain(val) +} + +func validateDomain(val string) (string, error) { alpha := regexp.MustCompile(`[a-zA-Z]`) if alpha.FindString(val) == "" { return "", fmt.Errorf("%s is not a valid domain", val) diff --git a/components/engine/opts/opts_test.go b/components/engine/opts/opts_test.go index 299cbfe503..b18088b934 100644 --- a/components/engine/opts/opts_test.go +++ b/components/engine/opts/opts_test.go @@ -23,8 +23,9 @@ func TestValidateIP4(t *testing.T) { } -func TestValidateDomain(t *testing.T) { +func TestValidateDnsSearch(t *testing.T) { valid := []string{ + `.`, `a`, `a.`, `1.foo`, @@ -49,7 +50,8 @@ func TestValidateDomain(t *testing.T) { invalid := []string{ ``, - `.`, + ` `, + ` `, `17`, `17.`, `.17`, @@ -65,14 +67,14 @@ func TestValidateDomain(t *testing.T) { } for _, domain := range valid { - if ret, err := ValidateDomain(domain); err != nil || ret == "" { - t.Fatalf("ValidateDomain(`"+domain+"`) got %s %s", ret, err) + if ret, err := ValidateDnsSearch(domain); err != nil || ret == "" { + t.Fatalf("ValidateDnsSearch(`"+domain+"`) got %s %s", ret, err) } } for _, domain := range invalid { - if ret, err := ValidateDomain(domain); err == nil || ret != "" { - t.Fatalf("ValidateDomain(`"+domain+"`) got %s %s", ret, err) + if ret, err := ValidateDnsSearch(domain); err == nil || ret != "" { + t.Fatalf("ValidateDnsSearch(`"+domain+"`) got %s %s", ret, err) } } } diff --git a/components/engine/pkg/networkfs/resolvconf/resolvconf.go b/components/engine/pkg/networkfs/resolvconf/resolvconf.go index d6854fb3b1..38ae56432d 100644 --- a/components/engine/pkg/networkfs/resolvconf/resolvconf.go +++ b/components/engine/pkg/networkfs/resolvconf/resolvconf.go @@ -78,8 +78,10 @@ func Build(path string, dns, dnsSearch []string) error { } } if len(dnsSearch) > 0 { - if _, err := content.WriteString("search " + strings.Join(dnsSearch, " ") + "\n"); err != nil { - return err + if searchString := strings.Join(dnsSearch, " "); strings.Trim(searchString, " ") != "." { + if _, err := content.WriteString("search " + searchString + "\n"); err != nil { + return err + } } } diff --git a/components/engine/pkg/networkfs/resolvconf/resolvconf_test.go b/components/engine/pkg/networkfs/resolvconf/resolvconf_test.go index fd20712376..6187acbae7 100644 --- a/components/engine/pkg/networkfs/resolvconf/resolvconf_test.go +++ b/components/engine/pkg/networkfs/resolvconf/resolvconf_test.go @@ -131,3 +131,28 @@ func TestBuild(t *testing.T) { t.Fatalf("Expected to find '%s' got '%s'", expected, content) } } + +func TestBuildWithZeroLengthDomainSearch(t *testing.T) { + file, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + defer os.Remove(file.Name()) + + err = Build(file.Name(), []string{"ns1", "ns2", "ns3"}, []string{"."}) + if err != nil { + t.Fatal(err) + } + + content, err := ioutil.ReadFile(file.Name()) + if err != nil { + t.Fatal(err) + } + + if expected := "nameserver ns1\nnameserver ns2\nnameserver ns3\n"; !bytes.Contains(content, []byte(expected)) { + t.Fatalf("Expected to find '%s' got '%s'", expected, content) + } + if notExpected := "search ."; bytes.Contains(content, []byte(notExpected)) { + t.Fatalf("Expected to not find '%s' got '%s'", notExpected, content) + } +} diff --git a/components/engine/runconfig/parse.go b/components/engine/runconfig/parse.go index dfd9f4ddd3..ae79186716 100644 --- a/components/engine/runconfig/parse.go +++ b/components/engine/runconfig/parse.go @@ -45,7 +45,7 @@ func parseRun(cmd *flag.FlagSet, args []string, sysInfo *sysinfo.SysInfo) (*Conf flPublish opts.ListOpts flExpose opts.ListOpts flDns opts.ListOpts - flDnsSearch = opts.NewListOpts(opts.ValidateDomain) + flDnsSearch = opts.NewListOpts(opts.ValidateDnsSearch) flVolumesFrom opts.ListOpts flLxcOpts opts.ListOpts flEnvFile opts.ListOpts From 5ec859dbc34cbbca639be06bf2ea5b6bb7e003a5 Mon Sep 17 00:00:00 2001 From: "hyeongkyu.lee" Date: Fri, 4 Jul 2014 17:59:26 +0900 Subject: [PATCH 036/516] Update README.md Docker-DCO-1.1-Signed-off-by: Hyeongkyu Lee (github: leeplay) Upstream-commit: 3f5ab612c8d0c1775a864e4414846b301e6498e9 Component: engine --- components/engine/daemonconfig/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/daemonconfig/README.md b/components/engine/daemonconfig/README.md index 488e7c7cac..b61df1449e 100644 --- a/components/engine/daemonconfig/README.md +++ b/components/engine/daemonconfig/README.md @@ -1,3 +1,3 @@ -This directory contains code pertaining to the configuration of the docker deamon +This directory contains code pertaining to the configuration of the docker daemon These are the configuration settings that you pass to the docker daemon when you launch it with say: `docker -d -e lxc` From 7f05b133742bcbd35fcbebcaa55fb5478dd1b171 Mon Sep 17 00:00:00 2001 From: "hyeongkyu.lee" Date: Fri, 4 Jul 2014 18:21:13 +0900 Subject: [PATCH 037/516] Update README.md Docker-DCO-1.1-Signed-off-by: Hyeongkyu Lee (github: leeplay) Upstream-commit: 649b50c0b71a21f8ea6b88c63008a0f909fe4113 Component: engine --- components/engine/api/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/api/README.md b/components/engine/api/README.md index 3ef33f8c29..453f61a1a1 100644 --- a/components/engine/api/README.md +++ b/components/engine/api/README.md @@ -1,5 +1,5 @@ This directory contains code pertaining to the Docker API: - - Used by the docker client when comunicating with the docker deamon + - Used by the docker client when communicating with the docker daemon - - Used by third party tools wishing to interface with the docker deamon + - Used by third party tools wishing to interface with the docker daemon From bb9d32707cf348928cf2ffdbf8922286300f4a9f Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Fri, 4 Jul 2014 15:56:23 +1000 Subject: [PATCH 038/516] add client OS and arch to docker version Docker-DCO-1.1-Signed-off-by: Sven Dowideit (github: SvenDowideit) Upstream-commit: eceacb8334250b9f241b263f28372d5f35ada71d Component: engine --- components/engine/api/client/commands.go | 1 + 1 file changed, 1 insertion(+) diff --git a/components/engine/api/client/commands.go b/components/engine/api/client/commands.go index df2125f5f3..c273895594 100644 --- a/components/engine/api/client/commands.go +++ b/components/engine/api/client/commands.go @@ -391,6 +391,7 @@ func (cli *DockerCli) CmdVersion(args ...string) error { if dockerversion.GITCOMMIT != "" { fmt.Fprintf(cli.out, "Git commit (client): %s\n", dockerversion.GITCOMMIT) } + fmt.Fprintf(cli.out, "OS/Arch (client): %s/%s\n", runtime.GOOS, runtime.GOARCH) body, _, err := readBody(cli.call("GET", "/version", nil, false)) if err != nil { From 5f0247a4bdbc51e77156daa211cc8af09674030f Mon Sep 17 00:00:00 2001 From: Andrew France Date: Fri, 4 Jul 2014 12:05:50 +0100 Subject: [PATCH 039/516] Fix "Error while reading file" Fish completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unbalanced single quote caused Fish to fail to load the completion file with `source: Error while reading file “/etc/fish/completions/docker.fish”`. Docker-DCO-1.1-Signed-off-by: Andrew France (github: Odaeus) Upstream-commit: 6ad5fe86590f757eca754d5bb4a599faec63ae30 Component: engine --- components/engine/contrib/completion/fish/docker.fish | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/contrib/completion/fish/docker.fish b/components/engine/contrib/completion/fish/docker.fish index a4a9365f92..ba83526c75 100644 --- a/components/engine/contrib/completion/fish/docker.fish +++ b/components/engine/contrib/completion/fish/docker.fish @@ -85,7 +85,7 @@ complete -c docker -A -f -n '__fish_seen_subcommand_from commit' -l run -d 'Conf complete -c docker -A -f -n '__fish_seen_subcommand_from commit' -a '(__fish_print_docker_containers all)' -d "Container" # cp -complete -c docker -f -n '__fish_docker_no_subcommand' -a cp -d 'Copy files/folders from a container's filesystem to the host path' +complete -c docker -f -n '__fish_docker_no_subcommand' -a cp -d "Copy files/folders from a container's filesystem to the host path" # diff complete -c docker -f -n '__fish_docker_no_subcommand' -a diff -d "Inspect changes on a container's filesystem" From 8174c1b3a93f066f1534a0f49dc469796781bfdb Mon Sep 17 00:00:00 2001 From: OddBloke Date: Thu, 3 Jul 2014 12:55:21 +0100 Subject: [PATCH 040/516] Explain ADD invalidation more accurately And also move it in to the `ADD` section, rather than being hidden in the `RUN` section. Docker-DCO-1.1-Signed-off-by: Daniel Watkins (github: OddBloke) Upstream-commit: 20a77aeeb8855ca0189fe9e3292d92b165b08905 Component: engine --- components/engine/docs/sources/reference/builder.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/engine/docs/sources/reference/builder.md b/components/engine/docs/sources/reference/builder.md index 91190933c9..da7f3ebff2 100644 --- a/components/engine/docs/sources/reference/builder.md +++ b/components/engine/docs/sources/reference/builder.md @@ -169,9 +169,8 @@ will be reused during the next build. The cache for `RUN` instructions can be invalidated by using the `--no-cache` flag, for example `docker build --no-cache`. -The first encountered `ADD` instruction will invalidate the cache for all -following instructions from the 'Dockerfile' if the contents of the context -have changed. This will also invalidate the cache for `RUN` instructions. +The cache for `RUN` instructions can be invalidated by `ADD` instructions. See +[below](#add) for details. ### Known Issues (RUN) @@ -285,6 +284,11 @@ In the case where `` is a remote file URL, the destination will have permis > or use another tool from within the container as ADD does not support > authentication. +> **Note**: +> The first encountered `ADD` instruction will invalidate the cache for all +> following instructions from the Dockerfile if the contents of `` have +> changed. This includes invalidating the cache for `RUN` instructions. + The copy obeys the following rules: - The `` path must be inside the *context* of the build; From 003b3ea1dffe68ad1538d5397098fa53e65e17ff Mon Sep 17 00:00:00 2001 From: Tom Fotherby Date: Fri, 4 Jul 2014 18:52:10 +0100 Subject: [PATCH 041/516] Add info to Mount a Host File as a Data Volume Docker-DCO-1.1-Signed-off-by: Tom Fotherby (github: tomfotherby) Upstream-commit: cfbe062eb2ba6f1d24ab3b6f9b965a65931d5b07 Component: engine --- .../docs/sources/userguide/dockervolumes.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/components/engine/docs/sources/userguide/dockervolumes.md b/components/engine/docs/sources/userguide/dockervolumes.md index 93ac37b1cc..fc9a7e0f82 100644 --- a/components/engine/docs/sources/userguide/dockervolumes.md +++ b/components/engine/docs/sources/userguide/dockervolumes.md @@ -71,6 +71,21 @@ read-only. Here we've mounted the same `/src/webapp` directory but we've added the `ro` option to specify that the mount should be read-only. +### Mount a Host File as a Data Volume + +As well as directories, the `-v` flag can be used to mount a single file from the host into a container. + + $ sudo docker run --rm -it -v ~/.bash_history:/.bash_history ubuntu /bin/bash + +This will drop you into a bash shell in a new container, you will have your bash history from your host and when +you exit the container the host will have the history of the commands typed while in the container. + +> **Note:** +> The two-way binding of the mounted file will only be preserved as long as the inode doesn't change. Many +> tools used to edit files including `vi` and `sed --in-place` may result in a inode change. In the case where you +> want to edit the file, it is often best to mount the parent directory. + + ## Creating and mounting a Data Volume Container If you have some persistent data that you want to share between From 82cbeaf60575f8fadb44c03d5d4b67105581cd6a Mon Sep 17 00:00:00 2001 From: Tom Fotherby Date: Sat, 5 Jul 2014 03:46:41 +0100 Subject: [PATCH 042/516] Update File mount info for docker v1.1.0 Docker-DCO-1.1-Signed-off-by: Tom Fotherby (github: tomfotherby) Upstream-commit: 87df57810d39bdd50167338e5ff8d0e6bc6e23e2 Component: engine --- .../engine/docs/sources/userguide/dockervolumes.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/components/engine/docs/sources/userguide/dockervolumes.md b/components/engine/docs/sources/userguide/dockervolumes.md index fc9a7e0f82..f0725d4ada 100644 --- a/components/engine/docs/sources/userguide/dockervolumes.md +++ b/components/engine/docs/sources/userguide/dockervolumes.md @@ -78,13 +78,12 @@ As well as directories, the `-v` flag can be used to mount a single file from th $ sudo docker run --rm -it -v ~/.bash_history:/.bash_history ubuntu /bin/bash This will drop you into a bash shell in a new container, you will have your bash history from your host and when -you exit the container the host will have the history of the commands typed while in the container. +you exit the container, the host will have the history of the commands typed while in the container. > **Note:** -> The two-way binding of the mounted file will only be preserved as long as the inode doesn't change. Many -> tools used to edit files including `vi` and `sed --in-place` may result in a inode change. In the case where you -> want to edit the file, it is often best to mount the parent directory. - +> Many tools used to edit files including `vi` and `sed --in-place` may result in a inode change. Since docker v1.1.0 +> this will produce a error such as "*sed: cannot rename ./sedKdJ9Dy: Device or resource busy*". In the case where you +> want to edit the mounted file, it is often easiest to instead mount the parent directory. ## Creating and mounting a Data Volume Container From 0fa89bda572878b16ad8efdb30e3f19b32accf7f Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Sat, 5 Jul 2014 16:20:14 -0400 Subject: [PATCH 043/516] docker save: fix the 'repositories' file For various use cases, the 'repositories' file does not match expected behavior. Like, docker save busybox:latest | tar t Before: [...] busybox:latest/ busybox:latest/VERSION busybox:latest/json busybox:latest/layer.tar # note, the layer name, and lack of 'repositories' file Now: [...] a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/ a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/VERSION a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/json a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/layer.tar repositories # and the repositories file is correct for the single tagged # image. #> {"busybox":{"latest":"a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721"}} and docker save a9eb17255234 | tar t Before: [...] a9eb17255234/ a9eb17255234/VERSION a9eb17255234/json a9eb17255234/layer.tar # Note the truncated layer name Now: [...] a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/ a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/VERSION a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/json a9eb172552348a9a49180694790b33a1097f546456d041b6e82e4d7716ddb721/layer.tar # There is no 'repositories' file, because there is no named repo Docker-DCO-1.1-Signed-off-by: Vincent Batts (github: vbatts) Upstream-commit: ac392bc0d775455f71da3c71956f9f6ae7a87f6d Component: engine --- components/engine/server/server.go | 35 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/components/engine/server/server.go b/components/engine/server/server.go index 7e453a3f0f..0d083ad9a4 100644 --- a/components/engine/server/server.go +++ b/components/engine/server/server.go @@ -351,29 +351,52 @@ func (srv *Server) ImageExport(job *engine.Job) engine.Status { utils.Debugf("Serializing %s", name) + rootRepoMap := map[string]graph.Repository{} rootRepo, err := srv.daemon.Repositories().Get(name) if err != nil { return job.Error(err) } if rootRepo != nil { + // this is a base repo name, like 'busybox' + for _, id := range rootRepo { if err := srv.exportImage(job.Eng, id, tempdir); err != nil { return job.Error(err) } } - - // write repositories - rootRepoMap := map[string]graph.Repository{} rootRepoMap[name] = rootRepo + } else { + img, err := srv.daemon.Repositories().LookupImage(name) + if err != nil { + return job.Error(err) + } + if img != nil { + // This is a named image like 'busybox:latest' + repoName, repoTag := utils.ParseRepositoryTag(name) + if err := srv.exportImage(job.Eng, img.ID, tempdir); err != nil { + return job.Error(err) + } + // check this length, because a lookup of a truncated has will not have a tag + // and will not need to be added to this map + if len(repoTag) > 0 { + rootRepoMap[repoName] = graph.Repository{repoTag: img.ID} + } + } else { + // this must be an ID that didn't get looked up just right? + if err := srv.exportImage(job.Eng, name, tempdir); err != nil { + return job.Error(err) + } + } + } + // write repositories, if there is something to write + if len(rootRepoMap) > 0 { rootRepoJson, _ := json.Marshal(rootRepoMap) if err := ioutil.WriteFile(path.Join(tempdir, "repositories"), rootRepoJson, os.FileMode(0644)); err != nil { return job.Error(err) } } else { - if err := srv.exportImage(job.Eng, name, tempdir); err != nil { - return job.Error(err) - } + utils.Debugf("There were no repositories to write") } fs, err := archive.Tar(tempdir, archive.Uncompressed) From 8ad8344e9f07a1fe1c3a67217b0dd24fd34e845b Mon Sep 17 00:00:00 2001 From: Ian Bull Date: Thu, 3 Jul 2014 22:40:02 -0700 Subject: [PATCH 044/516] Fix the parent/child relationship in the docs In the Docker Linking Docs, the parent child relationship was backwards. The Web container should be able to access the DB, not other way around. Furthermore, the output of 'docker ps' was wrong (it showed that the DB could access the Web). This fixes both typos. Docker-DCO-1.1-Signed-off-by: Ian Bull (github: irbull) Upstream-commit: 29a20672c216e2f8aa90af765ac5c47e3350b3d9 Component: engine --- .../engine/docs/sources/userguide/dockerlinks.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/components/engine/docs/sources/userguide/dockerlinks.md b/components/engine/docs/sources/userguide/dockerlinks.md index 20a5c1a179..91c6cb355b 100644 --- a/components/engine/docs/sources/userguide/dockerlinks.md +++ b/components/engine/docs/sources/userguide/dockerlinks.md @@ -149,16 +149,16 @@ Let's look at our linked containers using `docker ps`. $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 349169744e49 training/postgres:latest su postgres -c '/usr About a minute ago Up About a minute 5432/tcp db - aed84ee21bde training/webapp:latest python app.py 16 hours ago Up 2 minutes 0.0.0.0:49154->5000/tcp db/web,web + 349169744e49 training/postgres:latest su postgres -c '/usr About a minute ago Up About a minute 5432/tcp db, web/db + aed84ee21bde training/webapp:latest python app.py 16 hours ago Up 2 minutes 0.0.0.0:49154->5000/tcp web -We can see our named containers, `db` and `web`, and we can see that the `web` -containers also shows `db/web` in the `NAMES` column. This tells us that the +We can see our named containers, `db` and `web`, and we can see that the `db` +containers also shows `web/db` in the `NAMES` column. This tells us that the `web` container is linked to the `db` container in a parent/child relationship. So what does linking the containers do? Well we've discovered the link creates a parent-child relationship between the two containers. The parent container, -here `db`, can access information on the child container `web`. To do this +here `web`, can access information on the child container `db`. To do this Docker creates a secure tunnel between the containers without the need to expose any ports externally on the container. You'll note when we started the `db` container we did not use either of the `-P` or `-p` flags. As we're From 3c88fb5533ba8ef1754fee0bcc718d3c63794071 Mon Sep 17 00:00:00 2001 From: LK4D4 Date: Sun, 6 Jul 2014 15:36:34 +0400 Subject: [PATCH 045/516] Moved last build tests to integration-cli Also removed skipped tests on "viz" and "tree" because they blocked integration/buildfile_test.go removing. Docker-DCO-1.1-Signed-off-by: Alexandr Morozov (github: LK4D4) Upstream-commit: 1d4862b7a2d0c3b0641ce6d539eabecee686c794 Component: engine --- .../integration-cli/docker_cli_build_test.go | 167 +++++++ .../engine/integration/buildfile_test.go | 414 ------------------ .../engine/integration/commands_test.go | 128 ------ 3 files changed, 167 insertions(+), 542 deletions(-) delete mode 100644 components/engine/integration/buildfile_test.go diff --git a/components/engine/integration-cli/docker_cli_build_test.go b/components/engine/integration-cli/docker_cli_build_test.go index 2f2f2b6e0c..df0cfcc88e 100644 --- a/components/engine/integration-cli/docker_cli_build_test.go +++ b/components/engine/integration-cli/docker_cli_build_test.go @@ -1564,3 +1564,170 @@ func TestDockerignoringDockerfile(t *testing.T) { } logDone("build - test .dockerignore of Dockerfile") } + +func TestBuildLineBreak(t *testing.T) { + name := "testbuildlinebreak" + defer deleteImages(name) + _, err := buildImage(name, + `FROM busybox +RUN sh -c 'echo root:testpass \ + > /tmp/passwd' +RUN mkdir -p /var/run/sshd +RUN [ "$(cat /tmp/passwd)" = "root:testpass" ] +RUN [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ]`, + true) + if err != nil { + t.Fatal(err) + } + logDone("build - line break with \\") +} + +func TestBuildEOLInLine(t *testing.T) { + name := "testbuildeolinline" + defer deleteImages(name) + _, err := buildImage(name, + `FROM busybox +RUN sh -c 'echo root:testpass > /tmp/passwd' +RUN echo "foo \n bar"; echo "baz" +RUN mkdir -p /var/run/sshd +RUN [ "$(cat /tmp/passwd)" = "root:testpass" ] +RUN [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ]`, + true) + if err != nil { + t.Fatal(err) + } + logDone("build - end of line in dockerfile instruction") +} + +func TestBuildCommentsShebangs(t *testing.T) { + name := "testbuildcomments" + defer deleteImages(name) + _, err := buildImage(name, + `FROM busybox +# This is an ordinary comment. +RUN { echo '#!/bin/sh'; echo 'echo hello world'; } > /hello.sh +RUN [ ! -x /hello.sh ] +# comment with line break \ +RUN chmod +x /hello.sh +RUN [ -x /hello.sh ] +RUN [ "$(cat /hello.sh)" = $'#!/bin/sh\necho hello world' ] +RUN [ "$(/hello.sh)" = "hello world" ]`, + true) + if err != nil { + t.Fatal(err) + } + logDone("build - comments and shebangs") +} + +func TestBuildUsersAndGroups(t *testing.T) { + name := "testbuildusers" + defer deleteImages(name) + _, err := buildImage(name, + `FROM busybox + +# Make sure our defaults work +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)" = '0:0/root:root' ] + +# TODO decide if "args.user = strconv.Itoa(syscall.Getuid())" is acceptable behavior for changeUser in sysvinit instead of "return nil" when "USER" isn't specified (so that we get the proper group list even if that is the empty list, even in the default case of not supplying an explicit USER to run as, which implies USER 0) +USER root +RUN [ "$(id -G):$(id -Gn)" = '0 10:root wheel' ] + +# Setup dockerio user and group +RUN echo 'dockerio:x:1001:1001::/bin:/bin/false' >> /etc/passwd +RUN echo 'dockerio:x:1001:' >> /etc/group + +# Make sure we can switch to our user and all the information is exactly as we expect it to be +USER dockerio +RUN id -G +RUN id -Gn +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] + +# Switch back to root and double check that worked exactly as we might expect it to +USER root +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '0:0/root:root/0 10:root wheel' ] + +# Add a "supplementary" group for our dockerio user +RUN echo 'supplementary:x:1002:dockerio' >> /etc/group + +# ... and then go verify that we get it like we expect +USER dockerio +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001 1002:dockerio supplementary' ] +USER 1001 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001 1002:dockerio supplementary' ] + +# super test the new "user:group" syntax +USER dockerio:dockerio +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] +USER 1001:dockerio +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] +USER dockerio:1001 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] +USER 1001:1001 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1001/dockerio:dockerio/1001:dockerio' ] +USER dockerio:supplementary +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1002/dockerio:supplementary/1002:supplementary' ] +USER dockerio:1002 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1002/dockerio:supplementary/1002:supplementary' ] +USER 1001:supplementary +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1002/dockerio:supplementary/1002:supplementary' ] +USER 1001:1002 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1001:1002/dockerio:supplementary/1002:supplementary' ] + +# make sure unknown uid/gid still works properly +USER 1042:1043 +RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1042:1043/1042:1043/1043:1043' ]`, + true) + if err != nil { + t.Fatal(err) + } + logDone("build - users and groups") +} + +func TestBuildEnvUsage(t *testing.T) { + name := "testbuildenvusage" + defer deleteImages(name) + dockerfile := `FROM busybox +ENV FOO /foo/baz +ENV BAR /bar +ENV BAZ $BAR +ENV FOOPATH $PATH:$FOO +RUN [ "$BAR" = "$BAZ" ] +RUN [ "$FOOPATH" = "$PATH:/foo/baz" ] +ENV FROM hello/docker/world +ENV TO /docker/world/hello +ADD $FROM $TO +RUN [ "$(cat $TO)" = "hello" ]` + ctx, err := fakeContext(dockerfile, map[string]string{ + "hello/docker/world": "hello", + }) + if err != nil { + t.Fatal(err) + } + _, err = buildImageFromContext(name, ctx, true) + if err != nil { + t.Fatal(err) + } + logDone("build - environment variables usage") +} + +func TestBuildAddScript(t *testing.T) { + name := "testbuildaddscript" + defer deleteImages(name) + dockerfile := ` +FROM busybox +ADD test /test +RUN ["chmod","+x","/test"] +RUN ["/test"] +RUN [ "$(cat /testfile)" = 'test!' ]` + ctx, err := fakeContext(dockerfile, map[string]string{ + "test": "#!/bin/sh\necho 'test!' > /testfile", + }) + if err != nil { + t.Fatal(err) + } + _, err = buildImageFromContext(name, ctx, true) + if err != nil { + t.Fatal(err) + } + logDone("build - add and run script") +} diff --git a/components/engine/integration/buildfile_test.go b/components/engine/integration/buildfile_test.go deleted file mode 100644 index 147ae353a2..0000000000 --- a/components/engine/integration/buildfile_test.go +++ /dev/null @@ -1,414 +0,0 @@ -package docker - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/dotcloud/docker/archive" - "github.com/dotcloud/docker/engine" - "github.com/dotcloud/docker/image" - "github.com/dotcloud/docker/server" - "github.com/dotcloud/docker/utils" -) - -// A testContextTemplate describes a build context and how to test it -type testContextTemplate struct { - // Contents of the Dockerfile - dockerfile string - // Additional files in the context, eg [][2]string{"./passwd", "gordon"} - files [][2]string - // Additional remote files to host on a local HTTP server. - remoteFiles [][2]string -} - -func (context testContextTemplate) Archive(dockerfile string, t *testing.T) archive.Archive { - input := []string{"Dockerfile", dockerfile} - for _, pair := range context.files { - input = append(input, pair[0], pair[1]) - } - a, err := archive.Generate(input...) - if err != nil { - t.Fatal(err) - } - return a -} - -// A table of all the contexts to build and test. -// A new docker runtime will be created and torn down for each context. -var testContexts = []testContextTemplate{ - { - ` -from {IMAGE} -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd -run [ "$(cat /tmp/passwd)" = "root:testpass" ] -run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] -`, - nil, - nil, - }, - - // Exactly the same as above, except uses a line split with a \ to test - // multiline support. - { - ` -from {IMAGE} -run sh -c 'echo root:testpass \ - > /tmp/passwd' -run mkdir -p /var/run/sshd -run [ "$(cat /tmp/passwd)" = "root:testpass" ] -run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] -`, - nil, - nil, - }, - - // Line containing literal "\n" - { - ` -from {IMAGE} -run sh -c 'echo root:testpass > /tmp/passwd' -run echo "foo \n bar"; echo "baz" -run mkdir -p /var/run/sshd -run [ "$(cat /tmp/passwd)" = "root:testpass" ] -run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] -`, - nil, - nil, - }, - { - ` -from {IMAGE} -add foo /usr/lib/bla/bar -run [ "$(cat /usr/lib/bla/bar)" = 'hello' ] -add http://{SERVERADDR}/baz /usr/lib/baz/quux -run [ "$(cat /usr/lib/baz/quux)" = 'world!' ] -`, - [][2]string{{"foo", "hello"}}, - [][2]string{{"/baz", "world!"}}, - }, - - { - ` -from {IMAGE} -add f / -run [ "$(cat /f)" = "hello" ] -add f /abc -run [ "$(cat /abc)" = "hello" ] -add f /x/y/z -run [ "$(cat /x/y/z)" = "hello" ] -add f /x/y/d/ -run [ "$(cat /x/y/d/f)" = "hello" ] -add d / -run [ "$(cat /ga)" = "bu" ] -add d /somewhere -run [ "$(cat /somewhere/ga)" = "bu" ] -add d /anotherplace/ -run [ "$(cat /anotherplace/ga)" = "bu" ] -add d /somewheeeere/over/the/rainbooow -run [ "$(cat /somewheeeere/over/the/rainbooow/ga)" = "bu" ] -`, - [][2]string{ - {"f", "hello"}, - {"d/ga", "bu"}, - }, - nil, - }, - - { - ` -from {IMAGE} -add http://{SERVERADDR}/x /a/b/c -run [ "$(cat /a/b/c)" = "hello" ] -add http://{SERVERADDR}/x?foo=bar / -run [ "$(cat /x)" = "hello" ] -add http://{SERVERADDR}/x /d/ -run [ "$(cat /d/x)" = "hello" ] -add http://{SERVERADDR} /e -run [ "$(cat /e)" = "blah" ] -`, - nil, - [][2]string{{"/x", "hello"}, {"/", "blah"}}, - }, - - // Comments, shebangs, and executability, oh my! - { - ` -FROM {IMAGE} -# This is an ordinary comment. -RUN { echo '#!/bin/sh'; echo 'echo hello world'; } > /hello.sh -RUN [ ! -x /hello.sh ] -RUN chmod +x /hello.sh -RUN [ -x /hello.sh ] -RUN [ "$(cat /hello.sh)" = $'#!/bin/sh\necho hello world' ] -RUN [ "$(/hello.sh)" = "hello world" ] -`, - nil, - nil, - }, - - // Users and groups - { - ` -FROM {IMAGE} - -# Make sure our defaults work -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)" = '0:0/root:root' ] - -# TODO decide if "args.user = strconv.Itoa(syscall.Getuid())" is acceptable behavior for changeUser in sysvinit instead of "return nil" when "USER" isn't specified (so that we get the proper group list even if that is the empty list, even in the default case of not supplying an explicit USER to run as, which implies USER 0) -USER root -RUN [ "$(id -G):$(id -Gn)" = '0:root' ] - -# Setup dockerio user and group -RUN echo 'dockerio:x:1000:1000::/bin:/bin/false' >> /etc/passwd -RUN echo 'dockerio:x:1000:' >> /etc/group - -# Make sure we can switch to our user and all the information is exactly as we expect it to be -USER dockerio -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] - -# Switch back to root and double check that worked exactly as we might expect it to -USER root -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '0:0/root:root/0:root' ] - -# Add a "supplementary" group for our dockerio user -RUN echo 'supplementary:x:1001:dockerio' >> /etc/group - -# ... and then go verify that we get it like we expect -USER dockerio -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000 1001:dockerio supplementary' ] -USER 1000 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000 1001:dockerio supplementary' ] - -# super test the new "user:group" syntax -USER dockerio:dockerio -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] -USER 1000:dockerio -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] -USER dockerio:1000 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] -USER 1000:1000 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1000/dockerio:dockerio/1000:dockerio' ] -USER dockerio:supplementary -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] -USER dockerio:1001 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] -USER 1000:supplementary -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] -USER 1000:1001 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1000:1001/dockerio:supplementary/1001:supplementary' ] - -# make sure unknown uid/gid still works properly -USER 1042:1043 -RUN [ "$(id -u):$(id -g)/$(id -un):$(id -gn)/$(id -G):$(id -Gn)" = '1042:1043/1042:1043/1043:1043' ] -`, - nil, - nil, - }, - - // Environment variable - { - ` -from {IMAGE} -env FOO BAR -run [ "$FOO" = "BAR" ] -`, - nil, - nil, - }, - - // Environment overwriting - { - ` -from {IMAGE} -env FOO BAR -run [ "$FOO" = "BAR" ] -env FOO BAZ -run [ "$FOO" = "BAZ" ] -`, - nil, - nil, - }, - - { - ` -from {IMAGE} -ENTRYPOINT /bin/echo -CMD Hello world -`, - nil, - nil, - }, - - { - ` -from {IMAGE} -VOLUME /test -CMD Hello world -`, - nil, - nil, - }, - - { - ` -from {IMAGE} -env FOO /foo/baz -env BAR /bar -env BAZ $BAR -env FOOPATH $PATH:$FOO -run [ "$BAR" = "$BAZ" ] -run [ "$FOOPATH" = "$PATH:/foo/baz" ] -`, - nil, - nil, - }, - - { - ` -from {IMAGE} -env FOO /bar -env TEST testdir -env BAZ /foobar -add testfile $BAZ/ -add $TEST $FOO -run [ "$(cat /foobar/testfile)" = "test1" ] -run [ "$(cat /bar/withfile)" = "test2" ] -`, - [][2]string{ - {"testfile", "test1"}, - {"testdir/withfile", "test2"}, - }, - nil, - }, - - // JSON! - { - ` -FROM {IMAGE} -RUN ["/bin/echo","hello","world"] -CMD ["/bin/true"] -ENTRYPOINT ["/bin/echo","your command -->"] -`, - nil, - nil, - }, - { - ` -FROM {IMAGE} -ADD test /test -RUN ["chmod","+x","/test"] -RUN ["/test"] -RUN [ "$(cat /testfile)" = 'test!' ] -`, - [][2]string{ - {"test", "#!/bin/sh\necho 'test!' > /testfile"}, - }, - nil, - }, - { - ` -FROM {IMAGE} -# what \ -RUN mkdir /testing -RUN touch /testing/other -`, - nil, - nil, - }, -} - -// FIXME: test building with 2 successive overlapping ADD commands - -func constructDockerfile(template string, ip net.IP, port string) string { - serverAddr := fmt.Sprintf("%s:%s", ip, port) - replacer := strings.NewReplacer("{IMAGE}", unitTestImageID, "{SERVERADDR}", serverAddr) - return replacer.Replace(template) -} - -func mkTestingFileServer(files [][2]string) (*httptest.Server, error) { - mux := http.NewServeMux() - for _, file := range files { - name, contents := file[0], file[1] - mux.HandleFunc(name, func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte(contents)) - }) - } - - // This is how httptest.NewServer sets up a net.Listener, except that our listener must accept remote - // connections (from the container). - listener, err := net.Listen("tcp", ":0") - if err != nil { - return nil, err - } - - s := httptest.NewUnstartedServer(mux) - s.Listener = listener - s.Start() - return s, nil -} - -func TestBuild(t *testing.T) { - for _, ctx := range testContexts { - _, err := buildImage(ctx, t, nil, true) - if err != nil { - t.Fatal(err) - } - } -} - -func buildImage(context testContextTemplate, t *testing.T, eng *engine.Engine, useCache bool) (*image.Image, error) { - if eng == nil { - eng = NewTestEngine(t) - runtime := mkDaemonFromEngine(eng, t) - // FIXME: we might not need runtime, why not simply nuke - // the engine? - defer nuke(runtime) - } - srv := mkServerFromEngine(eng, t) - - httpServer, err := mkTestingFileServer(context.remoteFiles) - if err != nil { - t.Fatal(err) - } - defer httpServer.Close() - - idx := strings.LastIndex(httpServer.URL, ":") - if idx < 0 { - t.Fatalf("could not get port from test http server address %s", httpServer.URL) - } - port := httpServer.URL[idx+1:] - - iIP := eng.Hack_GetGlobalVar("httpapi.bridgeIP") - if iIP == nil { - t.Fatal("Legacy bridgeIP field not set in engine") - } - ip, ok := iIP.(net.IP) - if !ok { - panic("Legacy bridgeIP field in engine does not cast to net.IP") - } - dockerfile := constructDockerfile(context.dockerfile, ip, port) - - buildfile := server.NewBuildFile(srv, ioutil.Discard, ioutil.Discard, false, useCache, false, false, ioutil.Discard, utils.NewStreamFormatter(false), nil, nil) - id, err := buildfile.Build(context.Archive(dockerfile, t)) - if err != nil { - return nil, err - } - - job := eng.Job("image_inspect", id) - buffer := bytes.NewBuffer(nil) - image := &image.Image{} - job.Stdout.Add(buffer) - if err := job.Run(); err != nil { - return nil, err - } - err = json.NewDecoder(buffer).Decode(image) - return image, err -} diff --git a/components/engine/integration/commands_test.go b/components/engine/integration/commands_test.go index 47e9860052..28210e7536 100644 --- a/components/engine/integration/commands_test.go +++ b/components/engine/integration/commands_test.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "os" "path" - "regexp" "strings" "testing" "time" @@ -15,7 +14,6 @@ import ( "github.com/dotcloud/docker/api/client" "github.com/dotcloud/docker/daemon" "github.com/dotcloud/docker/engine" - "github.com/dotcloud/docker/image" "github.com/dotcloud/docker/pkg/term" "github.com/dotcloud/docker/utils" ) @@ -604,132 +602,6 @@ func TestRunErrorBindNonExistingSource(t *testing.T) { }) } -func TestImagesViz(t *testing.T) { - t.Skip("Image viz is deprecated") - stdout, stdoutPipe := io.Pipe() - - cli := client.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr, nil) - defer cleanup(globalEngine, t) - - image := buildTestImages(t, globalEngine) - - c := make(chan struct{}) - go func() { - defer close(c) - if err := cli.CmdImages("--viz"); err != nil { - t.Fatal(err) - } - stdoutPipe.Close() - }() - - setTimeout(t, "Reading command output time out", 2*time.Second, func() { - cmdOutputBytes, err := ioutil.ReadAll(bufio.NewReader(stdout)) - if err != nil { - t.Fatal(err) - } - cmdOutput := string(cmdOutputBytes) - - regexpStrings := []string{ - "digraph docker {", - fmt.Sprintf("base -> \"%s\" \\[style=invis]", unitTestImageIDShort), - fmt.Sprintf("label=\"%s\\\\n%s:latest\"", unitTestImageIDShort, unitTestImageName), - fmt.Sprintf("label=\"%s\\\\n%s:%s\"", utils.TruncateID(image.ID), "test", "latest"), - "base \\[style=invisible]", - } - - compiledRegexps := []*regexp.Regexp{} - for _, regexpString := range regexpStrings { - regexp, err := regexp.Compile(regexpString) - if err != nil { - fmt.Println("Error in regex string: ", err) - return - } - compiledRegexps = append(compiledRegexps, regexp) - } - - for _, regexp := range compiledRegexps { - if !regexp.MatchString(cmdOutput) { - t.Fatalf("images --viz content '%s' did not match regexp '%s'", cmdOutput, regexp) - } - } - }) -} - -func TestImagesTree(t *testing.T) { - t.Skip("Image tree is deprecated") - stdout, stdoutPipe := io.Pipe() - - cli := client.NewDockerCli(nil, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr, nil) - defer cleanup(globalEngine, t) - - image := buildTestImages(t, globalEngine) - - c := make(chan struct{}) - go func() { - defer close(c) - if err := cli.CmdImages("--tree"); err != nil { - t.Fatal(err) - } - stdoutPipe.Close() - }() - - setTimeout(t, "Reading command output time out", 2*time.Second, func() { - cmdOutputBytes, err := ioutil.ReadAll(bufio.NewReader(stdout)) - if err != nil { - t.Fatal(err) - } - cmdOutput := string(cmdOutputBytes) - regexpStrings := []string{ - fmt.Sprintf("└─%s Virtual Size: \\d+.\\d+ MB Tags: %s:latest", unitTestImageIDShort, unitTestImageName), - "(?m) └─[0-9a-f]+.*", - "(?m) └─[0-9a-f]+.*", - "(?m) └─[0-9a-f]+.*", - fmt.Sprintf("(?m)^ └─%s Virtual Size: \\d+.\\d+ MB Tags: test:latest", utils.TruncateID(image.ID)), - } - - compiledRegexps := []*regexp.Regexp{} - for _, regexpString := range regexpStrings { - regexp, err := regexp.Compile(regexpString) - if err != nil { - fmt.Println("Error in regex string: ", err) - return - } - compiledRegexps = append(compiledRegexps, regexp) - } - - for _, regexp := range compiledRegexps { - if !regexp.MatchString(cmdOutput) { - t.Fatalf("images --tree content '%s' did not match regexp '%s'", cmdOutput, regexp) - } - } - }) -} - -func buildTestImages(t *testing.T, eng *engine.Engine) *image.Image { - - var testBuilder = testContextTemplate{ - ` -from {IMAGE} -run sh -c 'echo root:testpass > /tmp/passwd' -run mkdir -p /var/run/sshd -run [ "$(cat /tmp/passwd)" = "root:testpass" ] -run [ "$(ls -d /var/run/sshd)" = "/var/run/sshd" ] -`, - nil, - nil, - } - image, err := buildImage(testBuilder, t, eng, true) - if err != nil { - t.Fatal(err) - } - - if err := eng.Job("tag", image.ID, "test").Run(); err != nil { - t.Fatal(err) - } - - return image -} - // #2098 - Docker cidFiles only contain short version of the containerId //sudo docker run --cidfile /tmp/docker_test.cid ubuntu echo "test" // TestRunCidFile tests that run --cidfile returns the longid From 0664d3e7f11cab07dfe036cbf61b453b63f230a1 Mon Sep 17 00:00:00 2001 From: Felix Rabe Date: Sun, 6 Jul 2014 14:47:37 +0200 Subject: [PATCH 046/516] run.md: Fix references to cli There are now no other occurrences of `#cli-` in this document. TODO: Find other places with wrong links. Upstream-commit: 95870ef3349920efc981a703d52f5832b79ee18e Component: engine --- components/engine/docs/sources/reference/run.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/docs/sources/reference/run.md b/components/engine/docs/sources/reference/run.md index a539ab0d18..82fe047cb3 100644 --- a/components/engine/docs/sources/reference/run.md +++ b/components/engine/docs/sources/reference/run.md @@ -11,7 +11,7 @@ its own networking, and its own isolated process tree. The defaults related to the binary to run, the networking to expose, and more, but `docker run` gives final control to the operator who starts the container from the image. That's the main reason -[*run*](/reference/commandline/cli/#cli-run) has more options than any +[*run*](/reference/commandline/cli/#run) has more options than any other `docker` command. ## General Form @@ -21,7 +21,7 @@ The basic `docker run` command takes this form: $ docker run [OPTIONS] IMAGE[:TAG] [COMMAND] [ARG...] To learn how to interpret the types of `[OPTIONS]`, -see [*Option types*](/reference/commandline/cli/#cli-options). +see [*Option types*](/reference/commandline/cli/#option-types). The list of `[OPTIONS]` breaks down into two groups: From 10fd0b6403aa37ff070b29a0fafdd46bfd684797 Mon Sep 17 00:00:00 2001 From: Tom Fotherby Date: Mon, 7 Jul 2014 12:01:04 +0100 Subject: [PATCH 047/516] Reformat to keep line length within 80 characters Docker-DCO-1.1-Signed-off-by: Tom Fotherby (github: tomfotherby) Upstream-commit: ebbfdf0a0f99aac67b8ad5d789d8a400be346c30 Component: engine --- .../docs/sources/userguide/dockervolumes.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/components/engine/docs/sources/userguide/dockervolumes.md b/components/engine/docs/sources/userguide/dockervolumes.md index f0725d4ada..959a01c502 100644 --- a/components/engine/docs/sources/userguide/dockervolumes.md +++ b/components/engine/docs/sources/userguide/dockervolumes.md @@ -73,17 +73,21 @@ option to specify that the mount should be read-only. ### Mount a Host File as a Data Volume -As well as directories, the `-v` flag can be used to mount a single file from the host into a container. +As well as directories, the `-v` flag can be used to mount a single file from +the host into a container. $ sudo docker run --rm -it -v ~/.bash_history:/.bash_history ubuntu /bin/bash -This will drop you into a bash shell in a new container, you will have your bash history from your host and when -you exit the container, the host will have the history of the commands typed while in the container. +This will drop you into a bash shell in a new container, you will have your bash +history from your host and when you exit the container, the host will have the +history of the commands typed while in the container. > **Note:** -> Many tools used to edit files including `vi` and `sed --in-place` may result in a inode change. Since docker v1.1.0 -> this will produce a error such as "*sed: cannot rename ./sedKdJ9Dy: Device or resource busy*". In the case where you -> want to edit the mounted file, it is often easiest to instead mount the parent directory. +> Many tools used to edit files including `vi` and `sed --in-place` may result +> in a inode change. Since docker v1.1.0 this will produce a error such as +> "*sed: cannot rename ./sedKdJ9Dy: Device or resource busy*". In the case where +> you want to edit the mounted file, it is often easiest to instead mount the +> parent directory. ## Creating and mounting a Data Volume Container From e2109c74a840159acba113eaa2b9d7c774e8b677 Mon Sep 17 00:00:00 2001 From: Andrea Turli Date: Mon, 7 Jul 2014 15:14:41 +0200 Subject: [PATCH 048/516] add jclouds-docker client reference Docker-DCO-1.1-Signed-off-by: Andrea Turli (github: andreaturli) Docker-DCO-1.1-Signed-off-by: Andrea Turli (github: ) Upstream-commit: 2f13d3a8e541b918135ea0bf0153f56927dd0511 Component: engine --- .../sources/reference/api/remote_api_client_libraries.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/components/engine/docs/sources/reference/api/remote_api_client_libraries.md b/components/engine/docs/sources/reference/api/remote_api_client_libraries.md index 3dcebeedaa..8f50804368 100644 --- a/components/engine/docs/sources/reference/api/remote_api_client_libraries.md +++ b/components/engine/docs/sources/reference/api/remote_api_client_libraries.md @@ -146,5 +146,11 @@ will add the libraries here. https://github.com/gesellix-docker/docker-client Active + + Java + jclouds-docker + https://github.com/jclouds/jclouds-labs/tree/master/docker + Active + From 199170408689259c4a3853a3cb0c5498fe4787c5 Mon Sep 17 00:00:00 2001 From: Chewey Date: Mon, 7 Jul 2014 17:52:18 +0400 Subject: [PATCH 049/516] Update running_riak_service.md Upstream-commit: 963ee6ce795ae6608ee9acce4bc6298356d7f02f Component: engine --- components/engine/docs/sources/examples/running_riak_service.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/docs/sources/examples/running_riak_service.md b/components/engine/docs/sources/examples/running_riak_service.md index 5909b7e2b0..59dba00639 100644 --- a/components/engine/docs/sources/examples/running_riak_service.md +++ b/components/engine/docs/sources/examples/running_riak_service.md @@ -14,7 +14,7 @@ Create an empty file called `Dockerfile`: $ touch Dockerfile Next, define the parent image you want to use to build your image on top -of. We'll use [Ubuntu](https://registry.hub.docker.cm/_/ubuntu/) (tag: +of. We'll use [Ubuntu](https://registry.hub.docker.com/_/ubuntu/) (tag: `latest`), which is available on [Docker Hub](https://hub.docker.com): # Riak From 9eda104cfe22b136f6d5bf800700503a49fe0ace Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 7 Jul 2014 08:09:25 -0600 Subject: [PATCH 050/516] Fix the systemd socket activation socket permissions Fixes #6836 Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 3589f5b9c2d9e5cb6d057713aa8d591295c5f523 Component: engine --- .../contrib/init/systemd/socket-activation/docker.socket | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/engine/contrib/init/systemd/socket-activation/docker.socket b/components/engine/contrib/init/systemd/socket-activation/docker.socket index 3635c89385..9db5049150 100644 --- a/components/engine/contrib/init/systemd/socket-activation/docker.socket +++ b/components/engine/contrib/init/systemd/socket-activation/docker.socket @@ -3,6 +3,9 @@ Description=Docker Socket for the API [Socket] ListenStream=/var/run/docker.sock +SocketMode=0660 +SocketUser=root +SocketGroup=docker [Install] WantedBy=sockets.target From 1107c3c89fdae1d67310168b8de0772de967ba0a Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 7 Jul 2014 08:32:09 -0600 Subject: [PATCH 051/516] Update the Travis GitHub username regex to match the GitHub username requirements Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: e895817ad72c681b265bc647d18f201ceced90c5 Component: engine --- components/engine/hack/make/validate-dco | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/components/engine/hack/make/validate-dco b/components/engine/hack/make/validate-dco index 6dbbe2250f..6e915c9aab 100644 --- a/components/engine/hack/make/validate-dco +++ b/components/engine/hack/make/validate-dco @@ -9,13 +9,22 @@ notDocs="$(validate_diff --numstat | awk '$3 !~ /^docs\// { print $3 }')" : ${adds:=0} : ${dels:=0} +# "Username may only contain alphanumeric characters or dashes and cannot begin with a dash" +githubUsernameRegex='[a-zA-Z][a-zA-Z-]+' + +# https://github.com/dotcloud/docker/blob/master/CONTRIBUTING.md#sign-your-work +dcoPrefix='Docker-DCO-1.1-Signed-off-by:' +dcoRegex="^$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)> \\(github: ($githubUsernameRegex)\\)$" + +check_dco() { + grep -qE "$dcoRegex" +} + if [ $adds -eq 0 -a $dels -eq 0 ]; then echo '0 adds, 0 deletions; nothing to validate! :)' elif [ -z "$notDocs" -a $adds -le 1 -a $dels -le 1 ]; then echo 'Congratulations! DCO small-patch-exception material!' else - dcoPrefix='Docker-DCO-1.1-Signed-off-by:' - dcoRegex="^$dcoPrefix ([^<]+) <([^<>@]+@[^<>]+)> \\(github: (\S+)\\)$" commits=( $(validate_log --format='format:%H%n') ) badCommits=() for commit in "${commits[@]}"; do @@ -23,7 +32,7 @@ else # no content (ie, Merge commit, etc) continue fi - if ! git log -1 --format='format:%B' "$commit" | grep -qE "$dcoRegex"; then + if ! git log -1 --format='format:%B' "$commit" | check_dco; then badCommits+=( "$commit" ) fi done From 03141b4b87ea1c9a3c2a9d7b0d9dfddf51d35987 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 7 Jul 2014 09:57:50 -0600 Subject: [PATCH 052/516] Fix DCO parsing for numeric github handles like cpuguy83 Turns out, "alphanumeric" actually means both "alpha" AND "numeric". Dur. Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: bbbf64242726dee184f82b3bb355327c9c85b274 Component: engine --- components/engine/hack/make/validate-dco | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/hack/make/validate-dco b/components/engine/hack/make/validate-dco index 6e915c9aab..8b7812982d 100644 --- a/components/engine/hack/make/validate-dco +++ b/components/engine/hack/make/validate-dco @@ -10,7 +10,7 @@ notDocs="$(validate_diff --numstat | awk '$3 !~ /^docs\// { print $3 }')" : ${dels:=0} # "Username may only contain alphanumeric characters or dashes and cannot begin with a dash" -githubUsernameRegex='[a-zA-Z][a-zA-Z-]+' +githubUsernameRegex='[a-zA-Z0-9][a-zA-Z0-9-]+' # https://github.com/dotcloud/docker/blob/master/CONTRIBUTING.md#sign-your-work dcoPrefix='Docker-DCO-1.1-Signed-off-by:' From 2f832eb65798b264f0b084c21a3352aa9fa5b485 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 7 Jul 2014 10:04:23 -0600 Subject: [PATCH 053/516] Update Travis to use pretty Docker-based workers These start up much faster and the only caveat is that we can't use "sudo" (which we don't currently use anyhow). Also, I've updated the Go version here to match what's in the Dockerfile. Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 3e2ad6ffee68be7f3462fd1311408c7880cd8d88 Component: engine --- components/engine/.travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/components/engine/.travis.yml b/components/engine/.travis.yml index ae03d6cde5..1e8d41184c 100644 --- a/components/engine/.travis.yml +++ b/components/engine/.travis.yml @@ -3,7 +3,12 @@ language: go -go: 1.2 +# This should match the version in the Dockerfile. +go: 1.2.1 + +# Let us have pretty experimental Docker-based Travis workers. +# (These spin up much faster than the VM-based ones.) +sudo: false # Disable the normal go build. install: true From e6a7274d4cd6fecc517c05c0eba3562951573603 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 7 Jul 2014 10:35:38 -0600 Subject: [PATCH 054/516] Standardize "curl" arguments across the repo to be or include "-sSL" - https://botbot.me/freenode/docker-dev/msg/17315692/ - https://botbot.me/freenode/docker-dev/msg/17315994/ Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: 9f61e233c6da72f692190f1f5eb672341253f3e7 Component: engine --- components/engine/CONTRIBUTING.md | 2 +- components/engine/Dockerfile | 2 +- components/engine/README.md | 2 +- components/engine/contrib/mkimage-alpine.sh | 4 ++-- .../engine/docs/sources/examples/running_riak_service.md | 2 +- components/engine/docs/sources/installation/google.md | 2 +- components/engine/docs/sources/installation/ubuntulinux.md | 4 ++-- components/engine/hack/RELEASE-CHECKLIST.md | 2 +- components/engine/hack/install.sh | 6 +++--- components/engine/hack/release.sh | 5 ++++- 10 files changed, 17 insertions(+), 14 deletions(-) diff --git a/components/engine/CONTRIBUTING.md b/components/engine/CONTRIBUTING.md index d07b972eb7..cd673b9980 100644 --- a/components/engine/CONTRIBUTING.md +++ b/components/engine/CONTRIBUTING.md @@ -175,7 +175,7 @@ One way to automate this, is customise your get ``commit.template`` by adding a ``prepare-commit-msg`` hook to your docker checkout: ``` -curl -o .git/hooks/prepare-commit-msg https://raw.githubusercontent.com/dotcloud/docker/master/contrib/prepare-commit-msg.hook && chmod +x .git/hooks/prepare-commit-msg +curl -sSL -o .git/hooks/prepare-commit-msg https://raw.githubusercontent.com/dotcloud/docker/master/contrib/prepare-commit-msg.hook && chmod +x .git/hooks/prepare-commit-msg ``` * Note: the above script expects to find your GitHub user name in ``git config --get github.user`` diff --git a/components/engine/Dockerfile b/components/engine/Dockerfile index 283e0a3262..d2e32a4ae5 100644 --- a/components/engine/Dockerfile +++ b/components/engine/Dockerfile @@ -60,7 +60,7 @@ RUN cd /usr/local/lvm2 && ./configure --enable-static_link && make device-mapper # see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL # Install Go -RUN curl -s https://go.googlecode.com/files/go1.2.1.src.tar.gz | tar -v -C /usr/local -xz +RUN curl -sSL https://go.googlecode.com/files/go1.2.1.src.tar.gz | tar -v -C /usr/local -xz ENV PATH /usr/local/go/bin:$PATH ENV GOPATH /go:/go/src/github.com/dotcloud/docker/vendor RUN cd /usr/local/go/src && ./make.bash --no-clean 2>&1 diff --git a/components/engine/README.md b/components/engine/README.md index 3c378de6f4..08d839c3cc 100644 --- a/components/engine/README.md +++ b/components/engine/README.md @@ -133,7 +133,7 @@ Here's a typical Docker build process: FROM ubuntu:12.04 RUN apt-get update RUN apt-get install -q -y python python-pip curl -RUN curl -L https://github.com/shykes/helloflask/archive/master.tar.gz | tar -xzv +RUN curl -sSL https://github.com/shykes/helloflask/archive/master.tar.gz | tar -xzv RUN cd helloflask-master && pip install -r requirements.txt ``` diff --git a/components/engine/contrib/mkimage-alpine.sh b/components/engine/contrib/mkimage-alpine.sh index 0bf328efa9..b9869ae61e 100755 --- a/components/engine/contrib/mkimage-alpine.sh +++ b/components/engine/contrib/mkimage-alpine.sh @@ -19,12 +19,12 @@ tmp() { } apkv() { - curl -s $REPO/$ARCH/APKINDEX.tar.gz | tar -Oxz | + curl -sSL $REPO/$ARCH/APKINDEX.tar.gz | tar -Oxz | grep '^P:apk-tools-static$' -A1 | tail -n1 | cut -d: -f2 } getapk() { - curl -s $REPO/$ARCH/apk-tools-static-$(apkv).apk | + curl -sSL $REPO/$ARCH/apk-tools-static-$(apkv).apk | tar -xz -C $TMP sbin/apk.static } diff --git a/components/engine/docs/sources/examples/running_riak_service.md b/components/engine/docs/sources/examples/running_riak_service.md index 5909b7e2b0..b2f9d99c49 100644 --- a/components/engine/docs/sources/examples/running_riak_service.md +++ b/components/engine/docs/sources/examples/running_riak_service.md @@ -59,7 +59,7 @@ After that, we install and setup a few dependencies: Next, we add Basho's APT repository: - RUN curl -s http://apt.basho.com/gpg/basho.apt.key | apt-key add -- + RUN curl -sSL http://apt.basho.com/gpg/basho.apt.key | apt-key add -- RUN echo "deb http://apt.basho.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/basho.list RUN apt-get update diff --git a/components/engine/docs/sources/installation/google.md b/components/engine/docs/sources/installation/google.md index b6c1b3d275..6c1e9b4f50 100644 --- a/components/engine/docs/sources/installation/google.md +++ b/components/engine/docs/sources/installation/google.md @@ -12,7 +12,7 @@ page_keywords: Docker, Docker documentation, installation, google, Google Comput 2. Download and configure the [Google Cloud SDK][3] to use your project with the following commands: - $ curl https://sdk.cloud.google.com | bash + $ curl -sSL https://sdk.cloud.google.com | bash $ gcloud auth login $ gcloud config set project diff --git a/components/engine/docs/sources/installation/ubuntulinux.md b/components/engine/docs/sources/installation/ubuntulinux.md index 5d1b6c3fbf..5ddc791f4f 100644 --- a/components/engine/docs/sources/installation/ubuntulinux.md +++ b/components/engine/docs/sources/installation/ubuntulinux.md @@ -63,7 +63,7 @@ continue installation.* > > There is also a simple `curl` script available to help with this process. > -> $ curl -s https://get.docker.io/ubuntu/ | sudo sh +> $ curl -sSL https://get.docker.io/ubuntu/ | sudo sh To verify that everything has worked as expected: @@ -134,7 +134,7 @@ continue installation.* > > There is also a simple `curl` script available to help with this process. > -> $ curl -s https://get.docker.io/ubuntu/ | sudo sh +> $ curl -sSL https://get.docker.io/ubuntu/ | sudo sh Now verify that the installation has worked by downloading the `ubuntu` image and launching a container. diff --git a/components/engine/hack/RELEASE-CHECKLIST.md b/components/engine/hack/RELEASE-CHECKLIST.md index 2fe1a3ce96..6dbff22bc4 100644 --- a/components/engine/hack/RELEASE-CHECKLIST.md +++ b/components/engine/hack/RELEASE-CHECKLIST.md @@ -181,7 +181,7 @@ Announcing on IRC in both `#docker` and `#docker-dev` is a great way to get help testing! An easy way to get some useful links for sharing: ```bash -echo "Ubuntu/Debian install script: curl -sLS https://test.docker.io/ | sh" +echo "Ubuntu/Debian: https://test.docker.io/ubuntu or curl -sSL https://test.docker.io/ | sh" echo "Linux 64bit binary: https://test.docker.io/builds/Linux/x86_64/docker-${VERSION#v}" echo "Darwin/OSX 64bit client binary: https://test.docker.io/builds/Darwin/x86_64/docker-${VERSION#v}" echo "Darwin/OSX 32bit client binary: https://test.docker.io/builds/Darwin/i386/docker-${VERSION#v}" diff --git a/components/engine/hack/install.sh b/components/engine/hack/install.sh index 43248cf2c0..641ec64155 100755 --- a/components/engine/hack/install.sh +++ b/components/engine/hack/install.sh @@ -2,7 +2,7 @@ set -e # # This script is meant for quick & easy install via: -# 'curl -sL https://get.docker.io/ | sh' +# 'curl -sSL https://get.docker.io/ | sh' # or: # 'wget -qO- https://get.docker.io/ | sh' # @@ -54,7 +54,7 @@ fi curl='' if command_exists curl; then - curl='curl -sL' + curl='curl -sSL' elif command_exists wget; then curl='wget -qO-' elif command_exists busybox && busybox --list-modules | grep -q wget; then @@ -133,7 +133,7 @@ case "$lsb_dist" in if [ -z "$curl" ]; then apt_get_update ( set -x; $sh_c 'sleep 3; apt-get install -y -q curl' ) - curl='curl -sL' + curl='curl -sSL' fi ( set -x diff --git a/components/engine/hack/release.sh b/components/engine/hack/release.sh index 8642a4edb9..2a6b3992ef 100755 --- a/components/engine/hack/release.sh +++ b/components/engine/hack/release.sh @@ -282,10 +282,13 @@ if [ ! -e /usr/lib/apt/methods/https ]; then apt-get update apt-get install -y apt-transport-https fi + # Add the repository to your APT sources echo deb $(s3_url)/ubuntu docker main > /etc/apt/sources.list.d/docker.list + # Then import the repository key apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 + # Install docker apt-get update ; apt-get install -y lxc-docker @@ -318,7 +321,7 @@ release_binaries() { cat < Date: Mon, 7 Jul 2014 10:50:48 -0600 Subject: [PATCH 055/516] Update Makefile to default BINDDIR to nothing if DOCKER_HOST is set If "DOCKER_HOST" is set, we can usually assume the user is connecting to a remote Docker and thus not bind mount anything by default (meaning the Makefile will more often DWIM for our users). Docker-DCO-1.1-Signed-off-by: Andrew Page (github: tianon) Upstream-commit: c2653b0a9ed4b371342fc2019c56d9433197171e Component: engine --- components/engine/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/engine/Makefile b/components/engine/Makefile index 2d07b39c3b..444ac3c6a6 100644 --- a/components/engine/Makefile +++ b/components/engine/Makefile @@ -1,7 +1,8 @@ .PHONY: all binary build cross default docs docs-build docs-shell shell test test-unit test-integration test-integration-cli validate # to allow `make BINDDIR=. shell` or `make BINDDIR= test` -BINDDIR := bundles +# (default to no bind mount if DOCKER_HOST is set) +BINDDIR := $(if $(DOCKER_HOST),,bundles) # to allow `make DOCSPORT=9000 docs` DOCSPORT := 8000 From b15cf92d6a388a69654202b9bed80db782b96f98 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Mon, 7 Jul 2014 14:58:27 -0400 Subject: [PATCH 056/516] docker save: more integration tests Docker-DCO-1.1-Signed-off-by: Vincent Batts (github: vbatts) Upstream-commit: 600f65b24786d15a382877da4c3ae77d88e252bb Component: engine --- .../docker_cli_save_load_test.go | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/components/engine/integration-cli/docker_cli_save_load_test.go b/components/engine/integration-cli/docker_cli_save_load_test.go index fb94cad9d8..c70769e034 100644 --- a/components/engine/integration-cli/docker_cli_save_load_test.go +++ b/components/engine/integration-cli/docker_cli_save_load_test.go @@ -59,6 +59,63 @@ func TestSaveAndLoadRepoStdout(t *testing.T) { logDone("load - load a repo using stdout") } +func TestSaveSingleTag(t *testing.T) { + repoName := "foobar-save-single-tag-test" + + tagCmdFinal := fmt.Sprintf("%v tag busybox:latest %v:latest", dockerBinary, repoName) + tagCmd := exec.Command("bash", "-c", tagCmdFinal) + out, _, err := runCommandWithOutput(tagCmd) + errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err)) + + idCmdFinal := fmt.Sprintf("%v images -q --no-trunc %v", dockerBinary, repoName) + idCmd := exec.Command("bash", "-c", idCmdFinal) + out, _, err = runCommandWithOutput(idCmd) + errorOut(err, t, fmt.Sprintf("failed to get repo ID: %v %v", out, err)) + + cleanedImageID := stripTrailingCharacters(out) + + saveCmdFinal := fmt.Sprintf("%v save %v:latest | tar t | grep -E '(^repositories$|%v)'", dockerBinary, repoName, cleanedImageID) + saveCmd := exec.Command("bash", "-c", saveCmdFinal) + out, _, err = runCommandWithOutput(saveCmd) + errorOut(err, t, fmt.Sprintf("failed to save repo with image ID and 'repositories' file: %v %v", out, err)) + + deleteImages(repoName) + + logDone("save - save a specific image:tag") +} + +func TestSaveImageId(t *testing.T) { + repoName := "foobar-save-image-id-test" + + tagCmdFinal := fmt.Sprintf("%v tag scratch:latest %v:latest", dockerBinary, repoName) + tagCmd := exec.Command("bash", "-c", tagCmdFinal) + out, _, err := runCommandWithOutput(tagCmd) + errorOut(err, t, fmt.Sprintf("failed to tag repo: %v %v", out, err)) + + idLongCmdFinal := fmt.Sprintf("%v images -q --no-trunc %v", dockerBinary, repoName) + idLongCmd := exec.Command("bash", "-c", idLongCmdFinal) + out, _, err = runCommandWithOutput(idLongCmd) + errorOut(err, t, fmt.Sprintf("failed to get repo ID: %v %v", out, err)) + + cleanedLongImageID := stripTrailingCharacters(out) + + idShortCmdFinal := fmt.Sprintf("%v images -q %v", dockerBinary, repoName) + idShortCmd := exec.Command("bash", "-c", idShortCmdFinal) + out, _, err = runCommandWithOutput(idShortCmd) + errorOut(err, t, fmt.Sprintf("failed to get repo short ID: %v %v", out, err)) + + cleanedShortImageID := stripTrailingCharacters(out) + + saveCmdFinal := fmt.Sprintf("%v save %v | tar t | grep %v", dockerBinary, cleanedShortImageID, cleanedLongImageID) + saveCmd := exec.Command("bash", "-c", saveCmdFinal) + out, _, err = runCommandWithOutput(saveCmd) + errorOut(err, t, fmt.Sprintf("failed to save repo with image ID: %v %v", out, err)) + + deleteImages(repoName) + + logDone("save - save a image by ID") +} + // save a repo and try to load it using flags func TestSaveAndLoadRepoFlags(t *testing.T) { runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true") From 41890809ceafe7655564eb2fc8ec28be7763413a Mon Sep 17 00:00:00 2001 From: Timothy Hobbs Date: Sun, 6 Jul 2014 10:38:30 +0000 Subject: [PATCH 057/516] We haven't required lxc and aufs for years now... Well, maybe not years, but internet years... Docker-DCO-1.1-Signed-off-by: Timothy Hobbs (github: timthelion) Upstream-commit: ea583fda97617a349ef026dd2d99a0d5e4b0980d Component: engine --- components/engine/hack/ROADMAP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/engine/hack/ROADMAP.md b/components/engine/hack/ROADMAP.md index c38be56f44..ff797b380c 100644 --- a/components/engine/hack/ROADMAP.md +++ b/components/engine/hack/ROADMAP.md @@ -31,9 +31,9 @@ We are working on a plugin API which will make Docker very, very customization-f ## Broader kernel support -Our goal is to make Docker run everywhere, but currently Docker requires Linux version 3.8 or higher with lxc and aufs support. If you’re deploying new machines for the purpose of running Docker, this is a fairly easy requirement to meet. However, if you’re adding Docker to an existing deployment, you may not have the flexibility to update and patch the kernel. +Our goal is to make Docker run everywhere, but currently Docker requires Linux version 3.8 or higher with cgroups support. If you’re deploying new machines for the purpose of running Docker, this is a fairly easy requirement to meet. However, if you’re adding Docker to an existing deployment, you may not have the flexibility to update and patch the kernel. -Expanding Docker’s kernel support is a priority. This includes running on older kernel versions, but also on kernels with no AUFS support, or with incomplete lxc capabilities. +Expanding Docker’s kernel support is a priority. This includes running on older kernel versions, specifically focusing on versions already popular in server deployments such as those used by RHEL and the OpenVZ stack. ## Cross-architecture support From f220709916e1ea4e50f7fbb6f81d5941bbabd37a Mon Sep 17 00:00:00 2001 From: Guilherme Salgado Date: Wed, 2 Jul 2014 10:01:19 +0200 Subject: [PATCH 058/516] Convert TestGetImagesJSON into several unit tests Docker-DCO-1.1-Signed-off-by: Guilherme Salgado (github: gsalgado) Upstream-commit: 627805f5f8895c3a3887c312f7b8f49a89e51732 Component: engine --- .../engine/api/server/server_unit_test.go | 172 ++++++++++++++++-- components/engine/integration/api_test.go | 104 ----------- 2 files changed, 157 insertions(+), 119 deletions(-) diff --git a/components/engine/api/server/server_unit_test.go b/components/engine/api/server/server_unit_test.go index f8068a29c5..c51ea6fc0d 100644 --- a/components/engine/api/server/server_unit_test.go +++ b/components/engine/api/server/server_unit_test.go @@ -7,11 +7,13 @@ import ( "io" "net/http" "net/http/httptest" + "reflect" "strings" "testing" "github.com/dotcloud/docker/api" "github.com/dotcloud/docker/engine" + "github.com/dotcloud/docker/pkg/version" ) func TestGetBoolParam(t *testing.T) { @@ -111,8 +113,105 @@ func TestGetInfo(t *testing.T) { if v.GetInt("Containers") != 1 { t.Fatalf("%#v\n", v) } - if r.HeaderMap.Get("Content-Type") != "application/json" { - t.Fatalf("%#v\n", r) + assertContentType(r, "application/json", t) +} + +func TestGetImagesJSON(t *testing.T) { + eng := engine.New() + var called bool + eng.Register("images", func(job *engine.Job) engine.Status { + called = true + v := createEnvFromGetImagesJSONStruct(sampleImage) + if _, err := v.WriteTo(job.Stdout); err != nil { + return job.Error(err) + } + return engine.StatusOK + }) + r := serveRequest("GET", "/images/json", nil, eng, t) + if !called { + t.Fatal("handler was not called") + } + assertHttpNotError(r, t) + assertContentType(r, "application/json", t) + var observed getImagesJSONStruct + if err := json.Unmarshal(r.Body.Bytes(), &observed); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(observed, sampleImage) { + t.Errorf("Expected %#v but got %#v", sampleImage, observed) + } +} + +func TestGetImagesJSONFilter(t *testing.T) { + eng := engine.New() + filter := "nothing" + eng.Register("images", func(job *engine.Job) engine.Status { + filter = job.Getenv("filter") + return engine.StatusOK + }) + serveRequest("GET", "/images/json?filter=aaaa", nil, eng, t) + if filter != "aaaa" { + t.Errorf("%#v", filter) + } +} + +func TestGetImagesJSONFilters(t *testing.T) { + eng := engine.New() + filter := "nothing" + eng.Register("images", func(job *engine.Job) engine.Status { + filter = job.Getenv("filters") + return engine.StatusOK + }) + serveRequest("GET", "/images/json?filters=nnnn", nil, eng, t) + if filter != "nnnn" { + t.Errorf("%#v", filter) + } +} + +func TestGetImagesJSONAll(t *testing.T) { + eng := engine.New() + allFilter := "-1" + eng.Register("images", func(job *engine.Job) engine.Status { + allFilter = job.Getenv("all") + return engine.StatusOK + }) + serveRequest("GET", "/images/json?all=1", nil, eng, t) + if allFilter != "1" { + t.Errorf("%#v", allFilter) + } +} + +func TestGetImagesJSONLegacyFormat(t *testing.T) { + eng := engine.New() + var called bool + eng.Register("images", func(job *engine.Job) engine.Status { + called = true + outsLegacy := engine.NewTable("Created", 0) + outsLegacy.Add(createEnvFromGetImagesJSONStruct(sampleImage)) + if _, err := outsLegacy.WriteListTo(job.Stdout); err != nil { + return job.Error(err) + } + return engine.StatusOK + }) + r := serveRequestUsingVersion("GET", "/images/json", "1.6", nil, eng, t) + if !called { + t.Fatal("handler was not called") + } + assertHttpNotError(r, t) + assertContentType(r, "application/json", t) + images := engine.NewTable("Created", 0) + if _, err := images.ReadListFrom(r.Body.Bytes()); err != nil { + t.Fatal(err) + } + if images.Len() != 1 { + t.Fatalf("Expected 1 image, %d found", images.Len()) + } + image := images.Data[0] + if image.Get("Tag") != "test-tag" { + t.Errorf("Expected tag 'test-tag', found '%s'", image.Get("Tag")) + } + if image.Get("Repository") != "test-name" { + t.Errorf("Expected repository 'test-name', found '%s'", image.Get("Repository")) } } @@ -123,12 +222,12 @@ func TestGetContainersByName(t *testing.T) { eng.Register("container_inspect", func(job *engine.Job) engine.Status { called = true if job.Args[0] != name { - t.Fatalf("name != '%s': %#v", name, job.Args[0]) + t.Errorf("name != '%s': %#v", name, job.Args[0]) } if api.APIVERSION.LessThan("1.12") && !job.GetenvBool("dirty") { - t.Fatal("dirty env variable not set") + t.Errorf("dirty env variable not set") } else if api.APIVERSION.GreaterThanOrEqualTo("1.12") && job.GetenvBool("dirty") { - t.Fatal("dirty env variable set when it shouldn't") + t.Errorf("dirty env variable set when it shouldn't") } v := &engine.Env{} v.SetBool("dirty", true) @@ -141,9 +240,7 @@ func TestGetContainersByName(t *testing.T) { if !called { t.Fatal("handler was not called") } - if r.HeaderMap.Get("Content-Type") != "application/json" { - t.Fatalf("%#v\n", r) - } + assertContentType(r, "application/json", t) var stdoutJson interface{} if err := json.Unmarshal(r.Body.Bytes(), &stdoutJson); err != nil { t.Fatalf("%#v", err) @@ -178,21 +275,19 @@ func TestGetEvents(t *testing.T) { if !called { t.Fatal("handler was not called") } - if r.HeaderMap.Get("Content-Type") != "application/json" { - t.Fatalf("%#v\n", r) - } + assertContentType(r, "application/json", t) var stdout_json struct { Since int Until int } if err := json.Unmarshal(r.Body.Bytes(), &stdout_json); err != nil { - t.Fatalf("%#v", err) + t.Fatal(err) } if stdout_json.Since != 1 { - t.Fatalf("since != 1: %#v", stdout_json.Since) + t.Errorf("since != 1: %#v", stdout_json.Since) } if stdout_json.Until != 0 { - t.Fatalf("until != 0: %#v", stdout_json.Until) + t.Errorf("until != 0: %#v", stdout_json.Until) } } @@ -357,12 +452,16 @@ func TestGetImagesByName(t *testing.T) { } func serveRequest(method, target string, body io.Reader, eng *engine.Engine, t *testing.T) *httptest.ResponseRecorder { + return serveRequestUsingVersion(method, target, api.APIVERSION, body, eng, t) +} + +func serveRequestUsingVersion(method, target string, version version.Version, body io.Reader, eng *engine.Engine, t *testing.T) *httptest.ResponseRecorder { r := httptest.NewRecorder() req, err := http.NewRequest(method, target, body) if err != nil { t.Fatal(err) } - if err := ServeRequest(eng, api.APIVERSION, r, req); err != nil { + if err := ServeRequest(eng, version, r, req); err != nil { t.Fatal(err) } return r @@ -388,3 +487,46 @@ func toJson(data interface{}, t *testing.T) io.Reader { } return &buf } + +func assertContentType(recorder *httptest.ResponseRecorder, content_type string, t *testing.T) { + if recorder.HeaderMap.Get("Content-Type") != content_type { + t.Fatalf("%#v\n", recorder) + } +} + +// XXX: Duplicated from integration/utils_test.go, but maybe that's OK as that +// should die as soon as we converted all integration tests? +// assertHttpNotError expect the given response to not have an error. +// Otherwise the it causes the test to fail. +func assertHttpNotError(r *httptest.ResponseRecorder, t *testing.T) { + // Non-error http status are [200, 400) + if r.Code < http.StatusOK || r.Code >= http.StatusBadRequest { + t.Fatal(fmt.Errorf("Unexpected http error: %v", r.Code)) + } +} + +func createEnvFromGetImagesJSONStruct(data getImagesJSONStruct) *engine.Env { + v := &engine.Env{} + v.SetList("RepoTags", data.RepoTags) + v.Set("Id", data.Id) + v.SetInt64("Created", data.Created) + v.SetInt64("Size", data.Size) + v.SetInt64("VirtualSize", data.VirtualSize) + return v +} + +type getImagesJSONStruct struct { + RepoTags []string + Id string + Created int64 + Size int64 + VirtualSize int64 +} + +var sampleImage getImagesJSONStruct = getImagesJSONStruct{ + RepoTags: []string{"test-name:test-tag"}, + Id: "ID", + Created: 999, + Size: 777, + VirtualSize: 666, +} diff --git a/components/engine/integration/api_test.go b/components/engine/integration/api_test.go index f5e5599e3f..0da96ccd1e 100644 --- a/components/engine/integration/api_test.go +++ b/components/engine/integration/api_test.go @@ -9,7 +9,6 @@ import ( "net" "net/http" "net/http/httptest" - "strings" "testing" "time" @@ -20,109 +19,6 @@ import ( "github.com/dotcloud/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar" ) -func TestGetImagesJSON(t *testing.T) { - eng := NewTestEngine(t) - defer mkDaemonFromEngine(eng, t).Nuke() - - job := eng.Job("images") - initialImages, err := job.Stdout.AddListTable() - if err != nil { - t.Fatal(err) - } - if err := job.Run(); err != nil { - t.Fatal(err) - } - - req, err := http.NewRequest("GET", "/images/json?all=0", nil) - if err != nil { - t.Fatal(err) - } - - r := httptest.NewRecorder() - - if err := server.ServeRequest(eng, api.APIVERSION, r, req); err != nil { - t.Fatal(err) - } - assertHttpNotError(r, t) - - images := engine.NewTable("Created", 0) - if _, err := images.ReadListFrom(r.Body.Bytes()); err != nil { - t.Fatal(err) - } - - if images.Len() != initialImages.Len() { - t.Errorf("Expected %d image, %d found", initialImages.Len(), images.Len()) - } - - found := false - for _, img := range images.Data { - if strings.Contains(img.GetList("RepoTags")[0], unitTestImageName) { - found = true - break - } - } - if !found { - t.Errorf("Expected image %s, %+v found", unitTestImageName, images) - } - - r2 := httptest.NewRecorder() - - // all=1 - - initialImages = getAllImages(eng, t) - - req2, err := http.NewRequest("GET", "/images/json?all=true", nil) - if err != nil { - t.Fatal(err) - } - if err := server.ServeRequest(eng, api.APIVERSION, r2, req2); err != nil { - t.Fatal(err) - } - assertHttpNotError(r2, t) - - images2 := engine.NewTable("Id", 0) - if _, err := images2.ReadListFrom(r2.Body.Bytes()); err != nil { - t.Fatal(err) - } - - if images2.Len() != initialImages.Len() { - t.Errorf("Expected %d image, %d found", initialImages.Len(), images2.Len()) - } - - found = false - for _, img := range images2.Data { - if img.Get("Id") == unitTestImageID { - found = true - break - } - } - if !found { - t.Errorf("Retrieved image Id differs, expected %s, received %+v", unitTestImageID, images2) - } - - r3 := httptest.NewRecorder() - - // filter=a - req3, err := http.NewRequest("GET", "/images/json?filter=aaaaaaaaaa", nil) - if err != nil { - t.Fatal(err) - } - - if err := server.ServeRequest(eng, api.APIVERSION, r3, req3); err != nil { - t.Fatal(err) - } - assertHttpNotError(r3, t) - - images3 := engine.NewTable("Id", 0) - if _, err := images3.ReadListFrom(r3.Body.Bytes()); err != nil { - t.Fatal(err) - } - - if images3.Len() != 0 { - t.Errorf("Expected 0 image, %d found", images3.Len()) - } -} - func TestGetContainersJSON(t *testing.T) { eng := NewTestEngine(t) defer mkDaemonFromEngine(eng, t).Nuke() From c0012fb00af7887f7bb569cbb6c1c3a560869a2f Mon Sep 17 00:00:00 2001 From: Roberto Gandolfo Hashioka Date: Mon, 7 Jul 2014 13:44:22 -0700 Subject: [PATCH 059/516] Update the footer version with Enterprise and Partners sections Docker-DCO-1.1-Signed-off-by: Roberto Hashioka (github: rogaha) Upstream-commit: fd66e7746519ab32bf3f6f336052a4ff763e213b Component: engine --- .../engine/docs/theme/mkdocs/footer.html | 49 ++++++++++++------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/components/engine/docs/theme/mkdocs/footer.html b/components/engine/docs/theme/mkdocs/footer.html index 1e2d73cf14..0b887b82d0 100644 --- a/components/engine/docs/theme/mkdocs/footer.html +++ b/components/engine/docs/theme/mkdocs/footer.html @@ -1,13 +1,13 @@