From 3c2cab11aae7e98c809f9154deddd1f86fdadf83 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Fri, 17 Apr 2015 15:30:22 -0700 Subject: [PATCH 1/5] Shallow clone using git to build images. Signed-off-by: David Calavera Upstream-commit: 36fbf4b86469ca6fe3677d47c9a1976bcdd111e4 Component: engine --- components/engine/builder/job.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/engine/builder/job.go b/components/engine/builder/job.go index 115d89a4b9..7ea1ae30b9 100644 --- a/components/engine/builder/job.go +++ b/components/engine/builder/job.go @@ -114,7 +114,7 @@ func Build(d *daemon.Daemon, buildConfig *Config) error { } defer os.RemoveAll(root) - if output, err := exec.Command("git", "clone", "--recursive", buildConfig.RemoteURL, root).CombinedOutput(); err != nil { + if output, err := exec.Command("git", "clone", "--depth", "1", "--recursive", buildConfig.RemoteURL, root).CombinedOutput(); err != nil { return fmt.Errorf("Error trying to use git: %s (%s)", err, output) } From d5b0b24bb84a3c1a654dcad8b85051e6e37def95 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Tue, 21 Apr 2015 10:58:38 -0700 Subject: [PATCH 2/5] Do not try to shallow git history when the protocol doesn't allow it. This only happens with the old git http dumb protocol, but that's what we use in our integration tests. We check the Content-Type header advertised in http requests to make sure the http transport is the git smart transport: See this commit as a reference: https://github.com/git/git/commit/4656bf47fca857df51b5d6f4b7b052192b3b2317 Signed-off-by: David Calavera Upstream-commit: 9fb7204a41804131c2492f9d50d7451e123a05e5 Component: engine --- components/engine/builder/job.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/components/engine/builder/job.go b/components/engine/builder/job.go index 7ea1ae30b9..8c031a8bd8 100644 --- a/components/engine/builder/job.go +++ b/components/engine/builder/job.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "io/ioutil" + "net/http" "os" "os/exec" "strings" @@ -114,7 +115,9 @@ func Build(d *daemon.Daemon, buildConfig *Config) error { } defer os.RemoveAll(root) - if output, err := exec.Command("git", "clone", "--depth", "1", "--recursive", buildConfig.RemoteURL, root).CombinedOutput(); err != nil { + clone := cloneArgs(buildConfig.RemoteURL, root) + + if output, err := exec.Command("git", clone...).CombinedOutput(); err != nil { return fmt.Errorf("Error trying to use git: %s (%s)", err, output) } @@ -239,3 +242,21 @@ func Commit(d *daemon.Daemon, name string, c *daemon.ContainerCommitConfig) (str return img.ID, nil } + +func cloneArgs(remoteURL, root string) []string { + args := []string{"clone", "--recursive"} + shallow := true + + if strings.HasPrefix(remoteURL, "http") { + res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL)) + if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" { + shallow = false + } + } + + if shallow { + args = append(args, "--depth", "1") + } + + return append(args, remoteURL, root) +} From 44cdfe63b1b7dffe2bae837d0d6236d2442e3074 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Tue, 21 Apr 2015 14:24:29 -0700 Subject: [PATCH 3/5] Test that we set the right arguments for git cloning depending on the transport. Signed-off-by: David Calavera Upstream-commit: 3117bf3ef562bc43ef007d8afe3815c58aac6e85 Component: engine --- components/engine/builder/job_test.go | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 components/engine/builder/job_test.go diff --git a/components/engine/builder/job_test.go b/components/engine/builder/job_test.go new file mode 100644 index 0000000000..79421a0688 --- /dev/null +++ b/components/engine/builder/job_test.go @@ -0,0 +1,56 @@ +package builder + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "testing" +) + +func TestCloneArgsSmartHttp(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + serverURL, _ := url.Parse(server.URL) + + serverURL.Path = "/repo.git" + gitURL := serverURL.String() + + mux.HandleFunc("/repo.git/info/refs", func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("service") + w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-advertisement", q)) + }) + + args := cloneArgs(gitURL, "/tmp") + exp := []string{"clone", "--recursive", "--depth", "1", gitURL, "/tmp"} + if !reflect.DeepEqual(args, exp) { + t.Fatalf("Expected %v, got %v", exp, args) + } +} + +func TestCloneArgsDumbHttp(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + serverURL, _ := url.Parse(server.URL) + + serverURL.Path = "/repo.git" + gitURL := serverURL.String() + + mux.HandleFunc("/repo.git/info/refs", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + }) + + args := cloneArgs(gitURL, "/tmp") + exp := []string{"clone", "--recursive", gitURL, "/tmp"} + if !reflect.DeepEqual(args, exp) { + t.Fatalf("Expected %v, got %v", exp, args) + } +} +func TestCloneArgsGit(t *testing.T) { + args := cloneArgs("git://github.com/docker/docker", "/tmp") + exp := []string{"clone", "--recursive", "--depth", "1", "git://github.com/docker/docker", "/tmp"} + if !reflect.DeepEqual(args, exp) { + t.Fatalf("Expected %v, got %v", exp, args) + } +} From ecfd095619a5839a95988e20cd1b9de48cb583cb Mon Sep 17 00:00:00 2001 From: David Calavera Date: Thu, 23 Apr 2015 09:35:19 -0700 Subject: [PATCH 4/5] Document the extra `depth` argument in git contexts. Signed-off-by: David Calavera Upstream-commit: 8bd5a95e1e39541dfc5dc635c5c8e7604fe10028 Component: engine --- .../docs/sources/reference/commandline/cli.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/components/engine/docs/sources/reference/commandline/cli.md b/components/engine/docs/sources/reference/commandline/cli.md index a871162049..26659c8ffa 100644 --- a/components/engine/docs/sources/reference/commandline/cli.md +++ b/components/engine/docs/sources/reference/commandline/cli.md @@ -636,12 +636,13 @@ refer to any of the files in the context. For example, your build can use an [*ADD*](/reference/builder/#add) instruction to reference a file in the context. -The `URL` parameter can specify the location of a Git repository; in this -case, the repository is the context. The Git repository is recursively -cloned with its submodules. The system does a fresh `git clone -recursive` -in a temporary directory on your local host. Then, this clone is sent to -the Docker daemon as the context. Local clones give you the ability to -access private repositories using local user credentials, VPN's, and so forth. +The `URL` parameter can specify the location of a Git repository; +the repository acts as the build context. The system recursively clones the repository +and its submodules using a `git clone --depth 1 --recursive` command. +This command runs in a temporary directory on your local host. +After the command succeeds, the directory is sent to the Docker daemon as the context. +Local clones give you the ability to access private repositories using +local user credentials, VPN's, and so forth. Instead of specifying a context, you can pass a single Dockerfile in the `URL` or pipe the file in via `STDIN`. To pipe a Dockerfile from `STDIN`: From dfd9ef2b6496fec134585adfc70c7419bd13a731 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Thu, 23 Apr 2015 10:19:34 -0700 Subject: [PATCH 5/5] Remove duplicated git clone logic. Signed-off-by: David Calavera Upstream-commit: 1cfb307d70558110e8e88a7391ae0e6cf6ebf174 Component: engine --- components/engine/api/client/build.go | 11 +---- components/engine/builder/job.go | 32 +------------ components/engine/utils/git.go | 47 +++++++++++++++++++ .../job_test.go => utils/git_test.go} | 2 +- 4 files changed, 51 insertions(+), 41 deletions(-) create mode 100644 components/engine/utils/git.go rename components/engine/{builder/job_test.go => utils/git_test.go} (98%) diff --git a/components/engine/api/client/build.go b/components/engine/api/client/build.go index 63cc63bc9e..eb39058b64 100644 --- a/components/engine/api/client/build.go +++ b/components/engine/api/client/build.go @@ -95,20 +95,11 @@ func (cli *DockerCli) CmdBuild(args ...string) error { } else { root := cmd.Arg(0) if urlutil.IsGitURL(root) { - remoteURL := cmd.Arg(0) - if !urlutil.IsGitTransport(remoteURL) { - remoteURL = "https://" + remoteURL - } - - root, err = ioutil.TempDir("", "docker-build-git") + root, err = utils.GitClone(root) if err != nil { return err } defer os.RemoveAll(root) - - if output, err := exec.Command("git", "clone", "--recursive", remoteURL, root).CombinedOutput(); err != nil { - return fmt.Errorf("Error trying to use git: %s (%s)", err, output) - } } if _, err := os.Stat(root); err != nil { return err diff --git a/components/engine/builder/job.go b/components/engine/builder/job.go index 8c031a8bd8..7991cba21c 100644 --- a/components/engine/builder/job.go +++ b/components/engine/builder/job.go @@ -5,9 +5,7 @@ import ( "fmt" "io" "io/ioutil" - "net/http" "os" - "os/exec" "strings" "sync" @@ -23,6 +21,7 @@ import ( "github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" + "github.com/docker/docker/utils" ) // whitelist of commands allowed for a commit/import @@ -106,21 +105,12 @@ func Build(d *daemon.Daemon, buildConfig *Config) error { if buildConfig.RemoteURL == "" { context = ioutil.NopCloser(buildConfig.Context) } else if urlutil.IsGitURL(buildConfig.RemoteURL) { - if !urlutil.IsGitTransport(buildConfig.RemoteURL) { - buildConfig.RemoteURL = "https://" + buildConfig.RemoteURL - } - root, err := ioutil.TempDir("", "docker-build-git") + root, err := utils.GitClone(buildConfig.RemoteURL) if err != nil { return err } defer os.RemoveAll(root) - clone := cloneArgs(buildConfig.RemoteURL, root) - - if output, err := exec.Command("git", clone...).CombinedOutput(); err != nil { - return fmt.Errorf("Error trying to use git: %s (%s)", err, output) - } - c, err := archive.Tar(root, archive.Uncompressed) if err != nil { return err @@ -242,21 +232,3 @@ func Commit(d *daemon.Daemon, name string, c *daemon.ContainerCommitConfig) (str return img.ID, nil } - -func cloneArgs(remoteURL, root string) []string { - args := []string{"clone", "--recursive"} - shallow := true - - if strings.HasPrefix(remoteURL, "http") { - res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL)) - if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" { - shallow = false - } - } - - if shallow { - args = append(args, "--depth", "1") - } - - return append(args, remoteURL, root) -} diff --git a/components/engine/utils/git.go b/components/engine/utils/git.go new file mode 100644 index 0000000000..18e002d184 --- /dev/null +++ b/components/engine/utils/git.go @@ -0,0 +1,47 @@ +package utils + +import ( + "fmt" + "io/ioutil" + "net/http" + "os/exec" + "strings" + + "github.com/docker/docker/pkg/urlutil" +) + +func GitClone(remoteURL string) (string, error) { + if !urlutil.IsGitTransport(remoteURL) { + remoteURL = "https://" + remoteURL + } + root, err := ioutil.TempDir("", "docker-build-git") + if err != nil { + return "", err + } + + clone := cloneArgs(remoteURL, root) + + if output, err := exec.Command("git", clone...).CombinedOutput(); err != nil { + return "", fmt.Errorf("Error trying to use git: %s (%s)", err, output) + } + + return root, nil +} + +func cloneArgs(remoteURL, root string) []string { + args := []string{"clone", "--recursive"} + shallow := true + + if strings.HasPrefix(remoteURL, "http") { + res, err := http.Head(fmt.Sprintf("%s/info/refs?service=git-upload-pack", remoteURL)) + if err != nil || res.Header.Get("Content-Type") != "application/x-git-upload-pack-advertisement" { + shallow = false + } + } + + if shallow { + args = append(args, "--depth", "1") + } + + return append(args, remoteURL, root) +} diff --git a/components/engine/builder/job_test.go b/components/engine/utils/git_test.go similarity index 98% rename from components/engine/builder/job_test.go rename to components/engine/utils/git_test.go index 79421a0688..a82841ae11 100644 --- a/components/engine/builder/job_test.go +++ b/components/engine/utils/git_test.go @@ -1,4 +1,4 @@ -package builder +package utils import ( "fmt"