Merge pull request #6265 from thaJeztah/28.x_backport_fork_remotecontext

[28.x backport] add local fork of github.com/docker/docker/builder/remotecontext
This commit is contained in:
Austin Vazquez
2025-08-15 05:51:05 -07:00
committed by GitHub
11 changed files with 503 additions and 42 deletions
+1 -1
View File
@@ -31,7 +31,7 @@ FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS build-base-alpine
ENV GOTOOLCHAIN=local
COPY --link --from=xx / /
RUN apk add --no-cache bash clang lld llvm file git
RUN apk add --no-cache bash clang lld llvm file git git-daemon
WORKDIR /go/src/github.com/docker/cli
FROM build-base-alpine AS build-alpine
+32 -36
View File
@@ -28,7 +28,6 @@ import (
buildtypes "github.com/docker/docker/api/types/build"
"github.com/docker/docker/api/types/container"
registrytypes "github.com/docker/docker/api/types/registry"
"github.com/docker/docker/builder/remotecontext/urlutil"
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/streamformatter"
"github.com/moby/go-archive"
@@ -76,12 +75,6 @@ func (o buildOptions) dockerfileFromStdin() bool {
return o.dockerfileName == "-"
}
// contextFromStdin returns true when the user specified that the build context
// should be read from stdin
func (o buildOptions) contextFromStdin() bool {
return o.context == "-"
}
func newBuildOptions() buildOptions {
ulimits := make(map[string]*container.Ulimit)
return buildOptions{
@@ -189,21 +182,24 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
buildCtx io.ReadCloser
dockerfileCtx io.ReadCloser
contextDir string
tempDir string
relDockerfile string
progBuff io.Writer
buildBuff io.Writer
remote string
)
contextType, err := build.DetectContextType(options.context)
if err != nil {
return err
}
if options.dockerfileFromStdin() {
if options.contextFromStdin() {
if contextType == build.ContextTypeStdin {
return errors.New("invalid argument: can't use stdin for both build context and dockerfile")
}
dockerfileCtx = dockerCli.In()
}
specifiedContext := options.context
progBuff = dockerCli.Out()
buildBuff = dockerCli.Out()
if options.quiet {
@@ -217,13 +213,19 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
}
}
switch {
case options.contextFromStdin():
switch contextType {
case build.ContextTypeStdin:
// buildCtx is tar archive. if stdin was dockerfile then it is wrapped
buildCtx, relDockerfile, err = build.GetContextFromReader(dockerCli.In(), options.dockerfileName)
case isLocalDir(specifiedContext):
contextDir, relDockerfile, err = build.GetContextFromLocalDir(specifiedContext, options.dockerfileName)
if err == nil && strings.HasPrefix(relDockerfile, ".."+string(filepath.Separator)) {
if err != nil {
return fmt.Errorf("unable to prepare context from STDIN: %w", err)
}
case build.ContextTypeLocal:
contextDir, relDockerfile, err = build.GetContextFromLocalDir(options.context, options.dockerfileName)
if err != nil {
return errors.Errorf("unable to prepare context: %s", err)
}
if strings.HasPrefix(relDockerfile, ".."+string(filepath.Separator)) {
// Dockerfile is outside of build-context; read the Dockerfile and pass it as dockerfileCtx
dockerfileCtx, err = os.Open(options.dockerfileName)
if err != nil {
@@ -231,24 +233,23 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
}
defer dockerfileCtx.Close()
}
case urlutil.IsGitURL(specifiedContext):
tempDir, relDockerfile, err = build.GetContextFromGitURL(specifiedContext, options.dockerfileName)
case urlutil.IsURL(specifiedContext):
buildCtx, relDockerfile, err = build.GetContextFromURL(progBuff, specifiedContext, options.dockerfileName)
default:
return errors.Errorf("unable to prepare context: path %q not found", specifiedContext)
}
if err != nil {
if options.quiet && urlutil.IsURL(specifiedContext) {
case build.ContextTypeGit:
var tempDir string
tempDir, relDockerfile, err = build.GetContextFromGitURL(options.context, options.dockerfileName)
if err != nil {
return errors.Errorf("unable to prepare context: %s", err)
}
defer func() {
_ = os.RemoveAll(tempDir)
}()
contextDir = tempDir
case build.ContextTypeRemote:
buildCtx, relDockerfile, err = build.GetContextFromURL(progBuff, options.context, options.dockerfileName)
if err != nil && options.quiet {
_, _ = fmt.Fprintln(dockerCli.Err(), progBuff)
}
return errors.Errorf("unable to prepare context: %s", err)
}
if tempDir != "" {
defer os.RemoveAll(tempDir)
contextDir = tempDir
default:
return errors.Errorf("unable to prepare context: path %q not found", options.context)
}
// read from a directory into tar archive
@@ -415,11 +416,6 @@ func runBuild(ctx context.Context, dockerCli command.Cli, options buildOptions)
return nil
}
func isLocalDir(c string) bool {
_, err := os.Stat(c)
return err == nil
}
type translatorFunc func(context.Context, reference.NamedTagged) (reference.Canonical, error)
// validateTag checks if the given image name can be resolved.
+1 -1
View File
@@ -16,7 +16,7 @@ import (
"strings"
"time"
"github.com/docker/docker/builder/remotecontext/git"
"github.com/docker/cli/cli/command/image/build/internal/git"
"github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/streamformatter"
"github.com/moby/go-archive"
+39
View File
@@ -0,0 +1,39 @@
package build
import (
"fmt"
"os"
"github.com/docker/cli/cli/command/image/build/internal/urlutil"
)
// ContextType describes the type (source) of build-context specified.
type ContextType string
const (
ContextTypeStdin ContextType = "stdin" // ContextTypeStdin indicates that the build-context is a TAR archive passed through STDIN.
ContextTypeLocal ContextType = "local" // ContextTypeLocal indicates that the build-context is a local directory.
ContextTypeRemote ContextType = "remote" // ContextTypeRemote indicates that the build-context is a remote URL.
ContextTypeGit ContextType = "git" // ContextTypeGit indicates that the build-context is a GIT URL.
)
// DetectContextType detects the type (source) of the build-context.
func DetectContextType(specifiedContext string) (ContextType, error) {
switch {
case specifiedContext == "-":
return ContextTypeStdin, nil
case isLocalDir(specifiedContext):
return ContextTypeLocal, nil
case urlutil.IsGitURL(specifiedContext):
return ContextTypeGit, nil
case urlutil.IsURL(specifiedContext):
return ContextTypeRemote, nil
default:
return "", fmt.Errorf("unable to prepare context: path %q not found", specifiedContext)
}
}
func isLocalDir(c string) bool {
_, err := os.Stat(c)
return err == nil
}
@@ -148,6 +148,9 @@ func supportsShallowClone(remoteURL string) bool {
// Try a HEAD request and fallback to a Get request on error
res, err := http.Head(serviceURL) // #nosec G107
if err == nil {
_ = res.Body.Close()
}
if err != nil || res.StatusCode != http.StatusOK {
res, err = http.Get(serviceURL) // #nosec G107
if err == nil {
@@ -0,0 +1,382 @@
package git
import (
"bytes"
"fmt"
"net/http"
"net/http/cgi"
"net/http/httptest"
"net/url"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
)
func TestParseRemoteURL(t *testing.T) {
tests := []struct {
doc string
url string
expected gitRepo
}{
{
doc: "git scheme uppercase, no url-fragment",
url: "GIT://github.com/user/repo.git",
expected: gitRepo{
remote: "git://github.com/user/repo.git",
ref: "master",
},
},
{
doc: "git scheme, no url-fragment",
url: "git://github.com/user/repo.git",
expected: gitRepo{
remote: "git://github.com/user/repo.git",
ref: "master",
},
},
{
doc: "git scheme, with url-fragment",
url: "git://github.com/user/repo.git#mybranch:mydir/mysubdir/",
expected: gitRepo{
remote: "git://github.com/user/repo.git",
ref: "mybranch",
subdir: "mydir/mysubdir/",
},
},
{
doc: "https scheme, no url-fragment",
url: "https://github.com/user/repo.git",
expected: gitRepo{
remote: "https://github.com/user/repo.git",
ref: "master",
},
},
{
doc: "https scheme, with url-fragment",
url: "https://github.com/user/repo.git#mybranch:mydir/mysubdir/",
expected: gitRepo{
remote: "https://github.com/user/repo.git",
ref: "mybranch",
subdir: "mydir/mysubdir/",
},
},
{
doc: "git@, no url-fragment",
url: "git@github.com:user/repo.git",
expected: gitRepo{
remote: "git@github.com:user/repo.git",
ref: "master",
},
},
{
doc: "git@, with url-fragment",
url: "git@github.com:user/repo.git#mybranch:mydir/mysubdir/",
expected: gitRepo{
remote: "git@github.com:user/repo.git",
ref: "mybranch",
subdir: "mydir/mysubdir/",
},
},
{
doc: "ssh, no url-fragment",
url: "ssh://github.com/user/repo.git",
expected: gitRepo{
remote: "ssh://github.com/user/repo.git",
ref: "master",
},
},
{
doc: "ssh, with url-fragment",
url: "ssh://github.com/user/repo.git#mybranch:mydir/mysubdir/",
expected: gitRepo{
remote: "ssh://github.com/user/repo.git",
ref: "mybranch",
subdir: "mydir/mysubdir/",
},
},
{
doc: "ssh, with url-fragment and user",
url: "ssh://foo%40barcorp.com@github.com/user/repo.git#mybranch:mydir/mysubdir/",
expected: gitRepo{
remote: "ssh://foo%40barcorp.com@github.com/user/repo.git",
ref: "mybranch",
subdir: "mydir/mysubdir/",
},
},
}
for _, tc := range tests {
t.Run(tc.doc, func(t *testing.T) {
repo, err := parseRemoteURL(tc.url)
assert.NilError(t, err)
assert.Check(t, is.DeepEqual(tc.expected, repo, cmp.AllowUnexported(gitRepo{})))
})
}
}
func TestCloneArgsSmartHttp(t *testing.T) {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
serverURL, err := url.Parse(server.URL)
assert.NilError(t, err)
serverURL.Path = "/repo.git"
mux.HandleFunc("/repo.git/info/refs", func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query().Get("service")
w.Header().Set("Content-Type", fmt.Sprintf("application/x-%s-advertisement", q))
})
args := fetchArgs(serverURL.String(), "master")
exp := []string{"fetch", "--depth", "1", "origin", "--", "master"}
assert.Check(t, is.DeepEqual(exp, args))
}
func TestCloneArgsDumbHttp(t *testing.T) {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
serverURL, _ := url.Parse(server.URL)
serverURL.Path = "/repo.git"
mux.HandleFunc("/repo.git/info/refs", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
})
args := fetchArgs(serverURL.String(), "master")
exp := []string{"fetch", "origin", "--", "master"}
assert.Check(t, is.DeepEqual(exp, args))
}
func TestCloneArgsGit(t *testing.T) {
args := fetchArgs("git://github.com/docker/docker", "master")
exp := []string{"fetch", "--depth", "1", "origin", "--", "master"}
assert.Check(t, is.DeepEqual(exp, args))
}
func gitGetConfig(name string) string {
b, err := gitRepo{}.gitWithinDir("", "config", "--get", name)
if err != nil {
// since we are interested in empty or non empty string,
// we can safely ignore the err here.
return ""
}
return strings.TrimSpace(string(b))
}
func TestCheckoutGit(t *testing.T) {
root := t.TempDir()
gitpath, err := exec.LookPath("git")
assert.NilError(t, err)
gitversion, _ := exec.Command(gitpath, "version").CombinedOutput()
t.Logf("%s", gitversion) // E.g. "git version 2.30.2"
// Serve all repositories under root using the Smart HTTP protocol so
// they can be cloned. The Dumb HTTP protocol is incompatible with
// shallow cloning but we unconditionally shallow-clone submodules, and
// we explicitly disable the file protocol.
// (Another option would be to use `git daemon` and the Git protocol,
// but that listens on a fixed port number which is a recipe for
// disaster in CI. Funnily enough, `git daemon --port=0` works but there
// is no easy way to discover which port got picked!)
// Associate git-http-backend logs with the current (sub)test.
// Incompatible with parallel subtests.
currentSubtest := t
githttp := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var logs bytes.Buffer
(&cgi.Handler{
Path: gitpath,
Args: []string{"http-backend"},
Dir: root,
Env: []string{
"GIT_PROJECT_ROOT=" + root,
"GIT_HTTP_EXPORT_ALL=1",
},
Stderr: &logs,
}).ServeHTTP(w, r)
if logs.Len() == 0 {
return
}
for {
line, err := logs.ReadString('\n')
currentSubtest.Log("git-http-backend: " + line)
if err != nil {
break
}
}
})
server := httptest.NewServer(&githttp)
defer server.Close()
eol := "\n"
autocrlf := gitGetConfig("core.autocrlf")
switch autocrlf {
case "true":
eol = "\r\n"
case "false", "input", "":
// accepted values
default:
t.Logf(`unknown core.autocrlf value: "%s"`, autocrlf)
}
must := func(out []byte, err error) {
t.Helper()
if len(out) > 0 {
t.Logf("%s", out)
}
assert.NilError(t, err)
}
gitDir := filepath.Join(root, "repo")
must(gitRepo{}.gitWithinDir(root, "-c", "init.defaultBranch=master", "init", gitDir))
must(gitRepo{}.gitWithinDir(gitDir, "config", "user.email", "test@docker.com"))
must(gitRepo{}.gitWithinDir(gitDir, "config", "user.name", "Docker test"))
assert.NilError(t, os.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch"), 0o644))
subDir := filepath.Join(gitDir, "subdir")
assert.NilError(t, os.Mkdir(subDir, 0o755))
assert.NilError(t, os.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 5000"), 0o644))
if runtime.GOOS != "windows" {
assert.NilError(t, os.Symlink("../subdir", filepath.Join(gitDir, "parentlink")))
assert.NilError(t, os.Symlink("/subdir", filepath.Join(gitDir, "absolutelink")))
}
must(gitRepo{}.gitWithinDir(gitDir, "add", "-A"))
must(gitRepo{}.gitWithinDir(gitDir, "commit", "-am", "First commit"))
must(gitRepo{}.gitWithinDir(gitDir, "checkout", "-b", "test"))
assert.NilError(t, os.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte("FROM scratch\nEXPOSE 3000"), 0o644))
assert.NilError(t, os.WriteFile(filepath.Join(subDir, "Dockerfile"), []byte("FROM busybox\nEXPOSE 5000"), 0o644))
must(gitRepo{}.gitWithinDir(gitDir, "add", "-A"))
must(gitRepo{}.gitWithinDir(gitDir, "commit", "-am", "Branch commit"))
must(gitRepo{}.gitWithinDir(gitDir, "checkout", "master"))
// set up submodule
subrepoDir := filepath.Join(root, "subrepo")
must(gitRepo{}.gitWithinDir(root, "-c", "init.defaultBranch=master", "init", subrepoDir))
must(gitRepo{}.gitWithinDir(subrepoDir, "config", "user.email", "test@docker.com"))
must(gitRepo{}.gitWithinDir(subrepoDir, "config", "user.name", "Docker test"))
assert.NilError(t, os.WriteFile(filepath.Join(subrepoDir, "subfile"), []byte("subcontents"), 0o644))
must(gitRepo{}.gitWithinDir(subrepoDir, "add", "-A"))
must(gitRepo{}.gitWithinDir(subrepoDir, "commit", "-am", "Subrepo initial"))
must(gitRepo{}.gitWithinDir(gitDir, "submodule", "add", server.URL+"/subrepo", "sub"))
must(gitRepo{}.gitWithinDir(gitDir, "add", "-A"))
must(gitRepo{}.gitWithinDir(gitDir, "commit", "-am", "With submodule"))
type singleCase struct {
frag string
exp string
fail bool
submodule bool
}
cases := []singleCase{
{"", "FROM scratch", false, true},
{"master", "FROM scratch", false, true},
{":subdir", "FROM scratch" + eol + "EXPOSE 5000", false, false},
{":nosubdir", "", true, false}, // missing directory error
{":Dockerfile", "", true, false}, // not a directory error
{"master:nosubdir", "", true, false},
{"master:subdir", "FROM scratch" + eol + "EXPOSE 5000", false, false},
{"master:../subdir", "", true, false},
{"test", "FROM scratch" + eol + "EXPOSE 3000", false, false},
{"test:", "FROM scratch" + eol + "EXPOSE 3000", false, false},
{"test:subdir", "FROM busybox" + eol + "EXPOSE 5000", false, false},
}
if runtime.GOOS != "windows" {
// Windows GIT (2.7.1 x64) does not support parentlink/absolutelink. Sample output below
// git --work-tree .\repo --git-dir .\repo\.git add -A
// error: readlink("absolutelink"): Function not implemented
// error: unable to index file absolutelink
// fatal: adding files failed
cases = append(cases, singleCase{frag: "master:absolutelink", exp: "FROM scratch" + eol + "EXPOSE 5000", fail: false})
cases = append(cases, singleCase{frag: "master:parentlink", exp: "FROM scratch" + eol + "EXPOSE 5000", fail: false})
}
for _, c := range cases {
t.Run(c.frag, func(t *testing.T) {
currentSubtest = t
ref, subdir := getRefAndSubdir(c.frag)
r, err := gitRepo{remote: server.URL + "/repo", ref: ref, subdir: subdir}.clone()
if c.fail {
assert.Check(t, is.ErrorContains(err, ""))
return
}
assert.NilError(t, err)
defer os.RemoveAll(r)
if c.submodule {
b, err := os.ReadFile(filepath.Join(r, "sub/subfile"))
assert.NilError(t, err)
assert.Check(t, is.Equal("subcontents", string(b)))
} else {
_, err := os.Stat(filepath.Join(r, "sub/subfile"))
assert.ErrorContains(t, err, "")
assert.Assert(t, os.IsNotExist(err))
}
b, err := os.ReadFile(filepath.Join(r, "Dockerfile"))
assert.NilError(t, err)
assert.Check(t, is.Equal(c.exp, string(b)))
})
}
}
func TestValidGitTransport(t *testing.T) {
gitUrls := []string{
"git://github.com/docker/docker",
"git@github.com:docker/docker.git",
"git@bitbucket.org:atlassianlabs/atlassian-docker.git",
"https://github.com/docker/docker.git",
"http://github.com/docker/docker.git",
"http://github.com/docker/docker.git#branch",
"http://github.com/docker/docker.git#:dir",
}
incompleteGitUrls := []string{
"github.com/docker/docker",
}
for _, u := range gitUrls {
if !isGitTransport(u) {
t.Fatalf("%q should be detected as valid Git prefix", u)
}
}
for _, u := range incompleteGitUrls {
if isGitTransport(u) {
t.Fatalf("%q should not be detected as valid Git prefix", u)
}
}
}
func TestGitInvalidRef(t *testing.T) {
gitUrls := []string{
"git://github.com/moby/moby#--foo bar",
"git@github.com/moby/moby#--upload-pack=sleep;:",
"git@g.com:a/b.git#-B",
"git@g.com:a/b.git#with space",
}
for _, u := range gitUrls {
_, err := Clone(u)
assert.Assert(t, err != nil)
// On Windows, git has different case for the "invalid refspec" error,
// so we can't use ErrorContains.
assert.Check(t, is.Contains(strings.ToLower(err.Error()), "invalid refspec"))
}
}
@@ -8,7 +8,7 @@ package urlutil
import (
"strings"
"github.com/docker/docker/internal/lazyregexp"
"github.com/docker/cli/internal/lazyregexp"
)
// urlPathWithFragmentSuffix matches fragments to use as Git reference and build
@@ -0,0 +1,42 @@
package urlutil
import "testing"
var (
gitUrls = []string{
"git://github.com/docker/docker",
"git@github.com:docker/docker.git",
"git@bitbucket.org:atlassianlabs/atlassian-docker.git",
"https://github.com/docker/docker.git",
"http://github.com/docker/docker.git",
"http://github.com/docker/docker.git#branch",
"http://github.com/docker/docker.git#:dir",
}
incompleteGitUrls = []string{
"github.com/docker/docker",
}
invalidGitUrls = []string{
"http://github.com/docker/docker.git:#branch",
"https://github.com/docker/dgit",
}
)
func TestIsGIT(t *testing.T) {
for _, url := range gitUrls {
if !IsGitURL(url) {
t.Fatalf("%q should be detected as valid Git url", url)
}
}
for _, url := range incompleteGitUrls {
if !IsGitURL(url) {
t.Fatalf("%q should be detected as valid Git url", url)
}
}
for _, url := range invalidGitUrls {
if IsGitURL(url) {
t.Fatalf("%q should not be detected as valid Git prefix", url)
}
}
}
+1
View File
@@ -50,6 +50,7 @@ RUN apk add --no-cache \
coreutils \
curl \
git \
git-daemon \
jq \
nano
+1 -1
View File
@@ -34,6 +34,7 @@ require (
github.com/moby/sys/capability v0.4.0
github.com/moby/sys/sequential v0.6.0
github.com/moby/sys/signal v0.7.1
github.com/moby/sys/symlink v0.3.0
github.com/moby/term v0.5.2
github.com/morikuni/aec v1.0.0
github.com/opencontainers/go-digest v1.0.0
@@ -83,7 +84,6 @@ require (
github.com/klauspost/compress v1.18.0 // indirect
github.com/miekg/pkcs11 v1.1.1 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/sys/symlink v0.3.0 // indirect
github.com/moby/sys/user v0.4.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-2
View File
@@ -89,8 +89,6 @@ github.com/docker/docker/api/types/system
github.com/docker/docker/api/types/time
github.com/docker/docker/api/types/versions
github.com/docker/docker/api/types/volume
github.com/docker/docker/builder/remotecontext/git
github.com/docker/docker/builder/remotecontext/urlutil
github.com/docker/docker/client
github.com/docker/docker/internal/lazyregexp
github.com/docker/docker/internal/multierror