From 8997667aa2ee10bd8dfba7e97efd9e6f7bd63bee Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 29 Nov 2018 01:06:10 +0100 Subject: [PATCH] Do not patch Dockerfiles in CI When building the Dockerfiles for development, those images are mainly used to create a reproducible build-environment. The source code is bind-mounted into the image at runtime; there is no need to create an image with the actual source code, and copying the source code into the image would lead to a new image being created for each code-change (possibly leading up to many "dangling" images for previous code-changes). However, when building (and using) the development images in CI, bind-mounting is not an option, because the daemon is running remotely. To make this work, the circle-ci script patched the Dockerfiles when CI is run; adding a `COPY` to the respective Dockerfiles. Patching Dockerfiles is not really a "best practice" and, even though the source code does not and up in the image, the source would still be _sent_ to the daemon for each build (unless BuildKit is used). This patch updates the makefiles, circle-ci script, and Dockerfiles; - When building the Dockerfiles locally, pipe the Dockerfile through stdin. Doing so, prevents the build-context from being sent to the daemon. This speeds up the build, and doesn't fill up the Docker "temp" directory with content that's not used - Now that no content is sent, add the COPY instructions to the Dockerfiles, and remove the code in the circle-ci script to "live patch" the Dockerfiles. Before this patch is applied (with cache): ``` $ time make -f docker.Makefile build_shell_validate_image docker build -t docker-cli-shell-validate -f ./dockerfiles/Dockerfile.shellcheck . Sending build context to Docker daemon 41MB Step 1/2 : FROM debian:stretch-slim ... Successfully built 81e14e8ad856 Successfully tagged docker-cli-shell-validate:latest 2.75 real 0.45 user 0.56 sys ``` After this patch is applied (with cache):: ``` $ time make -f docker.Makefile build_shell_validate_image cat ./dockerfiles/Dockerfile.shellcheck | docker build -t docker-cli-shell-validate - Sending build context to Docker daemon 2.048kB Step 1/2 : FROM debian:stretch-slim ... Successfully built 81e14e8ad856 Successfully tagged docker-cli-shell-validate:latest 0.33 real 0.07 user 0.08 sys ``` Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 166856ab1b1681de107b5e064ea89ea4b27154fd) Signed-off-by: Sebastiaan van Stijn --- circle.yml | 20 +++++--------------- docker.Makefile | 15 ++++++++++----- dockerfiles/Dockerfile.cross | 1 + dockerfiles/Dockerfile.dev | 1 + dockerfiles/Dockerfile.lint | 1 + dockerfiles/Dockerfile.shellcheck | 1 + 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/circle.yml b/circle.yml index 073d4cf69e..3b4d9259e9 100644 --- a/circle.yml +++ b/circle.yml @@ -16,9 +16,7 @@ jobs: - run: name: "Lint" command: | - dockerfile=dockerfiles/Dockerfile.lint - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-linter:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.lint --tag cli-linter:$CIRCLE_BUILD_NUM . docker run --rm cli-linter:$CIRCLE_BUILD_NUM cross: @@ -34,9 +32,7 @@ jobs: - run: name: "Cross" command: | - dockerfile=dockerfiles/Dockerfile.cross - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-builder:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.cross --tag cli-builder:$CIRCLE_BUILD_NUM . name=cross-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX docker run \ -e CROSS_GROUP=$CIRCLE_NODE_INDEX \ @@ -60,9 +56,7 @@ jobs: - run: name: "Unit Test with Coverage" command: | - dockerfile=dockerfiles/Dockerfile.dev - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-builder:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.dev --tag cli-builder:$CIRCLE_BUILD_NUM . docker run --name \ test-$CIRCLE_BUILD_NUM cli-builder:$CIRCLE_BUILD_NUM \ make test-coverage @@ -89,10 +83,8 @@ jobs: - run: name: "Validate Vendor, Docs, and Code Generation" command: | - dockerfile=dockerfiles/Dockerfile.dev - echo "COPY . ." >> $dockerfile rm -f .dockerignore # include .git - docker build -f $dockerfile --tag cli-builder-with-git:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.dev --tag cli-builder-with-git:$CIRCLE_BUILD_NUM . docker run --rm cli-builder-with-git:$CIRCLE_BUILD_NUM \ make ci-validate shellcheck: @@ -107,9 +99,7 @@ jobs: - run: name: "Run shellcheck" command: | - dockerfile=dockerfiles/Dockerfile.shellcheck - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-validator:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.shellcheck --tag cli-validator:$CIRCLE_BUILD_NUM . docker run --rm cli-validator:$CIRCLE_BUILD_NUM \ make shellcheck workflows: diff --git a/docker.Makefile b/docker.Makefile index 2ff2594a7f..628e989fb0 100644 --- a/docker.Makefile +++ b/docker.Makefile @@ -17,24 +17,29 @@ ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT -e PLATFORM # build docker image (dockerfiles/Dockerfile.build) .PHONY: build_docker_image build_docker_image: - docker build ${DOCKER_BUILD_ARGS} -t $(DEV_DOCKER_IMAGE_NAME) -f ./dockerfiles/Dockerfile.dev . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.dev | docker build ${DOCKER_BUILD_ARGS} -t $(DEV_DOCKER_IMAGE_NAME) - # build docker image having the linting tools (dockerfiles/Dockerfile.lint) .PHONY: build_linter_image build_linter_image: - docker build ${DOCKER_BUILD_ARGS} -t $(LINTER_IMAGE_NAME) -f ./dockerfiles/Dockerfile.lint . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.lint | docker build ${DOCKER_BUILD_ARGS} -t $(LINTER_IMAGE_NAME) - .PHONY: build_cross_image build_cross_image: - docker build ${DOCKER_BUILD_ARGS} -t $(CROSS_IMAGE_NAME) -f ./dockerfiles/Dockerfile.cross . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.cross | docker build ${DOCKER_BUILD_ARGS} -t $(CROSS_IMAGE_NAME) - .PHONY: build_shell_validate_image build_shell_validate_image: - docker build -t $(VALIDATE_IMAGE_NAME) -f ./dockerfiles/Dockerfile.shellcheck . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.shellcheck | docker build -t $(VALIDATE_IMAGE_NAME) - .PHONY: build_binary_native_image build_binary_native_image: - docker build -t $(BINARY_NATIVE_IMAGE_NAME) -f ./dockerfiles/Dockerfile.binary-native . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.binary-native | docker build -t $(BINARY_NATIVE_IMAGE_NAME) - .PHONY: build_e2e_image build_e2e_image: diff --git a/dockerfiles/Dockerfile.cross b/dockerfiles/Dockerfile.cross index 9ee67e4eef..240ff2bfb5 100644 --- a/dockerfiles/Dockerfile.cross +++ b/dockerfiles/Dockerfile.cross @@ -1,3 +1,4 @@ FROM dockercore/golang-cross:1.10.8@sha256:a93210f55a8137b4aa4b9f033ac7a80b66ab6337e98e7afb62abe93b4ad73cad ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 WORKDIR /go/src/github.com/docker/cli +COPY . . diff --git a/dockerfiles/Dockerfile.dev b/dockerfiles/Dockerfile.dev index c155587a7b..c8ce67ab06 100644 --- a/dockerfiles/Dockerfile.dev +++ b/dockerfiles/Dockerfile.dev @@ -22,3 +22,4 @@ ENV CGO_ENABLED=0 \ DISABLE_WARN_OUTSIDE_CONTAINER=1 WORKDIR /go/src/github.com/docker/cli CMD sh +COPY . . diff --git a/dockerfiles/Dockerfile.lint b/dockerfiles/Dockerfile.lint index 8be2d6114c..2a5354f432 100644 --- a/dockerfiles/Dockerfile.lint +++ b/dockerfiles/Dockerfile.lint @@ -15,3 +15,4 @@ ENV CGO_ENABLED=0 ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 ENTRYPOINT ["/usr/local/bin/gometalinter"] CMD ["--config=gometalinter.json", "./..."] +COPY . . diff --git a/dockerfiles/Dockerfile.shellcheck b/dockerfiles/Dockerfile.shellcheck index 43112b314d..4d0ddd3332 100644 --- a/dockerfiles/Dockerfile.shellcheck +++ b/dockerfiles/Dockerfile.shellcheck @@ -7,3 +7,4 @@ RUN apt-get update && \ WORKDIR /go/src/github.com/docker/cli ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 CMD bash +COPY . .