From 8997667aa2ee10bd8dfba7e97efd9e6f7bd63bee Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 29 Nov 2018 01:06:10 +0100 Subject: [PATCH 1/3] Do not patch Dockerfiles in CI When building the Dockerfiles for development, those images are mainly used to create a reproducible build-environment. The source code is bind-mounted into the image at runtime; there is no need to create an image with the actual source code, and copying the source code into the image would lead to a new image being created for each code-change (possibly leading up to many "dangling" images for previous code-changes). However, when building (and using) the development images in CI, bind-mounting is not an option, because the daemon is running remotely. To make this work, the circle-ci script patched the Dockerfiles when CI is run; adding a `COPY` to the respective Dockerfiles. Patching Dockerfiles is not really a "best practice" and, even though the source code does not and up in the image, the source would still be _sent_ to the daemon for each build (unless BuildKit is used). This patch updates the makefiles, circle-ci script, and Dockerfiles; - When building the Dockerfiles locally, pipe the Dockerfile through stdin. Doing so, prevents the build-context from being sent to the daemon. This speeds up the build, and doesn't fill up the Docker "temp" directory with content that's not used - Now that no content is sent, add the COPY instructions to the Dockerfiles, and remove the code in the circle-ci script to "live patch" the Dockerfiles. Before this patch is applied (with cache): ``` $ time make -f docker.Makefile build_shell_validate_image docker build -t docker-cli-shell-validate -f ./dockerfiles/Dockerfile.shellcheck . Sending build context to Docker daemon 41MB Step 1/2 : FROM debian:stretch-slim ... Successfully built 81e14e8ad856 Successfully tagged docker-cli-shell-validate:latest 2.75 real 0.45 user 0.56 sys ``` After this patch is applied (with cache):: ``` $ time make -f docker.Makefile build_shell_validate_image cat ./dockerfiles/Dockerfile.shellcheck | docker build -t docker-cli-shell-validate - Sending build context to Docker daemon 2.048kB Step 1/2 : FROM debian:stretch-slim ... Successfully built 81e14e8ad856 Successfully tagged docker-cli-shell-validate:latest 0.33 real 0.07 user 0.08 sys ``` Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 166856ab1b1681de107b5e064ea89ea4b27154fd) Signed-off-by: Sebastiaan van Stijn --- circle.yml | 20 +++++--------------- docker.Makefile | 15 ++++++++++----- dockerfiles/Dockerfile.cross | 1 + dockerfiles/Dockerfile.dev | 1 + dockerfiles/Dockerfile.lint | 1 + dockerfiles/Dockerfile.shellcheck | 1 + 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/circle.yml b/circle.yml index 073d4cf69e..3b4d9259e9 100644 --- a/circle.yml +++ b/circle.yml @@ -16,9 +16,7 @@ jobs: - run: name: "Lint" command: | - dockerfile=dockerfiles/Dockerfile.lint - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-linter:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.lint --tag cli-linter:$CIRCLE_BUILD_NUM . docker run --rm cli-linter:$CIRCLE_BUILD_NUM cross: @@ -34,9 +32,7 @@ jobs: - run: name: "Cross" command: | - dockerfile=dockerfiles/Dockerfile.cross - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-builder:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.cross --tag cli-builder:$CIRCLE_BUILD_NUM . name=cross-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX docker run \ -e CROSS_GROUP=$CIRCLE_NODE_INDEX \ @@ -60,9 +56,7 @@ jobs: - run: name: "Unit Test with Coverage" command: | - dockerfile=dockerfiles/Dockerfile.dev - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-builder:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.dev --tag cli-builder:$CIRCLE_BUILD_NUM . docker run --name \ test-$CIRCLE_BUILD_NUM cli-builder:$CIRCLE_BUILD_NUM \ make test-coverage @@ -89,10 +83,8 @@ jobs: - run: name: "Validate Vendor, Docs, and Code Generation" command: | - dockerfile=dockerfiles/Dockerfile.dev - echo "COPY . ." >> $dockerfile rm -f .dockerignore # include .git - docker build -f $dockerfile --tag cli-builder-with-git:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.dev --tag cli-builder-with-git:$CIRCLE_BUILD_NUM . docker run --rm cli-builder-with-git:$CIRCLE_BUILD_NUM \ make ci-validate shellcheck: @@ -107,9 +99,7 @@ jobs: - run: name: "Run shellcheck" command: | - dockerfile=dockerfiles/Dockerfile.shellcheck - echo "COPY . ." >> $dockerfile - docker build -f $dockerfile --tag cli-validator:$CIRCLE_BUILD_NUM . + docker build -f dockerfiles/Dockerfile.shellcheck --tag cli-validator:$CIRCLE_BUILD_NUM . docker run --rm cli-validator:$CIRCLE_BUILD_NUM \ make shellcheck workflows: diff --git a/docker.Makefile b/docker.Makefile index 2ff2594a7f..628e989fb0 100644 --- a/docker.Makefile +++ b/docker.Makefile @@ -17,24 +17,29 @@ ENVVARS = -e VERSION=$(VERSION) -e GITCOMMIT -e PLATFORM # build docker image (dockerfiles/Dockerfile.build) .PHONY: build_docker_image build_docker_image: - docker build ${DOCKER_BUILD_ARGS} -t $(DEV_DOCKER_IMAGE_NAME) -f ./dockerfiles/Dockerfile.dev . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.dev | docker build ${DOCKER_BUILD_ARGS} -t $(DEV_DOCKER_IMAGE_NAME) - # build docker image having the linting tools (dockerfiles/Dockerfile.lint) .PHONY: build_linter_image build_linter_image: - docker build ${DOCKER_BUILD_ARGS} -t $(LINTER_IMAGE_NAME) -f ./dockerfiles/Dockerfile.lint . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.lint | docker build ${DOCKER_BUILD_ARGS} -t $(LINTER_IMAGE_NAME) - .PHONY: build_cross_image build_cross_image: - docker build ${DOCKER_BUILD_ARGS} -t $(CROSS_IMAGE_NAME) -f ./dockerfiles/Dockerfile.cross . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.cross | docker build ${DOCKER_BUILD_ARGS} -t $(CROSS_IMAGE_NAME) - .PHONY: build_shell_validate_image build_shell_validate_image: - docker build -t $(VALIDATE_IMAGE_NAME) -f ./dockerfiles/Dockerfile.shellcheck . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.shellcheck | docker build -t $(VALIDATE_IMAGE_NAME) - .PHONY: build_binary_native_image build_binary_native_image: - docker build -t $(BINARY_NATIVE_IMAGE_NAME) -f ./dockerfiles/Dockerfile.binary-native . + # build dockerfile from stdin so that we don't send the build-context; source is bind-mounted in the development environment + cat ./dockerfiles/Dockerfile.binary-native | docker build -t $(BINARY_NATIVE_IMAGE_NAME) - .PHONY: build_e2e_image build_e2e_image: diff --git a/dockerfiles/Dockerfile.cross b/dockerfiles/Dockerfile.cross index 9ee67e4eef..240ff2bfb5 100644 --- a/dockerfiles/Dockerfile.cross +++ b/dockerfiles/Dockerfile.cross @@ -1,3 +1,4 @@ FROM dockercore/golang-cross:1.10.8@sha256:a93210f55a8137b4aa4b9f033ac7a80b66ab6337e98e7afb62abe93b4ad73cad ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 WORKDIR /go/src/github.com/docker/cli +COPY . . diff --git a/dockerfiles/Dockerfile.dev b/dockerfiles/Dockerfile.dev index c155587a7b..c8ce67ab06 100644 --- a/dockerfiles/Dockerfile.dev +++ b/dockerfiles/Dockerfile.dev @@ -22,3 +22,4 @@ ENV CGO_ENABLED=0 \ DISABLE_WARN_OUTSIDE_CONTAINER=1 WORKDIR /go/src/github.com/docker/cli CMD sh +COPY . . diff --git a/dockerfiles/Dockerfile.lint b/dockerfiles/Dockerfile.lint index 8be2d6114c..2a5354f432 100644 --- a/dockerfiles/Dockerfile.lint +++ b/dockerfiles/Dockerfile.lint @@ -15,3 +15,4 @@ ENV CGO_ENABLED=0 ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 ENTRYPOINT ["/usr/local/bin/gometalinter"] CMD ["--config=gometalinter.json", "./..."] +COPY . . diff --git a/dockerfiles/Dockerfile.shellcheck b/dockerfiles/Dockerfile.shellcheck index 43112b314d..4d0ddd3332 100644 --- a/dockerfiles/Dockerfile.shellcheck +++ b/dockerfiles/Dockerfile.shellcheck @@ -7,3 +7,4 @@ RUN apt-get update && \ WORKDIR /go/src/github.com/docker/cli ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 CMD bash +COPY . . From b59752479b72f8b6c7367f48d51e81ba8fef0024 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Thu, 29 Nov 2018 01:19:18 +0100 Subject: [PATCH 2/3] Use official shellcheck image This patch switches the shellcheck image to use the official image from Docker Hub. Note that this does not yet update shellcheck to the latest version (v0.5.x); Shellcheck v0.4.7 added some new checks, which makes CI currently fail, so will be done in a follow-up PR. Instead, the v0.4.6 version is used in this PR, which is closest to the same version as was installed in the image before this change; ``` docker run --rm docker-cli-shell-validate shellcheck --version ShellCheck - shell script analysis tool version: 0.4.4 license: GNU General Public License, version 3 website: http://www.shellcheck.net ``` Signed-off-by: Sebastiaan van Stijn (cherry picked from commit 388646eab0259c7e7410fe51679d6e4191cd8ab4) Signed-off-by: Sebastiaan van Stijn --- dockerfiles/Dockerfile.shellcheck | 11 ++++------- scripts/warn-outside-container | 4 ++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/dockerfiles/Dockerfile.shellcheck b/dockerfiles/Dockerfile.shellcheck index 4d0ddd3332..2507062ee9 100644 --- a/dockerfiles/Dockerfile.shellcheck +++ b/dockerfiles/Dockerfile.shellcheck @@ -1,10 +1,7 @@ -FROM debian:stretch-slim - -RUN apt-get update && \ - apt-get -y install make shellcheck && \ - apt-get clean - +FROM koalaman/shellcheck-alpine:v0.4.6 +RUN apk add --no-cache bash make WORKDIR /go/src/github.com/docker/cli ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 -CMD bash +ENTRYPOINT [""] +CMD ["/bin/sh"] COPY . . diff --git a/scripts/warn-outside-container b/scripts/warn-outside-container index e937caafeb..ecbecc0ea2 100755 --- a/scripts/warn-outside-container +++ b/scripts/warn-outside-container @@ -1,9 +1,9 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh set -eu target="${1:-}" -if [[ "$target" != "help" && -z "${DISABLE_WARN_OUTSIDE_CONTAINER:-}" ]]; then +if [ "$target" != "help" ] && [ -z "${DISABLE_WARN_OUTSIDE_CONTAINER:-}" ]; then ( echo echo From 9a5296c8f15e2dc1b83bc22327f70b499a19ab4c Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Tue, 26 Feb 2019 12:49:44 +0100 Subject: [PATCH 3/3] Update to shellcheck v0.6.0 Signed-off-by: Sebastiaan van Stijn (cherry picked from commit ff107b313a00c1ab8989f380e1ef65529b3f2e37) Signed-off-by: Sebastiaan van Stijn --- contrib/completion/bash/docker | 8 +++++--- dockerfiles/Dockerfile.shellcheck | 4 +--- scripts/gen/windows-resources | 8 ++++---- scripts/test/e2e/run | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index 192cdbf526..92d5740861 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -1,12 +1,14 @@ #!/usr/bin/env bash -# shellcheck disable=SC2016,SC2119,SC2155 +# shellcheck disable=SC2016,SC2119,SC2155,SC2206,SC2207 # # Shellcheck ignore list: # - SC2016: Expressions don't expand in single quotes, use double quotes for that. # - SC2119: Use foo "$@" if function's $1 should mean script's $1. # - SC2155: Declare and assign separately to avoid masking return values. -# -# You can find more details for each warning at the following page: +# - SC2206: Quote to prevent word splitting, or split robustly with mapfile or read -a. +# - SC2207: Prefer mapfile or read -a to split command output (or quote to avoid splitting). +# +# You can find more details for each warning at the following page: # https://github.com/koalaman/shellcheck/wiki/ # # bash completion file for core docker commands diff --git a/dockerfiles/Dockerfile.shellcheck b/dockerfiles/Dockerfile.shellcheck index 2507062ee9..dd3cab79b2 100644 --- a/dockerfiles/Dockerfile.shellcheck +++ b/dockerfiles/Dockerfile.shellcheck @@ -1,7 +1,5 @@ -FROM koalaman/shellcheck-alpine:v0.4.6 +FROM koalaman/shellcheck-alpine:v0.6.0 RUN apk add --no-cache bash make WORKDIR /go/src/github.com/docker/cli ENV DISABLE_WARN_OUTSIDE_CONTAINER=1 -ENTRYPOINT [""] -CMD ["/bin/sh"] COPY . . diff --git a/scripts/gen/windows-resources b/scripts/gen/windows-resources index 14b45e08e0..5fdc1b4ade 100755 --- a/scripts/gen/windows-resources +++ b/scripts/gen/windows-resources @@ -7,7 +7,7 @@ set -eu -o pipefail SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # shellcheck source=/go/src/github.com/docker/cli/scripts/build/.variables -source $SCRIPTDIR/../build/.variables +source "$SCRIPTDIR"/../build/.variables RESOURCES=$SCRIPTDIR/../winresources @@ -26,9 +26,9 @@ VERSION_QUAD=$(echo -n "$VERSION" | sed -re 's/^([0-9.]*).*$/\1/' | tr . ,) # Pass version and commit information into the resource compiler defs= -[ ! -z "$VERSION" ] && defs+=( "-D DOCKER_VERSION=\"$VERSION\"") -[ ! -z "$VERSION_QUAD" ] && defs+=( "-D DOCKER_VERSION_QUAD=$VERSION_QUAD") -[ ! -z "$GITCOMMIT" ] && defs+=( "-D DOCKER_COMMIT=\"$GITCOMMIT\"") +[ -n "$VERSION" ] && defs+=( "-D DOCKER_VERSION=\"$VERSION\"") +[ -n "$VERSION_QUAD" ] && defs+=( "-D DOCKER_VERSION_QUAD=$VERSION_QUAD") +[ -n "$GITCOMMIT" ] && defs+=( "-D DOCKER_COMMIT=\"$GITCOMMIT\"") function makeres { "$WINDRES" \ diff --git a/scripts/test/e2e/run b/scripts/test/e2e/run index 173397a703..b960709701 100755 --- a/scripts/test/e2e/run +++ b/scripts/test/e2e/run @@ -70,7 +70,7 @@ function runtests { GOPATH="$GOPATH" \ PATH="$PWD/build/:/usr/bin" \ HOME="$HOME" \ - "$(which go)" test -v ./e2e/... ${TESTFLAGS-} + "$(command -v go)" test -v ./e2e/... ${TESTFLAGS-} } export unique_id="${E2E_UNIQUE_ID:-cliendtoendsuite}"