Merge pull request #433 from glours/backport-master-to-19.03

Backport master latest commits to 19.03
Upstream-commit: 4814b6e5ef2bf510bd436158e0811db42a24e057
Component: packaging

Author: tiborvass

components/packaging/.gitignore

@@ -1,8 +1,5 @@
 build
 debbuild
 rpmbuild
-tmp
-artifacts
 sources
 *.tar
-image-linux*

components/packaging/.unir.yml

@@ -1,9 +1,10 @@
 # The list of people who can trigger an auto-merge with approvals
 whitelist:
-    - seemethere
-    - andrewhsu
-    - jose-bigio
-    - corbin-coleman
+  - andrewhsu
+  - silvin-lubecki
+  - StefanScherer
+  - thaJeztah
+  - zelahi
 
 # At least 2 approvals are needed for auto-merging
 approvals_needed: 2

components/packaging/Jenkinsfile

@@ -1,73 +1,56 @@
 #!groovy
 
-
-def genBranch(String arch, String branch) {
-	return [
-		"${arch}": { ->
-			stage("Build engine image on ${arch}") {
-				wrappedNode(label: "linux && ${arch}", cleanWorkspace: true) {
-					try {
-						checkout scm
-						sh("git clone https://github.com/docker/engine.git")
-						sh("git -C engine checkout $branch")
-						sh('make ENGINE_DIR=$(pwd)/engine image')
-					} finally {
-						sh('make ENGINE_DIR=$(pwd)/engine clean-image clean-engine')
-					}
-				}
-		}
-	}]
-}
-
 def branch = env.CHANGE_TARGET ?: env.BRANCH_NAME
 
 test_steps = [
 	'deb': { ->
 		stage('Ubuntu Xenial Debian Package') {
 			wrappedNode(label: 'ubuntu && x86_64', cleanWorkspace: true) {
-				checkout scm
-				sh('git clone https://github.com/docker/cli.git')
-				sh("git -C cli checkout $branch")
-				sh('git clone https://github.com/docker/engine.git')
-				sh("git -C engine checkout $branch")
-				sh('make VERSION=0.0.1-dev DOCKER_BUILD_PKGS=ubuntu-xenial ENGINE_DIR=$(pwd)/engine CLI_DIR=$(pwd)/cli deb')
+				try {
+					checkout scm
+					sh('git clone https://github.com/docker/cli.git')
+					sh("git -C cli checkout $branch")
+					sh('git clone https://github.com/docker/engine.git')
+					sh("git -C engine checkout $branch")
+					sh('make -C deb VERSION=0.0.1-dev ENGINE_DIR=$(pwd)/engine CLI_DIR=$(pwd)/cli ubuntu-xenial')
+				} finally {
+					sh('make ENGINE_DIR=$(pwd)/engine clean-engine')
+				}
 			}
 		}
 	},
 	'rpm': { ->
 		stage('Centos 7 RPM Package') {
 			wrappedNode(label: 'ubuntu && x86_64', cleanWorkspace: true) {
-				checkout scm
-				sh('git clone https://github.com/docker/cli.git')
-				sh("git -C cli checkout $branch")
-				sh('git clone https://github.com/docker/engine.git')
-				sh("git -C engine checkout $branch")
-				sh('make VERSION=0.0.1-dev DOCKER_BUILD_PKGS=centos-7 ENGINE_DIR=$(pwd)/engine CLI_DIR=$(pwd)/cli rpm')
+				try {
+					checkout scm
+					sh('git clone https://github.com/docker/cli.git')
+					sh("git -C cli checkout $branch")
+					sh('git clone https://github.com/docker/engine.git')
+					sh("git -C engine checkout $branch")
+					sh('make -C rpm VERSION=0.0.1-dev ENGINE_DIR=$(pwd)/engine CLI_DIR=$(pwd)/cli centos-7')
+				} finally {
+					sh('make ENGINE_DIR=$(pwd)/engine clean-engine')
+				}
 			}
 		}
 	},
 	'static': { ->
 		stage('Static Linux Binaries') {
 			wrappedNode(label: 'ubuntu && x86_64', cleanWorkspace: true) {
-				checkout scm
-				sh('git clone https://github.com/docker/cli.git')
-				sh("git -C cli checkout $branch")
-				sh('git clone https://github.com/docker/engine.git')
-				sh("git -C engine checkout $branch")
-				sh('make VERSION=0.0.1-dev DOCKER_BUILD_PKGS=static-linux ENGINE_DIR=$(pwd)/engine CLI_DIR=$(pwd)/cli static')
+				try {
+					checkout scm
+					sh('git clone https://github.com/docker/cli.git')
+					sh("git -C cli checkout $branch")
+					sh('git clone https://github.com/docker/engine.git')
+					sh("git -C engine checkout $branch")
+					sh('make VERSION=0.0.1-dev DOCKER_BUILD_PKGS=static-linux ENGINE_DIR=$(pwd)/engine CLI_DIR=$(pwd)/cli static')
+				} finally {
+					sh('make ENGINE_DIR=$(pwd)/engine clean-engine')
+				}
 			}
 		}
 	},
 ]
 
-arches = [
-	"x86_64",
-	"aarch64",
-	"armhf"
-]
-
-arches.each {
-	test_steps << genBranch(it, branch)
-}
-
 parallel(test_steps)

components/packaging/Makefile

@@ -13,51 +13,26 @@ help: ## show make targets
 
 .PHONY: clean-engine
 clean-engine:
+	[ ! -d $(ENGINE_DIR) ] || docker run --rm -v $(ENGINE_DIR):/v -w /v alpine chown -R $(shell id -u):$(shell id -g) /v
 	rm -rf $(ENGINE_DIR)
 
-.PHONY: clean-image
-clean-image:
-	$(MAKE) ENGINE_DIR=$(ENGINE_DIR) -C image clean
-
-
 .PHONY: clean
-clean: clean-image ## remove build artifacts
+clean: ## remove build artifacts
 	$(MAKE) -C rpm clean
 	$(MAKE) -C deb clean
 	$(MAKE) -C static clean
 
 .PHONY: rpm
-rpm: DOCKER_BUILD_PKGS:=$(shell find rpm -type d | grep ".*-.*" | sed 's/^rpm\///')
 rpm: ## build rpm packages
-	for p in $(DOCKER_BUILD_PKGS); do \
-		$(MAKE) -C $@ VERSION=$(VERSION) ENGINE_DIR=$(ENGINE_DIR) CLI_DIR=$(CLI_DIR) GO_VERSION=$(GO_VERSION) $${p}; \
-	done
+	$(MAKE) -C $@ VERSION=$(VERSION) ENGINE_DIR=$(ENGINE_DIR) CLI_DIR=$(CLI_DIR) GO_VERSION=$(GO_VERSION) rpm
 
 .PHONY: deb
-deb: DOCKER_BUILD_PKGS:=$(shell find deb -type d | grep ".*-.*" | sed 's/^deb\///')
 deb: ## build deb packages
-	for p in $(DOCKER_BUILD_PKGS); do \
-		$(MAKE) -C $@ VERSION=$(VERSION) ENGINE_DIR=$(ENGINE_DIR) CLI_DIR=$(CLI_DIR) GO_VERSION=$(GO_VERSION) $${p}; \
-	done
+	$(MAKE) -C $@ VERSION=$(VERSION) ENGINE_DIR=$(ENGINE_DIR) CLI_DIR=$(CLI_DIR) GO_VERSION=$(GO_VERSION) deb
 
 .PHONY: static
 static: DOCKER_BUILD_PKGS:=static-linux cross-mac cross-win cross-arm
 static: ## build static-compiled packages
 	for p in $(DOCKER_BUILD_PKGS); do \
 		$(MAKE) -C $@ VERSION=$(VERSION) ENGINE_DIR=$(ENGINE_DIR) CLI_DIR=$(CLI_DIR) GO_VERSION=$(GO_VERSION) $${p}; \
 	done
-
-# TODO - figure out multi-arch
-.PHONY: image
-image: DOCKER_BUILD_PKGS:=image-linux
-image: ## build static-compiled packages
-	for p in $(DOCKER_BUILD_PKGS); do \
-		$(MAKE) -C $@ VERSION=$(VERSION) ENGINE_DIR=$(ENGINE_DIR) CLI_DIR=$(CLI_DIR) GO_VERSION=$(GO_VERSION) $${p}; \
-	done
-
-engine-$(ARCH).tar:
-	$(MAKE) -C image $@
-
-.PHONY: release
-release:
-	$(MAKE) -C image $@

components/packaging/README.md

@@ -7,11 +7,15 @@ This repository is solely maintained by Docker, Inc.
 
 The scripts will build for this list of packages types:
 
+* DEB packages for Ubuntu 19.04 Disco
+* DEB packages for Ubuntu 18.10 Cosmic
 * DEB packages for Ubuntu 18.04 Bionic
 * DEB packages for Ubuntu 16.04 Xenial
 * DEB packages for Debian 10 Buster
 * DEB packages for Debian 9 Stretch
+* RPM packages for Fedora 31
+* RPM packages for Fedora 30
+* RPM packages for Fedora 29
 * RPM packages for Fedora 28
-* RPM packages for Fedora 27
 * RPM packages for CentOS 7
 * TGZ and ZIP files with static binaries

components/packaging/deb/Makefile

@@ -16,7 +16,8 @@ ifdef BUILD_IMAGE
 endif
 
 COMMON_FILES=common
-BUILD?=docker build \
+BUILD?=DOCKER_BUILDKIT=1 \
+	docker build \
 	$(BUILD_IMAGE_FLAG) \
 	--build-arg GO_IMAGE=$(GO_IMAGE) \
 	--build-arg COMMON_FILES=$(COMMON_FILES) \
@@ -34,9 +35,14 @@ RUN=docker run --rm -i \
 	$(RUN_FLAGS) \
 	debbuild-$@/$(ARCH)
 
-SOURCE_FILES=engine-image cli.tgz engine.tgz docker.service docker.socket distribution_based_engine.json plugin-installers.tgz
+SOURCE_FILES=cli.tgz engine.tgz docker.service docker.socket plugin-installers.tgz
 SOURCES=$(addprefix sources/, $(SOURCE_FILES))
 
+DEBIAN_VERSIONS := debian-stretch debian-buster
+UBUNTU_VERSIONS := ubuntu-xenial ubuntu-bionic ubuntu-cosmic ubuntu-disco
+RASPBIAN_VERSIONS := raspbian-stretch raspbian-buster
+DISTROS := $(DEBIAN_VERSIONS) $(UBUNTU_VERSIONS) $(RASPBIAN_VERSIONS)
+
 .PHONY: help
 help: ## show make targets
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf " \033[36m%-20s\033[0m  %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -47,41 +53,22 @@ clean: ## remove build artifacts
 	$(RM) -r debbuild
 	[ ! -d sources ] || $(CHOWN) -R $(shell id -u):$(shell id -g) sources
 	$(RM) -r sources
-	$(RM) engine-image
-
-engine-$(ARCH).tar:
-	$(MAKE) -C ../image image-linux
-	docker save -o $@ $$(cat ../image/image-linux)
 
 .PHONY: deb
-deb: ubuntu debian raspbian ## build all deb packages
+deb: ubuntu debian ## build all deb packages except for raspbian
 
 .PHONY: ubuntu
-ubuntu: ubuntu-bionic ubuntu-xenial ubuntu-disco ubuntu-eoan  ## build all ubuntu deb packages
+ubuntu: $(UBUNTU_VERSIONS) ## build all ubuntu deb packages
 
 .PHONY: debian
-debian: debian-stretch debian-buster ## build all debian deb packages
+debian: $(DEBIAN_VERSIONS) ## build all debian deb packages
 
 .PHONY: raspbian
-raspbian: raspbian-stretch raspbian-buster ## build all raspbian deb packages
-
-.PHONY: ubuntu-%
-## build ubuntu deb packages
-ubuntu-%: $(SOURCES)
-	$(BUILD)
-	$(RUN)
-	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
-
-.PHONY: debian-%
-## build debian deb packages
-debian-%: $(SOURCES)
-	$(BUILD)
-	$(RUN)
-	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
+raspbian: $(RASPBIAN_VERSIONS) ## build all raspbian deb packages
 
-.PHONY: raspbian-%
-## build raspbian deb packages
-raspbian-%: $(SOURCES)
+.PHONY: $(DISTROS)
+$(DISTROS): $(SOURCES)
+	@echo "== Building packages for $@ =="
 	$(BUILD)
 	$(RUN)
 	$(CHOWN) -R $(shell id -u):$(shell id -g) debbuild/$@
@@ -110,18 +97,6 @@ sources/docker.socket: ../systemd/docker.socket
 	mkdir -p $(@D)
 	cp $< $@
 
-# TODO: Figure out how to decouple this
-# TODO: These might just end up being static files that are hardcoded
-# TODO: FROM HERE <=====================
-sources/distribution_based_engine.json: sources/engine-image
-	mkdir -p $(@D)
-	echo '{"platform":"Docker Engine - Community","engine_image":"engine-community","containerd_min_version":"1.2.0-beta.1","runtime":"host_install"}' > $@
-
-sources/engine-image:
-	mkdir -p $(@D)
-	echo "docker.io/dockereng/engine-community:$(STATIC_VERSION)" > $@
-# TODO: TO HERE <=====================
-
 sources/plugin-installers.tgz: $(wildcard ../plugins/*)
 	docker run --rm -i -w /v \
 		-v $(PLUGINS_DIR):/plugins \

components/packaging/deb/common/control

@@ -28,7 +28,7 @@ Vcs-Git: git://github.com/docker/docker.git
 Package: docker-ce
 Architecture: linux-any
 Depends: docker-ce-cli, containerd.io (>= 1.2.2-3), iptables, libseccomp2 (>= 2.3.0), ${shlibs:Depends}
-Recommends: aufs-tools,
+Recommends: aufs-tools [amd64],
             ca-certificates,
             cgroupfs-mount | cgroup-lite,
             git,

components/packaging/deb/common/rules

@@ -49,7 +49,6 @@ override_dh_auto_install:
 	install -D -m 0755 $(shell readlink -e engine/bundles/dynbinary-daemon/dockerd) debian/docker-ce/usr/bin/dockerd
 	install -D -m 0755 /usr/local/bin/docker-proxy debian/docker-ce/usr/bin/docker-proxy
 	install -D -m 0755 /usr/local/bin/docker-init debian/docker-ce/usr/bin/docker-init
-	install -D -m 0644 /sources/distribution_based_engine.json debian/docker-ce/var/lib/docker-engine/distribution_based_engine.json
 
 override_dh_installinit:
 	# use "docker" as our service name, not "docker-ce"

components/packaging/deb/debian-buster/Dockerfile

@@ -1,12 +1,16 @@
 ARG GO_IMAGE
-ARG BUILD_IMAGE=debian:buster
-FROM ${GO_IMAGE} as golang
+ARG DISTRO=debian
+ARG SUITE=buster
+ARG BUILD_IMAGE=${DISTRO}:${SUITE}
+
+FROM ${GO_IMAGE} AS golang
 
 FROM ${BUILD_IMAGE}
 
 RUN apt-get update && apt-get install -y curl devscripts equivs git
 
-ARG GO_VERSION
+ENV GOPROXY=direct
+ENV GO111MODULE=off
 ENV GOPATH /go
 ENV PATH $PATH:/usr/local/go/bin:$GOPATH/bin
 ENV DOCKER_BUILDTAGS apparmor seccomp selinux
@@ -17,9 +21,10 @@ COPY ${COMMON_FILES} /root/build-deb/debian
 RUN mk-build-deps -t "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y" -i /root/build-deb/debian/control
 
 COPY sources/ /sources
-
-ENV DISTRO debian
-ENV SUITE buster
+ARG DISTRO
+ARG SUITE
+ENV DISTRO=${DISTRO}
+ENV SUITE=${SUITE}
 
 COPY --from=golang /usr/local/go /usr/local/go
 

components/packaging/deb/debian-stretch/Dockerfile

@@ -1,12 +1,16 @@
 ARG GO_IMAGE
-ARG BUILD_IMAGE=debian:stretch
-FROM ${GO_IMAGE} as golang
+ARG DISTRO=debian
+ARG SUITE=stretch
+ARG BUILD_IMAGE=${DISTRO}:${SUITE}
+
+FROM ${GO_IMAGE} AS golang
 
 FROM ${BUILD_IMAGE}
 
 RUN apt-get update && apt-get install -y curl devscripts equivs git
 
-ARG GO_VERSION
+ENV GOPROXY=direct
+ENV GO111MODULE=off
 ENV GOPATH /go
 ENV PATH $PATH:/usr/local/go/bin:$GOPATH/bin
 ENV DOCKER_BUILDTAGS apparmor seccomp selinux
@@ -17,9 +21,10 @@ COPY ${COMMON_FILES} /root/build-deb/debian
 RUN mk-build-deps -t "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends -y" -i /root/build-deb/debian/control
 
 COPY sources/ /sources
-
-ENV DISTRO debian
-ENV SUITE stretch
+ARG DISTRO
+ARG SUITE
+ENV DISTRO=${DISTRO}
+ENV SUITE=${SUITE}
 
 COPY --from=golang /usr/local/go /usr/local/go