[Buildroot] [PATCH v3 1/6] package/go: implement go modules integration

[Buildroot] [PATCH v3 1/6] package/go: implement go modules integration

[Christian Stewart]

This commit moves from the GOPATH mechanism to the new GO111MODULE approach for
Go based packages. Old Go packages compile with small tweaks.

The Go module system replaces the GOPATH mechanism by allowing the Go tool to
work with packages correctly without a GOPATH tree.

Reference: https://github.com/golang/go/wiki/Modules

Signed-off-by: Christian Stewart <christian@paral.in>

v2 -> v3:

 - cjs: cleaned up spelling and moved extract hook to configure step
 - cjs: applied fixes from vincent fazio related to host packages

Signed-off-by: Christian Stewart <christian@paral.in>
---
 package/go/go.mk      |  8 ++++++--
 package/pkg-golang.mk | 31 +++++++++++++++++++------------
 2 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/package/go/go.mk b/package/go/go.mk
index 376c3db7f0..1291874b1d 100644
--- a/package/go/go.mk
+++ b/package/go/go.mk
@@ -15,6 +15,8 @@ HOST_GO_DEPENDENCIES = host-go-bootstrap
 HOST_GO_HOST_CACHE = $(HOST_DIR)/usr/share/host-go-cache
 HOST_GO_ROOT = $(HOST_DIR)/lib/go
 HOST_GO_TARGET_CACHE = $(HOST_DIR)/usr/share/go-cache
+# used for go module downloads
+HOST_GO_GOPATH = $(DL_DIR)/go-module
 
 ifeq ($(BR2_PACKAGE_HOST_GO_TARGET_ARCH_SUPPORTS),y)
 
@@ -43,13 +45,15 @@ else ifeq ($(BR2_mips64el),y)
 GO_GOARCH = mips64le
 endif
 
-# For the convienience of target packages.
+# For the convenience of target packages.
 HOST_GO_TOOLDIR = $(HOST_GO_ROOT)/pkg/tool/linux_$(GO_GOARCH)
 HOST_GO_TARGET_ENV = \
-	GO111MODULE=off \
+	GO111MODULE=on \
 	GOARCH=$(GO_GOARCH) \
 	GOCACHE="$(HOST_GO_TARGET_CACHE)" \
+	GOPROXY=off \
 	GOROOT="$(HOST_GO_ROOT)" \
+	GOPATH="$(HOST_GO_GOPATH)" \
 	CC="$(TARGET_CC)" \
 	CXX="$(TARGET_CXX)" \
 	GOTOOLDIR="$(HOST_GO_TOOLDIR)"
diff --git a/package/pkg-golang.mk b/package/pkg-golang.mk
index 2494ce028c..7b5c263806 100644
--- a/package/pkg-golang.mk
+++ b/package/pkg-golang.mk
@@ -35,8 +35,16 @@ GO_TARGET_ENV = \
 	$(GO_COMMON_ENV)
 
 GO_HOST_ENV = \
+	CC=$(HOSTCC_NOCCACHE) \
+	CXX=$(HOSTCXX_NOCCACHE) \
 	CGO_CFLAGS="$(HOST_CFLAGS)" \
 	CGO_LDFLAGS="$(HOST_LDFLAGS)" \
+	GO111MODULE=on \
+	GOCACHE="$(HOST_GO_TARGET_CACHE)" \
+	GOPROXY=off \
+	GOROOT="$(HOST_GO_ROOT)" \
+	GOPATH="$(HOST_GO_GOPATH)" \
+	GOTOOLDIR="$(HOST_GO_TOOLDIR)" \
 	$(GO_COMMON_ENV)
 
 ################################################################################
@@ -60,6 +68,7 @@ $(2)_WORKSPACE ?= _gopath
 
 $(2)_BUILD_OPTS += \
 	-ldflags "$$($(2)_LDFLAGS)" \
+	-mod=vendor \
 	-tags "$$($(2)_TAGS)" \
 	-trimpath \
 	-p $(PARALLEL_JOBS)
@@ -79,23 +88,23 @@ endif
 
 $(2)_INSTALL_BINS ?= $(1)
 
-# Source files in Go should be extracted in a precise folder in the hierarchy
-# of GOPATH. It usually resolves around domain/vendor/software. By default, we
-# derive domain/vendor/software from the upstream URL of the project, but we
-# allow $(2)_SRC_SUBDIR to be overridden if needed.
+# Source files in Go usually use an import path resolved around
+# domain/vendor/software. We infer domain/vendor/software from the upstream URL
+# of the project. $(2)_GOMOD can be overridden.
 $(2)_SRC_DOMAIN = $$(call domain,$$($(2)_SITE))
 $(2)_SRC_VENDOR = $$(word 1,$$(subst /, ,$$(call notdomain,$$($(2)_SITE))))
 $(2)_SRC_SOFTWARE = $$(word 2,$$(subst /, ,$$(call notdomain,$$($(2)_SITE))))
+$(2)_SRC_PATH = $$(@D)
 
-$(2)_SRC_SUBDIR ?= $$($(2)_SRC_DOMAIN)/$$($(2)_SRC_VENDOR)/$$($(2)_SRC_SOFTWARE)
-$(2)_SRC_PATH = $$(@D)/$$($(2)_WORKSPACE)/src/$$($(2)_SRC_SUBDIR)
+$(2)_GOMOD ?= $$($(2)_SRC_DOMAIN)/$$($(2)_SRC_VENDOR)/$$($(2)_SRC_SOFTWARE)
 
 # Configure step. Only define it if not already defined by the package .mk
 # file.
 ifndef $(2)_CONFIGURE_CMDS
 define $(2)_CONFIGURE_CMDS
-	mkdir -p $$(dir $$($(2)_SRC_PATH))
-	ln -sf $$(@D) $$($(2)_SRC_PATH)
+	if [ ! -f $$($(2)_SRC_PATH)/go.mod ] && [ -n "$$($(2)_GOMOD)" ]; then \
+		printf "module $$($(2)_GOMOD)\n" > $$($(2)_SRC_PATH)/go.mod; \
+	fi
 endef
 endif
 
@@ -113,11 +122,10 @@ define $(2)_BUILD_CMDS
 	$$(foreach d,$$($(2)_BUILD_TARGETS),\
 		cd $$($(2)_SRC_PATH); \
 		$$(GO_TARGET_ENV) \
-			GOPATH="$$(@D)/$$($(2)_WORKSPACE)" \
 			$$($(2)_GO_ENV) \
 			$$(GO_BIN) build -v $$($(2)_BUILD_OPTS) \
 			-o $$(@D)/bin/$$(or $$($(2)_BIN_NAME),$$(notdir $$(d))) \
-			./$$(d)
+			$$(d)
 	)
 endef
 else
@@ -126,11 +134,10 @@ define $(2)_BUILD_CMDS
 	$$(foreach d,$$($(2)_BUILD_TARGETS),\
 		cd $$($(2)_SRC_PATH); \
 		$$(GO_HOST_ENV) \
-			GOPATH="$$(@D)/$$($(2)_WORKSPACE)" \
 			$$($(2)_GO_ENV) \
 			$$(GO_BIN) build -v $$($(2)_BUILD_OPTS) \
 			-o $$(@D)/bin/$$(or $$($(2)_BIN_NAME),$$(notdir $$(d))) \
-			./$$(d)
+			$$(d)
 	)
 endef
 endif
-- 
2.25.1

[Buildroot] [PATCH v3 2/6] package/runc: remove unnecessary workspace identifier

[Christian Stewart]

Signed-off-by: Christian Stewart <christian@paral.in>
---
 package/runc/runc.mk | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/package/runc/runc.mk b/package/runc/runc.mk
index 4c2f84ab16..aedfb5ef9e 100644
--- a/package/runc/runc.mk
+++ b/package/runc/runc.mk
@@ -9,10 +9,7 @@ RUNC_SITE = $(call github,opencontainers,runc,v$(RUNC_VERSION))
 RUNC_LICENSE = Apache-2.0
 RUNC_LICENSE_FILES = LICENSE
 
-RUNC_WORKSPACE = Godeps/_workspace
-
 RUNC_LDFLAGS = -X main.gitCommit=$(RUNC_VERSION)
-
 RUNC_TAGS = cgo static_build
 
 ifeq ($(BR2_PACKAGE_LIBSECCOMP),y)
-- 
2.25.1

[Buildroot] [PATCH v3 3/6] package/docker-containerd: fix go-module package identifier/targets

[Christian Stewart]

Signed-off-by: Christian Stewart <christian@paral.in>
---
 package/docker-containerd/docker-containerd.mk | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/package/docker-containerd/docker-containerd.mk b/package/docker-containerd/docker-containerd.mk
index 67de8d7468..0a971850d0 100644
--- a/package/docker-containerd/docker-containerd.mk
+++ b/package/docker-containerd/docker-containerd.mk
@@ -9,13 +9,14 @@ DOCKER_CONTAINERD_SITE = $(call github,containerd,containerd,v$(DOCKER_CONTAINER
 DOCKER_CONTAINERD_LICENSE = Apache-2.0
 DOCKER_CONTAINERD_LICENSE_FILES = LICENSE
 
-DOCKER_CONTAINERD_WORKSPACE = vendor
-
+DOCKER_CONTAINERD_GOMOD = github.com/containerd/containerd
 DOCKER_CONTAINERD_LDFLAGS = \
-	-X github.com/docker/containerd.GitCommit=$(DOCKER_CONTAINERD_VERSION)
-
-DOCKER_CONTAINERD_BUILD_TARGETS = cmd/ctr cmd/containerd cmd/containerd-shim
+	-X $(DOCKER_CONTAINERD_GOMOD).GitCommit=$(DOCKER_CONTAINERD_VERSION)
 
+DOCKER_CONTAINERD_BUILD_TARGETS = \
+	$(DOCKER_CONTAINERD_GOMOD)/cmd/ctr \
+	$(DOCKER_CONTAINERD_GOMOD)/cmd/containerd \
+	$(DOCKER_CONTAINERD_GOMOD)/cmd/containerd-shim
 DOCKER_CONTAINERD_INSTALL_BINS = containerd containerd-shim
 
 ifeq ($(BR2_PACKAGE_LIBSECCOMP),y)
-- 
2.25.1

[Buildroot] [PATCH v3 4/6] package/docker-engine: fix go-module package identifier

[Christian Stewart]

Signed-off-by: Christian Stewart <christian@paral.in>
---
 package/docker-engine/docker-engine.mk | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/package/docker-engine/docker-engine.mk b/package/docker-engine/docker-engine.mk
index eb3a7fd29f..e44d561b45 100644
--- a/package/docker-engine/docker-engine.mk
+++ b/package/docker-engine/docker-engine.mk
@@ -11,14 +11,14 @@ DOCKER_ENGINE_LICENSE = Apache-2.0
 DOCKER_ENGINE_LICENSE_FILES = LICENSE
 
 DOCKER_ENGINE_DEPENDENCIES = host-pkgconf
-DOCKER_ENGINE_SRC_SUBDIR = github.com/docker/docker
 
 DOCKER_ENGINE_LDFLAGS = \
 	-X main.GitCommit=$(DOCKER_ENGINE_VERSION) \
 	-X main.Version=$(DOCKER_ENGINE_VERSION)
-
 DOCKER_ENGINE_TAGS = cgo exclude_graphdriver_zfs autogen
-DOCKER_ENGINE_BUILD_TARGETS = cmd/dockerd
+
+DOCKER_ENGINE_GOMOD = github.com/docker/docker
+DOCKER_ENGINE_BUILD_TARGETS = $(DOCKER_ENGINE_GOMOD)/cmd/dockerd
 
 ifeq ($(BR2_PACKAGE_LIBSECCOMP),y)
 DOCKER_ENGINE_TAGS += seccomp
-- 
2.25.1

[Buildroot] [PATCH v3 5/6] package/docker-cli: fix go-module package identifier

[Christian Stewart]

Signed-off-by: Christian Stewart <christian@paral.in>
---
 package/docker-cli/docker-cli.mk | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/package/docker-cli/docker-cli.mk b/package/docker-cli/docker-cli.mk
index 4ad30e0278..12fb563885 100644
--- a/package/docker-cli/docker-cli.mk
+++ b/package/docker-cli/docker-cli.mk
@@ -13,12 +13,13 @@ DOCKER_CLI_LICENSE_FILES = LICENSE
 
 DOCKER_CLI_DEPENDENCIES = host-pkgconf
 
-DOCKER_CLI_TAGS = autogen
-DOCKER_CLI_BUILD_TARGETS = cmd/docker
-
+DOCKER_CLI_GOMOD = github.com/docker/cli
 DOCKER_CLI_LDFLAGS = \
-	-X github.com/docker/cli/cli.GitCommit=$(DOCKER_CLI_VERSION) \
-	-X github.com/docker/cli/cli.Version=$(DOCKER_CLI_VERSION)
+	-X $(DOCKER_CLI_GOMOD).GitCommit=$(DOCKER_CLI_VERSION) \
+	-X $(DOCKER_CLI_GOMOD).Version=$(DOCKER_CLI_VERSION)
+
+DOCKER_CLI_BUILD_TARGETS = $(DOCKER_CLI_GOMOD)/cmd/docker
+DOCKER_CLI_TAGS = autogen
 
 ifeq ($(BR2_PACKAGE_DOCKER_CLI_STATIC),y)
 DOCKER_CLI_LDFLAGS += -extldflags '-static'
-- 
2.25.1

[Buildroot] [PATCH v3 6/6] package/docker-proxy: fix go-module package identifier

[Christian Stewart]

Signed-off-by: Christian Stewart <christian@paral.in>
---
 package/docker-proxy/docker-proxy.mk | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/package/docker-proxy/docker-proxy.mk b/package/docker-proxy/docker-proxy.mk
index 8843266c30..41ae07da09 100644
--- a/package/docker-proxy/docker-proxy.mk
+++ b/package/docker-proxy/docker-proxy.mk
@@ -12,9 +12,7 @@ DOCKER_PROXY_LICENSE_FILES = LICENSE
 
 DOCKER_PROXY_DEPENDENCIES = host-pkgconf
 
-DOCKER_PROXY_WORKSPACE = gopath
-
-DOCKER_PROXY_BUILD_TARGETS = cmd/proxy
+DOCKER_PROXY_BUILD_TARGETS = github.com/docker/libnetwork/cmd/proxy
 
 define DOCKER_PROXY_INSTALL_TARGET_CMDS
 	$(INSTALL) -D -m 0755 $(@D)/bin/proxy $(TARGET_DIR)/usr/bin/docker-proxy
-- 
2.25.1

[Buildroot] [PATCH v3 1/6] package/go: implement go modules integration

[Christian Stewart]

Hi all,


On Sat, Feb 29, 2020 at 11:52 PM Christian Stewart <christian@paral.in> wrote:
> This commit moves from the GOPATH mechanism to the new GO111MODULE approach for
> Go based packages. Old Go packages compile with small tweaks.
>
> The Go module system replaces the GOPATH mechanism by allowing the Go tool to
> work with packages correctly without a GOPATH tree.

I have been using this series for everything, including my daily
driver, since submitting this patch.

It works fine and most importantly works with both the old vendor/
approach as well as the new go.mod approach.

In this version of the series the Go compiler is never asked to
download anything. It merely replaces the old "gopath" hacks that were
in place before. A future patch series on top of this one would add
support for Go downloading packages as well, however, for now that is
not part of this series. This is as minimal as possible.

I recommend merging the series as is (in v3) as soon as possible.

Best regards,
Christian Stewart

[Buildroot] [PATCH v3 1/6] package/go: implement go modules integration

[Christian Stewart]

Hi all,

Was there any particular blocker / issue with this transition to Go
modules which needs addressing?

It should be fairly straightforward: this initial pass of the series
will allow the Go compiler to automatically resolve the "vendor"
directories rather than needing to build a symlinked GOPATH directory.
Everything that worked before should continue to work now, otherwise.

As reported on the Go release docs:

Module support in the go command is now ready for production use, and
we encourage all users to migrate to Go modules for dependency
management. If you are unable to migrate due to a problem in the Go
toolchain, please ensure that the problem has an open issue filed.

Best,
Christian Stewart

On Sat, Jun 6, 2020 at 4:22 PM Christian Stewart <christian@paral.in> wrote:
>
> Hi all,
>
>
> On Sat, Feb 29, 2020 at 11:52 PM Christian Stewart <christian@paral.in> wrote:
> > This commit moves from the GOPATH mechanism to the new GO111MODULE approach for
> > Go based packages. Old Go packages compile with small tweaks.
> >
> > The Go module system replaces the GOPATH mechanism by allowing the Go tool to
> > work with packages correctly without a GOPATH tree.
>
> I have been using this series for everything, including my daily
> driver, since submitting this patch.
>
> It works fine and most importantly works with both the old vendor/
> approach as well as the new go.mod approach.
>
> In this version of the series the Go compiler is never asked to
> download anything. It merely replaces the old "gopath" hacks that were
> in place before. A future patch series on top of this one would add
> support for Go downloading packages as well, however, for now that is
> not part of this series. This is as minimal as possible.
>
> I recommend merging the series as is (in v3) as soon as possible.
>
> Best regards,
> Christian Stewart