[Buildroot] [RFC v2 1/4] package/go: fixing crosscompilation settings

[Buildroot] [RFC v2 1/4] package/go: fixing crosscompilation settings

[Angelo Compagnucci]

This patch fixes a bug with the BR2_TOOLCHAIN_HAS_THREADS variable
handling which causes CGO_ENABLED to be always 0.

Furthermore, it fixes the cross compilation options for the go compiler:
setting CGO_ENABLED should be done only for the target compiler not the
host one.

Signed-off-by: Angelo Compagnucci <angelo.compagnucci@gmail.com>
---
 package/go/go.mk | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/package/go/go.mk b/package/go/go.mk
index 0781dff..6035164 100644
--- a/package/go/go.mk
+++ b/package/go/go.mk
@@ -52,7 +52,7 @@ HOST_GO_TARGET_ENV = \
 # set, build in cgo support for any go programs that may need it.  Note that
 # any target package needing cgo support must include
 # 'depends on BR2_TOOLCHAIN_HAS_THREADS' in its config file.
-ifeq (BR2_TOOLCHAIN_HAS_THREADS,y)
+ifeq ($(BR2_TOOLCHAIN_HAS_THREADS),y)
 HOST_GO_CGO_ENABLED = 1
 else
 HOST_GO_CGO_ENABLED = 0
@@ -74,8 +74,8 @@ HOST_GO_MAKE_ENV = \
 	GOARCH=$(GO_GOARCH) \
 	$(if $(GO_GOARM),GOARM=$(GO_GOARM)) \
 	GOOS=linux \
-	CGO_ENABLED=$(HOST_GO_CGO_ENABLED) \
-	CC=$(HOSTCC_NOCCACHE)
+	CC=$(HOSTCC_NOCCACHE) \
+	CXX=$(HOSTCXX_NOCCACHE)
 
 HOST_GO_TARGET_CC = \
 	CC_FOR_TARGET="$(TARGET_CC)" \
@@ -83,16 +83,18 @@ HOST_GO_TARGET_CC = \
 
 HOST_GO_HOST_CC = \
 	CC_FOR_TARGET=$(HOSTCC_NOCCACHE) \
-	CXX_FOR_TARGET=$(HOSTCC_NOCCACHE)
+	CXX_FOR_TARGET=$(HOSTCXX_NOCCACHE)
 
 HOST_GO_TMP = $(@D)/host-go-tmp
 
 define HOST_GO_BUILD_CMDS
-	cd $(@D)/src && $(HOST_GO_MAKE_ENV) $(HOST_GO_HOST_CC) ./make.bash
+	cd $(@D)/src && \
+		$(HOST_GO_MAKE_ENV) $(HOST_GO_HOST_CC) CGO_ENABLED=0 ./make.bash
 	mkdir -p $(HOST_GO_TMP)
 	mv $(@D)/pkg/tool $(HOST_GO_TMP)/
 	mv $(@D)/bin/ $(HOST_GO_TMP)/
-	cd $(@D)/src && $(HOST_GO_MAKE_ENV) $(HOST_GO_TARGET_CC) ./make.bash
+	cd $(@D)/src && \
+		$(HOST_GO_MAKE_ENV) $(HOST_GO_TARGET_CC) CGO_ENABLED=$(HOST_GO_CGO_ENABLED) ./make.bash
 endef
 
 define HOST_GO_INSTALL_CMDS
-- 
2.7.4

[Buildroot] [RFC v2 2/4] package/pkg-golang: new package infrastructure

[Angelo Compagnucci]

This patch adds a new infrastructure for golang based packages.

Signed-off-by: Angelo Compagnucci <angelo.compagnucci@gmail.com>
---
 package/Makefile.in   |   1 +
 package/pkg-golang.mk | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+)
 create mode 100644 package/pkg-golang.mk

diff --git a/package/Makefile.in b/package/Makefile.in
index a1a5316..60d98d0 100644
--- a/package/Makefile.in
+++ b/package/Makefile.in
@@ -429,3 +429,4 @@ include package/pkg-kconfig.mk
 include package/pkg-rebar.mk
 include package/pkg-kernel-module.mk
 include package/pkg-waf.mk
+include package/pkg-golang.mk
diff --git a/package/pkg-golang.mk b/package/pkg-golang.mk
new file mode 100644
index 0000000..c4e6410
--- /dev/null
+++ b/package/pkg-golang.mk
@@ -0,0 +1,121 @@
+################################################################################
+# Golang package infrastructure
+#
+# This file implements an infrastructure that eases development of
+# package .mk files for Go packages. It should be used for all
+# packages that are written in go.
+#
+# See the Buildroot documentation for details on the usage of this
+# infrastructure
+#
+#
+# In terms of implementation, this golang infrastructure requires
+# the .mk file to only specify metadata information about the
+# package: name, version, download URL, etc.
+#
+# We still allow the package .mk file to override what the different
+# steps are doing, if needed. For example, if <PKG>_BUILD_CMDS is
+# already defined, it is used as the list of commands to perform to
+# build the package, instead of the default golang behaviour. The
+# package can also define some post operation hooks.
+#
+################################################################################
+
+################################################################################
+# inner-golang-package -- defines how the configuration, compilation and
+# installation of a Go package should be done, implements a few hooks to
+# tune the build process for Go specifities and calls the generic package
+# infrastructure to generate the necessary make targets
+#
+#  argument 1 is the lowercase package name
+#  argument 2 is the uppercase package name, including a HOST_ prefix
+#             for host packages
+#  argument 3 is the uppercase package name, without the HOST_ prefix
+#             for host packages
+#  argument 4 is the type (target or host)
+################################################################################
+
+define inner-golang-package
+
+ifndef $(2)_MAKE_ENV
+define $(2)_MAKE_ENV
+	$$(HOST_GO_TARGET_ENV) \
+	GOPATH="$$(@D)/gopath" \
+	CGO_ENABLED=$$(HOST_GO_CGO_ENABLED)
+endef
+endif
+
+# Target packages need the Go compiler on the host.
+$(2)_DEPENDENCIES += host-go
+
+#
+# go install command doesn't work well when cross compilation is enabled,
+# we set the executable output of the compilation to a specific location.
+# We set this variable here to be used by packages if needed.
+#
+$(2)_EXECUTABLE = $$(@D)/gopath/bin/$(1)
+
+#
+# Source files in Go should be uncompressed in a precise folder in the
+# hiearchy of GOPATH. It usually resolves around domain/vendor/software.
+#
+$(1)_src_path ?= $$(call domain,$($(2)_SITE))/$$(firstword $$(subst /, ,$$(call notdomain,$($(2)_SITE))))
+$(2)_SRC_PATH  = $$(@D)/gopath/src/$$($(1)_src_path)/$(1)
+
+#
+# Configure step. Only define it if not already defined by the package
+# .mk file. And take care of the differences between host and target
+# packages.
+#
+ifndef $(2)_CONFIGURE_CMDS
+define $(2)_CONFIGURE_CMDS
+	mkdir -p $$(@D)/gopath/bin
+	mkdir -p $$(@D)/gopath/src/$$($(1)_src_path)
+	ln -sf $$(@D) $$($(2)_SRC_PATH)
+endef
+endif
+
+#
+# Build step. Only define it if not already defined by the package .mk file.
+# There is no differences between host and target packages.
+#
+ifndef $(2)_BUILD_CMDS
+define $(2)_BUILD_CMDS
+	cd $$($(2)_SRC_PATH) && $$($(2)_MAKE_ENV) $(HOST_DIR)/bin/go build \
+	-o $$($(2)_EXECUTABLE) -v $$($(2)_BUILD_OPTS)
+endef
+endif
+
+#
+# Host installation step. Only define it if not already defined by the
+# package .mk file.
+#
+ifndef $(2)_INSTALL_CMDS
+define $(2)_INSTALL_CMDS
+	$(INSTALL) -D -m 0755 $$($(2)_EXECUTABLE) $(HOST_DIR)/usr/bin/
+endef
+endif
+
+#
+# Target installation step. Only define it if not already defined by the
+# package .mk file.
+#
+ifndef $(2)_INSTALL_TARGET_CMDS
+define $(2)_INSTALL_TARGET_CMDS
+	$(INSTALL) -D -m 0755 $$($(2)_EXECUTABLE) $(TARGET_DIR)/usr/bin/
+endef
+endif
+
+# Call the generic package infrastructure to generate the necessary make
+# targets
+$(call inner-generic-package,$(1),$(2),$(3),$(4))
+
+endef # inner-golang-package
+
+################################################################################
+# golang-package -- the target generator macro for Go packages
+################################################################################
+
+golang-package = $(call inner-golang-package,$(pkgname),$(call UPPERCASE,$(pkgname)),$(call UPPERCASE,$(pkgname)),target)
+host-golang-package = $(call inner-golang-package,host-$(pkgname),$(call UPPERCASE,host-$(pkgname)),$(call UPPERCASE,$(pkgname)),host)
+
-- 
2.7.4

[Buildroot] [RFC v2 3/4] docs/manual: adding documentation for the golang infrastructure

[Angelo Compagnucci]

This patch adds the documentation for the golang infrastructure.

Signed-off-by: Angelo Compagnucci <angelo.compagnucci@gmail.com>
---
 docs/manual/adding-packages-golang.txt | 111 +++++++++++++++++++++++++++++++++
 docs/manual/adding-packages.txt        |   2 +
 2 files changed, 113 insertions(+)
 create mode 100644 docs/manual/adding-packages-golang.txt

diff --git a/docs/manual/adding-packages-golang.txt b/docs/manual/adding-packages-golang.txt
new file mode 100644
index 0000000..2ba45f0
--- /dev/null
+++ b/docs/manual/adding-packages-golang.txt
@@ -0,0 +1,111 @@
+// -*- mode:doc; -*-
+// vim: set syntax=asciidoc:
+
+=== Infrastructure for Go packages
+
+This infrastructure applies to Go packages that use the standard
+build system and use bundled dependencies.
+
+[[golang-package-tutorial]]
+
+==== +golang-package+ tutorial
+
+First, let's see how to write a +.mk+ file for a go package,
+with an example :
+
+------------------------
+01: ################################################################################
+02: #
+03: # go-foo
+04: #
+05: ################################################################################
+06:
+07: GO_FOO_VERSION = 1.0
+08: GO_FOO_SOURCE = go-foo-$(GO_FOO_VERSION).tar.xz
+09: GO_FOO_SITE = http://www.foosoftware.org/download
+10: GO_FOO_LICENSE = BSD-3-Clause
+11: GO_FOO_LICENSE_FILES = LICENSE
+12:
+13: $(eval $(golang-package))
+------------------------
+
+On line 7, we declare the version of the package.
+
+On line 8 and 9, we declare the name of the tarball (xz-ed tarball
+recommended) and the location of the tarball on the Web. Buildroot
+will automatically download the tarball from this location.
+
+On line 10 and 11, we give licensing details about the package (its
+license on line 10, and the file containing the license text on line
+11).
+
+Finally, on line 13, we invoke the +golang-package+ macro that
+generates all the Makefile rules that actually allow the package to be
+built.
+
+[[golang-package-reference]]
+
+==== +golang-package+ reference
+
+As a policy packages can freely choose their name (existing example in 
+Buildroot is +flannel+).
+
+In their +Config.in+ file, they should depend on 
++BR2_PACKAGE_HOST_GO_ARCH_SUPPORTS+ and
++BR2_PACKAGE_HOST_GO_CGO_LINKING_SUPPORTS+ cause host-go will compile when Buildroot will add the
+dependency automatically.
+
+The main macro of the Go package infrastructure is
++golang-package+. It is similar to the +generic-package+ macro. It is
+also possible to create Go host packages with the
++host-golang-package+ macro.
+
+Just like the generic infrastructure, the Go infrastructure works
+by defining a number of variables before calling the +golang-package+
+or +host-golang-package+ macros.
+
+All the package metadata information variables that exist in the
+xref:generic-package-reference[generic package infrastructure] also
+exist in the Go infrastructure: +GO_FOO_VERSION+,
++GO_FOO_SOURCE+, +GO_FOO_PATCH+, +GO_FOO_SITE+,
++GO_FOO_SUBDIR+, +GO_FOO_DEPENDENCIES+, +GO_FOO_LICENSE+,
++GO_FOO_LICENSE_FILES+, +GO_FOO_INSTALL_STAGING+, etc.
+
+Note that:
+
+ * It is not necessary to add +go+ or +host-go+ in the
+   +GO_FOO_DEPENDENCIES+ variable of a package, since these basic
+   dependencies are automatically added as needed by the Go
+   package infrastructure.
+
+A few additional variables, specific to the Go infrastructure, can
+optionally be defined, depending on the package's needs. Many of them
+are only useful in very specific cases, typical packages will
+therefore only use a few of them, or none.
+
+* +GO_FOO_GO_SRC_PATH+: go sources should be compiled inside GOPATH.
+  The golang package infrastructure tries to guess the correct 
+  GOPATH subfolder to compile in. If guessing is not correct or your
+  package behaves differently, you can use this variable to
+  adjust the path.
+  
+
+* +GO_FOO_BUILD_OPTS+, to specify additional options to pass to the
+  Go +setup.py+ script during the build step. For target distutils
+  packages, the +PKG_GO_DISTUTILS_BUILD_OPTS+ options are already
+  passed automatically by the infrastructure.
+
+With the Go infrastructure, all the steps required to build and
+install the packages are already defined, and they generally work well
+for most Go-based packages. However, when required, it is still
+possible to customize what is done in any particular step:
+
+* By adding a post-operation hook (after extract, patch, configure,
+  build or install). See xref:hooks[] for details.
+
+* By overriding one of the steps. For example, even if the Go
+  infrastructure is used, if the package +.mk+ file defines its own
+  +GO_FOO_BUILD_CMDS+ variable, it will be used instead of the
+  default Go one. However, using this method should be restricted
+  to very specific cases. Do not use it in the general case.
+
diff --git a/docs/manual/adding-packages.txt b/docs/manual/adding-packages.txt
index d577ff0..be7468b 100644
--- a/docs/manual/adding-packages.txt
+++ b/docs/manual/adding-packages.txt
@@ -34,6 +34,8 @@ include::adding-packages-rebar.txt[]
 
 include::adding-packages-waf.txt[]
 
+include::adding-packages-golang.txt[]
+
 include::adding-packages-kernel-module.txt[]
 
 include::adding-packages-asciidoc.txt[]
-- 
2.7.4

[Buildroot] [RFC v2 4/4] package/flannel: converting to golang infrastructure

[Angelo Compagnucci]

This patch converts the flannel package to the new golang
infrastructure.

Signed-off-by: Angelo Compagnucci <angelo.compagnucci@gmail.com>
---
 package/flannel/flannel.mk | 25 ++++---------------------
 1 file changed, 4 insertions(+), 21 deletions(-)

diff --git a/package/flannel/flannel.mk b/package/flannel/flannel.mk
index bbb2c72..641bb97 100644
--- a/package/flannel/flannel.mk
+++ b/package/flannel/flannel.mk
@@ -11,14 +11,6 @@ FLANNEL_SOURCE = $(FLANNEL_VERSION).tar.gz
 FLANNEL_LICENSE = Apache-2.0
 FLANNEL_LICENSE_FILES = LICENSE
 
-FLANNEL_DEPENDENCIES = host-go
-
-FLANNEL_MAKE_ENV = \
-	$(HOST_GO_TARGET_ENV) \
-	GOBIN="$(@D)/bin" \
-	GOPATH="$(@D)/gopath" \
-	CGO_ENABLED=1
-
 FLANNEL_GLDFLAGS = \
 	-X github.com/coreos/flannel/version.Version=$(FLANNEL_VERSION)
 
@@ -26,21 +18,12 @@ ifeq ($(BR2_STATIC_LIBS),y)
 FLANNEL_GLDFLAGS += -extldflags '-static'
 endif
 
-define FLANNEL_CONFIGURE_CMDS
-	# Put sources at prescribed GOPATH location.
-	mkdir -p $(@D)/gopath/src/github.com/coreos
-	ln -s $(@D) $(@D)/gopath/src/github.com/coreos/flannel
-endef
-
-define FLANNEL_BUILD_CMDS
-	cd $(@D) && $(FLANNEL_MAKE_ENV) $(HOST_DIR)/bin/go \
-		build -v -o $(@D)/bin/flanneld -ldflags "$(FLANNEL_GLDFLAGS)" .
-endef
+FLANNEL_BUILD_OPTS = -ldflags "$(FLANNEL_GLDFLAGS)"
 
+# Install flannel to its well known location.
 define FLANNEL_INSTALL_TARGET_CMDS
-	# Install flannel to its well known location.
-	$(INSTALL) -D -m 0755 $(@D)/bin/flanneld $(TARGET_DIR)/opt/bin/flanneld
+	$(INSTALL) -D -m 0755 $(FLANNEL_EXECUTABLE) $(TARGET_DIR)/opt/bin/flanneld
 	$(INSTALL) -D -m 0755 $(@D)/dist/mk-docker-opts.sh $(TARGET_DIR)/opt/bin/mk-docker-opts.sh
 endef
 
-$(eval $(generic-package))
+$(eval $(golang-package))
-- 
2.7.4

[Buildroot] [RFC v2 1/4] package/go: fixing crosscompilation settings

[Thomas Petazzoni]

Hello,

On Mon, 16 Oct 2017 19:08:45 +0200, Angelo Compagnucci wrote:
> This patch fixes a bug with the BR2_TOOLCHAIN_HAS_THREADS variable
> handling which causes CGO_ENABLED to be always 0.
> 
> Furthermore, it fixes the cross compilation options for the go compiler:
> setting CGO_ENABLED should be done only for the target compiler not the
> host one.
> 
> Signed-off-by: Angelo Compagnucci <angelo.compagnucci@gmail.com>
> ---
>  package/go/go.mk | 14 ++++++++------
>  1 file changed, 8 insertions(+), 6 deletions(-)

Thanks applied the patch, with Christian's Acked-by.

Thomas
-- 
Thomas Petazzoni, CTO, Free Electrons
Embedded Linux, Kernel and Android engineering
http://free-electrons.com

[Buildroot] [RFC v2 2/4] package/pkg-golang: new package infrastructure

[Thomas Petazzoni]

Hello,

On Mon, 16 Oct 2017 19:08:46 +0200, Angelo Compagnucci wrote:
> This patch adds a new infrastructure for golang based packages.
> 
> Signed-off-by: Angelo Compagnucci <angelo.compagnucci@gmail.com>

As Christian said, it seems that this infrastructure isn't "general"
enough to really be useful. We would to see it applied to several
packages and see that there is a real benefit to several packages to
apply it.

So I'd be happy to take a go package infrastructure, but it needs to be
usable by more than one package in a useful way.

> +# Target packages need the Go compiler on the host.
> +$(2)_DEPENDENCIES += host-go
> +
> +#
> +# go install command doesn't work well when cross compilation is enabled,
> +# we set the executable output of the compilation to a specific location.
> +# We set this variable here to be used by packages if needed.
> +#
> +$(2)_EXECUTABLE = $$(@D)/gopath/bin/$(1)

This seems to make the assumption that a given package can only install
a single executable, and that this executable is named after the
package. It is true for flannel, but what about other Go packages ?

> +# Source files in Go should be uncompressed in a precise folder in the
> +# hiearchy of GOPATH. It usually resolves around domain/vendor/software.
> +#
> +$(1)_src_path ?= $$(call domain,$($(2)_SITE))/$$(firstword $$(subst /, ,$$(call notdomain,$($(2)_SITE))))
> +$(2)_SRC_PATH  = $$(@D)/gopath/src/$$($(1)_src_path)/$(1)

We don't use lower-case variables in Buildroot, especially if there is
a variable with the same name upper-case.

Best regards,

Thomas Petazzoni
-- 
Thomas Petazzoni, CTO, Free Electrons
Embedded Linux and Kernel engineering
http://free-electrons.com

[Buildroot] [RFC v2 1/4] package/go: fixing crosscompilation settings

[Peter Korsgaard]

>>>>> "Thomas" == Thomas Petazzoni <thomas.petazzoni@free-electrons.com> writes:

 > Hello,
 > On Mon, 16 Oct 2017 19:08:45 +0200, Angelo Compagnucci wrote:
 >> This patch fixes a bug with the BR2_TOOLCHAIN_HAS_THREADS variable
 >> handling which causes CGO_ENABLED to be always 0.
 >> 
 >> Furthermore, it fixes the cross compilation options for the go compiler:
 >> setting CGO_ENABLED should be done only for the target compiler not the
 >> host one.
 >> 
 >> Signed-off-by: Angelo Compagnucci <angelo.compagnucci@gmail.com>
 >> ---
 >> package/go/go.mk | 14 ++++++++------
 >> 1 file changed, 8 insertions(+), 6 deletions(-)

 > Thanks applied the patch, with Christian's Acked-by.

Committed to 2017.08.x, thanks.

-- 
Bye, Peter Korsgaard

[Buildroot] [RFC v2 2/4] package/pkg-golang: new package infrastructure

[Angelo Compagnucci]

Dear Thomas,

2017-10-21 21:55 GMT+02:00 Thomas Petazzoni
<thomas.petazzoni@free-electrons.com>:
> Hello,
>
> On Mon, 16 Oct 2017 19:08:46 +0200, Angelo Compagnucci wrote:
>> This patch adds a new infrastructure for golang based packages.
>>
>> Signed-off-by: Angelo Compagnucci <angelo.compagnucci@gmail.com>
>
> As Christian said, it seems that this infrastructure isn't "general"
> enough to really be useful. We would to see it applied to several
> packages and see that there is a real benefit to several packages to
> apply it.
>
> So I'd be happy to take a go package infrastructure, but it needs to be
> usable by more than one package in a useful way.
>
>> +# Target packages need the Go compiler on the host.
>> +$(2)_DEPENDENCIES += host-go
>> +
>> +#
>> +# go install command doesn't work well when cross compilation is enabled,
>> +# we set the executable output of the compilation to a specific location.
>> +# We set this variable here to be used by packages if needed.
>> +#
>> +$(2)_EXECUTABLE = $$(@D)/gopath/bin/$(1)
>
> This seems to make the assumption that a given package can only install
> a single executable, and that this executable is named after the
> package. It is true for flannel, but what about other Go packages ?

It will be addressed in the respin of the series.

>
>> +# Source files in Go should be uncompressed in a precise folder in the
>> +# hiearchy of GOPATH. It usually resolves around domain/vendor/software.
>> +#
>> +$(1)_src_path ?= $$(call domain,$($(2)_SITE))/$$(firstword $$(subst /, ,$$(call notdomain,$($(2)_SITE))))
>> +$(2)_SRC_PATH  = $$(@D)/gopath/src/$$($(1)_src_path)/$(1)
>
> We don't use lower-case variables in Buildroot, especially if there is
> a variable with the same name upper-case.

Fixed, thanks!

>
> Best regards,
>
> Thomas Petazzoni
> --
> Thomas Petazzoni, CTO, Free Electrons
> Embedded Linux and Kernel engineering
> http://free-electrons.com



-- 
Profile: http://it.linkedin.com/in/compagnucciangelo

[Buildroot] [RFC v2 1/4] package/go: fixing crosscompilation settings

[Peter Korsgaard]

>>>>> "Thomas" == Thomas Petazzoni <thomas.petazzoni@free-electrons.com> writes:

 > Hello,
 > On Mon, 16 Oct 2017 19:08:45 +0200, Angelo Compagnucci wrote:
 >> This patch fixes a bug with the BR2_TOOLCHAIN_HAS_THREADS variable
 >> handling which causes CGO_ENABLED to be always 0.
 >> 
 >> Furthermore, it fixes the cross compilation options for the go compiler:
 >> setting CGO_ENABLED should be done only for the target compiler not the
 >> host one.
 >> 
 >> Signed-off-by: Angelo Compagnucci <angelo.compagnucci@gmail.com>
 >> ---
 >> package/go/go.mk | 14 ++++++++------
 >> 1 file changed, 8 insertions(+), 6 deletions(-)

 > Thanks applied the patch, with Christian's Acked-by.

Committed to 2017.02.x, thanks.

-- 
Bye, Peter Korsgaard