[Buildroot] [PATCH 0 of 3] gentargets: add scp and Mercurial download helpers

[Buildroot] [PATCH 0 of 3] gentargets: add scp and Mercurial download helpers

[Thomas De Schampheleire]

This patch series adds download methods for scp and Mercurial, and updates
the documentation (including some typo fixes)

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
 Config.in                   |  12 ++++++++++++
 docs/buildroot.html         |  29 +++++++++++++++++------------
 package/Makefile.package.in |  62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
 3 files changed, 86 insertions(+), 17 deletions(-)

[Buildroot] [PATCH 1 of 3] GENTARGETS: add support for scp://

[Thomas De Schampheleire]

This patch adds support for scp:// both for use in the package Makefiles, as for
the BR2_PRIMARY_SITE variable.

This patch was based on the work of Richard Guy Briggs
(see https://bugs.busybox.net/show_bug.cgi?id=3343).

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
 Config.in                   |   8 ++++++++
 package/Makefile.package.in |  34 +++++++++++++++++++++++++++++-----
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/Config.in b/Config.in
--- a/Config.in
+++ b/Config.in
@@ -32,6 +32,14 @@
 	string "Git command"
 	default "git"
 
+config BR2_SCP
+	string "Secure copy (scp) command"
+	default "scp"
+
+config BR2_SSH
+	string "Secure shell (ssh) command"
+	default "ssh"
+
 config BR2_ZCAT
 	string "zcat command"
 	default "gzip -d -c"
diff --git a/package/Makefile.package.in b/package/Makefile.package.in
--- a/package/Makefile.package.in
+++ b/package/Makefile.package.in
@@ -74,6 +74,8 @@
 SVN:=$(call qstrip,$(BR2_SVN)) $(QUIET)
 BZR:=$(call qstrip,$(BR2_BZR)) $(QUIET)
 GIT:=$(call qstrip,$(BR2_GIT)) $(QUIET)
+SCP:=$(call qstrip,$(BR2_SCP)) $(QUIET)
+SSH:=$(call qstrip,$(BR2_SSH)) $(QUIET)
 
 # Default spider mode is 'DOWNLOAD'. Other possible values are 'SOURCE_CHECK'
 # used by the _source-check target and 'SHOW_EXTERNAL_DEPS', used by the
@@ -85,19 +87,24 @@
 DL_DIR:=$(TOPDIR)/dl
 endif
 
+# URI scheme helper functions
+geturischeme=$(firstword $(subst ://, ,$(call qstrip,$(1))))
+stripurischeme=$(lastword $(subst ://, ,$(call qstrip,$(1))))
+
 ################################################################################
 # The DOWNLOAD_{GIT,SVN,BZR} helpers are in charge of getting a
 # working copy of the source repository for their corresponding SCM,
 # checking out the requested version / commit / tag, and create an
-# archive out of it. DOWNLOAD_WGET is the normal wget-based download
+# archive out of it. DOWNLOAD_SCP uses scp to obtain a remote file with
+# ssh authentication. DOWNLOAD_WGET is the normal wget-based download
 # mechanism.
 #
-# The SOURCE_CHECK_{GIT,SVN,BZR,WGET} helpers are in charge of simply
+# The SOURCE_CHECK_{GIT,SVN,BZR,SCP,WGET} helpers are in charge of simply
 # checking that the source is available for download. This can be used
 # to make sure one will be able to get all the sources needed for
 # one's build configuration.
 #
-# The SHOW_EXTERNAL_DEPS_{GIT,SVN,BZR,WGET} helpers simply output to
+# The SHOW_EXTERNAL_DEPS_{GIT,SVN,BZR,SCP,WGET} helpers simply output to
 # the console the names of the files that will be downloaded, or path
 # and revision of the source repositories, producing a list of all the
 # "external dependencies" of a given build configuration.
@@ -158,6 +165,20 @@
 endef
 
 
+define DOWNLOAD_SCP
+	test -e $(DL_DIR)/$(2) || \
+	$(SCP) $(call stripurischeme,$(call qstrip,$(1)))/$(2) $(DL_DIR)
+endef
+
+define SOURCE_CHECK_SCP
+	$(SSH) $(call stripurischeme,`echo "$(call qstrip,$(1))" | cut -d/ -f1`) ls /`echo "$(call qstrip,$(1))/$(2)" | cut -d/ -f2-` > /dev/null
+endef
+
+define SHOW_EXTERNAL_DEPS_SCP
+	echo "$($(PKG)_SITE) [scp: $($(PKG)_DL_VERSION)]"
+endef
+
+
 define DOWNLOAD_WGET
 	test -e $(DL_DIR)/$(2) || \
 	$(WGET) -P $(DL_DIR) $(call qstrip,$(1))/$(2)
@@ -186,13 +207,17 @@
 
 define DOWNLOAD
 	$(Q)if test -n "$(call qstrip,$(BR2_PRIMARY_SITE))" ; then \
-		$(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ; \
+		case "$(call geturischeme,$(BR2_PRIMARY_SITE))" in \
+			scp) $(call $(DL_MODE)_SCP,$(BR2_PRIMARY_SITE),$(2)) && exit ;; \
+			*) $(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ;; \
+		esac ; \
 	fi ; \
 	if test -n "$(1)" ; then \
 		case "$($(PKG)_SITE_METHOD)" in \
 			git) $($(DL_MODE)_GIT) && exit ;; \
 			svn) $($(DL_MODE)_SVN) && exit ;; \
 			bzr) $($(DL_MODE)_BZR) && exit ;; \
+			scp) $($(DL_MODE)_SCP) && exit ;; \
 			*) $(call $(DL_MODE)_WGET,$(1),$(2)) && exit ;; \
 		esac ; \
 	fi ; \
@@ -548,6 +573,8 @@
 DL_TOOLS_DEPENDENCIES += git
 else ifeq ($$($(2)_SITE_METHOD),bzr)
 DL_TOOLS_DEPENDENCIES += bzr
+else ifeq ($$($(2)_SITE_METHOD),scp)
+DL_TOOLS_DEPENDENCIES += scp ssh
 endif # SITE_METHOD
 
 endif # $(2)_KCONFIG_VAR

[Buildroot] [PATCH 2 of 3] Add support for packages stored in Mercurial (hg) repositories

[Thomas De Schampheleire]

Add support for packages stored in Mercurial (hg) repositories.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>

---
 Config.in                   |   4 ++++
 package/Makefile.package.in |  31 ++++++++++++++++++++++++++++---
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/Config.in b/Config.in
--- a/Config.in
+++ b/Config.in
@@ -40,6 +40,10 @@
 	string "Secure shell (ssh) command"
 	default "ssh"
 
+config BR2_HG
+	string "Mercurial (hg) command"
+	default "hg"
+
 config BR2_ZCAT
 	string "zcat command"
 	default "gzip -d -c"
diff --git a/package/Makefile.package.in b/package/Makefile.package.in
--- a/package/Makefile.package.in
+++ b/package/Makefile.package.in
@@ -76,6 +76,7 @@
 GIT:=$(call qstrip,$(BR2_GIT)) $(QUIET)
 SCP:=$(call qstrip,$(BR2_SCP)) $(QUIET)
 SSH:=$(call qstrip,$(BR2_SSH)) $(QUIET)
+HG:=$(call qstrip,$(BR2_HG)) $(QUIET)
 
 # Default spider mode is 'DOWNLOAD'. Other possible values are 'SOURCE_CHECK'
 # used by the _source-check target and 'SHOW_EXTERNAL_DEPS', used by the
@@ -92,19 +93,19 @@
 stripurischeme=$(lastword $(subst ://, ,$(call qstrip,$(1))))
 
 ################################################################################
-# The DOWNLOAD_{GIT,SVN,BZR} helpers are in charge of getting a
+# The DOWNLOAD_{GIT,SVN,BZR,HG} helpers are in charge of getting a
 # working copy of the source repository for their corresponding SCM,
 # checking out the requested version / commit / tag, and create an
 # archive out of it. DOWNLOAD_SCP uses scp to obtain a remote file with
 # ssh authentication. DOWNLOAD_WGET is the normal wget-based download
 # mechanism.
 #
-# The SOURCE_CHECK_{GIT,SVN,BZR,SCP,WGET} helpers are in charge of simply
+# The SOURCE_CHECK_{GIT,SVN,BZR,HG,SCP,WGET} helpers are in charge of simply
 # checking that the source is available for download. This can be used
 # to make sure one will be able to get all the sources needed for
 # one's build configuration.
 #
-# The SHOW_EXTERNAL_DEPS_{GIT,SVN,BZR,SCP,WGET} helpers simply output to
+# The SHOW_EXTERNAL_DEPS_{GIT,SVN,BZR,HG,SCP,WGET} helpers simply output to
 # the console the names of the files that will be downloaded, or path
 # and revision of the source repositories, producing a list of all the
 # "external dependencies" of a given build configuration.
@@ -179,6 +180,27 @@
 endef
 
 
+define DOWNLOAD_HG
+	test -e $(DL_DIR)/$($(PKG)_SOURCE) || \
+	(pushd $(DL_DIR) > /dev/null && \
+	$(HG) clone --noupdate --rev $($(PKG)_DL_VERSION) $($(PKG)_SITE) $($(PKG)_BASE_NAME) && \
+	$(HG) archive --repository $($(PKG)_BASE_NAME) --type tgz --prefix $($(PKG)_BASE_NAME)/ \
+	              --rev $($(PKG)_DL_VERSION) $(DL_DIR)/$($(PKG)_SOURCE) && \
+	rm -rf $($(PKG)_DL_DIR) && \
+	popd > /dev/null)
+endef
+
+# TODO: improve to check that the given PKG_DL_VERSION exists on the remote
+# repository
+define SOURCE_CHECK_HG
+  $(HG) incoming --force -l1 $($(PKG)_SITE) > /dev/null
+endef
+
+define SHOW_EXTERNAL_DEPS_HG
+  echo "$($(PKG)_SITE) [hg: $($(PKG)_DL_VERSION)]"
+endef
+
+
 define DOWNLOAD_WGET
 	test -e $(DL_DIR)/$(2) || \
 	$(WGET) -P $(DL_DIR) $(call qstrip,$(1))/$(2)
@@ -218,6 +240,7 @@
 			svn) $($(DL_MODE)_SVN) && exit ;; \
 			bzr) $($(DL_MODE)_BZR) && exit ;; \
 			scp) $($(DL_MODE)_SCP) && exit ;; \
+			hg) $($(DL_MODE)_HG) && exit ;; \
 			*) $(call $(DL_MODE)_WGET,$(1),$(2)) && exit ;; \
 		esac ; \
 	fi ; \
@@ -575,6 +598,8 @@
 DL_TOOLS_DEPENDENCIES += bzr
 else ifeq ($$($(2)_SITE_METHOD),scp)
 DL_TOOLS_DEPENDENCIES += scp ssh
+else ifeq ($$($(2)_SITE_METHOD),hg)
+DL_TOOLS_DEPENDENCIES += hg
 endif # SITE_METHOD
 
 endif # $(2)_KCONFIG_VAR

[Buildroot] [PATCH 3 of 3] Update documentation for scp and Mercurial support

[Thomas De Schampheleire]

Update documentation for scp and Mercurial support.
Along with the documentation update, a few typos were fixed as well.

Signed-off-by: Thomas De Schampheleire <thomas.de.schampheleire@gmail.com>
---
 docs/buildroot.html |  29 +++++++++++++++++------------
 1 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/docs/buildroot.html b/docs/buildroot.html
--- a/docs/buildroot.html
+++ b/docs/buildroot.html
@@ -1025,8 +1025,8 @@
       version of the package. Note that
       if <code>HOST_LIBFOO_VERSION</code> doesn't exist, it is assumed
       to be the same as <code>LIBFOO_VERSION</code>. It can also be a
-      Subversion or Git branch or tag, for packages that are fetched
-      directly from their revision control system.<br/>
+      Subversion, Git, Bazaar or Mercurial branch or tag, for packages
+      that are fetched directly from their revision control system.<br/>
       Example: <code>LIBFOO_VERSION = 0.1.2</code></li>
 
       <li><code>LIBFOO_SOURCE</code> may contain the name of the tarball of
@@ -1047,7 +1047,8 @@
 
       <li><code>LIBFOO_SITE</code> may contain the Internet location
       of the package. It can either be the HTTP or FTP location of a
-      tarball, or the URL of a Git or Subversion repository
+      tarball, an SCP URI of a tarball accessible with Secure Copy, or
+      the URL of a Git, Subversion, Bazaar or Mercurial repository.
       (see <code>LIBFOO_SITE_METHOD</code>
       below). If <code>HOST_LIBFOO_SITE</code> is not specified, it
       defaults to <code>LIBFOO_SITE</code>. If none are specified,
@@ -1055,22 +1056,26 @@
       <code>http://$$(BR2_SOURCEFORGE_MIRROR).dl.sourceforge.net/sourceforge/packagename</code>.
       <br/>Examples:<br/>
       <code>LIBFOO_SITE=http://www.libfoosoftware.org/libfoo</code><br/>
-      <code>LIBFOO_SITE=http://svn.xiph.org/trunk/Tremor/</code></li>
+      <code>LIBFOO_SITE=http://svn.xiph.org/trunk/Tremor/</code><br/>
+      <code>LIBFOO_SITE=scp://machinename:/path/to/tarball</code><br/>
+      <code>LIBFOO_SITE=git://github.com/user/test.git</code></li>
 
       <li><code>LIBFOO_SITE_METHOD</code> may contain the method to
       fetch the package source code. It can either
       be <code>wget</code> (for normal FTP/HTTP downloads of
-      tarballs), <code>svn</code>, <code>git</code> or <code>bzr</code>.
+      tarballs), <code>scp</code>, <code>svn</code>, <code>git</code>,
+      <code>bzr</code> or <code>hg</code>.
       When not specified, it is guessed from the URL given
       in <code>LIBFOO_SITE</code>: <code>svn://</code>, <code>git://</code>
-      and <code>bzr://</code> URLs will use the <code>svn</code>,
-      <code>git</code> and <code>bzr</code> methods respectively. All other
+      <code>bzr://</code> and <code>scp://</code> URLs will use the
+      <code>svn</code>, <code>git</code>, <code>bzr</code> and
+      <code>scp</code>methods respectively. All other
       URL-types will use the <code>wget</code> method. So for example, in the
       case of a package whose source code is available through
       Subversion repository on HTTP, one <i>must</i>
-      specifiy <code>LIBFOO_SITE_METHOD=svn</code>. For <code>svn</code>
-      and <code>git</code> methods, what Buildroot does is a
-      checkout/clone of the repository which is then tarballed and
+      specify <code>LIBFOO_SITE_METHOD=svn</code>. For <code>svn</code>,
+      <code>git</code>, <code>bzr</code> and <code>hg</code> methods, what
+      Buildroot does is a checkout/clone of the repository which is then tarballed and
       stored into the download cache. Next builds will not
       checkout/clone again, but will use the tarball
       directly. When <code>HOST_LIBFOO_SITE_METHOD</code> is not
@@ -1184,7 +1189,7 @@
     However, since they are provided by the generic infrastructure, they are
     documented here. The exception is <code>LIBFOO_POST_PATCH_HOOKS</code>.
     Patching the package is not user definable, so
-    <code>LIBFOO_POST_PATCH_HOOKS</code> will be userful for generic packages.
+    <code>LIBFOO_POST_PATCH_HOOKS</code> will be useful for generic packages.
     </p>
 
     <p>The following hook points are available:</p>
@@ -1776,7 +1781,7 @@
     network...</code></h3>
 
     <p>If the boot process seems to hang after the following messages
-    (messages not necessarly exactly similar, depending on the list of
+    (messages not necessarily exactly similar, depending on the list of
     packages selected):</p>
 
     <pre>Freeing init memory: 3972K

[Buildroot] [PATCH 3 of 3] Update documentation for scp and Mercurial support

[Thomas Petazzoni]

Hello Thomas,

Le Wed, 21 Sep 2011 08:42:14 +0200,
Thomas De Schampheleire <patrickdepinguin+buildroot@gmail.com> a ?crit :

> Update documentation for scp and Mercurial support.
> Along with the documentation update, a few typos were fixed as well.

Please don't make any changes to the HTML documentation, as we are
moving to the AsciiDoc version. I'd prefer to have the AsciiDoc version
merge, and from there update the documentation with all the latest
developments of the package infrastructure.

Regards,

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com

[Buildroot] [PATCH 3 of 3] Update documentation for scp and Mercurial support

[Thomas De Schampheleire]

Hi Thomas,

On Wed, Sep 21, 2011 at 9:24 AM, Thomas Petazzoni
<thomas.petazzoni@free-electrons.com> wrote:
> Hello Thomas,
>
> Le Wed, 21 Sep 2011 08:42:14 +0200,
> Thomas De Schampheleire <patrickdepinguin+buildroot@gmail.com> a ?crit :
>
>> Update documentation for scp and Mercurial support.
>> Along with the documentation update, a few typos were fixed as well.
>
> Please don't make any changes to the HTML documentation, as we are
> moving to the AsciiDoc version. I'd prefer to have the AsciiDoc version
> merge, and from there update the documentation with all the latest
> developments of the package infrastructure.

I am aware of your AsciiDoc patches, but given that they are not yet
merged, I though I'd still update the HTML docs.

What is blocking the merge of the AsciiDoc patches?

Thanks,
Thomas

[Buildroot] [PATCH 1 of 3] GENTARGETS: add support for scp://

[Thomas Petazzoni]

Le Wed, 21 Sep 2011 08:42:12 +0200,
Thomas De Schampheleire <patrickdepinguin+buildroot@gmail.com> a ?crit :

> This patch adds support for scp:// both for use in the package Makefiles, as for
> the BR2_PRIMARY_SITE variable.

I'm ok with the principle. Some implementation comments below.

> +define DOWNLOAD_SCP
> +	test -e $(DL_DIR)/$(2) || \
> +	$(SCP) $(call stripurischeme,$(call qstrip,$(1)))/$(2) $(DL_DIR)
> +endef
> +
> +define SOURCE_CHECK_SCP
> +	$(SSH) $(call stripurischeme,`echo "$(call qstrip,$(1))" | cut -d/ -f1`) ls /`echo "$(call qstrip,$(1))/$(2)" | cut -d/ -f2-` > /dev/null
> +endef

Ouch, this looks ugly. I guess you can use "test -f" instead of using
ls. Maybe something like:

scphost=$(firstword $(subst /, ,$(call stripurischeme,$(1))))
scpfile=$(patsubst $(call scphost,$(1))/%,/%/$(2),$(call stripurischeme,$(1)))

define SOURCE_CHECK_SCP
	$(SSH) $(call scphost,$(1),$(2)) test -f $(call scpfile,$(1),$(2))
endef

Maybe it's possible to do better and/or to make those scphost/scpfile
macros a bit more generic. But the general comment is :

 * Try to use make functions instead of shell functions when possible.
   Every shell function requires a fork+exec and is very costly
   compared to make functions. In the past, we had a single shell
   function call for every package, and this was causing a 5-10 seconds
   startup delay for the build process.

 * Try to split in several small macros, whose name allows to
   understand what it is doing.

> +define SHOW_EXTERNAL_DEPS_SCP
> +	echo "$($(PKG)_SITE) [scp: $($(PKG)_DL_VERSION)]"
> +endef

As per a recent discussion with Peter, external-deps shouldn't show the
URL of the tarball of the package, but rather simply the tarball name,
as found in the $(DL_DIR). I know this is not consistent with the
current GIT/SVN/BZR helpers, but I have sent patches to fix this.

>  define DOWNLOAD_WGET
>  	test -e $(DL_DIR)/$(2) || \
>  	$(WGET) -P $(DL_DIR) $(call qstrip,$(1))/$(2)
> @@ -186,13 +207,17 @@
>  
>  define DOWNLOAD
>  	$(Q)if test -n "$(call qstrip,$(BR2_PRIMARY_SITE))" ; then \
> -		$(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ; \
> +		case "$(call geturischeme,$(BR2_PRIMARY_SITE))" in \
> +			scp) $(call $(DL_MODE)_SCP,$(BR2_PRIMARY_SITE),$(2)) && exit ;; \
> +			*) $(call $(DL_MODE)_WGET,$(BR2_PRIMARY_SITE),$(2)) && exit ;; \
> +		esac ; \

It would be good to update the BR2_PRIMARY_SITE help text to mention
which URL types are possible.

Regards,

Thomas
-- 
Thomas Petazzoni, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com