[Buildroot] [RFC/PATCH] download/git: support Git LFS

* [Buildroot] [RFC/PATCH] download/git: support Git LFS
@ 2018-04-26 17:36 John Keeping
  2018-04-26 20:24 ` Yann E. MORIN
  0 siblings, 1 reply; 5+ messages in thread
From: John Keeping @ 2018-04-26 17:36 UTC (permalink / raw)
  To: buildroot

Git Large File Storage replaces large files with text pointers in the
Git repository while storing the contents on a remote server.  If a
repository is using this extension, then git-lfs must be used to
checkout the large files before the source archive is generated.

Signed-off-by: John Keeping <john@metanate.com>
---
Currently this relies on git-lfs being installed on the host system and
I'm not sure if that's considered acceptable or not.

Unfortunately, building it as a host package opens a can of worms
because it is written in Go and pkg-golang only supports target packages
at the moment.

 docs/manual/adding-packages-generic.txt | 4 ++++
 package/pkg-download.mk                 | 1 +
 support/download/dl-wrapper             | 9 +++++----
 support/download/git                    | 9 +++++++++
 4 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/docs/manual/adding-packages-generic.txt b/docs/manual/adding-packages-generic.txt
index 62906d92bb..63b5bf70c9 100644
--- a/docs/manual/adding-packages-generic.txt
+++ b/docs/manual/adding-packages-generic.txt
@@ -327,6 +327,10 @@ information is (assuming the package name is +libfoo+) :
   submodules when they contain bundled libraries, in which case we
   prefer to use those libraries from their own package.
 
+* +LIBFOO_GIT_LFS+ should be set to +YES+ if the Git repository uses
+  Git LFS to store large files out of band.  This is only available for
+  packages downloaded with git (i.e. when +LIBFOO_SITE_METHOD=git+).
+
 * +LIBFOO_STRIP_COMPONENTS+ is the number of leading components
   (directories) that tar must strip from file names on extraction.
   The tarball for most packages has one leading component named
diff --git a/package/pkg-download.mk b/package/pkg-download.mk
index 2c4ad3ba2c..a634d5bd59 100644
--- a/package/pkg-download.mk
+++ b/package/pkg-download.mk
@@ -104,6 +104,7 @@ define DOWNLOAD
 		-N '$($(PKG)_RAWNAME)' \
 		-o '$($(PKG)_DL_DIR)/$(notdir $(1))' \
 		$(if $($(PKG)_GIT_SUBMODULES),-r) \
+		$(if $($(PKG)_GIT_LFS),-l) \
 		$(DOWNLOAD_URIS) \
 		$(QUIET) \
 		-- \
diff --git a/support/download/dl-wrapper b/support/download/dl-wrapper
index 8d6365e08d..7a98c89aa5 100755
--- a/support/download/dl-wrapper
+++ b/support/download/dl-wrapper
@@ -19,15 +19,15 @@
 # We want to catch any unexpected failure, and exit immediately.
 set -e
 
-export BR_BACKEND_DL_GETOPTS=":hc:d:o:n:N:H:ru:qf:e"
+export BR_BACKEND_DL_GETOPTS=":hc:d:o:n:N:H:lru:qf:e"
 
 main() {
     local OPT OPTARG
-    local backend output hfile recurse quiet rc
+    local backend output hfile large_file recurse quiet rc
     local -a uris
 
     # Parse our options; anything after '--' is for the backend
-    while getopts ":hc:d:D:o:n:N:H:rf:u:q" OPT; do
+    while getopts ":hc:d:D:o:n:N:H:lrf:u:q" OPT; do
         case "${OPT}" in
         h)  help; exit 0;;
         c)  cset="${OPTARG}";;
@@ -37,6 +37,7 @@ main() {
         n)  raw_base_name="${OPTARG}";;
         N)  base_name="${OPTARG}";;
         H)  hfile="${OPTARG}";;
+        l)  large_file="-l";;
         r)  recurse="-r";;
         f)  filename="${OPTARG}";;
         u)  uris+=( "${OPTARG}" );;
@@ -129,7 +130,7 @@ main() {
                 -f "${filename}" \
                 -u "${uri}" \
                 -o "${tmpf}" \
-                ${quiet} ${recurse} -- "${@}"
+                ${quiet} ${large_file} ${recurse} -- "${@}"
         then
             # cd back to keep path coherence
             cd "${OLDPWD}"
diff --git a/support/download/git b/support/download/git
index bf05c595a5..fc0039a6e2 100755
--- a/support/download/git
+++ b/support/download/git
@@ -17,10 +17,12 @@ set -e
 #   GIT      : the git command to call
 
 verbose=
+large_file=0
 recurse=0
 while getopts "${BR_BACKEND_DL_GETOPTS}" OPT; do
     case "${OPT}" in
     q)  verbose=-q; exec >/dev/null;;
+    l)  large_file=1;;
     r)  recurse=1;;
     o)  output="${OPTARG}";;
     u)  uri="${OPTARG}";;
@@ -109,6 +111,13 @@ if [ ${recurse} -eq 1 ]; then
     _git submodule update --init --recursive
 fi
 
+# If there are large files then fetch them.
+if [ ${large_file} -eq 1 ]; then
+    _git lfs install
+    _git lfs fetch
+    _git lfs checkout
+fi
+
 # Generate the archive, sort with the C locale so that it is reproducible.
 # We do not want the .git dir; we keep other .git files, in case they are the
 # only files in their directory.
-- 
2.17.0

^ permalink raw reply related	[flat|nested] 5+ messages in thread