[Qemu-devel] [PULL v1 0/6] Merge build 2017/11/07

[Qemu-devel] [PULL v1 0/6] Merge build 2017/11/07

[Daniel P. Berrange]

The following changes since commit ec7a8bf0b8f7dc7288fe8745464ee8217528cc6c:

  Merge remote-tracking branch 'remotes/bonzini/tags/for-upstream' into staging (2017-11-06 10:04:16 +0000)

are available in the git repository at:

  git://github.com/berrange/qemu tags/pull-build-2017-11-07-1

for you to fetch changes up to c4b01c7c5171e9f77ed4ed6d9b4f63845410f641:

  build: remove use of MAKELEVEL optimization in submodule handling (2017-11-06 12:49:20 +0000)

----------------------------------------------------------------
Merge build 2017/11/07 v1

----------------------------------------------------------------

Daniel P. Berrange (6):
  build: allow setting a custom GIT binary for transparent proxying
  build: don't create temporary files in source dir
  build: allow automatic git submodule updates to be disabled
  build: don't fail if given a git submodule which does not exist
  build: delay check for empty git submodule list
  build: remove use of MAKELEVEL optimization in submodule handling

 Makefile                 | 23 +++++++++++-----
 configure                | 12 ++++++++
 scripts/git-submodule.sh | 72 ++++++++++++++++++++++++++++++++++++++----------
 3 files changed, 86 insertions(+), 21 deletions(-)

-- 
2.13.6

[Qemu-devel] [PULL v1 1/6] build: allow setting a custom GIT binary for transparent proxying

[Daniel P. Berrange]

Some users can't run a bare 'git' command, due to need for a transparent
proxying solution such as 'tsocks'. This adds an argument to configure to
let users specify such a thing:

  ./configure --with-git="tsocks git"

The submodule script is also updated to give the user a hint about using this
flag, if we fail to checkout modules.

Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
---
 Makefile                 |  4 ++--
 configure                |  5 +++++
 scripts/git-submodule.sh | 29 ++++++++++++++++++++++++-----
 3 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index ec73acfa9a..d218f8f3c8 100644
--- a/Makefile
+++ b/Makefile
@@ -21,14 +21,14 @@ git-submodule-update:
 ifeq (0,$(MAKELEVEL))
   git_module_status := $(shell \
     cd '$(SRC_PATH)' && \
-    ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
+    GIT="$(GIT)" ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
     echo $$?; \
   )
 
 ifeq (1,$(git_module_status))
 git-submodule-update:
 	$(call quiet-command, \
-          (cd $(SRC_PATH) && ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
+          (cd $(SRC_PATH) && GIT="$(GIT)" ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
           "GIT","$(GIT_SUBMODULES)")
 endif
 endif
diff --git a/configure b/configure
index e31d6a7fee..5af463fed6 100755
--- a/configure
+++ b/configure
@@ -271,6 +271,7 @@ then
 else
     git_submodules=""
 fi
+git="git"
 
 # Don't accept a target_list environment variable.
 unset target_list
@@ -1303,6 +1304,8 @@ for opt do
   ;;
   --enable-capstone=system) capstone="system"
   ;;
+  --with-git=*) git="$optarg"
+  ;;
   *)
       echo "ERROR: unknown option $opt"
       echo "Try '$0 --help' for more information"
@@ -5413,6 +5416,7 @@ echo "local state directory   queried at runtime"
 echo "Windows SDK       $win_sdk"
 fi
 echo "Source path       $source_path"
+echo "GIT binary        $git"
 echo "GIT submodules    $git_submodules"
 echo "C compiler        $cc"
 echo "Host C compiler   $host_cc"
@@ -5604,6 +5608,7 @@ echo "extra_cxxflags=$EXTRA_CXXFLAGS" >> $config_host_mak
 echo "extra_ldflags=$EXTRA_LDFLAGS" >> $config_host_mak
 echo "qemu_localedir=$qemu_localedir" >> $config_host_mak
 echo "libs_softmmu=$libs_softmmu" >> $config_host_mak
+echo "GIT=$git" >> $config_host_mak
 echo "GIT_SUBMODULES=$git_submodules" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak
diff --git a/scripts/git-submodule.sh b/scripts/git-submodule.sh
index 08932a35f0..039f782d81 100755
--- a/scripts/git-submodule.sh
+++ b/scripts/git-submodule.sh
@@ -3,14 +3,27 @@
 # This code is licensed under the GPL version 2 or later.  See
 # the COPYING file in the top-level directory.
 
-set -e
-
 substat=".git-submodule-status"
 
 command=$1
 shift
 modules="$@"
 
+test -z "$GIT" && GIT=git
+
+error() {
+    echo "$0: $*"
+    echo
+    echo "Unable to automatically checkout GIT submodules '$modules'."
+    echo "If you require use of an alternative GIT binary (for example to"
+    echo "enable use of a transparent proxy), then please specify it by"
+    echo "running configure by with the '--with-git' argument. e.g."
+    echo
+    echo " $ ./configure --with-git='tsocks git'"
+    echo
+    exit 1
+}
+
 if test -z "$modules"
 then
     test -e $substat || touch $substat
@@ -27,12 +40,18 @@ case "$command" in
 status)
     test -f "$substat" || exit 1
     trap "rm -f ${substat}.tmp" EXIT
-    git submodule status $modules > "${substat}.tmp"
+    $GIT submodule status $modules > "${substat}.tmp"
+    test $? -ne 0 && error "failed to query git submodule status"
     diff "${substat}" "${substat}.tmp" >/dev/null
     exit $?
     ;;
 update)
-    git submodule update --init $modules 1>/dev/null
-    git submodule status $modules > "${substat}"
+    $GIT submodule update --init $modules 1>/dev/null
+    test $? -ne 0 && error "failed to update modules"
+
+    $GIT submodule status $modules > "${substat}"
+    test $? -ne 0 && error "failed to save git submodule status" >&2
     ;;
 esac
+
+exit 0
-- 
2.13.6

[Qemu-devel] [PULL v1 2/6] build: don't create temporary files in source dir

[Daniel P. Berrange]

There are cases where users do VPATH builds with the source directory being on
a read-only volume. In such a case they have to manually run the command
'git-submodule.sh ...modules...' ahead of time. When checking for status we
should not then write into the source dir.

Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
---
 scripts/git-submodule.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scripts/git-submodule.sh b/scripts/git-submodule.sh
index 039f782d81..21909ea413 100755
--- a/scripts/git-submodule.sh
+++ b/scripts/git-submodule.sh
@@ -39,10 +39,9 @@ fi
 case "$command" in
 status)
     test -f "$substat" || exit 1
-    trap "rm -f ${substat}.tmp" EXIT
-    $GIT submodule status $modules > "${substat}.tmp"
-    test $? -ne 0 && error "failed to query git submodule status"
-    diff "${substat}" "${substat}.tmp" >/dev/null
+    CURSTATUS=`$GIT submodule status $modules`
+    OLDSTATUS=`cat $substat`
+    test "$CURSTATUS" = "$OLDSTATUS"
     exit $?
     ;;
 update)
-- 
2.13.6

[Qemu-devel] [PULL v1 3/6] build: allow automatic git submodule updates to be disabled

[Daniel P. Berrange]

Some people building QEMU use VPATH builds where the source directory is on a
read-only volume. In such a case 'scripts/git-submodules.sh update' will always
fail and users are required to run it manually themselves on their original
writable source directory.

While this is already supported, it is nice to give users a command line flag
to configure to permanently disable automatic submodule updates, as it means
they won't get hard to diagnose failures from git-submodules.sh at an arbitrary
later date.

This patch thus introduces a flag '--disable-git-update' which will prevent
'make' from ever running 'scripts/git-submodules.sh update'. It will still run
the 'status' command to determine if a submodule update is needed, but when it
does this it'll simply stop and print a message instructing the developer what
todo. eg

$ ./configure  --target-list=x86_64-softmmu --disable-git-update
...snip...

$ make
  GEN     config-host.h
  GEN     trace/generated-tcg-tracers.h
  GEN     trace/generated-helpers-wrappers.h
  GEN     trace/generated-helpers.h
  GEN     trace/generated-helpers.c
  GEN     module_block.h

GIT submodule checkout is out of date. Please run
  scripts/git-submodule.sh update ui/keycodemapdb
from the source directory checkout /home/berrange/src/virt/qemu

make: *** [Makefile:31: git-submodule-update] Error 1

Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
---
 Makefile                 | 11 +++++++++++
 configure                |  7 +++++++
 scripts/git-submodule.sh |  9 +++++++++
 3 files changed, 27 insertions(+)

diff --git a/Makefile b/Makefile
index d218f8f3c8..a92d7b8e14 100644
--- a/Makefile
+++ b/Makefile
@@ -26,12 +26,23 @@ ifeq (0,$(MAKELEVEL))
   )
 
 ifeq (1,$(git_module_status))
+ifeq (no,$(GIT_UPDATE))
+git-submodule-update:
+	$(call quiet-command, \
+            echo && \
+            echo "GIT submodule checkout is out of date. Please run" && \
+            echo "  scripts/git-submodule.sh update $(GIT_SUBMODULES)" && \
+            echo "from the source directory checkout $(SRC_PATH)" && \
+            echo && \
+            exit 1)
+else
 git-submodule-update:
 	$(call quiet-command, \
           (cd $(SRC_PATH) && GIT="$(GIT)" ./scripts/git-submodule.sh update $(GIT_SUBMODULES)), \
           "GIT","$(GIT_SUBMODULES)")
 endif
 endif
+endif
 
 .git-submodule-status: git-submodule-update config-host.mak
 
diff --git a/configure b/configure
index 5af463fed6..a6055c0710 100755
--- a/configure
+++ b/configure
@@ -267,8 +267,10 @@ stack_protector=""
 
 if test -e "$source_path/.git"
 then
+    git_update=yes
     git_submodules="ui/keycodemapdb"
 else
+    git_update=no
     git_submodules=""
 fi
 git="git"
@@ -1306,6 +1308,10 @@ for opt do
   ;;
   --with-git=*) git="$optarg"
   ;;
+  --enable-git-update) git_update=yes
+  ;;
+  --disable-git-update) git_update=no
+  ;;
   *)
       echo "ERROR: unknown option $opt"
       echo "Try '$0 --help' for more information"
@@ -5610,6 +5616,7 @@ echo "qemu_localedir=$qemu_localedir" >> $config_host_mak
 echo "libs_softmmu=$libs_softmmu" >> $config_host_mak
 echo "GIT=$git" >> $config_host_mak
 echo "GIT_SUBMODULES=$git_submodules" >> $config_host_mak
+echo "GIT_UPDATE=$git_update" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak
 
diff --git a/scripts/git-submodule.sh b/scripts/git-submodule.sh
index 21909ea413..63ace6fc55 100755
--- a/scripts/git-submodule.sh
+++ b/scripts/git-submodule.sh
@@ -21,6 +21,15 @@ error() {
     echo
     echo " $ ./configure --with-git='tsocks git'"
     echo
+    echo "Alternatively you may disable automatic GIT submodule checkout"
+    echo "with:"
+    echo
+    echo " $ ./configure --disable-git-update'"
+    echo
+    echo "and then manually update submodules prior to running make, with:"
+    echo
+    echo " $ scripts/git-sbumodule.sh update $modules"
+    echo
     exit 1
 }
 
-- 
2.13.6

[Qemu-devel] [PULL v1 4/6] build: don't fail if given a git submodule which does not exist

[Daniel P. Berrange]

If going back in time in git history, across a commit that introduces a new
submodule, the 'git-submodule.sh' script will fail, causing rebuild to fail.

This is because config-host.mak contains a GIT_SUBMODULES variable that lists
a submodule that only exists in the later commit. config-host.mak won't get
repopulated until config.status is invoked, but make won't get this far due to
the submodule error.

This change makes 'git-submodule.sh' check whether each module is known to git
and drops any which are not present. A warning message will be printed when any
submodule is dropped in this manner.

Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
---
 scripts/git-submodule.sh | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/scripts/git-submodule.sh b/scripts/git-submodule.sh
index 63ace6fc55..3683bc9a04 100755
--- a/scripts/git-submodule.sh
+++ b/scripts/git-submodule.sh
@@ -7,7 +7,7 @@ substat=".git-submodule-status"
 
 command=$1
 shift
-modules="$@"
+maybe_modules="$@"
 
 test -z "$GIT" && GIT=git
 
@@ -33,12 +33,24 @@ error() {
     exit 1
 }
 
-if test -z "$modules"
+if test -z "$maybe_modules"
 then
     test -e $substat || touch $substat
     exit 0
 fi
 
+modules=""
+for m in $maybe_modules
+do
+    $GIT submodule status $m 1> /dev/null 2>&1
+    if test $? = 0
+    then
+        modules="$modules $m"
+    else
+        echo "warn: ignoring non-existent submodule $m"
+    fi
+done
+
 if ! test -e ".git"
 then
     echo "$0: unexpectedly called with submodules but no git checkout exists"
-- 
2.13.6

[Qemu-devel] [PULL v1 5/6] build: delay check for empty git submodule list

[Daniel P. Berrange]

We short circuit the git submodule update when passed an empty module list.
This accidentally causes the 'status' command to write to the status file. The
test needs to be delayed into the individual commands to avoid this premature
writing of the status file.

Tested-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
---
 scripts/git-submodule.sh | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/scripts/git-submodule.sh b/scripts/git-submodule.sh
index 3683bc9a04..030617b4ac 100755
--- a/scripts/git-submodule.sh
+++ b/scripts/git-submodule.sh
@@ -33,12 +33,6 @@ error() {
     exit 1
 }
 
-if test -z "$maybe_modules"
-then
-    test -e $substat || touch $substat
-    exit 0
-fi
-
 modules=""
 for m in $maybe_modules
 do
@@ -51,7 +45,7 @@ do
     fi
 done
 
-if ! test -e ".git"
+if test -n "$maybe_modules" && ! test -e ".git"
 then
     echo "$0: unexpectedly called with submodules but no git checkout exists"
     exit 1
@@ -59,6 +53,11 @@ fi
 
 case "$command" in
 status)
+    if test -z "$maybe_modules"
+    then
+         test -s ${substat} && exit 1 || exit 0
+    fi
+
     test -f "$substat" || exit 1
     CURSTATUS=`$GIT submodule status $modules`
     OLDSTATUS=`cat $substat`
@@ -66,6 +65,12 @@ status)
     exit $?
     ;;
 update)
+    if test -z "$maybe_modules"
+    then
+        test -e $substat || touch $substat
+        exit 0
+    fi
+
     $GIT submodule update --init $modules 1>/dev/null
     test $? -ne 0 && error "failed to update modules"
 
-- 
2.13.6

[Qemu-devel] [PULL v1 6/6] build: remove use of MAKELEVEL optimization in submodule handling

[Daniel P. Berrange]

The Makefile attempts to optimize the handling of submodules by using MAKELEVEL
to only check the submodule status when running from the top level make
invokation. This causes problems for people who are using a makefile of their
own to in turn invoke QEMU's makefile, as MAKELEVEL is already set to 1 (or
more) when QEMU's makefile runs.

This optimization should not really be needed, since the git-submodule.sh
script is already used to detect if a submodule update is required. This by
removing the MAKELEVEL check, we at most add an extra 'git-submodule.sh status'
call to each make level, the overhead of which is lost in noise of building
QEMU.

Reviewed-by: Greg Kurz <groug@kaod.org>
Tested-by: Greg Kurz <groug@kaod.org>
Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
---
 Makefile | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index a92d7b8e14..0496c5057a 100644
--- a/Makefile
+++ b/Makefile
@@ -18,12 +18,11 @@ git-submodule-update:
 
 .PHONY: git-submodule-update
 
-ifeq (0,$(MAKELEVEL))
-  git_module_status := $(shell \
-    cd '$(SRC_PATH)' && \
-    GIT="$(GIT)" ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
-    echo $$?; \
-  )
+git_module_status := $(shell \
+  cd '$(SRC_PATH)' && \
+  GIT="$(GIT)" ./scripts/git-submodule.sh status $(GIT_SUBMODULES); \
+  echo $$?; \
+)
 
 ifeq (1,$(git_module_status))
 ifeq (no,$(GIT_UPDATE))
@@ -42,7 +41,6 @@ git-submodule-update:
           "GIT","$(GIT_SUBMODULES)")
 endif
 endif
-endif
 
 .git-submodule-status: git-submodule-update config-host.mak
 
-- 
2.13.6

Re: [Qemu-devel] [PULL v1 0/6] Merge build 2017/11/07

[Peter Maydell]

On 7 November 2017 at 10:16, Daniel P. Berrange <berrange@redhat.com> wrote:
> The following changes since commit ec7a8bf0b8f7dc7288fe8745464ee8217528cc6c:
>
>   Merge remote-tracking branch 'remotes/bonzini/tags/for-upstream' into staging (2017-11-06 10:04:16 +0000)
>
> are available in the git repository at:
>
>   git://github.com/berrange/qemu tags/pull-build-2017-11-07-1
>
> for you to fetch changes up to c4b01c7c5171e9f77ed4ed6d9b4f63845410f641:
>
>   build: remove use of MAKELEVEL optimization in submodule handling (2017-11-06 12:49:20 +0000)
>
> ----------------------------------------------------------------
> Merge build 2017/11/07 v1
>
> ----------------------------------------------------------------
>

Applied, thanks.

-- PMM