[PATCH 0/4] gitlab-ci: Allow using FreeBSD runners

[PATCH 0/4] gitlab-ci: Allow using FreeBSD runners

[Philippe Mathieu-Daudé]

Minor shell fixes to use FreeBSD runners.

Based-on: <20210503104456.1036472-1-thuth@redhat.com>
https://gitlab.com/thuth/qemu.git tags/pull-request-2021-05-03

Philippe Mathieu-Daudé (4):
  gitlab-ci: Extract &environment_variables template
  gitlab-ci: Adapt JOBS variable for FreeBSD runners
  gitlab-ci: Run GNU make via the $MAKE variable
  gitlab-ci: Add ccache in $PATH and display statistics

 .gitlab-ci.yml | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

-- 
2.26.3

[PATCH 1/4] gitlab-ci: Extract &environment_variables template

[Philippe Mathieu-Daudé]

To be able to set the same environment variables to multiple jobs,
extract what we currently have as a template.

Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
---
 .gitlab-ci.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f01a1dbd7c9..598a8fb096f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -13,11 +13,14 @@ include:
   - local: '/.gitlab-ci.d/containers.yml'
   - local: '/.gitlab-ci.d/crossbuilds.yml'
 
+.environment_variables_template:
+  before_script:
+    - JOBS=$(expr $(nproc) + 1)
+
 .native_build_job_template:
   stage: build
   image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
-  before_script:
-    - JOBS=$(expr $(nproc) + 1)
+  extends: .environment_variables_template
   script:
     - mkdir build
     - cd build
-- 
2.26.3

[PATCH 2/4] gitlab-ci: Adapt JOBS variable for FreeBSD runners

[Philippe Mathieu-Daudé]

'nproc' is not available on FreeBSD:

  $ JOBS=$(expr $(nproc) + 1)
  bash: line 119: nproc: command not found
  expr: syntax error

Instead, use 'sysctl -n hw.ncpu'.

Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
---
 .gitlab-ci.yml | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 598a8fb096f..aceaac5a398 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,7 +15,17 @@ include:
 
 .environment_variables_template:
   before_script:
-    - JOBS=$(expr $(nproc) + 1)
+    - if
+        test $(uname) = "FreeBSD"
+        ;
+      then
+        JOBS=$(sysctl -n hw.ncpu)
+        ;
+      else
+        JOBS=$(expr $(nproc) + 1)
+        ;
+      fi
+    - echo "=== Using $JOBS simultaneous jobs ==="
 
 .native_build_job_template:
   stage: build
-- 
2.26.3

[PATCH 3/4] gitlab-ci: Run GNU make via the $MAKE variable

[Philippe Mathieu-Daudé]

Add the $MAKE variable to call GNU make, and set it to 'gmake'
on FreeBSD to avoid:

  $ make -j"$JOBS"
  make: Unknown modifier ','
  make: "/builds/dTyar424/0/qemu/build/Makefile" line 3: Need an operator
  make: "/builds/dTyar424/0/qemu/build/Makefile" line 4: Missing dependency operator

Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
---
 .gitlab-ci.yml | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index aceaac5a398..a33e51bf453 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,9 +19,11 @@ include:
         test $(uname) = "FreeBSD"
         ;
       then
+        MAKE=gmake
         JOBS=$(sysctl -n hw.ncpu)
         ;
       else
+        MAKE=make
         JOBS=$(expr $(nproc) + 1)
         ;
       fi
@@ -44,22 +46,23 @@ include:
       then
         meson configure . -Dbackend_max_links="$LD_JOBS" ;
       fi || exit 1;
-    - make -j"$JOBS"
+    - $MAKE -j"$JOBS"
     - if test -n "$MAKE_CHECK_ARGS";
       then
-        make -j"$JOBS" $MAKE_CHECK_ARGS ;
+        $MAKE -j"$JOBS" $MAKE_CHECK_ARGS ;
       fi
 
 .native_test_job_template:
   stage: test
   image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
+  extends: .environment_variables_template
   script:
     - scripts/git-submodule.sh update
         $(sed -n '/GIT_SUBMODULES=/ s/.*=// p' build/config-host.mak)
     - cd build
     - find . -type f -exec touch {} +
     # Avoid recompiling by hiding ninja with NINJA=":"
-    - make NINJA=":" $MAKE_CHECK_ARGS
+    - $MAKE NINJA=":" $MAKE_CHECK_ARGS
 
 .acceptance_template: &acceptance_definition
   cache:
@@ -833,17 +836,18 @@ build-tools-and-docs-debian:
 pages:
   image: $CI_REGISTRY_IMAGE/qemu/debian-amd64:latest
   stage: test
+  extends: .environment_variables_template
   needs:
     - job: build-tools-and-docs-debian
   script:
     - mkdir -p public
     # HTML-ised source tree
-    - make gtags
+    - $MAKE gtags
     - htags -anT --tree-view=filetree -m qemu_init
         -t "Welcome to the QEMU sourcecode"
     - mv HTML public/src
     # Project documentation
-    - make -C build install DESTDIR=$(pwd)/temp-install
+    - $MAKE -C build install DESTDIR=$(pwd)/temp-install
     - mv temp-install/usr/local/share/doc/qemu/* public/
   artifacts:
     paths:
-- 
2.26.3

[PATCH 4/4] gitlab-ci: Add ccache in $PATH and display statistics

[Philippe Mathieu-Daudé]

If a runner has ccache installed, use it and display statistics
at the end of the build.

Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
---
 .gitlab-ci.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a33e51bf453..d9a0252c54d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -21,13 +21,18 @@ include:
       then
         MAKE=gmake
         JOBS=$(sysctl -n hw.ncpu)
+        PATH=/usr/local/libexec/ccache:$PATH
         ;
       else
         MAKE=make
         JOBS=$(expr $(nproc) + 1)
+        PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH
         ;
       fi
     - echo "=== Using $JOBS simultaneous jobs ==="
+    - if command -v ccache > /dev/null ; then ccache --zero-stats ; fi
+  after_script:
+    - if command -v ccache > /dev/null ; then ccache --show-stats ; fi
 
 .native_build_job_template:
   stage: build
-- 
2.26.3

Re: [PATCH 3/4] gitlab-ci: Run GNU make via the $MAKE variable

[Philippe Mathieu-Daudé]

On 5/10/21 5:22 PM, Philippe Mathieu-Daudé wrote:
> Add the $MAKE variable to call GNU make, and set it to 'gmake'
> on FreeBSD to avoid:
> 
>   $ make -j"$JOBS"
>   make: Unknown modifier ','
>   make: "/builds/dTyar424/0/qemu/build/Makefile" line 3: Need an operator
>   make: "/builds/dTyar424/0/qemu/build/Makefile" line 4: Missing dependency operator
> 
> Signed-off-by: Philippe Mathieu-Daudé <f4bug@amsat.org>
> ---
>  .gitlab-ci.yml | 14 +++++++++-----
>  1 file changed, 9 insertions(+), 5 deletions(-)
> 
> diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
> index aceaac5a398..a33e51bf453 100644
> --- a/.gitlab-ci.yml
> +++ b/.gitlab-ci.yml
> @@ -19,9 +19,11 @@ include:
>          test $(uname) = "FreeBSD"
>          ;
>        then
> +        MAKE=gmake
>          JOBS=$(sysctl -n hw.ncpu)
>          ;
>        else
> +        MAKE=make
>          JOBS=$(expr $(nproc) + 1)
>          ;
>        fi
> @@ -44,22 +46,23 @@ include:
>        then
>          meson configure . -Dbackend_max_links="$LD_JOBS" ;
>        fi || exit 1;
> -    - make -j"$JOBS"
> +    - $MAKE -j"$JOBS"
>      - if test -n "$MAKE_CHECK_ARGS";
>        then
> -        make -j"$JOBS" $MAKE_CHECK_ARGS ;
> +        $MAKE -j"$JOBS" $MAKE_CHECK_ARGS ;
>        fi
>  
>  .native_test_job_template:
>    stage: test
>    image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest
> +  extends: .environment_variables_template
>    script:
>      - scripts/git-submodule.sh update
>          $(sed -n '/GIT_SUBMODULES=/ s/.*=// p' build/config-host.mak)
>      - cd build
>      - find . -type f -exec touch {} +
>      # Avoid recompiling by hiding ninja with NINJA=":"
> -    - make NINJA=":" $MAKE_CHECK_ARGS
> +    - $MAKE NINJA=":" $MAKE_CHECK_ARGS
>  
>  .acceptance_template: &acceptance_definition
>    cache:

I forgot this hunk:

-- >8 --
@@ -70,6 +70,7 @@ include:
     - $MAKE NINJA=":" $MAKE_CHECK_ARGS

 .acceptance_template: &acceptance_definition
+  extends: .environment_variables_template
   cache:
     key: "${CI_JOB_NAME}-cache"
     paths:
---

> @@ -833,17 +836,18 @@ build-tools-and-docs-debian:
>  pages:
>    image: $CI_REGISTRY_IMAGE/qemu/debian-amd64:latest
>    stage: test
> +  extends: .environment_variables_template
>    needs:
>      - job: build-tools-and-docs-debian
>    script:
>      - mkdir -p public
>      # HTML-ised source tree
> -    - make gtags
> +    - $MAKE gtags
>      - htags -anT --tree-view=filetree -m qemu_init
>          -t "Welcome to the QEMU sourcecode"
>      - mv HTML public/src
>      # Project documentation
> -    - make -C build install DESTDIR=$(pwd)/temp-install
> +    - $MAKE -C build install DESTDIR=$(pwd)/temp-install
>      - mv temp-install/usr/local/share/doc/qemu/* public/
>    artifacts:
>      paths:
>