[XEN PATCH 0/4] Some improvement for gitlab CI jobs

[XEN PATCH 0/4] Some improvement for gitlab CI jobs

[Anthony PERARD]

Patch series available in this git branch:
https://xenbits.xen.org/git-http/people/aperard/xen-unstable.git br.gitlab-ci-improvement-v1

Allow build and test jobs to run concurently.

Avoid running "test artifact" jobs on branch "master" and other, when test jobs
aren't runned.

Anthony PERARD (4):
  automation: fix typo in .gcc-tmpl
  automation: add a templates for test jobs
  automation: only run test artifact jobs when needed
  automation: use "needs" instead of "dependencies" for test jobs

 automation/gitlab-ci/build.yaml | 18 ++++---
 automation/gitlab-ci/test.yaml  | 89 +++++++++------------------------
 2 files changed, 37 insertions(+), 70 deletions(-)

-- 
Anthony PERARD

[XEN PATCH 1/4] automation: fix typo in .gcc-tmpl

[Anthony PERARD]

The name of the field doesn't matter because it's use as a YAML achor,
but it's nicer to have the proper spelling.

Signed-off-by: Anthony PERARD <anthony.perard@citrix.com>
---
 automation/gitlab-ci/build.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/automation/gitlab-ci/build.yaml b/automation/gitlab-ci/build.yaml
index cc36428cf5..26dcfb1c25 100644
--- a/automation/gitlab-ci/build.yaml
+++ b/automation/gitlab-ci/build.yaml
@@ -17,7 +17,7 @@
     - /^stable-.*/
 
 .gcc-tmpl:
-  variabes: &gcc
+  variables: &gcc
     CC: gcc
     CXX: g++
 
-- 
Anthony PERARD

[XEN PATCH 2/4] automation: add a templates for test jobs

[Anthony PERARD]

Allow to set common configuration from a single place for all tests
jobs.

Signed-off-by: Anthony PERARD <anthony.perard@citrix.com>
---
 automation/gitlab-ci/test.yaml | 73 ++++++++--------------------------
 1 file changed, 17 insertions(+), 56 deletions(-)

diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
index 47e8704df3..7cf21b7493 100644
--- a/automation/gitlab-ci/test.yaml
+++ b/automation/gitlab-ci/test.yaml
@@ -1,7 +1,15 @@
-# Test jobs
-build-each-commit-gcc:
+.test-jobs-common:
   stage: test
   image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  except:
+    - master
+    - smoke
+    - /^coverity-tested\/.*/
+    - /^stable-.*/
+
+# Test jobs
+build-each-commit-gcc:
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
     XEN_TARGET_ARCH: x86_64
@@ -16,15 +24,9 @@ build-each-commit-gcc:
   dependencies: []
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-alpine-arm64-gcc:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:unstable-arm64v8
   script:
@@ -41,15 +43,9 @@ qemu-alpine-arm64-gcc:
     when: always
   tags:
     - arm64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-alpine-x86_64-gcc:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
   script:
@@ -65,15 +61,9 @@ qemu-alpine-x86_64-gcc:
     when: always
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-smoke-arm64-gcc:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:unstable-arm64v8
   script:
@@ -89,15 +79,9 @@ qemu-smoke-arm64-gcc:
     when: always
   tags:
     - arm64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-smoke-x86-64-gcc:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
   script:
@@ -111,15 +95,9 @@ qemu-smoke-x86-64-gcc:
     - debian-stretch-gcc-debug
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-smoke-x86-64-clang:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
   script:
@@ -133,15 +111,9 @@ qemu-smoke-x86-64-clang:
     - debian-unstable-clang-debug
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-smoke-x86-64-gcc-pvh:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
   script:
@@ -155,15 +127,9 @@ qemu-smoke-x86-64-gcc-pvh:
     - debian-stretch-gcc-debug
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
 
 qemu-smoke-x86-64-clang-pvh:
-  stage: test
-  image: registry.gitlab.com/xen-project/xen/${CONTAINER}
+  extends: .test-jobs-common
   variables:
     CONTAINER: debian:stretch
   script:
@@ -177,8 +143,3 @@ qemu-smoke-x86-64-clang-pvh:
     - debian-unstable-clang-debug
   tags:
     - x86_64
-  except:
-    - master
-    - smoke
-    - /^coverity-tested\/.*/
-    - /^stable-.*/
-- 
Anthony PERARD

[XEN PATCH 3/4] automation: only run test artifact jobs when needed

[Anthony PERARD]

Share the same "except" as the one used for tests.

Signed-off-by: Anthony PERARD <anthony.perard@citrix.com>
---
 automation/gitlab-ci/build.yaml | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/automation/gitlab-ci/build.yaml b/automation/gitlab-ci/build.yaml
index 26dcfb1c25..cbbe0b8346 100644
--- a/automation/gitlab-ci/build.yaml
+++ b/automation/gitlab-ci/build.yaml
@@ -567,10 +567,16 @@ alpine-3.12-gcc-debug-arm64:
     CONTAINER: alpine:3.12-arm64v8
 
 
+## Test artifacts common
+
+.test-jobs-artifact-common:
+  stage: build
+  except: !reference [.test-jobs-common, except]
+
 # Arm test artifacts
 
 alpine-3.12-arm64-rootfs-export:
-  stage: build
+  extends: .test-jobs-artifact-common
   image: registry.gitlab.com/xen-project/xen/tests-artifacts/alpine:3.12-arm64v8
   script:
     - mkdir binaries && cp /initrd.tar.gz binaries/initrd.tar.gz
@@ -581,7 +587,7 @@ alpine-3.12-arm64-rootfs-export:
     - arm64
 
 kernel-5.9.9-arm64-export:
-  stage: build
+  extends: .test-jobs-artifact-common
   image: registry.gitlab.com/xen-project/xen/tests-artifacts/kernel:5.9.9-arm64v8
   script:
     - mkdir binaries && cp /Image binaries/Image
@@ -592,7 +598,7 @@ kernel-5.9.9-arm64-export:
     - arm64
 
 qemu-system-aarch64-5.2.0-arm64-export:
-  stage: build
+  extends: .test-jobs-artifact-common
   image: registry.gitlab.com/xen-project/xen/tests-artifacts/qemu-system-aarch64:5.2.0-arm64v8
   script:
     - mkdir binaries && cp /qemu-system-aarch64 binaries/qemu-system-aarch64
@@ -606,7 +612,7 @@ qemu-system-aarch64-5.2.0-arm64-export:
 # x86_64 test artifacts
 
 alpine-3.12-rootfs-export:
-  stage: build
+  extends: .test-jobs-artifact-common
   image: registry.gitlab.com/xen-project/xen/tests-artifacts/alpine:3.12
   script:
     - mkdir binaries && cp /initrd.tar.gz binaries/initrd.tar.gz
@@ -617,7 +623,7 @@ alpine-3.12-rootfs-export:
     - x86_64
 
 kernel-5.10.74-export:
-  stage: build
+  extends: .test-jobs-artifact-common
   image: registry.gitlab.com/xen-project/xen/tests-artifacts/kernel:5.10.74
   script:
     - mkdir binaries && cp /bzImage binaries/bzImage
-- 
Anthony PERARD

[XEN PATCH 4/4] automation: use "needs" instead of "dependencies" for test jobs

[Anthony PERARD]

Like with "dependencies", the jobs will get artifacts from the jobs
listed in "needs". But the test jobs can run as soon as the build jobs
listed have finished.

Signed-off-by: Anthony PERARD <anthony.perard@citrix.com>
---
 automation/gitlab-ci/test.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/automation/gitlab-ci/test.yaml b/automation/gitlab-ci/test.yaml
index 7cf21b7493..56747fb335 100644
--- a/automation/gitlab-ci/test.yaml
+++ b/automation/gitlab-ci/test.yaml
@@ -21,7 +21,7 @@ build-each-commit-gcc:
     paths:
       - '*.log'
     when: always
-  dependencies: []
+  needs: []
   tags:
     - x86_64
 
@@ -31,7 +31,7 @@ qemu-alpine-arm64-gcc:
     CONTAINER: debian:unstable-arm64v8
   script:
     - ./automation/scripts/qemu-alpine-arm64.sh 2>&1 | tee qemu-smoke-arm64.log
-  dependencies:
+  needs:
     - alpine-3.12-gcc-arm64
     - alpine-3.12-arm64-rootfs-export
     - kernel-5.9.9-arm64-export
@@ -50,7 +50,7 @@ qemu-alpine-x86_64-gcc:
     CONTAINER: debian:stretch
   script:
     - ./automation/scripts/qemu-alpine-x86_64.sh 2>&1 | tee qemu-smoke-x86_64.log
-  dependencies:
+  needs:
     - alpine-3.12-gcc
     - alpine-3.12-rootfs-export
     - kernel-5.10.74-export
@@ -68,7 +68,7 @@ qemu-smoke-arm64-gcc:
     CONTAINER: debian:unstable-arm64v8
   script:
     - ./automation/scripts/qemu-smoke-arm64.sh 2>&1 | tee qemu-smoke-arm64.log
-  dependencies:
+  needs:
     - debian-unstable-gcc-arm64
     - kernel-5.9.9-arm64-export
     - qemu-system-aarch64-5.2.0-arm64-export
@@ -91,7 +91,7 @@ qemu-smoke-x86-64-gcc:
       - smoke.serial
       - '*.log'
     when: always
-  dependencies:
+  needs:
     - debian-stretch-gcc-debug
   tags:
     - x86_64
@@ -107,7 +107,7 @@ qemu-smoke-x86-64-clang:
       - smoke.serial
       - '*.log'
     when: always
-  dependencies:
+  needs:
     - debian-unstable-clang-debug
   tags:
     - x86_64
@@ -123,7 +123,7 @@ qemu-smoke-x86-64-gcc-pvh:
       - smoke.serial
       - '*.log'
     when: always
-  dependencies:
+  needs:
     - debian-stretch-gcc-debug
   tags:
     - x86_64
@@ -139,7 +139,7 @@ qemu-smoke-x86-64-clang-pvh:
       - smoke.serial
       - '*.log'
     when: always
-  dependencies:
+  needs:
     - debian-unstable-clang-debug
   tags:
     - x86_64
-- 
Anthony PERARD