All of lore.kernel.org
 help / color / mirror / Atom feed
* [Buildroot] [PATCH 1/3] utils/test-pkg: add support for externals
@ 2021-12-01 23:08 Matthew Weber via buildroot
  2021-12-01 23:08 ` [Buildroot] [PATCH 2/3] utils/test-pkg: add ability to fail when skipped Matthew Weber via buildroot
                   ` (2 more replies)
  0 siblings, 3 replies; 6+ messages in thread
From: Matthew Weber via buildroot @ 2021-12-01 23:08 UTC (permalink / raw)
  To: buildroot; +Cc: Matthew Weber

Add support to test-pkg to allow the usage br2-externals, so that
it may be leveraged for packages outside of upstream buildroot as well.

Signed-off-by: Matthew Weber <matthew.weber@collins.com>
---
 utils/test-pkg | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/utils/test-pkg b/utils/test-pkg
index 526b95b1fb..d0472f176b 100755
--- a/utils/test-pkg
+++ b/utils/test-pkg
@@ -3,6 +3,7 @@ set -e
 
 TOOLCHAINS_CSV='support/config-fragments/autobuild/toolchain-configs.csv'
 TEMP_CONF=""
+BR2_EXTERNALS=""
 
 do_clean() {
     if [ ! -z "${TEMP_CONF}" ]; then
@@ -17,8 +18,8 @@ main() {
     local -a toolchains
     local pkg_br_name
 
-    o='hakc:d:n:p:r:t:'
-    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:'
+    o='hakc:d:n:p:r:t:e:'
+    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:,externals:,'
     opts="$(getopt -n "${my_name}" -o "${o}" -l "${O}" -- "${@}")"
     eval set -- "${opts}"
 
@@ -61,6 +62,9 @@ main() {
         (-t|--toolchains-csv)
             toolchains_csv="${2}"; shift 2
             ;;
+        (-e|--externals)
+            BR2_EXTERNALS="${2}"; shift 2
+            ;;
         (--)
             shift; break
             ;;
@@ -155,7 +159,7 @@ build_one() {
 
     mkdir -p "${dir}"
 
-    CONFIG_= support/kconfig/merge_config.sh -O "${dir}" \
+    CONFIG_= support/kconfig/merge_config.sh -e "${BR2_EXTERNALS}" -O "${dir}" \
         "${toolchainconfig}" "support/config-fragments/minimal.config" "${cfg}" \
         >> "${dir}/logfile" 2>&1
     # We want all the options from the snippet to be present as-is (set
@@ -181,7 +185,7 @@ build_one() {
     fi
 
     if [ -n "${pkg}" ]; then
-        if ! make O="${dir}" "${pkg}-dirclean" >> "${dir}/logfile" 2>&1; then
+        if ! make BR2_EXTERNAL="${BR2_EXTERNALS}" O="${dir}" "${pkg}-dirclean" >> "${dir}/logfile" 2>&1; then
             return 2
         fi
     fi
@@ -193,14 +197,14 @@ build_one() {
 
     # legal-info done systematically, because some packages have different
     # sources depending on the configuration (e.g. lua-5.2 vs. lua-5.3)
-    if ! make O="${dir}" legal-info >> "${dir}/logfile" 2>&1; then
+    if ! make BR2_EXTERNAL="${BR2_EXTERNALS}" O="${dir}" legal-info >> "${dir}/logfile" 2>&1; then
         return 3
     fi
 
     # If we get here, the build was successful. Clean up the build/host
     # directories to save disk space, unless 'keep' was set.
     if [ ${keep} -ne 1 ]; then
-        make O="${dir}" clean >> "${dir}/logfile" 2>&1
+        make BR2_EXTERNAL="${BR2_EXTERNALS}" O="${dir}" clean >> "${dir}/logfile" 2>&1
     fi
 }
 
@@ -271,6 +275,10 @@ Options:
         Only prepare the .config files, but do not build them. Output the
         list of build directories to stdout, and the status on stderr.
 
+    -e, --externals
+        Externals to be used as part of the build process. Packages from
+        within these externals may be tested.
+
 Example:
 
     Testing libcec would require a config snippet that contains:
-- 
2.17.1

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

^ permalink raw reply related	[flat|nested] 6+ messages in thread
* [Buildroot] [PATCH 2/3] utils/test-pkg: add ability to fail when skipped
  2021-12-01 23:08 [Buildroot] [PATCH 1/3] utils/test-pkg: add support for externals Matthew Weber via buildroot
@ 2021-12-01 23:08 ` Matthew Weber via buildroot
  2022-01-01 21:08   ` Yann E. MORIN
  2021-12-01 23:08 ` [Buildroot] [PATCH 3/3] docs/manual: add section describing testing external applications Matthew Weber via buildroot
  2022-01-03 20:59 ` [Buildroot] [PATCH 1/3] utils/test-pkg: add support for externals Yann E. MORIN
  2 siblings, 1 reply; 6+ messages in thread
From: Matthew Weber via buildroot @ 2021-12-01 23:08 UTC (permalink / raw)
  To: buildroot; +Cc: Matthew Weber

Add flag to allow the return value to be incremented when a skip
happens, treating the overall run as a failure.

This flag could be used when using test-pkg for local validation of
downstream packages, for example in a user's br2-external. An example of
this could be a user choosing to use test-pkg as part of a CI
infrastructure where the fragment and toolchains are tightly controlled
to test a package against multiple toolchains easily.

In this case, the user would want to lower the rate of false-positives
of that fragment/toolchain configuration if a build is instead skipped
(which may happen in the case of buildroot version bumps where
dependencies could change).

Signed-off-by: Matthew Weber <matthew.weber@collins.com>
---
 utils/test-pkg | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/utils/test-pkg b/utils/test-pkg
index d0472f176b..97957008fa 100755
--- a/utils/test-pkg
+++ b/utils/test-pkg
@@ -18,8 +18,8 @@ main() {
     local -a toolchains
     local pkg_br_name
 
-    o='hakc:d:n:p:r:t:e:'
-    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:,externals:,'
+    o='hakc:d:n:p:r:t:e:f'
+    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:,externals:,fail-on-skip'
     opts="$(getopt -n "${my_name}" -o "${o}" -l "${O}" -- "${@}")"
     eval set -- "${opts}"
 
@@ -29,6 +29,7 @@ main() {
     number=0
     mode=0
     prepare_only=0
+    fail_on_skip=0
     toolchains_csv="${TOOLCHAINS_CSV}"
     while [ ${#} -gt 0 ]; do
         case "${1}" in
@@ -65,6 +66,9 @@ main() {
         (-e|--externals)
             BR2_EXTERNALS="${2}"; shift 2
             ;;
+        (-f|--fail-on-skip)
+            fail_on_skip=1; shift 1
+            ;;
         (--)
             shift; break
             ;;
@@ -147,7 +151,12 @@ main() {
     printf "%d builds, %d skipped, %d build failed, %d legal-info failed\n" \
         ${nb} ${nb_skip} ${nb_fail} ${nb_legal}
 
-    return $((nb_fail + nb_legal))
+    nb_result=$((nb_fail + nb_legal))
+    if [ ${fail_on_skip} ]; then
+        nb_result=$((nb_result + nb_skip))
+    fi
+
+    return $nb_result
 }
 
 build_one() {
@@ -279,6 +288,9 @@ Options:
         Externals to be used as part of the build process. Packages from
         within these externals may be tested.
 
+    -f, --fail-on-skip
+        If any builds are skipped, return a negative exit value.
+
 Example:
 
     Testing libcec would require a config snippet that contains:
-- 
2.17.1

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

^ permalink raw reply related	[flat|nested] 6+ messages in thread
* [Buildroot] [PATCH 3/3] docs/manual: add section describing testing external applications
  2021-12-01 23:08 [Buildroot] [PATCH 1/3] utils/test-pkg: add support for externals Matthew Weber via buildroot
  2021-12-01 23:08 ` [Buildroot] [PATCH 2/3] utils/test-pkg: add ability to fail when skipped Matthew Weber via buildroot
@ 2021-12-01 23:08 ` Matthew Weber via buildroot
  2022-01-03 20:59 ` [Buildroot] [PATCH 1/3] utils/test-pkg: add support for externals Yann E. MORIN
  2 siblings, 0 replies; 6+ messages in thread
From: Matthew Weber via buildroot @ 2021-12-01 23:08 UTC (permalink / raw)
  To: buildroot; +Cc: Matthew Weber

Buildroot-provided utilities can easily be leveraged to improve the
testing of applications across multiple toolchains. Add section which
describes considerations to take about reducing false positives if this
approach is taken in your br2-external.

Signed-off-by: Matthew Weber <matthew.weber@collins.com>
---
 docs/manual/customize-outside-br.txt | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/docs/manual/customize-outside-br.txt b/docs/manual/customize-outside-br.txt
index 348f2b089c..e7aeeed8c1 100644
--- a/docs/manual/customize-outside-br.txt
+++ b/docs/manual/customize-outside-br.txt
@@ -228,6 +228,21 @@ Additional Linux kernel extensions (see xref:linux-kernel-ext[]) can
 be added by storing them in the `linux/` directory at the root of a
 br2-external tree.
 
+===== Leveraging test infrastructure for your packages
+
+Buildroot provides utilities for testing packages across multiple toolchain
+conditions, as described in xref:testing-package[] below. Packages that live
+exclusively in an external environment may not be candidates for upstream and
+only have specific toolchain targets due to being internal by nature.
+Leveraging these utilities with a list of toolchains you wish to guarantee
+support (instead of depending on the utility picking toolchains for you
+randomly) is an easy way to improve the test surface of your application.
+
+In this configuration, ideally, false positives should be minimized, and
++utils/test-pkg+ should be run with the +--fail-on-skip+ flag. This option
+covers the case of a configuration dependency change (for example, between
+Buildroot versions).
+
 ===== Example layout
 
 Here is an example layout using all features of br2-external (the sample
-- 
2.17.1

_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

^ permalink raw reply related	[flat|nested] 6+ messages in thread
* Re: [Buildroot] [PATCH 2/3] utils/test-pkg: add ability to fail when skipped
  2021-12-01 23:08 ` [Buildroot] [PATCH 2/3] utils/test-pkg: add ability to fail when skipped Matthew Weber via buildroot
@ 2022-01-01 21:08   ` Yann E. MORIN
  2022-01-31 20:54     ` Arnout Vandecappelle
  0 siblings, 1 reply; 6+ messages in thread
From: Yann E. MORIN @ 2022-01-01 21:08 UTC (permalink / raw)
  To: Matthew Weber; +Cc: buildroot

Matthew, All,

On 2021-12-01 17:08 -0600, Matthew Weber via buildroot spake thusly:
> Add flag to allow the return value to be incremented when a skip
> happens, treating the overall run as a failure.
> 
> This flag could be used when using test-pkg for local validation of
> downstream packages, for example in a user's br2-external. An example of
> this could be a user choosing to use test-pkg as part of a CI
> infrastructure where the fragment and toolchains are tightly controlled
> to test a package against multiple toolchains easily.
> 
> In this case, the user would want to lower the rate of false-positives
> of that fragment/toolchain configuration if a build is instead skipped
> (which may happen in the case of buildroot version bumps where
> dependencies could change).

I am a bit skeptical about this use-case.

As I understand it, wht is most interesting in a CI setup for internal
development, is that *all* internal packages do build togetrher and form
a working sysem *as a whole*.

As such, a CI setup would build the defconfig file(s) for a project (and
run the integration test-suite), rather than build each package
individually

Building each package individually with test-pkg would have the
disadvantage that it takes more time overall, as all the common deps are
build as many times as there are packages to test, and package are also
dependencies one of the others, so most packages would be built more
than once as well...

So, I am really not convinced...

Regards,
Yann E. MORIN.

> Signed-off-by: Matthew Weber <matthew.weber@collins.com>
> ---
>  utils/test-pkg | 18 +++++++++++++++---
>  1 file changed, 15 insertions(+), 3 deletions(-)
> 
> diff --git a/utils/test-pkg b/utils/test-pkg
> index d0472f176b..97957008fa 100755
> --- a/utils/test-pkg
> +++ b/utils/test-pkg
> @@ -18,8 +18,8 @@ main() {
>      local -a toolchains
>      local pkg_br_name
>  
> -    o='hakc:d:n:p:r:t:e:'
> -    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:,externals:,'
> +    o='hakc:d:n:p:r:t:e:f'
> +    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:,externals:,fail-on-skip'
>      opts="$(getopt -n "${my_name}" -o "${o}" -l "${O}" -- "${@}")"
>      eval set -- "${opts}"
>  
> @@ -29,6 +29,7 @@ main() {
>      number=0
>      mode=0
>      prepare_only=0
> +    fail_on_skip=0
>      toolchains_csv="${TOOLCHAINS_CSV}"
>      while [ ${#} -gt 0 ]; do
>          case "${1}" in
> @@ -65,6 +66,9 @@ main() {
>          (-e|--externals)
>              BR2_EXTERNALS="${2}"; shift 2
>              ;;
> +        (-f|--fail-on-skip)
> +            fail_on_skip=1; shift 1
> +            ;;
>          (--)
>              shift; break
>              ;;
> @@ -147,7 +151,12 @@ main() {
>      printf "%d builds, %d skipped, %d build failed, %d legal-info failed\n" \
>          ${nb} ${nb_skip} ${nb_fail} ${nb_legal}
>  
> -    return $((nb_fail + nb_legal))
> +    nb_result=$((nb_fail + nb_legal))
> +    if [ ${fail_on_skip} ]; then
> +        nb_result=$((nb_result + nb_skip))
> +    fi
> +
> +    return $nb_result
>  }
>  
>  build_one() {
> @@ -279,6 +288,9 @@ Options:
>          Externals to be used as part of the build process. Packages from
>          within these externals may be tested.
>  
> +    -f, --fail-on-skip
> +        If any builds are skipped, return a negative exit value.
> +
>  Example:
>  
>      Testing libcec would require a config snippet that contains:
> -- 
> 2.17.1
> 
> _______________________________________________
> buildroot mailing list
> buildroot@buildroot.org
> https://lists.buildroot.org/mailman/listinfo/buildroot

-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 561 099 427 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

^ permalink raw reply	[flat|nested] 6+ messages in thread
* Re: [Buildroot] [PATCH 1/3] utils/test-pkg: add support for externals
  2021-12-01 23:08 [Buildroot] [PATCH 1/3] utils/test-pkg: add support for externals Matthew Weber via buildroot
  2021-12-01 23:08 ` [Buildroot] [PATCH 2/3] utils/test-pkg: add ability to fail when skipped Matthew Weber via buildroot
  2021-12-01 23:08 ` [Buildroot] [PATCH 3/3] docs/manual: add section describing testing external applications Matthew Weber via buildroot
@ 2022-01-03 20:59 ` Yann E. MORIN
  2 siblings, 0 replies; 6+ messages in thread
From: Yann E. MORIN @ 2022-01-03 20:59 UTC (permalink / raw)
  To: Matthew Weber; +Cc: buildroot

Matthew, All,

On 2021-12-01 17:08 -0600, Matthew Weber via buildroot spake thusly:
> Add support to test-pkg to allow the usage br2-externals, so that
> it may be leveraged for packages outside of upstream buildroot as well.

Why is it even needed? Just pass BR2_EXTERNAL in the environment:

    BR2_EXTERNAL=/path/to/foo:/path/to/bar ./utils/test-pkg

Maybe we should change the manual to state that BR2_EXTERNAL can be set
in the environment as well, and the help for ./utils/test-pkg could also
mention that.

Oh, and by the way: we should not have added the -e option to
support/kconfig/merge_config.sh either, because even in that case,
BR2_EXTERNAL in the environment works as well.

Regards,
Yann E. MORIN.

> Signed-off-by: Matthew Weber <matthew.weber@collins.com>
> ---
>  utils/test-pkg | 20 ++++++++++++++------
>  1 file changed, 14 insertions(+), 6 deletions(-)
> 
> diff --git a/utils/test-pkg b/utils/test-pkg
> index 526b95b1fb..d0472f176b 100755
> --- a/utils/test-pkg
> +++ b/utils/test-pkg
> @@ -3,6 +3,7 @@ set -e
>  
>  TOOLCHAINS_CSV='support/config-fragments/autobuild/toolchain-configs.csv'
>  TEMP_CONF=""
> +BR2_EXTERNALS=""
>  
>  do_clean() {
>      if [ ! -z "${TEMP_CONF}" ]; then
> @@ -17,8 +18,8 @@ main() {
>      local -a toolchains
>      local pkg_br_name
>  
> -    o='hakc:d:n:p:r:t:'
> -    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:'
> +    o='hakc:d:n:p:r:t:e:'
> +    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:,externals:,'
>      opts="$(getopt -n "${my_name}" -o "${o}" -l "${O}" -- "${@}")"
>      eval set -- "${opts}"
>  
> @@ -61,6 +62,9 @@ main() {
>          (-t|--toolchains-csv)
>              toolchains_csv="${2}"; shift 2
>              ;;
> +        (-e|--externals)
> +            BR2_EXTERNALS="${2}"; shift 2
> +            ;;
>          (--)
>              shift; break
>              ;;
> @@ -155,7 +159,7 @@ build_one() {
>  
>      mkdir -p "${dir}"
>  
> -    CONFIG_= support/kconfig/merge_config.sh -O "${dir}" \
> +    CONFIG_= support/kconfig/merge_config.sh -e "${BR2_EXTERNALS}" -O "${dir}" \
>          "${toolchainconfig}" "support/config-fragments/minimal.config" "${cfg}" \
>          >> "${dir}/logfile" 2>&1
>      # We want all the options from the snippet to be present as-is (set
> @@ -181,7 +185,7 @@ build_one() {
>      fi
>  
>      if [ -n "${pkg}" ]; then
> -        if ! make O="${dir}" "${pkg}-dirclean" >> "${dir}/logfile" 2>&1; then
> +        if ! make BR2_EXTERNAL="${BR2_EXTERNALS}" O="${dir}" "${pkg}-dirclean" >> "${dir}/logfile" 2>&1; then
>              return 2
>          fi
>      fi
> @@ -193,14 +197,14 @@ build_one() {
>  
>      # legal-info done systematically, because some packages have different
>      # sources depending on the configuration (e.g. lua-5.2 vs. lua-5.3)
> -    if ! make O="${dir}" legal-info >> "${dir}/logfile" 2>&1; then
> +    if ! make BR2_EXTERNAL="${BR2_EXTERNALS}" O="${dir}" legal-info >> "${dir}/logfile" 2>&1; then
>          return 3
>      fi
>  
>      # If we get here, the build was successful. Clean up the build/host
>      # directories to save disk space, unless 'keep' was set.
>      if [ ${keep} -ne 1 ]; then
> -        make O="${dir}" clean >> "${dir}/logfile" 2>&1
> +        make BR2_EXTERNAL="${BR2_EXTERNALS}" O="${dir}" clean >> "${dir}/logfile" 2>&1
>      fi
>  }
>  
> @@ -271,6 +275,10 @@ Options:
>          Only prepare the .config files, but do not build them. Output the
>          list of build directories to stdout, and the status on stderr.
>  
> +    -e, --externals
> +        Externals to be used as part of the build process. Packages from
> +        within these externals may be tested.
> +
>  Example:
>  
>      Testing libcec would require a config snippet that contains:
> -- 
> 2.17.1
> 
> _______________________________________________
> buildroot mailing list
> buildroot@buildroot.org
> https://lists.buildroot.org/mailman/listinfo/buildroot

-- 
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 561 099 427 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

^ permalink raw reply	[flat|nested] 6+ messages in thread
* Re: [Buildroot] [PATCH 2/3] utils/test-pkg: add ability to fail when skipped
  2022-01-01 21:08   ` Yann E. MORIN
@ 2022-01-31 20:54     ` Arnout Vandecappelle
  0 siblings, 0 replies; 6+ messages in thread
From: Arnout Vandecappelle @ 2022-01-31 20:54 UTC (permalink / raw)
  To: Yann E. MORIN, Matthew Weber; +Cc: buildroot



On 01/01/2022 22:08, Yann E. MORIN wrote:
> Matthew, All,
> 
> On 2021-12-01 17:08 -0600, Matthew Weber via buildroot spake thusly:
>> Add flag to allow the return value to be incremented when a skip
>> happens, treating the overall run as a failure.
>>
>> This flag could be used when using test-pkg for local validation of
>> downstream packages, for example in a user's br2-external. An example of
>> this could be a user choosing to use test-pkg as part of a CI
>> infrastructure where the fragment and toolchains are tightly controlled
>> to test a package against multiple toolchains easily.
>>
>> In this case, the user would want to lower the rate of false-positives
>> of that fragment/toolchain configuration if a build is instead skipped
>> (which may happen in the case of buildroot version bumps where
>> dependencies could change).
> 
> I am a bit skeptical about this use-case. >
> As I understand it, wht is most interesting in a CI setup for internal
> development, is that *all* internal packages do build togetrher and form
> a working sysem *as a whole*.
> 
> As such, a CI setup would build the defconfig file(s) for a project (and
> run the integration test-suite), rather than build each package
> individually

  I don't think the intention is necessarily to build each package individually. 
Although it actually does make a whole lot of sense to do that, because it helps 
to check if the dependencies are set correctly.

  So, the obvious way to take the approach that Yann proposes is to have a list 
of defconfigs that enable all the packages in the external with the various 
toolchains. However, that still doesn't solve the issue that Matt faces, because 
any package can silently disappear from the config if its dependencies are no 
longer met.

  But indeed, it would make a *whole* lot of sense to add a check to %_defconfig 
that everything in the defconfig still appears in the generated .config file. If 
it doesn't, there is something really wrong...

  So yeah, I tend to agree that this is not the approach we should have upstream.

  Note that in CI it's very easy to check the output of test-pkg and search for 
SKIPPED lines. So it's not that this feature is really *needed* to land upstream.


  Therefore, I've marked the series as Rejected.

  As always, you're welcome to argue why we should include it after all.

  Regards,
  Arnout


> 
> Building each package individually with test-pkg would have the
> disadvantage that it takes more time overall, as all the common deps are
> build as many times as there are packages to test, and package are also
> dependencies one of the others, so most packages would be built more
> than once as well...
> 
> So, I am really not convinced...
> 
> Regards,
> Yann E. MORIN.
> 
>> Signed-off-by: Matthew Weber <matthew.weber@collins.com>
>> ---
>>   utils/test-pkg | 18 +++++++++++++++---
>>   1 file changed, 15 insertions(+), 3 deletions(-)
>>
>> diff --git a/utils/test-pkg b/utils/test-pkg
>> index d0472f176b..97957008fa 100755
>> --- a/utils/test-pkg
>> +++ b/utils/test-pkg
>> @@ -18,8 +18,8 @@ main() {
>>       local -a toolchains
>>       local pkg_br_name
>>   
>> -    o='hakc:d:n:p:r:t:e:'
>> -    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:,externals:,'
>> +    o='hakc:d:n:p:r:t:e:f'
>> +    O='help,all,keep,prepare-only,config-snippet:,build-dir:,number:,package:,random:,toolchains-csv:,externals:,fail-on-skip'
>>       opts="$(getopt -n "${my_name}" -o "${o}" -l "${O}" -- "${@}")"
>>       eval set -- "${opts}"
>>   
>> @@ -29,6 +29,7 @@ main() {
>>       number=0
>>       mode=0
>>       prepare_only=0
>> +    fail_on_skip=0
>>       toolchains_csv="${TOOLCHAINS_CSV}"
>>       while [ ${#} -gt 0 ]; do
>>           case "${1}" in
>> @@ -65,6 +66,9 @@ main() {
>>           (-e|--externals)
>>               BR2_EXTERNALS="${2}"; shift 2
>>               ;;
>> +        (-f|--fail-on-skip)
>> +            fail_on_skip=1; shift 1
>> +            ;;
>>           (--)
>>               shift; break
>>               ;;
>> @@ -147,7 +151,12 @@ main() {
>>       printf "%d builds, %d skipped, %d build failed, %d legal-info failed\n" \
>>           ${nb} ${nb_skip} ${nb_fail} ${nb_legal}
>>   
>> -    return $((nb_fail + nb_legal))
>> +    nb_result=$((nb_fail + nb_legal))
>> +    if [ ${fail_on_skip} ]; then
>> +        nb_result=$((nb_result + nb_skip))
>> +    fi
>> +
>> +    return $nb_result
>>   }
>>   
>>   build_one() {
>> @@ -279,6 +288,9 @@ Options:
>>           Externals to be used as part of the build process. Packages from
>>           within these externals may be tested.
>>   
>> +    -f, --fail-on-skip
>> +        If any builds are skipped, return a negative exit value.
>> +
>>   Example:
>>   
>>       Testing libcec would require a config snippet that contains:
>> -- 
>> 2.17.1
>>
>> _______________________________________________
>> buildroot mailing list
>> buildroot@buildroot.org
>> https://lists.buildroot.org/mailman/listinfo/buildroot
> 
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot

^ permalink raw reply	[flat|nested] 6+ messages in thread
end of thread, other threads:[~2022-01-31 20:54 UTC | newest]

Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-12-01 23:08 [Buildroot] [PATCH 1/3] utils/test-pkg: add support for externals Matthew Weber via buildroot
2021-12-01 23:08 ` [Buildroot] [PATCH 2/3] utils/test-pkg: add ability to fail when skipped Matthew Weber via buildroot
2022-01-01 21:08   ` Yann E. MORIN
2022-01-31 20:54     ` Arnout Vandecappelle
2021-12-01 23:08 ` [Buildroot] [PATCH 3/3] docs/manual: add section describing testing external applications Matthew Weber via buildroot
2022-01-03 20:59 ` [Buildroot] [PATCH 1/3] utils/test-pkg: add support for externals Yann E. MORIN

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.