[PATCH 1/4] meta: correct collections vs collections.abc deprecation

[PATCH 1/4] meta: correct collections vs collections.abc deprecation

[Alexander Kanavin]

collections.abc is available since 3.3 and doesn't need special handling.

Signed-off-by: Alexander Kanavin <alex@linutronix.de>
---
 meta/lib/oe/maketype.py          | 7 +------
 scripts/lib/build_perf/report.py | 3 ++-
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/meta/lib/oe/maketype.py b/meta/lib/oe/maketype.py
index d929c8b3e54..d36082c5351 100644
--- a/meta/lib/oe/maketype.py
+++ b/meta/lib/oe/maketype.py
@@ -10,12 +10,7 @@ the arguments of the type's factory for details.
 
 import inspect
 import oe.types as types
-try:
-    # Python 3.7+
-    from collections.abc import Callable
-except ImportError:
-    # Python < 3.7
-    from collections import Callable
+from collections.abc import Callable
 
 available_types = {}
 
diff --git a/scripts/lib/build_perf/report.py b/scripts/lib/build_perf/report.py
index 4e8e2a8a939..ab77424cc74 100644
--- a/scripts/lib/build_perf/report.py
+++ b/scripts/lib/build_perf/report.py
@@ -4,7 +4,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 """Handling of build perf test reports"""
-from collections import OrderedDict, Mapping, namedtuple
+from collections import OrderedDict, namedtuple
+from collections.abc import Mapping
 from datetime import datetime, timezone
 from numbers import Number
 from statistics import mean, stdev, variance
-- 
2.20.1

[PATCH 2/4] wic: keep rootfs_size as integer

[Alexander Kanavin]

The corrected line accidentally converted it to float,
which causes problems later on with python 3.10:

|   File "/home/alex/development/poky/scripts/lib/wic/partition.py", line 278, in prepare_rootfs_ext
|     os.ftruncate(sparse.fileno(), rootfs_size * 1024)
| TypeError: 'float' object cannot be interpreted as an integer

Signed-off-by: Alexander Kanavin <alex@linutronix.de>
---
 scripts/lib/wic/partition.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py
index ab304f1b2a8..a25834048e8 100644
--- a/scripts/lib/wic/partition.py
+++ b/scripts/lib/wic/partition.py
@@ -105,7 +105,7 @@ class Partition():
                 extra_blocks = self.extra_space
 
             rootfs_size = actual_rootfs_size + extra_blocks
-            rootfs_size *= self.overhead_factor
+            rootfs_size = int(rootfs_size * self.overhead_factor)
 
             logger.debug("Added %d extra blocks to %s to get to %d total blocks",
                          extra_blocks, self.mountpoint, rootfs_size)
-- 
2.20.1

[PATCH 3/4] testimage: copy the task log and qemu console log to tmp/log/oeqa

[Alexander Kanavin]

This makes it easier for the AB scripts (particularly, collect-results)
to access and archive these items, as they can contain useful information
when ptests or other qemu tests fail.

[YOCTO #14518]

Signed-off-by: Alexander Kanavin <alex@linutronix.de>
---
 meta/classes/testimage.bbclass | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 3c689aec913..305e26d24d6 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -201,6 +201,7 @@ def testimage_main(d):
     import json
     import signal
     import logging
+    import shutil
 
     from bb.utils import export_proxies
     from oeqa.core.utils.misc import updateTestData
@@ -408,10 +409,17 @@ def testimage_main(d):
                         get_testimage_result_id(configuration),
                         dump_streams=d.getVar('TESTREPORT_FULLLOGS'))
         results.logSummary(pn)
+
+    # Copy additional logs to tmp/log/oeqa so it's easier to find them
+    targetdir = os.path.join(get_testimage_json_result_dir(d), d.getVar("PN"))
+    os.makedirs(targetdir, exist_ok=True)
+    shutil.copy(bootlog, targetdir)
+    shutil.copy(d.getVar("BB_LOGFILE"), targetdir)
+
     if not results or not complete:
-        bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
+        bb.fatal('%s - FAILED - tests were interrupted during execution, check the logs in %s' % (pn, d.getVar("LOG_DIR")), forcelog=True)
     if not results.wasSuccessful():
-        bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
+        bb.fatal('%s - FAILED - also check the logs in %s' % (pn, d.getVar("LOG_DIR")), forcelog=True)
 
 def get_runtime_paths(d):
     """
-- 
2.20.1

[PATCH 4/4] cpan-base.bbclass: use raw string for regexp

[Alexander Kanavin]

This eliminates a deprecation warning and a future error as stipulated by
https://docs.python.org/3/library/re.html

Signed-off-by: Alexander Kanavin <alex@linutronix.de>
---
 meta/classes/cpan-base.bbclass | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/meta/classes/cpan-base.bbclass b/meta/classes/cpan-base.bbclass
index 9758065bfcb..93d11e1bee4 100644
--- a/meta/classes/cpan-base.bbclass
+++ b/meta/classes/cpan-base.bbclass
@@ -20,7 +20,7 @@ PERLLIBDIRS:class-native = "${libdir}/perl5"
 def cpan_upstream_check_pattern(d):
     for x in (d.getVar('SRC_URI') or '').split(' '):
         if x.startswith("https://cpan.metacpan.org"):
-            _pattern = x.split('/')[-1].replace(d.getVar('PV'), '(?P<pver>\d+.\d+)')
+            _pattern = x.split('/')[-1].replace(d.getVar('PV'), r'(?P<pver>\d+.\d+)')
             return _pattern
     return ''
 
-- 
2.20.1

Re: [OE-core] [PATCH 3/4] testimage: copy the task log and qemu console log to tmp/log/oeqa

[Richard Purdie]

On Fri, 2021-09-17 at 14:59 +0200, Alexander Kanavin wrote:
> This makes it easier for the AB scripts (particularly, collect-results)
> to access and archive these items, as they can contain useful information
> when ptests or other qemu tests fail.
> 
> [YOCTO #14518]
> 
> Signed-off-by: Alexander Kanavin <alex@linutronix.de>
> ---
>  meta/classes/testimage.bbclass | 12 ++++++++++--
>  1 file changed, 10 insertions(+), 2 deletions(-)
> 
> diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
> index 3c689aec913..305e26d24d6 100644
> --- a/meta/classes/testimage.bbclass
> +++ b/meta/classes/testimage.bbclass
> @@ -201,6 +201,7 @@ def testimage_main(d):
>      import json
>      import signal
>      import logging
> +    import shutil
>  
>      from bb.utils import export_proxies
>      from oeqa.core.utils.misc import updateTestData
> @@ -408,10 +409,17 @@ def testimage_main(d):
>                          get_testimage_result_id(configuration),
>                          dump_streams=d.getVar('TESTREPORT_FULLLOGS'))
>          results.logSummary(pn)
> +
> +    # Copy additional logs to tmp/log/oeqa so it's easier to find them
> +    targetdir = os.path.join(get_testimage_json_result_dir(d), d.getVar("PN"))
> +    os.makedirs(targetdir, exist_ok=True)
> +    shutil.copy(bootlog, targetdir)
> +    shutil.copy(d.getVar("BB_LOGFILE"), targetdir)
> +
>      if not results or not complete:
> -        bb.fatal('%s - FAILED - tests were interrupted during execution' % pn, forcelog=True)
> +        bb.fatal('%s - FAILED - tests were interrupted during execution, check the logs in %s' % (pn, d.getVar("LOG_DIR")), forcelog=True)
>      if not results.wasSuccessful():
> -        bb.fatal('%s - FAILED - check the task log and the ssh log' % pn, forcelog=True)
> +        bb.fatal('%s - FAILED - also check the logs in %s' % (pn, d.getVar("LOG_DIR")), forcelog=True)
>  
>  def get_runtime_paths(d):
>      """

I'm a little torn on this too as I dislike copies of files in multiple places.
Part of me wonders if we should move the files to this location at generation
instead. That would break any existing users though and make it less obvious
when looking at WORKDIR for a given failure and I think having WORKDIR remain as
the place for a given task's execution is important for user expectations.
Symlinking them may be the lesser of several issues. Perhaps they should really
be in ${T} (in WORKDIR) too.

I did wonder about moving and standardising on tmp/log for more logfiles a while
back but came down against as it would undermine the usefulness and user
expectations of WORKDIR.

So symlinking here may be the best option?

Cheers,

Richard