[1.46][PATCH 0/8] Backport fixes for Python 3.10 compatibility

[1.46][PATCH 0/8] Backport fixes for Python 3.10 compatibility

[Justin Bronder]

With Python 3.10.0 and the following patches bitbake-selftest passed for
me and I was able to run through my standard uses of bitbake.  Note that
all but the last commit are direct cherry-picks from master.

 lib/bb/compat.py              | 10 ----------
 lib/bb/data_smart.py          |  4 ++--
 lib/bb/event.py               | 16 ++++++++--------
 lib/bb/persist_data.py        | 13 +++++++------
 lib/bb/server/process.py      |  2 +-
 lib/bb/tests/event.py         | 17 +++++++++--------
 lib/bb/tests/fetch.py         |  6 +++---
 lib/bblayers/query.py         |  6 +++---
 lib/hashserv/server.py        |  4 ++--
 lib/layerindexlib/__init__.py |  1 -
 10 files changed, 35 insertions(+), 44 deletions(-)

[1.46][PATCH 1/8] compat.py: remove file since it no longer actually implements anything

[Justin Bronder]

From: Chris Laplante <chris.laplante@agilent.com>

Now that compat.py just imports Python standard library stuff, get rid
of the layer of indirection.

Signed-off-by: Chris Laplante <chris.laplante@agilent.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Justin Bronder <jsbronder@cold-front.org>
---
 lib/bb/compat.py       | 10 ----------
 lib/bb/event.py        | 16 ++++++++--------
 lib/bb/persist_data.py |  8 ++++----
 lib/bb/tests/event.py  | 17 +++++++++--------
 4 files changed, 21 insertions(+), 30 deletions(-)
 delete mode 100644 lib/bb/compat.py

diff --git a/lib/bb/compat.py b/lib/bb/compat.py
deleted file mode 100644
index 49356681..00000000
--- a/lib/bb/compat.py
+++ /dev/null
@@ -1,10 +0,0 @@
-#
-# SPDX-License-Identifier: GPL-2.0-only
-#
-
-"""Code pulled from future python versions, here for compatibility"""
-
-from collections import MutableMapping, KeysView, ValuesView, ItemsView, OrderedDict
-from functools import total_ordering
-
-
diff --git a/lib/bb/event.py b/lib/bb/event.py
index d1359f01..cb0b3b33 100644
--- a/lib/bb/event.py
+++ b/lib/bb/event.py
@@ -10,17 +10,17 @@ BitBake build tools.
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import sys
-import pickle
-import logging
-import atexit
-import traceback
 import ast
+import atexit
+import collections
+import logging
+import pickle
+import sys
 import threading
+import traceback
 
-import bb.utils
-import bb.compat
 import bb.exceptions
+import bb.utils
 
 # This is the pid for which we should generate the event. This is set when
 # the runqueue forks off.
@@ -56,7 +56,7 @@ def set_class_handlers(h):
     _handlers = h
 
 def clean_class_handlers():
-    return bb.compat.OrderedDict()
+    return collections.OrderedDict()
 
 # Internal
 _handlers = clean_class_handlers()
diff --git a/lib/bb/persist_data.py b/lib/bb/persist_data.py
index 7357ab2d..5f4fbe35 100644
--- a/lib/bb/persist_data.py
+++ b/lib/bb/persist_data.py
@@ -12,14 +12,14 @@ currently, providing a key/value store accessed by 'domain'.
 #
 
 import collections
+import contextlib
+import functools
 import logging
 import os.path
+import sqlite3
 import sys
 import warnings
-from bb.compat import total_ordering
 from collections import Mapping
-import sqlite3
-import contextlib
 
 sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
@@ -28,7 +28,7 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
 
 logger = logging.getLogger("BitBake.PersistData")
 
-@total_ordering
+@functools.total_ordering
 class SQLTable(collections.MutableMapping):
     class _Decorators(object):
         @staticmethod
diff --git a/lib/bb/tests/event.py b/lib/bb/tests/event.py
index 9229b63d..9ca7e9bc 100644
--- a/lib/bb/tests/event.py
+++ b/lib/bb/tests/event.py
@@ -6,17 +6,18 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import unittest
-import bb
-import logging
-import bb.compat
-import bb.event
+import collections
 import importlib
+import logging
+import pickle
 import threading
 import time
-import pickle
+import unittest
 from unittest.mock import Mock
 from unittest.mock import call
+
+import bb
+import bb.event
 from bb.msg import BBLogFormatter
 
 
@@ -75,7 +76,7 @@ class EventHandlingTest(unittest.TestCase):
 
     def _create_test_handlers(self):
         """ Method used to create a test handler ordered dictionary """
-        test_handlers = bb.compat.OrderedDict()
+        test_handlers = collections.OrderedDict()
         test_handlers["handler1"] = self._test_process.handler1
         test_handlers["handler2"] = self._test_process.handler2
         return test_handlers
@@ -96,7 +97,7 @@ class EventHandlingTest(unittest.TestCase):
 
     def test_clean_class_handlers(self):
         """ Test clean_class_handlers method """
-        cleanDict = bb.compat.OrderedDict()
+        cleanDict = collections.OrderedDict()
         self.assertEqual(cleanDict,
                          bb.event.clean_class_handlers())
 
-- 
2.33.0

[1.46][PATCH 2/8] bitbake: correct the collections vs collections.abc deprecation

[Justin Bronder]

From: Alexander Kanavin <alex.kanavin@gmail.com>

This becomes a hard error in python 3.10.

Signed-off-by: Alexander Kanavin <alex@linutronix.de>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Justin Bronder <jsbronder@cold-front.org>
---
 lib/bb/data_smart.py   | 2 +-
 lib/bb/persist_data.py | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 68770832..c8d2a437 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -17,7 +17,7 @@ BitBake build tools.
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
 import copy, re, sys, traceback
-from collections import MutableMapping
+from collections.abc import MutableMapping
 import logging
 import hashlib
 import bb, bb.codeparser
diff --git a/lib/bb/persist_data.py b/lib/bb/persist_data.py
index 5f4fbe35..56c983f8 100644
--- a/lib/bb/persist_data.py
+++ b/lib/bb/persist_data.py
@@ -12,6 +12,7 @@ currently, providing a key/value store accessed by 'domain'.
 #
 
 import collections
+import collections.abc
 import contextlib
 import functools
 import logging
@@ -19,7 +20,7 @@ import os.path
 import sqlite3
 import sys
 import warnings
-from collections import Mapping
+from collections.abc import Mapping
 
 sqlversion = sqlite3.sqlite_version_info
 if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
@@ -29,7 +30,7 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
 logger = logging.getLogger("BitBake.PersistData")
 
 @functools.total_ordering
-class SQLTable(collections.MutableMapping):
+class SQLTable(collections.abc.MutableMapping):
     class _Decorators(object):
         @staticmethod
         def retry(*, reconnect=True):
-- 
2.33.0

[1.46][PATCH 3/8] bitbake: fix regexp deprecation warnings

[Justin Bronder]

From: Alexander Kanavin <alex.kanavin@gmail.com>

See here for details:
https://docs.python.org/3/library/re.html

Signed-off-by: Alexander Kanavin <alex@linutronix.de>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Justin Bronder <jsbronder@cold-front.org>
---
 lib/bblayers/query.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/bblayers/query.py b/lib/bblayers/query.py
index fb6f550e..652a3acc 100644
--- a/lib/bblayers/query.py
+++ b/lib/bblayers/query.py
@@ -433,10 +433,10 @@ NOTE: .bbappend files can impact the dependencies.
                     line = fnfile.readline()
 
         # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass
-        conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$")
-        inc_re = re.compile(".*\.inc$")
+        conf_re = re.compile(r".*/conf/machine/[^\/]*\.conf$")
+        inc_re = re.compile(r".*\.inc$")
         # The "inherit xxx" in .bbclass
-        bbclass_re = re.compile(".*\.bbclass$")
+        bbclass_re = re.compile(r".*\.bbclass$")
         for layerdir in self.bblayers:
             layername = self.get_layer_name(layerdir)
             for dirpath, dirnames, filenames in os.walk(layerdir):
-- 
2.33.0

[1.46][PATCH 4/8] bitbake: do not import imp in layerindexlib

[Justin Bronder]

From: Alexander Kanavin <alex.kanavin@gmail.com>

The module is deprecated and unused.

Signed-off-by: Alexander Kanavin <alex@linutronix.de>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Justin Bronder <jsbronder@cold-front.org>
---
 lib/layerindexlib/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/layerindexlib/__init__.py b/lib/layerindexlib/__init__.py
index 77196b40..f30ee9e2 100644
--- a/lib/layerindexlib/__init__.py
+++ b/lib/layerindexlib/__init__.py
@@ -6,7 +6,6 @@
 import datetime
 
 import logging
-import imp
 
 from collections import OrderedDict
 from layerindexlib.plugin import LayerIndexPluginUrlError
-- 
2.33.0

[1.46][PATCH 5/8] bitbake: adjust parser error check for python 3.10 compatibility

[Justin Bronder]

From: Alexander Kanavin <alex.kanavin@gmail.com>

The change was introduced in
https://github.com/python/cpython/commit/a698d52c3975c80b45b139b2f08402ec514dce75

Signed-off-by: Alexander Kanavin <alex@linutronix.de>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Justin Bronder <jsbronder@cold-front.org>
---
 lib/bb/data_smart.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index c8d2a437..c46d3f0a 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -403,7 +403,7 @@ class DataSmart(MutableMapping):
                     s = __expand_python_regexp__.sub(varparse.python_sub, s)
                 except SyntaxError as e:
                     # Likely unmatched brackets, just don't expand the expression
-                    if e.msg != "EOL while scanning string literal":
+                    if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"):
                         raise
                 if s == olds:
                     break
-- 
2.33.0

[1.46][PATCH 6/8] bitbake: correct deprecation warning in process.py

[Justin Bronder]

From: Alexander Kanavin <alex.kanavin@gmail.com>

Signed-off-by: Alexander Kanavin <alex@linutronix.de>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Justin Bronder <jsbronder@cold-front.org>
---
 lib/bb/server/process.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/bb/server/process.py b/lib/bb/server/process.py
index 45f2e863..43061eb3 100644
--- a/lib/bb/server/process.py
+++ b/lib/bb/server/process.py
@@ -587,7 +587,7 @@ class BBUIEventQueue:
         self.reader = ConnectionReader(readfd)
 
         self.t = threading.Thread()
-        self.t.setDaemon(True)
+        self.t.daemon = True
         self.t.run = self.startCallbackHandler
         self.t.start()
 
-- 
2.33.0

[1.46][PATCH 7/8] test/fetch: Update urls to match upstream branch name changes

[Justin Bronder]

From: Richard Purdie <richard.purdie@linuxfoundation.org>

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Justin Bronder <jsbronder@cold-front.org>
---
 lib/bb/tests/fetch.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index a0787952..8ad10708 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -937,7 +937,7 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_git_submodule_CLI11(self):
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -952,12 +952,12 @@ class FetcherNetworkTest(FetcherTest):
     @skipIfNoNetwork()
     def test_git_submodule_update_CLI11(self):
         """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
 
         # CLI11 that pulls in a newer nlohmann-json
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
-- 
2.33.0

[1.46][PATCH 8/8] hashserv: let asyncio discover the running loop

[Justin Bronder]

From 3.10 documentation [1]:
    Deprecated since version 3.8, removed in version 3.10: The loop
    parameter. This function has been implicitly getting the current
    running loop since 3.7

This is fixed in master as a side-effect of
cf9bc0310b0092bf52b61057405aeb51c86ba137 which is more intrusive but
likewise drops the loop parameter.

1. https://docs.python.org/3/library/asyncio-stream.html#asyncio.open_connection

Signed-off-by: Justin Bronder <jsbronder@cold-front.org>
---
 lib/hashserv/server.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/hashserv/server.py b/lib/hashserv/server.py
index 81050715..56f354bd 100644
--- a/lib/hashserv/server.py
+++ b/lib/hashserv/server.py
@@ -420,7 +420,7 @@ class Server(object):
 
     def start_tcp_server(self, host, port):
         self.server = self.loop.run_until_complete(
-            asyncio.start_server(self.handle_client, host, port, loop=self.loop)
+            asyncio.start_server(self.handle_client, host, port)
         )
 
         for s in self.server.sockets:
@@ -445,7 +445,7 @@ class Server(object):
             # Work around path length limits in AF_UNIX
             os.chdir(os.path.dirname(path))
             self.server = self.loop.run_until_complete(
-                asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop)
+                asyncio.start_unix_server(self.handle_client, os.path.basename(path))
             )
         finally:
             os.chdir(cwd)
-- 
2.33.0