[PATCH 1/5] data_smart: Fix overrides file/line message additions

[PATCH 1/5] data_smart: Fix overrides file/line message additions

[Richard Purdie]

The overrides warning message is meant to show filename and line
numbers but the variable names are incorrect and this wasn't working.
Fix it.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/data_smart.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 543372d153..242476be0e 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -494,10 +494,10 @@ class DataSmart(MutableMapping):
 
         if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var):
             info = "%s" % var
-            if "filename" in loginfo:
-                info += " file: %s" % loginfo[filename]
-            if "lineno" in loginfo:
-                info += " line: %s" % loginfo[lineno]
+            if "file" in loginfo:
+                info += " file: %s" % loginfo["file"]
+            if "line" in loginfo:
+                info += " line: %s" % loginfo["line"]
             bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info)
 
         self.expand_cache = {}
-- 
2.32.0

[PATCH 2/5] cooker: Improve parsing failure from handled exception usability

[Richard Purdie]

When a recipe raises a BBHandledException, it means the error was already
shown to the user. Adding an additional one here isn't helpful. What is
helpful is to mention that parsing was halted.

Tweak the code to do this with improves the messages the user sees
and helps understand what happened.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/cooker.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 08e45e79d0..d1d4e32595 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -2167,6 +2167,8 @@ class CookerParser(object):
                                             self.total)
 
             bb.event.fire(event, self.cfgdata)
+        else:
+            bb.error("Parsing halted due to errors")
 
         for process in self.processes:
             self.parser_quit.put(None)
@@ -2257,7 +2259,7 @@ class CookerParser(object):
             return False
         except bb.BBHandledException as exc:
             self.error += 1
-            logger.error('Failed to parse recipe: %s' % exc.recipe)
+            logger.debug('Failed to parse recipe: %s' % exc.recipe)
             self.shutdown(clean=False, force=True)
             return False
         except ParsingFailure as exc:
-- 
2.32.0

[PATCH 3/5] msg: Add a bb.warnonce() log method

[Richard Purdie]

This adds a log level and logging function call to use it where the
warning will only be displayed once, regardless of how many times the
message is logged.

This has to be done either in the cooker or on the UI side. I've opted
for the UI side since display control is really a UI issue but it uses
a common library filter function to enable it which can be reused
elsewhere.

The knotty message displayed as the build summary is tweaked to
make sense when the numbers won't match since it will still count
the number of times it was logged and this is probably helpful
for debugging in some cases so I've deliberately left it that way.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/__init__.py  |  6 ++++++
 lib/bb/msg.py       | 15 +++++++++++++++
 lib/bb/ui/knotty.py | 17 ++++++++++-------
 3 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py
index e01b8d5256..4b6ad5cbc8 100644
--- a/lib/bb/__init__.py
+++ b/lib/bb/__init__.py
@@ -71,6 +71,9 @@ class BBLoggerMixin(object):
     def verbnote(self, msg, *args, **kwargs):
         return self.log(logging.INFO + 2, msg, *args, **kwargs)
 
+    def warnonce(self, msg, *args, **kwargs):
+        return self.log(logging.WARNING - 1, msg, *args, **kwargs)
+
 Logger = logging.getLoggerClass()
 class BBLogger(Logger, BBLoggerMixin):
     def __init__(self, name, *args, **kwargs):
@@ -157,6 +160,9 @@ def verbnote(*args):
 def warn(*args):
     mainlogger.warning(''.join(args))
 
+def warnonce(*args):
+    mainlogger.warnonce(''.join(args))
+
 def error(*args, **kwargs):
     mainlogger.error(''.join(args), extra=kwargs)
 
diff --git a/lib/bb/msg.py b/lib/bb/msg.py
index 291b38ff7f..ae079c4596 100644
--- a/lib/bb/msg.py
+++ b/lib/bb/msg.py
@@ -31,6 +31,7 @@ class BBLogFormatter(logging.Formatter):
     VERBNOTE = logging.INFO + 2
     ERROR = logging.ERROR
     WARNING = logging.WARNING
+    WARNONCE = logging.WARNING - 1
     CRITICAL = logging.CRITICAL
 
     levelnames = {
@@ -42,6 +43,7 @@ class BBLogFormatter(logging.Formatter):
         PLAIN  : '',
         VERBNOTE: 'NOTE',
         WARNING : 'WARNING',
+        WARNONCE : 'WARNING',
         ERROR   : 'ERROR',
         CRITICAL: 'ERROR',
     }
@@ -58,6 +60,7 @@ class BBLogFormatter(logging.Formatter):
         PLAIN   : BASECOLOR,
         VERBNOTE: BASECOLOR,
         WARNING : YELLOW,
+        WARNONCE : YELLOW,
         ERROR   : RED,
         CRITICAL: RED,
     }
@@ -121,6 +124,18 @@ class BBLogFilter(object):
             return True
         return False
 
+class LogFilterWarnOnce(logging.Filter):
+    def __init__(self):
+        self.seen_warnings = set()
+
+    def filter(self, record):
+        msg = record.msg
+        if record.levelno == bb.msg.BBLogFormatter.WARNONCE:
+            if record.msg in self.seen_warnings:
+                return False
+            self.seen_warnings.add(record.msg)
+        return True
+
 class LogFilterGEQLevel(logging.Filter):
     def __init__(self, level):
         self.strlevel = str(level)
diff --git a/lib/bb/ui/knotty.py b/lib/bb/ui/knotty.py
index 484545a684..1c46ec843b 100644
--- a/lib/bb/ui/knotty.py
+++ b/lib/bb/ui/knotty.py
@@ -418,7 +418,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "formatter": "BitBake.consoleFormatter",
                 "level": console_loglevel,
                 "stream": "ext://sys.stdout",
-                "filters": ["BitBake.stdoutFilter"],
+                "filters": ["BitBake.warnonceFilter", "BitBake.stdoutFilter"],
                 ".": {
                     "is_console": True,
                 },
@@ -428,7 +428,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "formatter": "BitBake.consoleFormatter",
                 "level": loglevel,
                 "stream": "ext://sys.stderr",
-                "filters": ["BitBake.stderrFilter"],
+                "filters": ["BitBake.warnonceFilter", "BitBake.stderrFilter"],
                 ".": {
                     "is_console": True,
                 },
@@ -443,7 +443,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "formatter": "BitBake.consoleFormatter",
                 "level": 1,
                 "stream": "ext://sys.stdout",
-                "filters": ["BitBake.verbconsoleFilter"],
+                "filters": ["BitBake.warnonceFilter", "BitBake.verbconsoleFilter"],
                 ".": {
                     "is_console": True,
                 },
@@ -476,6 +476,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "()": "bb.msg.LogFilterLTLevel",
                 "level": console_loglevel
             },
+            "BitBake.warnonceFilter": {
+                "()": "bb.msg.LogFilterWarnOnce",
+            },
         },
         "loggers": {
             "BitBake": {
@@ -661,10 +664,10 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                         continue
 
                     # Prefix task messages with recipe/task
-                    if event.taskpid in helper.pidmap and event.levelno != bb.msg.BBLogFormatter.PLAIN:
+                    if event.taskpid in helper.pidmap and event.levelno != bb.msg.BBLogFormatter.PLAIN and event.levelno != bb.msg.BBLogFormatter.WARNONCE:
                         taskinfo = helper.running_tasks[helper.pidmap[event.taskpid]]
                         event.msg = taskinfo['title'] + ': ' + event.msg
-                if hasattr(event, 'fn'):
+                if hasattr(event, 'fn') and event.levelno != bb.msg.BBLogFormatter.WARNONCE:
                     event.msg = event.fn + ': ' + event.msg
                 logging.getLogger(event.name).handle(event)
                 continue
@@ -875,8 +878,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             for failure in taskfailures:
                 summary += "\n  %s" % failure
         if warnings:
-            summary += pluralise("\nSummary: There was %s WARNING message shown.",
-                                 "\nSummary: There were %s WARNING messages shown.", warnings)
+            summary += pluralise("\nSummary: There was %s WARNING message.",
+                                 "\nSummary: There were %s WARNING messages.", warnings)
         if return_value and errors:
             summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
                                  "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
-- 
2.32.0

[PATCH 4/5] msg: Add bb.erroronce() to logging

[Richard Purdie]

Similar to the warnonce addition in the previous commit, add a similarly
behaving version for error messages.

Tweak the log filter further rather than adding an additional one.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/__init__.py  |  7 +++++++
 lib/bb/msg.py       | 10 +++++++++-
 lib/bb/ui/knotty.py | 20 ++++++++++----------
 3 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/lib/bb/__init__.py b/lib/bb/__init__.py
index 4b6ad5cbc8..269f65edce 100644
--- a/lib/bb/__init__.py
+++ b/lib/bb/__init__.py
@@ -74,6 +74,10 @@ class BBLoggerMixin(object):
     def warnonce(self, msg, *args, **kwargs):
         return self.log(logging.WARNING - 1, msg, *args, **kwargs)
 
+    def erroronce(self, msg, *args, **kwargs):
+        return self.log(logging.ERROR - 1, msg, *args, **kwargs)
+
+
 Logger = logging.getLoggerClass()
 class BBLogger(Logger, BBLoggerMixin):
     def __init__(self, name, *args, **kwargs):
@@ -166,6 +170,9 @@ def warnonce(*args):
 def error(*args, **kwargs):
     mainlogger.error(''.join(args), extra=kwargs)
 
+def erroronce(*args):
+    mainlogger.erroronce(''.join(args))
+
 def fatal(*args, **kwargs):
     mainlogger.critical(''.join(args), extra=kwargs)
     raise BBHandledException()
diff --git a/lib/bb/msg.py b/lib/bb/msg.py
index ae079c4596..c2bd8aaf40 100644
--- a/lib/bb/msg.py
+++ b/lib/bb/msg.py
@@ -30,6 +30,7 @@ class BBLogFormatter(logging.Formatter):
     PLAIN = logging.INFO + 1
     VERBNOTE = logging.INFO + 2
     ERROR = logging.ERROR
+    ERRORONCE = logging.ERROR - 1
     WARNING = logging.WARNING
     WARNONCE = logging.WARNING - 1
     CRITICAL = logging.CRITICAL
@@ -45,6 +46,7 @@ class BBLogFormatter(logging.Formatter):
         WARNING : 'WARNING',
         WARNONCE : 'WARNING',
         ERROR   : 'ERROR',
+        ERRORONCE   : 'ERROR',
         CRITICAL: 'ERROR',
     }
 
@@ -62,6 +64,7 @@ class BBLogFormatter(logging.Formatter):
         WARNING : YELLOW,
         WARNONCE : YELLOW,
         ERROR   : RED,
+        ERRORONCE : RED,
         CRITICAL: RED,
     }
 
@@ -124,9 +127,10 @@ class BBLogFilter(object):
             return True
         return False
 
-class LogFilterWarnOnce(logging.Filter):
+class LogFilterShowOnce(logging.Filter):
     def __init__(self):
         self.seen_warnings = set()
+        self.seen_errors = set()
 
     def filter(self, record):
         msg = record.msg
@@ -134,6 +138,10 @@ class LogFilterWarnOnce(logging.Filter):
             if record.msg in self.seen_warnings:
                 return False
             self.seen_warnings.add(record.msg)
+        if record.levelno == bb.msg.BBLogFormatter.ERRORONCE:
+            if record.msg in self.seen_errors:
+                return False
+            self.seen_errors.add(record.msg)
         return True
 
 class LogFilterGEQLevel(logging.Filter):
diff --git a/lib/bb/ui/knotty.py b/lib/bb/ui/knotty.py
index 1c46ec843b..93eb4b56cb 100644
--- a/lib/bb/ui/knotty.py
+++ b/lib/bb/ui/knotty.py
@@ -418,7 +418,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "formatter": "BitBake.consoleFormatter",
                 "level": console_loglevel,
                 "stream": "ext://sys.stdout",
-                "filters": ["BitBake.warnonceFilter", "BitBake.stdoutFilter"],
+                "filters": ["BitBake.showonceFilter", "BitBake.stdoutFilter"],
                 ".": {
                     "is_console": True,
                 },
@@ -428,7 +428,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "formatter": "BitBake.consoleFormatter",
                 "level": loglevel,
                 "stream": "ext://sys.stderr",
-                "filters": ["BitBake.warnonceFilter", "BitBake.stderrFilter"],
+                "filters": ["BitBake.showonceFilter", "BitBake.stderrFilter"],
                 ".": {
                     "is_console": True,
                 },
@@ -443,7 +443,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "formatter": "BitBake.consoleFormatter",
                 "level": 1,
                 "stream": "ext://sys.stdout",
-                "filters": ["BitBake.warnonceFilter", "BitBake.verbconsoleFilter"],
+                "filters": ["BitBake.showonceFilter", "BitBake.verbconsoleFilter"],
                 ".": {
                     "is_console": True,
                 },
@@ -476,8 +476,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 "()": "bb.msg.LogFilterLTLevel",
                 "level": console_loglevel
             },
-            "BitBake.warnonceFilter": {
-                "()": "bb.msg.LogFilterWarnOnce",
+            "BitBake.showonceFilter": {
+                "()": "bb.msg.LogFilterShowOnce",
             },
         },
         "loggers": {
@@ -650,7 +650,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             if isinstance(event, logging.LogRecord):
                 lastprint = time.time()
                 printinterval = 5000
-                if event.levelno >= bb.msg.BBLogFormatter.ERROR:
+                if event.levelno >= bb.msg.BBLogFormatter.ERRORONCE:
                     errors = errors + 1
                     return_value = 1
                 elif event.levelno == bb.msg.BBLogFormatter.WARNING:
@@ -664,10 +664,10 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                         continue
 
                     # Prefix task messages with recipe/task
-                    if event.taskpid in helper.pidmap and event.levelno != bb.msg.BBLogFormatter.PLAIN and event.levelno != bb.msg.BBLogFormatter.WARNONCE:
+                    if event.taskpid in helper.pidmap and event.levelno not in [bb.msg.BBLogFormatter.PLAIN, bb.msg.BBLogFormatter.WARNONCE, bb.msg.BBLogFormatter.ERRORONCE]:
                         taskinfo = helper.running_tasks[helper.pidmap[event.taskpid]]
                         event.msg = taskinfo['title'] + ': ' + event.msg
-                if hasattr(event, 'fn') and event.levelno != bb.msg.BBLogFormatter.WARNONCE:
+                if hasattr(event, 'fn') and event.levelno not in [bb.msg.BBLogFormatter.WARNONCE, bb.msg.BBLogFormatter.ERRORONCE]:
                     event.msg = event.fn + ': ' + event.msg
                 logging.getLogger(event.name).handle(event)
                 continue
@@ -881,8 +881,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             summary += pluralise("\nSummary: There was %s WARNING message.",
                                  "\nSummary: There were %s WARNING messages.", warnings)
         if return_value and errors:
-            summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
-                                 "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
+            summary += pluralise("\nSummary: There was %s ERROR message, returning a non-zero exit code.",
+                                 "\nSummary: There were %s ERROR messages, returning a non-zero exit code.", errors)
         if summary and params.options.quiet == 0:
             print(summary)
 
-- 
2.32.0

[PATCH 5/5] data_smart: Add hasOverrides method to public datastore API

[Richard Purdie]

There are some cases where the metadata needs to check if a variable
has any overrides set, even if they are currently inactive. That code
currently pokes into datastore internals. Add API instead to replace
and avoid that.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/data_smart.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/bb/data_smart.py b/lib/bb/data_smart.py
index 242476be0e..cb2d4b620f 100644
--- a/lib/bb/data_smart.py
+++ b/lib/bb/data_smart.py
@@ -488,6 +488,8 @@ class DataSmart(MutableMapping):
         else:
             self.initVar(var)
 
+    def hasOverrides(self, var):
+        return var in self.overridedata
 
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
-- 
2.32.0