[toaster][PATCHv3 3/3] toaster: Update eventreplay functionality for new eventlog file structure

[Marlon Rodriguez Garcia <marlon.rodriguez-garcia@savoirfairelinux.com>]

Added class EventPlayer to list of libraries under bitbake/bb/ui/
Update file read functionality to match new eventlog format

Exclude listing of files that don't contain the allvariables definitions used to replay builds
This part of the feature should be revisited. Over a long period of time, the BB_DEFAULT_EVENTLOG
will exponentially increase the size of the log file and cause bottlenecks when importing.

Signed-off-by: Marlon Rodriguez Garcia <marlon.rodriguez-garcia@savoirfairelinux.com>
---
 lib/bb/ui/eventreplay.py        |  85 +++++++++++++++++++++
 lib/toaster/toastergui/views.py | 131 ++++++++++++--------------------
 2 files changed, 132 insertions(+), 84 deletions(-)
 create mode 100644 lib/bb/ui/eventreplay.py

diff --git a/lib/bb/ui/eventreplay.py b/lib/bb/ui/eventreplay.py
new file mode 100644
index 00000000..b5999a1b
--- /dev/null
+++ b/lib/bb/ui/eventreplay.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file re-uses code spread throughout other Bitbake source files.
+# As such, all other copyrights belong to their own right holders.
+#
+
+
+import os
+import sys
+import json
+import pickle
+import codecs
+
+
+class EventPlayer:
+    """Emulate a connection to a bitbake server."""
+
+    def __init__(self, eventfile, variables):
+        self.eventfile = eventfile
+        self.variables = variables
+        self.eventmask = []
+
+    def waitEvent(self, _timeout):
+        """Read event from the file."""
+        line = self.eventfile.readline().strip()
+        if not line:
+            return
+        try:
+            decodedline = json.loads(line)
+            if 'allvariables' in decodedline:
+                return
+            if not 'vars' in decodedline:
+                raise ValueError
+            event_str = decodedline['vars'].encode('utf-8')
+            event = pickle.loads(codecs.decode(event_str, 'base64'))
+            event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
+            if event_name not in self.eventmask:
+                return
+            return event
+        except ValueError as err:
+            print("Failed loading ", line)
+            raise err
+
+    def runCommand(self, command_line):
+        """Emulate running a command on the server."""
+        name = command_line[0]
+
+        if name == "getVariable":
+            var_name = command_line[1]
+            variable = self.variables.get(var_name)
+            if variable:
+                return variable['v'], None
+            return None, "Missing variable %s" % var_name
+
+        elif name == "getAllKeysWithFlags":
+            dump = {}
+            flaglist = command_line[1]
+            for key, val in self.variables.items():
+                try:
+                    if not key.startswith("__"):
+                        dump[key] = {
+                            'v': val['v'],
+                            'history' : val['history'],
+                        }
+                        for flag in flaglist:
+                            dump[key][flag] = val[flag]
+                except Exception as err:
+                    print(err)
+            return (dump, None)
+
+        elif name == 'setEventMask':
+            self.eventmask = command_line[-1]
+            return True, None
+
+        else:
+            raise Exception("Command %s not implemented" % command_line[0])
+
+    def getEventHandle(self):
+        """
+        This method is called by toasterui.
+        The return value is passed to self.runCommand but not used there.
+        """
+        pass
\ No newline at end of file
diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py
index 2e34c8a2..cac72d36 100644
--- a/lib/toaster/toastergui/views.py
+++ b/lib/toaster/toastergui/views.py
@@ -8,12 +8,12 @@
 
 import ast
 import re
-import pickle
-import codecs
 import subprocess
+import sys
 
 import bb.cooker
 from bb.ui import toasterui
+from bb.ui import eventreplay
 
 from django.db.models import F, Q, Sum
 from django.db import IntegrityError
@@ -1957,71 +1957,6 @@ if True:
         except (ObjectDoesNotExist, IOError):
             return toaster_render(request, "unavailable_artifact.html")
 
-class EventPlayer:
-    """Emulate a connection to a bitbake server."""
-
-    def __init__(self, eventfile, variables):
-        self.eventfile = eventfile
-        self.variables = variables
-        self.eventmask = []
-
-    def waitEvent(self, _timeout):
-        """Read event from the file."""
-        line = self.eventfile.readline().strip()
-        if not line:
-            return
-        try:
-            event_str = json.loads(line)['vars'].encode('utf-8')
-            event = pickle.loads(codecs.decode(event_str, 'base64'))
-            event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
-            if event_name not in self.eventmask:
-                return
-            return event
-        except ValueError as err:
-            print("Failed loading ", line)
-            raise err
-
-    def runCommand(self, command_line):
-        """Emulate running a command on the server."""
-        name = command_line[0]
-
-        if name == "getVariable":
-            var_name = command_line[1]
-            variable = self.variables.get(var_name)
-            if variable:
-                return variable['v'], None
-            return None, "Missing variable %s" % var_name
-
-        elif name == "getAllKeysWithFlags":
-            dump = {}
-            flaglist = command_line[1]
-            for key, val in self.variables.items():
-                try:
-                    if not key.startswith("__"):
-                        dump[key] = {
-                            'v': val['v'],
-                            'history' : val['history'],
-                        }
-                        for flag in flaglist:
-                            dump[key][flag] = val[flag]
-                except Exception as err:
-                    print(err)
-            return (dump, None)
-
-        elif name == 'setEventMask':
-            self.eventmask = command_line[-1]
-            return True, None
-
-        else:
-            raise Exception("Command %s not implemented" % command_line[0])
-
-    def getEventHandle(self):
-        """
-        This method is called by toasterui.
-        The return value is passed to self.runCommand but not used there.
-        """
-        pass
-
 
 class CommandLineBuilds(TemplateView):
     model = EventLogsImports
@@ -2038,7 +1973,14 @@ class CommandLineBuilds(TemplateView):
             files_list = []
 
             # Filter files that end with ".json"
-            event_files = [file for file in files if file.endswith(".json")]
+            event_files = []
+            for file in files:
+                if file.endswith(".json"):
+                    # because BB_DEFAULT_EVENTLOG is a directory, we need to check if the file is a valid eventlog
+                    with open("{}/{}".format(logs_dir, file)) as efile:
+                        content = efile.read()
+                        if 'allvariables' in content:
+                            event_files.append(file)
 
             #build dict for template using db data
             for event_file in event_files:
@@ -2077,7 +2019,6 @@ class CommandLineBuilds(TemplateView):
         imported_files = EventLogsImports.objects.all()
         try:
             if all_files == 'true':
-
                 # use of session variable to deactivate icon for builds in progress
                 request.session['all_builds'] = True
                 request.session.modified = True
@@ -2090,10 +2031,17 @@ class CommandLineBuilds(TemplateView):
                     else:
                         with open("{}/{}".format(logs_dir, file.get('name'))) as eventfile:
                             # load variables from the first line
-                            variables = json.loads(eventfile.readline().strip())['allvariables']
-
+                            variables = None
+                            while line := eventfile.readline().strip():
+                                try:
+                                    variables = json.loads(line)['allvariables']
+                                    break
+                                except (KeyError, json.JSONDecodeError):
+                                    continue
+                            if not variables:
+                                raise Exception("File content missing  build variables")
                             params = namedtuple('ConfigParams', ['observe_only'])(True)
-                            player = EventPlayer(eventfile, variables)
+                            player = eventreplay.EventPlayer(eventfile, variables)
 
                             toasterui.main(player, player, params)
                         event_log_import = EventLogsImports.objects.create(name=file.get('name'), imported=True)
@@ -2104,20 +2052,27 @@ class CommandLineBuilds(TemplateView):
                     file = self.request.FILES['eventlog_file']
                 else:
                     file = request.POST.get('file')
-
-                # use of session variable to deactivate icon for build in progress 
-                request.session['file'] = file
-                request.session['all_builds'] = False
-                request.session.modified = True
-                request.session.save()
+                    # use of session variable to deactivate icon for build in progress 
+                    request.session['file'] = file
+                    request.session['all_builds'] = False
+                    request.session.modified = True
+                    request.session.save()
 
                 if imported_files.filter(name=file).exists():
                     imported_files.filter(name=file)[0].imported = True
                 else: 
                     if isinstance(file, InMemoryUploadedFile) or isinstance(file, TemporaryUploadedFile):
-                        variables = json.loads(file.readline().strip())['allvariables']
+                        variables = None
+                        while line := file.readline().strip():
+                            try:
+                                variables = json.loads(line)['allvariables']
+                                break
+                            except (KeyError, json.JSONDecodeError):
+                                continue
+                        if not variables:
+                            raise Exception("File content missing  build variables")
                         params = namedtuple('ConfigParams', ['observe_only'])(True)
-                        player = EventPlayer(file, variables)
+                        player = eventreplay.EventPlayer(file, variables)
                         if not os.path.exists('{}/{}'.format(logs_dir, file.name)):
                             fs = FileSystemStorage(location=logs_dir)
                             fs.save(file.name, file)
@@ -2125,18 +2080,26 @@ class CommandLineBuilds(TemplateView):
                     else:
                         with open("{}/{}".format(logs_dir, file)) as eventfile:
                             # load variables from the first line
-                            variables = json.loads(eventfile.readline().strip())['allvariables']
+                            variables = None
+                            while line := eventfile.readline().strip():
+                                try:
+                                    variables = json.loads(line)['allvariables']
+                                    break
+                                except (KeyError, json.JSONDecodeError):
+                                    continue
+                            if not variables:
+                                raise Exception("File content missing  build variables")
                             params = namedtuple('ConfigParams', ['observe_only'])(True)
-                            player = EventPlayer(eventfile, variables)
+                            player = eventreplay.EventPlayer(eventfile, variables)
                             toasterui.main(player, player, params)
                     event_log_import = EventLogsImports.objects.create(name=file, imported=True)
                     event_log_import.build_id = Build.objects.last().id
                     event_log_import.save()
                     request.session['file'] = ""
-        except json.decoder.JSONDecodeError:
+        except Exception:
             messages.add_message(
                 self.request,
-                messages.SUCCESS,
+                messages.ERROR,
                 "The file content is not in the correct format. Update file content or upload a different file."
             )
             return HttpResponseRedirect("/toastergui/cmdline/")
-- 
2.34.1