[PATCH 0/2] AUH patchset

[PATCH 0/2] AUH patchset

[Marius Avram]

Two small features for the auto upgrade helper.
Contrib branch can be found here:
http://git.yoctoproject.org/cgit/cgit.cgi/poky-contrib/log/?h=mavram/auh_features

Marius Avram (2):
  Send emails for a list of recipes
  Kill suprocesses when pressing ctrl+c

 upgradehelper.py |  205 +++++++++++++++++++++++++++---------------------------
 1 file changed, 104 insertions(+), 101 deletions(-)

-- 
1.7.9.5

[PATCH 1/2] Send emails for a list of recipes

[Marius Avram]

If the --emails or -e command line argument is active and a single
or a list of recipes are given to the script, emails with the results
and patches will be sent to their maintainers or to the overriden
addresses from the configuration file.

Also the status email will be sent in the case a selection of
at least two recipes has been given to the script for upgrading.

Signed-off-by: Marius Avram <marius.avram@intel.com>
---
 upgradehelper.py |  199 +++++++++++++++++++++++++++---------------------------
 1 file changed, 98 insertions(+), 101 deletions(-)

diff --git a/upgradehelper.py b/upgradehelper.py
index cc90958..771c981 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -71,6 +71,8 @@ def parse_cmdline():
                         help="disable interactive mode")
     parser.add_argument("-d", "--debug-level", type=int, default=4, choices=range(1, 6),
                         help="set the debug level: CRITICAL=1, ERROR=2, WARNING=3, INFO=4, DEBUG=5")
+    parser.add_argument("-e", "--emails", type=bool, default=False,
+                        help="send emails to recipe maintainers")
     parser.add_argument("-s", "--skip-compilation", action="store_true", default=False,
                         help="do not compile, just change the checksums, remove PR, and commit")
     parser.add_argument("-c", "--config-file", default=None,
@@ -116,7 +118,27 @@ def parse_config_file(config_file):
     return (settings, maintainer_override)
 
 class Updater(object):
-    def __init__(self, auto_mode=False, skip_compilation=False):
+    mail_header = \
+        "Hello,\n\nYou are receiving this email because you are the maintainer\n" \
+        "of *%s* recipe and this is to let you know that the automatic attempt\n" \
+        "to upgrade the recipe to *%s* has %s.\n\n"
+
+    next_steps_info = \
+        "The recipe has been successfully compiled for all major architectures.\n\n" \
+        "Next steps:\n" \
+        "    - apply the patch: git am %s\n" \
+        "    - check that required patches have not been removed from the recipe\n" \
+        "    - compile an image that contains the package\n" \
+        "    - perform some basic sanity tests\n" \
+        "    - amend the patch and sign it off: git commit -s --reset-author --amend\n" \
+        "    - send it to the list\n\n" \
+
+    mail_footer = \
+        "Attached are the patch and the logs (+ license file diff) in case of failure.\n\n" \
+        "Regards,\nThe Upgrade Helper"
+
+
+    def __init__(self, auto_mode=False, send_email=False, skip_compilation=False):
 
         self.uh_dir = get_build_dir() + "/upgrade-helper"
         if not os.path.exists(self.uh_dir):
@@ -125,10 +147,10 @@ class Updater(object):
         self.bb = Bitbake(get_build_dir())
         self.buildhistory = BuildHistory(get_build_dir())
         self.git = None
-
-        self.author = None
+        self.author = "Upgrade Helper <uh@not.set>"
         self.skip_compilation = skip_compilation
         self.interactive = not auto_mode
+        self.send_email = send_email
 
         self.machines = ["qemux86", "qemux86-64", "qemuarm", "qemumips", "qemuppc"]
 
@@ -144,6 +166,7 @@ class Updater(object):
             (self._compile, None)
         ]
 
+        self.email_handler = Email(settings)
         self.statistics = Statistics()
 
 
@@ -307,8 +330,9 @@ class Updater(object):
 
     # this function will be called at the end of each recipe upgrade
     def pkg_upgrade_handler(self, err):
-        if err is not None and self.patch_file is not None:
+        if err and self.patch_file:
             answer = "N"
+            status_msg = str(err)
             if self.interactive:
                 I(" %s: Do you want to keep the changes? (y/N)" % self.pn)
                 answer = sys.stdin.readline().strip().upper()
@@ -317,6 +341,53 @@ class Updater(object):
                 I(" %s: Dropping changes from git ..." % self.pn)
                 self.git.reset_hard(1)
                 self.git.clean_untracked()
+                return
+        elif not err:
+            status_msg = "Succeeded"
+
+        status = type(err).__name__
+
+        # drop last upgrade from git. It's safer this way if the upgrade has
+        # problems and other recipes depend on it. Give the other recipes a
+        # chance...
+        if ("drop_previous_commits" in settings and
+                settings["drop_previous_commits"] == "yes" and
+                err) or (err and self.patch_file):
+            I(" %s: Dropping changes from git ..." % self.pn)
+            self.git.reset_hard(1)
+            self.git.clean_untracked()
+
+        if self.send_email:
+            # don't bother maintainer with mail if the recipe is already up to date
+            if status == "UpgradeNotNeededError":
+                return
+
+            if self.maintainer in maintainer_override:
+                to_addr = maintainer_override[self.maintainer]
+            else:
+                to_addr = self.maintainer
+
+            subject = "[AUH] " + self.pn + ": upgrading to " + self.new_ver
+            if err is None:
+                subject += " SUCCEEDED"
+            else:
+                subject += " FAILED"
+
+            msg_body = self.mail_header % (self.pn, self.new_ver, status_msg)
+
+            if err is None:
+                msg_body += self.next_steps_info % os.path.basename(self.patch_file)
+
+            msg_body += self.mail_footer
+
+            # Add possible attachments to email
+            attachments = []
+            for attachment in os.listdir(self.workdir):
+                attachment_fullpath = os.path.join(self.workdir, attachment)
+                if os.path.isfile(attachment_fullpath):
+                    attachments.append(attachment_fullpath)
+
+            self.email_handler.send_email(to_addr, subject, msg_body, attachments)
 
     def _commit_changes(self):
         try:
@@ -355,6 +426,22 @@ class Updater(object):
 
         return ordered_list
 
+    def send_status_mail(self):
+        if "status_recipients" not in settings:
+            E("Could not send status email, no recipients set!")
+            return -1
+
+        to_list = settings["status_recipients"].split()
+
+        subject = "[AUH] Upgrade status: " + date.isoformat(date.today())
+
+        msg = self.statistics.pkg_stats() + self.statistics.maintainer_stats()
+
+        if self.statistics.total_attempted:
+            self.email_handler.send_email(to_list, subject, msg)
+        else:
+            W("No recipes attempted, not sending status mail!")
+
     def run(self, package_list=None):
 #[lp]        pkgs_to_upgrade = self._order_list(self._get_packages_to_upgrade(package_list))
         pkgs_to_upgrade = self._get_packages_to_upgrade(package_list)
@@ -399,32 +486,12 @@ class Updater(object):
 
         if (attempted_pkgs > 1):
             print("%s" % self.statistics.pkg_stats())
+            if self.send_email:
+                self.send_status_mail()
 
-
-class UniverseUpdater(Updater, Email):
-    mail_header = \
-        "Hello,\n\nYou are receiving this email because you are the maintainer\n" \
-        "of *%s* recipe and this is to let you know that the automatic attempt\n" \
-        "to upgrade the recipe to *%s* has %s.\n\n"
-
-    next_steps_info = \
-        "The recipe has been successfully compiled for all major architectures.\n\n" \
-        "Next steps:\n" \
-        "    - apply the patch: git am %s\n" \
-        "    - check that required patches have not been removed from the recipe\n" \
-        "    - compile an image that contains the package\n" \
-        "    - perform some basic sanity tests\n" \
-        "    - amend the patch and sign it off: git commit -s --reset-author --amend\n" \
-        "    - send it to the list\n\n" \
-
-    mail_footer = \
-        "Attached are the patch and the logs (+ license file diff) in case of failure.\n\n" \
-        "Regards,\nThe Upgrade Helper"
-
+class UniverseUpdater(Updater):
     def __init__(self):
-        Updater.__init__(self, True)
-        Email.__init__(self, settings)
-        self.author = "Upgrade Helper <uh@not.set>"
+        Updater.__init__(self, True, True)
         self.git = Git(os.path.dirname(os.getenv('PATH', False).split(':')[0]))
 
         # read history file
@@ -570,84 +637,14 @@ class UniverseUpdater(Updater, Email):
 
     # overriding the base method
     def pkg_upgrade_handler(self, err):
-        if err is None:
-            status_msg = "Succeeded"
-        else:
-            status_msg = str(err)
-
-        status = type(err).__name__
-
-        # drop last upgrade from git. It's safer this way if the upgrade has
-        # problems and other recipes depend on it. Give the other recipes a
-        # chance...
-        if ("drop_previous_commits" in settings and
-                settings["drop_previous_commits"] == "yes" and
-                err is None) or (err is not None and self.patch_file is not None):
-            I(" %s: Dropping changes from git ..." % self.pn)
-            self.git.reset_hard(1)
-            self.git.clean_untracked()
-
-        self.update_history(self.pn, self.new_ver, self.maintainer,
-                            status_msg)
-
-        # don't bother maintainer with mails for unknown errors, unsuported
-        # protocol or if the recipe is already up to date
-        if status == "Error" or status == "UnsupportedProtocolError" or \
-                status == "UpgradeNotNeededError":
-            return
-
-        if self.maintainer in maintainer_override:
-            to_addr = maintainer_override[self.maintainer]
-        else:
-            to_addr = self.maintainer
-
-        subject = "[AUH] " + self.pn + ": upgrading to " + self.new_ver
-        if err is None:
-            subject += " SUCCEEDED"
-        else:
-            subject += " FAILED"
-
-        msg_body = self.mail_header % (self.pn, self.new_ver, status_msg)
-
-        if err is None:
-            msg_body += self.next_steps_info % os.path.basename(self.patch_file)
-
-        msg_body += self.mail_footer
-
-        # Add possible attachments to list
-        attachments = []
-        for attachment in os.listdir(self.workdir):
-            attachment_fullpath = os.path.join(self.workdir, attachment)
-            if os.path.isfile(attachment_fullpath):
-                attachments.append(attachment_fullpath)
-
-        self.send_email(to_addr, subject, msg_body, attachments)
-
-    def send_status_mail(self):
-        if "status_recipients" not in settings:
-            E("Could not send status email, no recipients set!")
-            return -1
-
-        to_list = settings["status_recipients"].split()
-
-        subject = "[AUH] Upgrade status: " + date.isoformat(date.today())
-
-        msg = self.statistics.pkg_stats() + self.statistics.maintainer_stats()
-
-        if self.statistics.total_attempted:
-            self.send_email(to_list, subject, msg)
-        else:
-            W("No recipes attempted, not sending status mail!")
+        super(UniverUpdate, self).pkg_upgrade_handler(self)
+        self.update_history(self.pn, self.new_ver, self.maintainer, status_msg)
 
     def run(self):
         self.update_master()
-
         self.prepare()
-
         super(UniverseUpdater, self).run()
 
-        self.send_status_mail()
-
 
 if __name__ == "__main__":
     global settings
@@ -675,5 +672,5 @@ if __name__ == "__main__":
             for pkg in args.recipe:
                 pkg_list.append((pkg, None, None))
 
-        updater = Updater(args.auto_mode, args.skip_compilation)
+        updater = Updater(args.auto_mode, args.emails, args.skip_compilation)
         updater.run(pkg_list)
-- 
1.7.9.5

[PATCH 2/2] Kill suprocesses when pressing ctrl+c

[Marius Avram]

This way the created bitbake subprocesses from the upgrade script
are killed as well when you press ctrl+c.

Signed-off-by: Marius Avram <marius.avram@intel.com>
---
 upgradehelper.py |    6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/upgradehelper.py b/upgradehelper.py
index 771c981..7db89a7 100755
--- a/upgradehelper.py
+++ b/upgradehelper.py
@@ -35,6 +35,7 @@ from logging import warning as W
 from logging import error as E
 from logging import critical as C
 import re
+import signal
 import sys
 import ConfigParser as cp
 from datetime import datetime
@@ -645,11 +646,16 @@ class UniverseUpdater(Updater):
         self.prepare()
         super(UniverseUpdater, self).run()
 
+def close_child_processes(signal_id, frame):
+    pid = os.getpid()
+    os.killpg(pid, signal.SIGKILL)
 
 if __name__ == "__main__":
     global settings
     global maintainer_override
 
+    signal.signal(signal.SIGINT, close_child_processes)
+
     debug_levels = [log.CRITICAL, log.ERROR, log.WARNING, log.INFO, log.DEBUG]
     args = parse_cmdline()
     log.basicConfig(format='%(levelname)s:%(message)s',
-- 
1.7.9.5

Re: [PATCH 2/2] Kill suprocesses when pressing ctrl+c

[Paul Eggleton]

Hi Marius,

On Thursday 21 August 2014 15:00:31 Marius Avram wrote:
> This way the created bitbake subprocesses from the upgrade script
> are killed as well when you press ctrl+c.
> 
> Signed-off-by: Marius Avram <marius.avram@intel.com>
> ---
>  upgradehelper.py |    6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/upgradehelper.py b/upgradehelper.py
> index 771c981..7db89a7 100755
> --- a/upgradehelper.py
> +++ b/upgradehelper.py
> @@ -35,6 +35,7 @@ from logging import warning as W
>  from logging import error as E
>  from logging import critical as C
>  import re
> +import signal
>  import sys
>  import ConfigParser as cp
>  from datetime import datetime
> @@ -645,11 +646,16 @@ class UniverseUpdater(Updater):
>          self.prepare()
>          super(UniverseUpdater, self).run()
> 
> +def close_child_processes(signal_id, frame):
> +    pid = os.getpid()
> +    os.killpg(pid, signal.SIGKILL)

I could be wrong but I think this ought to be:

    pgid = os.getpgrp()
    os.killpg(pgid, signal.SIGKILL)

> 
>  if __name__ == "__main__":
>      global settings
>      global maintainer_override
> 
> +    signal.signal(signal.SIGINT, close_child_processes)
> +
>      debug_levels = [log.CRITICAL, log.ERROR, log.WARNING, log.INFO,
> log.DEBUG] args = parse_cmdline()
>      log.basicConfig(format='%(levelname)s:%(message)s',

-- 

Paul Eggleton
Intel Open Source Technology Centre

Re: [PATCH 1/2] Send emails for a list of recipes

[Paul Eggleton]

Hi Marius,

On Thursday 21 August 2014 15:00:30 Marius Avram wrote:
> If the --emails or -e command line argument is active and a single
> or a list of recipes are given to the script, emails with the results
> and patches will be sent to their maintainers or to the overriden
> addresses from the configuration file.
> 
> Also the status email will be sent in the case a selection of
> at least two recipes has been given to the script for upgrading.
> 
> Signed-off-by: Marius Avram <marius.avram@intel.com>
> ---
>  upgradehelper.py |  199
> +++++++++++++++++++++++++++--------------------------- 1 file changed, 98
> insertions(+), 101 deletions(-)
> 
> diff --git a/upgradehelper.py b/upgradehelper.py
> index cc90958..771c981 100755
> --- a/upgradehelper.py
> +++ b/upgradehelper.py
> @@ -71,6 +71,8 @@ def parse_cmdline():
>                          help="disable interactive mode")
>      parser.add_argument("-d", "--debug-level", type=int, default=4,
> choices=range(1, 6), help="set the debug level: CRITICAL=1, ERROR=2,
> WARNING=3, INFO=4, DEBUG=5") 
> +    parser.add_argument("-e", "--emails", type=bool, default=False, 
> +                        help="send emails to recipe maintainers") 

Picky I know, but could you make the long option --send-emails?

> @@ -125,10 +147,10 @@ class Updater(object):
>          self.bb = Bitbake(get_build_dir())
>          self.buildhistory = BuildHistory(get_build_dir())
>          self.git = None
> -
> -        self.author = None
> +        self.author = "Upgrade Helper <uh@not.set>"

Does this mean that the author value is now forced to be this even if
I just want to upgrade a few recipes locally with the tool (and therefore
my own author info should be used)?

> -        if self.statistics.total_attempted:
> -            self.send_email(to_list, subject, msg)
> -        else:
> -            W("No recipes attempted, not sending status mail!")
> +        super(UniverUpdate, self).pkg_upgrade_handler(self)

Shouldn't this be UniverseUpdater?

Cheers,
Paul

-- 

Paul Eggleton
Intel Open Source Technology Centre