[PATCH V3 0/5] Changes regarding log checking and license checksums

[PATCH V3 0/5] Changes regarding log checking and license checksums

[Chen Qi]

The following changes since commit 0752c79282b1cc9699743e719518e6c341d50a3a:

  systemd: fix /var/log/journal ownership (2015-03-16 17:38:51 +0000)

are available in the git repository at:

  git://git.openembedded.org/openembedded-core-contrib ChenQi/log-check-license-warn
  http://cgit.openembedded.org/cgit.cgi/openembedded-core-contrib/log/?h=ChenQi/log-check-license-warn

Chen Qi (5):
  rootfs.py: two changes regarding log checking
  rootfs.py: add log checking ability for deb and ipk
  license.bbclass: skip license checking if the package contains no file
  os-release: add LIC_FILES_CHKSUM
  glibc-collateral.inc: add LIC_FILES_CHKSUM

 meta/classes/license.bbclass                 |  8 ++++
 meta/lib/oe/rootfs.py                        | 64 +++++++++++++++++++---------
 meta/recipes-core/glibc/glibc-collateral.inc |  2 +
 meta/recipes-core/os-release/os-release.bb   |  1 +
 4 files changed, 56 insertions(+), 19 deletions(-)

-- 
1.9.1

[PATCH V3 1/5] rootfs.py: two changes regarding log checking

[Chen Qi]

This patch involves two changes.

1. Extend the regular expression to also catch '^WARNING:' in _log_check_warn.
   Warnings from bb.note or bbnote begin with 'WARNING:'. So if we decide to
   catch warnings at rootfs time, we should not ignore those produced by
   the build system itself.

2. Delay _log_check in rootfs process so that more warnings are likely to be
   catched. Note that we should at least delay the _log_check after the
   execution of ROOTFS_POSTPROCESS_COMMANDS, because we want to catch warnings
   there.

Signed-off-by: Chen Qi <Qi.Chen@windriver.com>
---
 meta/lib/oe/rootfs.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index 7e8d5d1..3a77e86 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -128,6 +128,7 @@ class Rootfs(object):
             self._generate_kernel_module_deps()
 
         self._cleanup()
+        self._log_check()
 
     def _uninstall_unneeded(self):
         # Remove unneeded init script symlinks
@@ -327,8 +328,6 @@ class RpmRootfs(Rootfs):
 
         self.pm.install_complementary()
 
-        self._log_check()
-
         if self.inc_rpm_image_gen == "1":
             self.pm.backup_packaging_data()
 
@@ -355,7 +354,7 @@ class RpmRootfs(Rootfs):
         pass
 
     def _log_check_warn(self):
-        r = re.compile('^(warn|Warn|NOTE: warn|NOTE: Warn)')
+        r = re.compile('^(warn|Warn|NOTE: warn|NOTE: Warn|WARNING:)')
         log_path = self.d.expand("${T}/log.do_rootfs")
         with open(log_path, 'r') as log:
             for line in log.read().split('\n'):
-- 
1.9.1

[PATCH V3 2/5] rootfs.py: add log checking ability for deb and ipk

[Chen Qi]

Extract the common codes of log checking and add the ability of log
checking for deb and ipk package backend.

Signed-off-by: Chen Qi <Qi.Chen@windriver.com>
---
 meta/lib/oe/rootfs.py | 61 +++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 44 insertions(+), 17 deletions(-)

diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index 3a77e86..f887a98 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -40,6 +40,43 @@ class Rootfs(object):
     def _log_check(self):
         pass
 
+    def _log_check_warn(self):
+        r = re.compile('^(warn|Warn|NOTE: warn|NOTE: Warn|WARNING:)')
+        log_path = self.d.expand("${T}/log.do_rootfs")
+        with open(log_path, 'r') as log:
+            for line in log.read().split('\n'):
+                if 'log_check' in line or 'NOTE:' in line:
+                    continue
+
+                m = r.search(line)
+                if m:
+                    bb.warn('log_check: There is a warn message in the logfile')
+                    bb.warn('log_check: Matched keyword: [%s]' % m.group())
+                    bb.warn('log_check: %s\n' % line)
+
+    def _log_check_error(self):
+        r = re.compile(self.log_check_regex)
+        log_path = self.d.expand("${T}/log.do_rootfs")
+        with open(log_path, 'r') as log:
+            found_error = 0
+            message = "\n"
+            for line in log.read().split('\n'):
+                if 'log_check' in line:
+                    continue
+
+                m = r.search(line)
+                if m:
+                    found_error = 1
+                    bb.warn('log_check: There were error messages in the logfile')
+                    bb.warn('log_check: Matched keyword: [%s]\n\n' % m.group())
+
+                if found_error >= 1 and found_error <= 5:
+                    message += line + '\n'
+                    found_error += 1
+
+                if found_error == 6:
+                    bb.fatal(message)
+
     def _insert_feed_uris(self):
         if bb.utils.contains("IMAGE_FEATURES", "package-management",
                          True, False, self.d):
@@ -256,7 +293,7 @@ class Rootfs(object):
 class RpmRootfs(Rootfs):
     def __init__(self, d, manifest_dir):
         super(RpmRootfs, self).__init__(d)
-
+        self.log_check_regex = '(unpacking of archive failed|Cannot find package|exit 1|ERR|Fail)'
         self.manifest = RpmManifest(d, manifest_dir)
 
         self.pm = RpmPM(d,
@@ -353,20 +390,6 @@ class RpmRootfs(Rootfs):
         # already saved in /etc/rpm-postinsts
         pass
 
-    def _log_check_warn(self):
-        r = re.compile('^(warn|Warn|NOTE: warn|NOTE: Warn|WARNING:)')
-        log_path = self.d.expand("${T}/log.do_rootfs")
-        with open(log_path, 'r') as log:
-            for line in log.read().split('\n'):
-                if 'log_check' in line or 'NOTE:' in line:
-                    continue
-
-                m = r.search(line)
-                if m:
-                    bb.warn('log_check: There is a warn message in the logfile')
-                    bb.warn('log_check: Matched keyword: [%s]' % m.group())
-                    bb.warn('log_check: %s\n' % line)
-
     def _log_check_error(self):
         r = re.compile('(unpacking of archive failed|Cannot find package|exit 1|ERR|Fail)')
         log_path = self.d.expand("${T}/log.do_rootfs")
@@ -414,6 +437,7 @@ class RpmRootfs(Rootfs):
 class DpkgRootfs(Rootfs):
     def __init__(self, d, manifest_dir):
         super(DpkgRootfs, self).__init__(d)
+        self.log_check_regex = '^E:'
 
         bb.utils.remove(self.image_rootfs, True)
         bb.utils.remove(self.d.getVar('MULTILIB_TEMP_ROOTFS', True), True)
@@ -485,7 +509,8 @@ class DpkgRootfs(Rootfs):
         self.pm.mark_packages("unpacked", registered_pkgs.split())
 
     def _log_check(self):
-        pass
+        self._log_check_warn()
+        self._log_check_error()
 
     def _cleanup(self):
         pass
@@ -494,6 +519,7 @@ class DpkgRootfs(Rootfs):
 class OpkgRootfs(Rootfs):
     def __init__(self, d, manifest_dir):
         super(OpkgRootfs, self).__init__(d)
+        self.log_check_regex = '(exit 1|Collected errors)'
 
         self.manifest = OpkgManifest(d, manifest_dir)
         self.opkg_conf = self.d.getVar("IPKGCONF_TARGET", True)
@@ -755,7 +781,8 @@ class OpkgRootfs(Rootfs):
         self.pm.mark_packages("unpacked", registered_pkgs.split())
 
     def _log_check(self):
-        pass
+        self._log_check_warn()
+        self._log_check_error()
 
     def _cleanup(self):
         pass
-- 
1.9.1

[PATCH V3 3/5] license.bbclass: skip license checking if the package contains no file

[Chen Qi]

If the package doesn't contain any file, then the license isn't relevant
as far as the final image is concerned. So we skip the license checking
in license_create_manifest if such case.

Signed-off-by: Chen Qi <Qi.Chen@windriver.com>
---
 meta/classes/license.bbclass | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index 95e0121..73a0e97 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -50,6 +50,7 @@ license_create_manifest() {
 		pkged_pv="$(sed -n 's/^PV: //p' ${filename})"
 		pkged_name="$(basename $(readlink ${filename}))"
 		pkged_lic="$(sed -n "/^LICENSE_${pkged_name}: /{ s/^LICENSE_${pkged_name}: //; p }" ${filename})"
+		pkged_size="$(sed -n "/^PKGSIZE_${pkged_name}: /{ s/^PKGSIZE_${pkged_name}: //; p }" ${filename})"
 		if [ -z "${pkged_lic}" ]; then
 			# fallback checking value of LICENSE
 			pkged_lic="$(sed -n "/^LICENSE: /{ s/^LICENSE: //; p }" ${filename})"
@@ -61,6 +62,13 @@ license_create_manifest() {
 		echo "LICENSE:" ${pkged_lic} >> ${LICENSE_MANIFEST}
 		echo "" >> ${LICENSE_MANIFEST}
 
+		# If the package doesn't contain any file, that is, its size is 0, the license
+		# isn't relevant as far as the final image is concerned. So doing license check
+		# doesn't make much sense, skip it.
+		if [ "$pkged_size" = "0" ]; then
+			continue
+		fi
+
 		lics="$(echo ${pkged_lic} | sed "s/[|&()*]/ /g" | sed "s/  */ /g" )"
 		for lic in ${lics}; do
 			# to reference a license file trim trailing + symbol
-- 
1.9.1

[PATCH V3 4/5] os-release: add LIC_FILES_CHKSUM

[Chen Qi]

Add LIC_FILES_CHKSUM to avoid the warning below at rootfs time.

    WARNING: The license listed MIT was not in the licenses collected for os-release

Signed-off-by: Chen Qi <Qi.Chen@windriver.com>
---
 meta/recipes-core/os-release/os-release.bb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/meta/recipes-core/os-release/os-release.bb b/meta/recipes-core/os-release/os-release.bb
index 33e9581..87fea6f 100644
--- a/meta/recipes-core/os-release/os-release.bb
+++ b/meta/recipes-core/os-release/os-release.bb
@@ -3,6 +3,7 @@ inherit allarch
 SUMMARY = "Operating system identification"
 DESCRIPTION = "The /etc/os-release file contains operating system identification data."
 LICENSE = "MIT"
+LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
 INHIBIT_DEFAULT_DEPS = "1"
 
 do_fetch[noexec] = "1"
-- 
1.9.1

[PATCH V3 5/5] glibc-collateral.inc: add LIC_FILES_CHKSUM

[Chen Qi]

Add LIC_FILES_CHKSUM to avoid warnings like below at rootfs time.

    WARNING: The license listed GPLv2 was not in the licenses collected for glibc-locale

Signed-off-by: Chen Qi <Qi.Chen@windriver.com>
---
 meta/recipes-core/glibc/glibc-collateral.inc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/meta/recipes-core/glibc/glibc-collateral.inc b/meta/recipes-core/glibc/glibc-collateral.inc
index dfcebae..e7258b9 100644
--- a/meta/recipes-core/glibc/glibc-collateral.inc
+++ b/meta/recipes-core/glibc/glibc-collateral.inc
@@ -1,5 +1,7 @@
 INHIBIT_DEFAULT_DEPS = "1"
 LICENSE = "GPLv2 & LGPLv2.1"
+LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/GPL-2.0;md5=801f80980d171dd6425610833a22dbe6 \
+                    file://${COMMON_LICENSE_DIR}/LGPL-2.1;md5=1a6d268fd218675ffea8be556788b780"
 HOMEPAGE = "http://www.gnu.org/software/libc/index.html"
 
 # This needs to match with glibc.inc, otherwise glibc-scripts and glibc-locale
-- 
1.9.1