[PATCH] cooker: Fix/improve collections handling

[PATCH] cooker: Fix/improve collections handling

[Richard Purdie]

Code changes for FILE_LAYERNAME exposed lifecycle issues around the collections
objection in Cooker which only appeared in devtool usage in eSDK. Move
the collections setup to an earlier stage after parsing compeltes to
avoid any kind of race around it.

Also stop overwriting the code variable in MatchFiles. Ultimately we
need to combine these codepaths but that is for another patch.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 lib/bb/cooker.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
index 3e298c478b..0a21f1c2f8 100644
--- a/lib/bb/cooker.py
+++ b/lib/bb/cooker.py
@@ -504,6 +504,9 @@ class BBCooker:
             self.recipecaches[mc] = bb.cache.CacheData(self.caches_array)
 
         self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS"))
+        self.collections = {}
+        for mc in self.multiconfigs:
+            self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
 
         self._parsecache_set(False)
 
@@ -1379,8 +1382,8 @@ class BBCooker:
         if bf.startswith("/") or bf.startswith("../"):
             bf = os.path.abspath(bf)
 
-        self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
-        filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
+        collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
+        filelist, masked, searchdirs = collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
         try:
             os.stat(bf)
             bf = os.path.abspath(bf)
@@ -1678,13 +1681,10 @@ class BBCooker:
                 for dep in self.configuration.extra_assume_provided:
                     self.recipecaches[mc].ignored_dependencies.add(dep)
 
-            self.collections = {}
-
             mcfilelist = {}
             total_masked = 0
             searchdirs = set()
             for mc in self.multiconfigs:
-                self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
                 (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
 
                 mcfilelist[mc] = filelist
-- 
2.39.2

RE: [bitbake-devel] [PATCH] cooker: Fix/improve collections handling

[Peter Kjellerstedt]

> -----Original Message-----
> From: bitbake-devel@lists.openembedded.org <bitbake-devel@lists.openembedded.org> On Behalf Of Richard Purdie
> Sent: den 10 maj 2023 22:35
> To: bitbake-devel@lists.openembedded.org
> Subject: [bitbake-devel] [PATCH] cooker: Fix/improve collections handling
> 
> Code changes for FILE_LAYERNAME exposed lifecycle issues around the collections
> objection in Cooker which only appeared in devtool usage in eSDK. Move

Should that be "object" or "objects", rather than "objection"?

> the collections setup to an earlier stage after parsing compeltes to

compeltes -> completes

> avoid any kind of race around it.
> 
> Also stop overwriting the code variable in MatchFiles. Ultimately we
> need to combine these codepaths but that is for another patch.
> 
> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
> ---
>  lib/bb/cooker.py | 10 +++++-----
>  1 file changed, 5 insertions(+), 5 deletions(-)
> 
> diff --git a/lib/bb/cooker.py b/lib/bb/cooker.py
> index 3e298c478b..0a21f1c2f8 100644
> --- a/lib/bb/cooker.py
> +++ b/lib/bb/cooker.py
> @@ -504,6 +504,9 @@ class BBCooker:
>              self.recipecaches[mc] = bb.cache.CacheData(self.caches_array)
> 
>          self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS"))
> +        self.collections = {}
> +        for mc in self.multiconfigs:
> +            self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
> 
>          self._parsecache_set(False)
> 
> @@ -1379,8 +1382,8 @@ class BBCooker:
>          if bf.startswith("/") or bf.startswith("../"):
>              bf = os.path.abspath(bf)
> 
> -        self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
> -        filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
> +        collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
> +        filelist, masked, searchdirs = collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
>          try:
>              os.stat(bf)
>              bf = os.path.abspath(bf)
> @@ -1678,13 +1681,10 @@ class BBCooker:
>                  for dep in self.configuration.extra_assume_provided:
>                      self.recipecaches[mc].ignored_dependencies.add(dep)
> 
> -            self.collections = {}
> -
>              mcfilelist = {}
>              total_masked = 0
>              searchdirs = set()
>              for mc in self.multiconfigs:
> -                self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
>                  (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
> 
>                  mcfilelist[mc] = filelist
> --
> 2.39.2

//Peter