[PATCH] populate_sdk_ext: Avoid copying and producing .pyc files

[PATCH] populate_sdk_ext: Avoid copying and producing .pyc files

[Mark Hatle]

Since pyc cache files are really system specific, no real reason to copy or
generate them during the eSDK build process.  Also generating them has the
possibility of re-using inodes that pseudo may have been tracking, leading
a build failure.

Signed-off-by: Mark Hatle <mark.hatle@kernel.crashing.org>
---
 meta/classes/populate_sdk_ext.bbclass | 4 +++-
 meta/lib/oe/copy_buildsystem.py       | 6 +++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass
index 9112ab6c5e..14689ec6ac 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -251,7 +251,9 @@ python copy_buildsystem () {
 
     # Create a layer for new recipes / appends
     bbpath = d.getVar('BBPATH')
-    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')])
+    env = os.environ.copy()
+    env['PYTHONDONTWRITEBYTECODE'] = '1'
+    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath, 'workspace')], env=env)
 
     # Create bblayers.conf
     bb.utils.mkdirhier(baseoutpath + '/conf')
diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py
index 31a84f5b06..d97bf9d1b9 100644
--- a/meta/lib/oe/copy_buildsystem.py
+++ b/meta/lib/oe/copy_buildsystem.py
@@ -20,7 +20,7 @@ def _smart_copy(src, dest):
     mode = os.stat(src).st_mode
     if stat.S_ISDIR(mode):
         bb.utils.mkdirhier(dest)
-        cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \
+        cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -chf - -C %s -p . \
         | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
         subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
     else:
@@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cac
     bb.note('Generating sstate-cache...')
 
     nativelsbstring = d.getVar('NATIVELSBSTRING')
-    bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
+    bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
     if fixedlsbstring and nativelsbstring != fixedlsbstring:
         nativedir = output_sstate_cache + '/' + nativelsbstring
         if os.path.isdir(nativedir):
@@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, lo
         logparam = '-l %s' % logfile
     else:
         logparam = ''
-    cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
+    cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
     env = dict(d.getVar('BB_ORIGENV', False))
     env.pop('BUILDDIR', '')
     env.pop('BBPATH', '')
-- 
2.17.1

Re: [OE-core] [PATCH] populate_sdk_ext: Avoid copying and producing .pyc files

[Alexander Kanavin]

[-- Attachment #1: Type: text/plain, Size: 3776 bytes --]

Wait, I have to say neither reason is convincing. Pyc files are not
system-specific, they are the same on all systems. And pseudo issues should
be dealt with by fixing the issue, not working around the symptoms, no?

Alex

On Thu, 25 Mar 2021 at 23:44, Mark Hatle <mark.hatle@kernel.crashing.org>
wrote:

> Since pyc cache files are really system specific, no real reason to copy or
> generate them during the eSDK build process.  Also generating them has the
> possibility of re-using inodes that pseudo may have been tracking, leading
> a build failure.
>
> Signed-off-by: Mark Hatle <mark.hatle@kernel.crashing.org>
> ---
>  meta/classes/populate_sdk_ext.bbclass | 4 +++-
>  meta/lib/oe/copy_buildsystem.py       | 6 +++---
>  2 files changed, 6 insertions(+), 4 deletions(-)
>
> diff --git a/meta/classes/populate_sdk_ext.bbclass
> b/meta/classes/populate_sdk_ext.bbclass
> index 9112ab6c5e..14689ec6ac 100644
> --- a/meta/classes/populate_sdk_ext.bbclass
> +++ b/meta/classes/populate_sdk_ext.bbclass
> @@ -251,7 +251,9 @@ python copy_buildsystem () {
>
>      # Create a layer for new recipes / appends
>      bbpath = d.getVar('BBPATH')
> -    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath',
> baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath,
> 'workspace')])
> +    env = os.environ.copy()
> +    env['PYTHONDONTWRITEBYTECODE'] = '1'
> +    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath',
> baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath,
> 'workspace')], env=env)
>
>      # Create bblayers.conf
>      bb.utils.mkdirhier(baseoutpath + '/conf')
> diff --git a/meta/lib/oe/copy_buildsystem.py
> b/meta/lib/oe/copy_buildsystem.py
> index 31a84f5b06..d97bf9d1b9 100644
> --- a/meta/lib/oe/copy_buildsystem.py
> +++ b/meta/lib/oe/copy_buildsystem.py
> @@ -20,7 +20,7 @@ def _smart_copy(src, dest):
>      mode = os.stat(src).st_mode
>      if stat.S_ISDIR(mode):
>          bb.utils.mkdirhier(dest)
> -        cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf -
> -C %s -p . \
> +        cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs
> --xattrs-include='*' -chf - -C %s -p . \
>          | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
>          subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
>      else:
> @@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs,
> input_sstate_cache, output_sstate_cac
>      bb.note('Generating sstate-cache...')
>
>      nativelsbstring = d.getVar('NATIVELSBSTRING')
> -    bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs,
> input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
> +    bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s
> %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache,
> nativelsbstring, filterfile or ''))
>      if fixedlsbstring and nativelsbstring != fixedlsbstring:
>          nativedir = output_sstate_cache + '/' + nativelsbstring
>          if os.path.isdir(nativedir):
> @@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile,
> cmdprefix='', cwd=None, lo
>          logparam = '-l %s' % logfile
>      else:
>          logparam = ''
> -    cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s
> -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
> +    cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1
> PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets,
> filteroutfile, logparam)
>      env = dict(d.getVar('BB_ORIGENV', False))
>      env.pop('BUILDDIR', '')
>      env.pop('BBPATH', '')
> --
> 2.17.1
>
>
> 
>
>

[-- Attachment #2: Type: text/html, Size: 4698 bytes --]

Re: [OE-core] [PATCH] populate_sdk_ext: Avoid copying and producing .pyc files

[Mark Hatle]

On 3/26/21 12:54 AM, Alexander Kanavin wrote:
> Wait, I have to say neither reason is convincing. Pyc files are not
> system-specific, they are the same on all systems. And pseudo issues should be
> dealt with by fixing the issue, not working around the symptoms, no?

These are pyc files that are generated by the host system python.  As such, they
contain versioning information of the particular host they were constructed on.

The next user (of the eSDK) would regenerate the pyc files, assuming their
python was a different version/configuration anyway.

The pseudo issue is simply caused by the generation of these files outside of
the pseudo context.  The three places modified need to run without pseudo
control in order to setup the bitbake (host) environment properly, as well as
verify that the previous files copied/setup are infact correct.

So we're not working around symptoms.. we are indeed fixing the underlying
issues in the implementation.  It just happened to be that I was able to
reproduce the build failure reliably for a time vs it just going away on it's own.

--Mark

> Alex
> 
> On Thu, 25 Mar 2021 at 23:44, Mark Hatle <mark.hatle@kernel.crashing.org
> <mailto:mark.hatle@kernel.crashing.org>> wrote:
> 
>     Since pyc cache files are really system specific, no real reason to copy or
>     generate them during the eSDK build process.  Also generating them has the
>     possibility of re-using inodes that pseudo may have been tracking, leading
>     a build failure.
> 
>     Signed-off-by: Mark Hatle <mark.hatle@kernel.crashing.org
>     <mailto:mark.hatle@kernel.crashing.org>>
>     ---
>      meta/classes/populate_sdk_ext.bbclass | 4 +++-
>      meta/lib/oe/copy_buildsystem.py       | 6 +++---
>      2 files changed, 6 insertions(+), 4 deletions(-)
> 
>     diff --git a/meta/classes/populate_sdk_ext.bbclass
>     b/meta/classes/populate_sdk_ext.bbclass
>     index 9112ab6c5e..14689ec6ac 100644
>     --- a/meta/classes/populate_sdk_ext.bbclass
>     +++ b/meta/classes/populate_sdk_ext.bbclass
>     @@ -251,7 +251,9 @@ python copy_buildsystem () {
> 
>          # Create a layer for new recipes / appends
>          bbpath = d.getVar('BBPATH')
>     -    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath',
>     baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath,
>     'workspace')])
>     +    env = os.environ.copy()
>     +    env['PYTHONDONTWRITEBYTECODE'] = '1'
>     +    bb.process.run(['devtool', '--bbpath', bbpath, '--basepath',
>     baseoutpath, 'create-workspace', '--create-only', os.path.join(baseoutpath,
>     'workspace')], env=env)
> 
>          # Create bblayers.conf
>          bb.utils.mkdirhier(baseoutpath + '/conf')
>     diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py
>     index 31a84f5b06..d97bf9d1b9 100644
>     --- a/meta/lib/oe/copy_buildsystem.py
>     +++ b/meta/lib/oe/copy_buildsystem.py
>     @@ -20,7 +20,7 @@ def _smart_copy(src, dest):
>          mode = os.stat(src).st_mode
>          if stat.S_ISDIR(mode):
>              bb.utils.mkdirhier(dest)
>     -        cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C
>     %s -p . \
>     +        cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs
>     --xattrs-include='*' -chf - -C %s -p . \
>              | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
>              subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
>          else:
>     @@ -259,7 +259,7 @@ def create_locked_sstate_cache(lockedsigs,
>     input_sstate_cache, output_sstate_cac
>          bb.note('Generating sstate-cache...')
> 
>          nativelsbstring = d.getVar('NATIVELSBSTRING')
>     -    bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs,
>     input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
>     +    bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s
>     %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache,
>     nativelsbstring, filterfile or ''))
>          if fixedlsbstring and nativelsbstring != fixedlsbstring:
>              nativedir = output_sstate_cache + '/' + nativelsbstring
>              if os.path.isdir(nativedir):
>     @@ -286,7 +286,7 @@ def check_sstate_task_list(d, targets, filteroutfile,
>     cmdprefix='', cwd=None, lo
>              logparam = '-l %s' % logfile
>          else:
>              logparam = ''
>     -    cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s
>     -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
>     +    cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1
>     PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets,
>     filteroutfile, logparam)
>          env = dict(d.getVar('BB_ORIGENV', False))
>          env.pop('BUILDDIR', '')
>          env.pop('BBPATH', '')
>     -- 
>     2.17.1
> 
> 
> 
> 
> 
> 
> 
>