From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from mail-qt1-f179.google.com (mail-qt1-f179.google.com [209.85.160.179]) by mx.groups.io with SMTP id smtpd.web10.974.1617057935423618527 for ; Mon, 29 Mar 2021 15:45:35 -0700 Authentication-Results: mx.groups.io; dkim=pass header.i=@gmail.com header.s=20161025 header.b=JXGIfzcd; spf=pass (domain: gmail.com, ip: 209.85.160.179, mailfrom: devendra.tewari@gmail.com) Received: by mail-qt1-f179.google.com with SMTP id l13so10570249qtu.9 for ; Mon, 29 Mar 2021 15:45:35 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:message-id:mime-version:subject:date:in-reply-to:cc:to :references; bh=Po2lTWCqcXeItyiOp6INjYrW03y6bp5P3/wOGTYCXT4=; b=JXGIfzcdSHiJn/1IKrg0QzGuU2T5uTFbs4eiosynhgWf+1KkyugkcumdKnNONlLJ+p W0luJmJDPWHHnlO9l1Prz5fkggXn9aA2MLQL2yoeITEp3owGoXA35dBSfrXp7UIWK3FB lqo/C7oJKFsFkPTWNIQrgr6mTO1dpgYzaOa+ECMFDWKA21/xaUlHv9OcuLKUqY5QbhF+ 9FlfxMA+GVc0e3cVMmAGPeWrmOMQr4XNYr3E4Tt1NBpV0K31cO2emYDpsvxKngX8m4H4 6GagRxsLvawo0e/KwF/B+UM+v/XyCxiggRqdDe04xlyzXlsq/C7Q7YK2Bd3SXkmyeJdh KMDQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:message-id:mime-version:subject:date :in-reply-to:cc:to:references; bh=Po2lTWCqcXeItyiOp6INjYrW03y6bp5P3/wOGTYCXT4=; b=MVG3t05/vAzrpbRW5flXN0UM8z1k8MtxWSPAxJ6aVn0B2aGbL1D7fI5wMeN+aWmhqs 18tbT+HLe6+ZSHqGJQ9nljiNW1O4DoKzzFFX+kjLNMjN25if0l/kmx2cnUHcVupB8bN6 fpyVYrTZRzdPlAR8m3uuFHiY75ONZKu9CNb16HkiMzJkYDuFuUQnMjDkfgKPDgXkEwc7 JsPIlx092zGBc5Rm6/4cTAbvaOsnFyadU4NcxCrzk4WHNZtEY53iDEgspNzK8auzT/qx NY9idoo0d7pW7iXg95+sON0fEwCowmnQIxgdaUNasnJ7/LIFroxKG9Un51gPnVDhTPFN BN1w== X-Gm-Message-State: AOAM533aabaDK9q3+y1UqjZ3NGfI/aZmJSKT+HQbZ4o3NS2d1CsNFg7q NR2T/of1GFNNx4S7UDSy0wQ= X-Google-Smtp-Source: ABdhPJxtnFHvcP1Wwjrshl0YevDGdWWYhD/u5khFiIrLK5UW4C2EYu7Is0NKtOrTwZOYvkjS6YCvAA== X-Received: by 2002:ac8:4d8b:: with SMTP id a11mr24397545qtw.302.1617057934373; Mon, 29 Mar 2021 15:45:34 -0700 (PDT) Return-Path: Received: from ?IPv6:2804:14d:5487:a7d6:28cd:51dd:6cf0:14e0? ([2804:14d:5487:a7d6:28cd:51dd:6cf0:14e0]) by smtp.gmail.com with ESMTPSA id r35sm11954690qtd.95.2021.03.29.15.45.32 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Mon, 29 Mar 2021 15:45:34 -0700 (PDT) From: "Devendra Tewari" Message-Id: Mime-Version: 1.0 (Mac OS X Mail 14.0 \(3654.60.0.2.21\)) Subject: Re: [OE-core] [PATCH] Use shutil.move when os.rename fails Date: Mon, 29 Mar 2021 19:45:29 -0300 In-Reply-To: <9e81b71466b771c55d8859e7d156b662e1c3d4f7.camel@linuxfoundation.org> Cc: openembedded-core@lists.openembedded.org To: Richard Purdie References: <20210329151442.86306-1-devendra.tewari@gmail.com> <9e81b71466b771c55d8859e7d156b662e1c3d4f7.camel@linuxfoundation.org> X-Mailer: Apple Mail (2.3654.60.0.2.21) Content-Type: multipart/alternative; boundary="Apple-Mail=_3EA19A8A-C7A8-4A73-9023-B32DE523E10B" --Apple-Mail=_3EA19A8A-C7A8-4A73-9023-B32DE523E10B Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset=utf-8 Thanks! My bad. The example I looked up in Python docs had a break and I ju= st realized it was a looping example. Here=E2=80=99s the updated patch (or should I submit it again via git send= -email?) >From 9deb390dcdcaef66cec2fae39454c7fb3c81c4e4 Mon Sep 17 00:00:00 2001 From: Devendra Tewari Date: Mon, 29 Mar 2021 19:41:02 -0300 Subject: [PATCH] Use shutil.move when os.rename fails Incremental build in Docker fails with OSError: [Errno 18] Invalid cross-device link When source and destination are on different overlay filesystems. This change handles the error with os.rename and retries with shutil.move. --- meta/classes/sstate.bbclass | 22 ++++++++++++++++++---- vscode-bitbake-build/executeBitBakeCmd.sh | 3 +++ 2 files changed, 21 insertions(+), 4 deletions(-) create mode 100755 vscode-bitbake-build/executeBitBakeCmd.sh diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass index f579168162..301dfc27db 100644 --- a/meta/classes/sstate.bbclass +++ b/meta/classes/sstate.bbclass @@ -384,6 +384,7 @@ def sstate_installpkg(ss, d): def sstate_installpkgdir(ss, d): import oe.path import subprocess + import shutil =20 sstateinst =3D d.getVar("SSTATE_INSTDIR") d.setVar('SSTATE_FIXMEDIR', ss['fixmedir']) @@ -401,7 +402,10 @@ def sstate_installpkgdir(ss, d): =20 for state in ss['dirs']: prepdir(state[1]) - os.rename(sstateinst + state[0], state[1]) + try: + os.rename(sstateinst + state[0], state[1]) + except OSError: + shutil.move(sstateinst + state[0], state[1]) sstate_install(ss, d) =20 for plain in ss['plaindirs']: @@ -413,7 +417,10 @@ def sstate_installpkgdir(ss, d): dest =3D plain bb.utils.mkdirhier(src) prepdir(dest) - os.rename(src, dest) + try: + os.rename(src, dest) + except OSError: + shutil.move(src, dest) =20 return True =20 @@ -638,6 +645,7 @@ python sstate_hardcode_path () { =20 def sstate_package(ss, d): import oe.path + import shutil =20 tmpdir =3D d.getVar('TMPDIR') =20 @@ -664,7 +672,10 @@ def sstate_package(ss, d): continue bb.error("sstate found an absolute path symlink %s pointi= ng at %s. Please replace this with a relative link." % (srcpath, link)) bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], = sstatebuild + state[0])) - os.rename(state[1], sstatebuild + state[0]) + try: + os.rename(state[1], sstatebuild + state[0]) + except OSError: + shutil.move(state[1], sstatebuild + state[0]) =20 workdir =3D d.getVar('WORKDIR') sharedworkdir =3D os.path.join(d.getVar('TMPDIR'), "work-shared") @@ -674,7 +685,10 @@ def sstate_package(ss, d): pdir =3D plain.replace(sharedworkdir, sstatebuild) bb.utils.mkdirhier(plain) bb.utils.mkdirhier(pdir) - os.rename(plain, pdir) + try: + os.rename(plain, pdir) + except OSError: + shutil.move(plain, pdir) =20 d.setVar('SSTATE_BUILDDIR', sstatebuild) d.setVar('SSTATE_INSTDIR', sstatebuild) diff --git a/vscode-bitbake-build/executeBitBakeCmd.sh b/vscode-bitbake-bu= ild/executeBitBakeCmd.sh new file mode 100755 index 0000000000..d7a4c5a5aa --- /dev/null +++ b/vscode-bitbake-build/executeBitBakeCmd.sh @@ -0,0 +1,3 @@ +#!/bin/bash +. ./oe-init-build-env vscode-bitbake-build > /dev/null +bitbake-layers show-layers \ No newline at end of file --=20 2.29.2 > On 29 Mar 2021, at 17:38, Richard Purdie wrote: >=20 > On Mon, 2021-03-29 at 12:14 -0300, Devendra Tewari wrote: >> --- >> meta/classes/sstate.bbclass | 26 ++++++++++++++++++++++---- >> 1 file changed, 22 insertions(+), 4 deletions(-) >>=20 >> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass >> index f579168162..f94aa96d70 100644 >> --- a/meta/classes/sstate.bbclass >> +++ b/meta/classes/sstate.bbclass >> @@ -384,6 +384,7 @@ def sstate_installpkg(ss, d): >> def sstate_installpkgdir(ss, d): >> import oe.path >> import subprocess >> + import shutil >> >>=20 >> sstateinst =3D d.getVar("SSTATE_INSTDIR") >> d.setVar('SSTATE_FIXMEDIR', ss['fixmedir']) >> @@ -401,7 +402,11 @@ def sstate_installpkgdir(ss, d): >> >>=20 >> for state in ss['dirs']: >> prepdir(state[1]) >> - os.rename(sstateinst + state[0], state[1]) >> + try: >> + os.rename(sstateinst + state[0], state[1]) >> + break >=20 > That break should definitely not be there, that changes behaviour. >=20 >> + except OSError: >> + shutil.move(sstateinst + state[0], state[1]) >> sstate_install(ss, d) >> >>=20 >> for plain in ss['plaindirs']: >> @@ -413,7 +418,11 @@ def sstate_installpkgdir(ss, d): >> dest =3D plain >> bb.utils.mkdirhier(src) >> prepdir(dest) >> - os.rename(src, dest) >> + try: >> + os.rename(src, dest) >> + break >=20 > Same here... >=20 >> + except OSError: >> + shutil.move(src, dest) >> >>=20 >> return True >> >>=20 >> @@ -638,6 +647,7 @@ python sstate_hardcode_path () { >> >>=20 >> def sstate_package(ss, d): >> import oe.path >> + import shutil >> >>=20 >> tmpdir =3D d.getVar('TMPDIR') >> >>=20 >> @@ -664,7 +674,11 @@ def sstate_package(ss, d): >> continue >> bb.error("sstate found an absolute path symlink %s poi= nting at %s. Please replace this with a relative link." % (srcpath, link)) >> bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1= ], sstatebuild + state[0])) >> - os.rename(state[1], sstatebuild + state[0]) >> + try: >> + os.rename(state[1], sstatebuild + state[0]) >> + break >=20 > and again... >=20 >> + except OSError: >> + shutil.move(state[1], sstatebuild + state[0]) >> >>=20 >> workdir =3D d.getVar('WORKDIR') >> sharedworkdir =3D os.path.join(d.getVar('TMPDIR'), "work-shared") >> @@ -674,7 +688,11 @@ def sstate_package(ss, d): >> pdir =3D plain.replace(sharedworkdir, sstatebuild) >> bb.utils.mkdirhier(plain) >> bb.utils.mkdirhier(pdir) >> - os.rename(plain, pdir) >> + try: >> + os.rename(plain, pdir) >> + break >=20 >=20 > and again... >=20 >> + except OSError: >> + shutil.move(plain, pdir) >> >>=20 >> d.setVar('SSTATE_BUILDDIR', sstatebuild) >> d.setVar('SSTATE_INSTDIR', sstatebuild) >>=20 --Apple-Mail=_3EA19A8A-C7A8-4A73-9023-B32DE523E10B Content-Transfer-Encoding: quoted-printable Content-Type: text/html; charset=utf-8 Thanks! My bad. The examp= le I looked up in Python docs had a break and I just realized it was a loop= ing example.

Here=E2=80= = =99s the updated patch (or should I submit it again via git send-email?)

From 9deb390dcdcaef66cec2fae3945= 4c7fb3c81c4e4 Mon Sep 17 00:00:00 2001
From: Devendra Tewari <devendra.tewari@gmail.com&g= t;
Date: Mon, 29 Mar 2021 19:41:02 -0300
Subject: [PATCH] Use shutil.move whe= n os.rename fails

Incremental build in Docker= fails with

OSError: [Errno 18] Invalid cross= -device link

When source and destination are = on different overlay filesystems.

This change= handles the error with os.rename and retries with shutil.move.
---
 meta/= classes/sstate.bbclass               | 2= 2 ++++++++++++++++++----
 vscode-bitbake-build/executeBitBakeCmd.sh |&nb= sp; 3 +++
 2 files changed, 21 insertions(+), 4 deletions(-)
 create = mode 100755 vscode-bitbake-build/executeBitBakeCmd.sh
<= span style=3D"font-variant-ligatures: no-common-ligatures" class=3D"">
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbc= lass
index f579168162..301dfc27db 100644
--- a/meta/classes/sstate.bbclass
+++ b/= meta/classes/sstate.bbclass
@@ -384,6 +384,7 @@ def sstate_installpkg(ss, d):=
&= nbsp;def sstate_installpkgdir(ss, d):
     import oe.path
 =     import subprocess
+    import shutil

&n= bsp;

     sstateinst =3D = d.getVar("SSTATE_INSTDIR")
     d.setVar('SSTATE_FIXMEDIR', ss= ['fixmedir'])
@@ -401,7 +402,10 @@ def sstate_installpkgdir(ss, d):

 

     for= state in ss['dirs']:
         prepdir(state[1])
<= span style=3D"font-variant-ligatures: no-common-ligatures" class=3D"">-&nbs= p;       os.rename(sstateinst + state[0], state[1])
+  &n= bsp;     try:
+            os.rename(= sstateinst + state[0], state[1])
+        except OSError:=
+=             shutil.move(sstateinst + state[0]= , state[1])
     sstate_install(ss, d)

 

     for plain in ss['plain= dirs']:
@@ -413,7 +417,10 @@ def sstate_installpkgdir(ss, d):
   &= nbsp;     dest =3D plain
         bb.utils= .mkdirhier(src)
         prepdir(dest)
<= div style=3D"margin: 0px; font-stretch: normal; font-size: 11px; line-heigh= t: normal; font-family: Menlo; color: rgb(0, 0, 0);" class=3D"">-   =     os.rename(src, dest)
+        try:<= /div>
+  &= nbsp;         os.rename(src, dest)
+      =   except OSError:
+            shutil.move= (src, dest)

 

 &= nbsp;   return True

 

@@ -638,6 +645,7 @@ python sstate_hardcode_path () {

&= nbsp;

 def sstate_package(ss, d):
&nb= sp;    import oe.path
+    import shutil

&n= bsp;

     tmpdir =3D d.ge= tVar('TMPDIR')

 

= @@ -= 664,7 +672,10 @@ def sstate_package(ss, d):
        =             continue
     &nb= sp;           bb.error("sstate found an absolute p= ath symlink %s pointing at %s. Please replace this with a relative link." %= (srcpath, link))
         bb.debug(2, "Preparing tr= ee %s for packaging at %s" % (state[1], sstatebuild + state[0]))
-  &nbs= p;     os.rename(state[1], sstatebuild + state[0])
+   =     try:
+            os.rename(stat= e[1], sstatebuild + state[0])
+        except OSError:
= +&nb= sp;           shutil.move(state[1], sstatebuild + = state[0])

 

 &nb= sp;   workdir =3D d.getVar('WORKDIR')
     sharedworkdir = = =3D os.path.join(d.getVar('TMPDIR'), "work-shared")
@@ -674,7 +685,10 @@ de= f sstate_package(ss, d):
             pdir= =3D plain.replace(sharedworkdir, sstatebuild)
       &nb= sp; bb.utils.mkdirhier(plain)
         bb.utils.mkdi= rhier(pdir)
-        os.rename(plain, pdir)
<= div style=3D"margin: 0px; font-stretch: normal; font-size: 11px; line-heigh= t: normal; font-family: Menlo; color: rgb(0, 0, 0);" class=3D"">+   =     try:
+            os.rename(plai= n, pdir)
+        except OSError:
+      &= nbsp;     shutil.move(plain, pdir)

 <= br class=3D"webkit-block-placeholder">

     d.setVar('SSTATE_BUILDDIR',= sstatebuild)
     d.setVar('SSTATE_INSTDIR', sstatebuild)
<= span style=3D"font-variant-ligatures: no-common-ligatures" class=3D"">diff = --git a/vscode-bitbake-build/executeBitBakeCmd.sh b/vscode-bitbake-build/ex= ecuteBitBakeCmd.sh
new file mode 100755
index 0000000000..d7a4c5a5aa
--- /dev/nul= l
+++ b/vscode-bitbake-build/executeBitBakeCmd.sh
@@ -0,0 +1,3 @@<= /div>
+#!/bin/b= ash
+. ./oe-init-build-env vscode-bitbake-build > /dev/null
<= div style=3D"margin: 0px; font-stretch: normal; font-size: 11px; line-heigh= t: normal; font-family: Menlo; color: rgb(0, 0, 0);" class=3D"">+bitbake-layer= s show-layers
\ No newline at end of file
-- 
2.29.2


On 29 Mar 2021, at 17:38, Richard Purdie <<= a href=3D"mailto:richard.purdie@linuxfoundation.org" class=3D"">richard.pur= die@linuxfoundation.org> wrote:

On Mon, = 2021-03-29 at 12:14 -0300, Devendra Tewari wrote:
---
 meta/classe= s/sstate.bbclass | 26 ++++++++++++++++++++++----
 1 file= changed, 22 insertions(+), 4 deletions(-)

dif= f --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index f579168162..f94aa96d70 100644
--- a/meta/classes= /sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -384,6 +384,7 @@ def sstate_installpkg(ss, d):
 de= f sstate_installpkgdir(ss, d):
     = import oe.path
     import subproces= s
+    import shutil
 

     sstateinst =3D d.get= Var("SSTATE_INSTDIR")
     d.setVar(= 'SSTATE_FIXMEDIR', ss['fixmedir'])
@@ -401,7 +402,11 @@ def s= state_installpkgdir(ss, d):
 

     for state in ss['dirs']:
&nb= sp;        prepdir(state[1])
-        os.rename(sstateinst + = state[0], state[1])
+       &nb= sp;try:
+         &nb= sp;  os.rename(sstateinst + state[0], state[1])
+ &= nbsp;          break

Tha= t break should definitely not be there, that changes behaviour.

+        except OSErro= r:
+          &n= bsp; shutil.move(sstateinst + state[0], state[1])
 =     sstate_install(ss, d)
 

     for plain in ss['plaind= irs']:
@@ -413,7 +418,11 @@ def sstate_installpkgdir(ss, d):<= br class=3D"">         dest = =3D plain
        &n= bsp;bb.utils.mkdirhier(src)
     &nb= sp;   prepdir(dest)
-     =    os.rename(src, dest)
+    &n= bsp;   try:
+      &n= bsp;     os.rename(src, dest)
+ &nbs= p;          break

Same here= ...

+       &nbs= p;except OSError:
+        = ;    shutil.move(src, dest)
 

     return True
 

@@ -638,6 +647,7 @@ python sstat= e_hardcode_path () {
 

&nbs= p;def sstate_package(ss, d):
     im= port oe.path
+    import shutil
=  

     tmpdir = =3D d.getVar('TMPDIR')
 

@= @ -664,7 +674,11 @@ def sstate_package(ss, d):
  &n= bsp;            = ;      continue
  &nb= sp;            =   bb.error("sstate found an absolute path symlink %s pointing at = %s. Please replace this with a relative link." % (srcpath, link))
         bb.debug(2, "P= reparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))=
-        os.rename(state[= 1], sstatebuild + state[0])
+      &= nbsp; try:
+        &= nbsp;   os.rename(state[1], sstatebuild + state[0])
+            = break

and again...

+    &nbs= p;   except OSError:
+     = ;       shutil.move(state[1], sstatebuil= d + state[0])
 

  = ;   workdir =3D d.getVar('WORKDIR')
 &nbs= p;   sharedworkdir =3D os.path.join(d.getVar('TMPDIR'), "wor= k-shared")
@@ -674,7 +688,11 @@ def sstate_package(ss, d):          &nb= sp;  pdir =3D plain.replace(sharedworkdir, sstatebuild)
         bb.utils.mkdir= hier(plain)
        &= nbsp;bb.utils.mkdirhier(pdir)
-      = ;  os.rename(plain, pdir)
+     = ;   try:
+       = ;     os.rename(plain, pdir)
+  = ;          break


and again...

+ &= nbsp;      except OSError:
+ &n= bsp;          shutil.move= (plain, pdir)
 

  = ;   d.setVar('SSTATE_BUILDDIR', sstatebuild)
&= nbsp;    d.setVar('SSTATE_INSTDIR', sstatebuild)

<= /html> --Apple-Mail=_3EA19A8A-C7A8-4A73-9023-B32DE523E10B--