Re: [PATCH] fetch/git: fix per-branch unpacking

Re: [PATCH] fetch/git: fix per-branch unpacking

[Alexander Kanavin]

On 10/17/2017 02:27 PM, David Vincent wrote:
> Create branches for each user-supplied name when running checkout and
> set start point of branch to the resolved revision.

Can you provide an example of how this should be used please (in the 
commit message)?

Alex

[PATCH] fetch/git: fix per-branch unpacking

[David Vincent]

Create branches for each user-supplied name when running checkout and
set start point of branch to the resolved revision.

Signed-off-by: David Vincent <freesilicon@gmail.com>
---
 lib/bb/fetch2/git.py  | 12 +++++++-----
 lib/bb/tests/fetch.py | 27 +++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index 5ef8cd69..e71551ff 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -482,11 +482,13 @@ class Git(FetchMethod):
                             workdir=destdir)
                 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
             elif not ud.nobranch:
-                branchname =  ud.branches[ud.names[0]]
-                runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
-                            ud.revisions[ud.names[0]]), d, workdir=destdir)
-                runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
-                            branchname), d, workdir=destdir)
+                for idx, name in enumerate(ud.names):
+                    checkoutcmd = 'checkout -B' if not idx else 'branch -f'
+                    branchname =  ud.branches[name]
+                    runfetchcmd("%s %s %s %s" % (ud.basecmd, checkoutcmd, branchname, \
+                                ud.revisions[name]), d, workdir=destdir)
+                    runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
+                                branchname), d, workdir=destdir)
             else:
                 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
 
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 209b13f6..217cd88b 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -647,6 +647,33 @@ class FetcherNetworkTest(FetcherTest):
             self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir))
             self.gitfetcher(dummyurl, dummyurl)
 
+        def test_gitfetch_multi_one_uri(self):
+            rev = "be393f247a08c0a4a50a6a76b8fd57f78295d2a1"
+            rev1 = "85596c9af3bb6407159c6c8de229cbe275aa74ea"
+            rev2 = "28249c42701f9156a0b3153d72d7e46dacab37cb"
+            self.d.setVar("SRCREV_rev1", rev1)
+            self.d.setVar("SRCREV_rev2", rev2)
+            url = "git://git.openembedded.org/bitbake;name=rev1,rev2;branch=1.34,1.36"
+            fetcher = bb.fetch.Fetch([url], self.d);
+            fetcher.download()
+            fetcher.unpack(self.unpackdir)
+            stdout = bb.process.run("git rev-parse master",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev);
+            stdout = bb.process.run("git rev-parse HEAD",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev1);
+            stdout = bb.process.run("git rev-parse 1.34",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev1);
+            stdout = bb.process.run("git rev-parse 1.36",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev2);
+
         def test_git_submodule(self):
             fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d)
             fetcher.download()
-- 
2.14.2

Re: [PATCH] fetch/git: fix per-branch unpacking

[David Vincent]

On mardi 17 octobre 2017 13:26:03 CEST Alexander Kanavin wrote:
> On 10/17/2017 02:27 PM, David Vincent wrote:
> > Create branches for each user-supplied name when running checkout and
> > set start point of branch to the resolved revision.
> 
> Can you provide an example of how this should be used please (in the
> commit message)?
> 
> Alex
First of all, sorry for the multipost, I resent the patch too quickly after 
seeing the mailman notification.

I don't clearly see the example I need to provide since this behavior is 
already documented in the fetcher and this patch is simply a bugfix :

- branch
   The git branch to retrieve from. The default is "master"

   This option also supports multiple branch fetching, with branches
   separated by commas.  In multiple branches case, the name option
   must have the same number of names to match the branches, which is
   used to specify the SRC_REV for the branch
   e.g:
   SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
   SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
   SRCREV_nameY = “YYYYYYYYYYYYYYYYYYYY”

Maybe I can provide some background behind this fix and you tell me if I must 
complete my commit message.
When working on a linux-yocto kernel, I tried to merge a feature branch in my 
machine branch using the merge functionality, therefore I needed to checkout 
two branches in a specific revision. Problem is that my machine branch was 
correctly checked out with the SRCREV provided in the recipe but the feature 
branch was not, it was checked out to the tip of the remote branch. When 
investigating a little further, I did find that bitbake only checks out the 
first branch and does not handle other branches (why it did work in the first 
place seems like some bbclass magic to me). The point is that with this patch, 
all my branches are correctly checked out and my kernel got the good revision.

--
David

Re: [PATCH] fetch/git: fix per-branch unpacking

[Alexander Kanavin]

On 10/17/2017 02:59 PM, David Vincent wrote:

> I don't clearly see the example I need to provide since this behavior is
> already documented in the fetcher and this patch is simply a bugfix :
> 
> - branch
>     The git branch to retrieve from. The default is "master"
> 
>     This option also supports multiple branch fetching, with branches
>     separated by commas.  In multiple branches case, the name option
>     must have the same number of names to match the branches, which is
>     used to specify the SRC_REV for the branch
>     e.g:
>     SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
>     SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
>     SRCREV_nameY = “YYYYYYYYYYYYYYYYYYYY”
> 
> Maybe I can provide some background behind this fix and you tell me if I must
> complete my commit message.
> When working on a linux-yocto kernel, I tried to merge a feature branch in my
> machine branch using the merge functionality, therefore I needed to checkout
> two branches in a specific revision. Problem is that my machine branch was
> correctly checked out with the SRCREV provided in the recipe but the feature
> branch was not, it was checked out to the tip of the remote branch. When
> investigating a little further, I did find that bitbake only checks out the
> first branch and does not handle other branches (why it did work in the first
> place seems like some bbclass magic to me). The point is that with this patch,
> all my branches are correctly checked out and my kernel got the good revision.

Thanks - actually Yocto documentation does not mention anything about 
multiple branches; you need to read the source code for the fetcher. So 
technically, it's not officially supported. And how do you pick a 
specific branch to be built?

Anyway, the kind of information you provided above can definitely go 
into the commit message. And can you file a bug to fix this in the docs 
please?

Alex

[PATCH v2 0/2] fetch/git: fix per-branch unpacking

[David Vincent]

Create branches for each user-supplied name when running checkout and
set start point of branch to the resolved revision.
Also, update documentation to better describe this functionality.

Changes in v2:
  - clarify commit message
  - update documentation

David Vincent (2):
  fetch/git: fix per-branch unpacking
  bitbake-user-manual: update git fetcher

 .../bitbake-user-manual-fetching.xml               | 34 ++++++++++++++++++++--
 lib/bb/fetch2/git.py                               | 12 ++++----
 lib/bb/tests/fetch.py                              | 27 +++++++++++++++++
 3 files changed, 66 insertions(+), 7 deletions(-)

-- 
2.14.2

[PATCH v2 1/2] fetch/git: fix per-branch unpacking

[David Vincent]

Create branches for each user-supplied name when running checkout and
set start point of branch to the resolved revision.

This fixes the following scenario:
When working with a Yocto Linux kernel, it may be required to merge
feature branches in a machine branch (using scc 'merge' functionality).
The current code checks out the machine branch correctly but feature
branches are checked out to the tip of their remote counterpart not to
the provided SRCREV. This is due to the fact that git fetcher does not
provide the same behavior for all branches and only handles the first
one.

Signed-off-by: David Vincent <freesilicon@gmail.com>
---
 lib/bb/fetch2/git.py  | 12 +++++++-----
 lib/bb/tests/fetch.py | 27 +++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index 5ef8cd69..e71551ff 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -482,11 +482,13 @@ class Git(FetchMethod):
                             workdir=destdir)
                 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
             elif not ud.nobranch:
-                branchname =  ud.branches[ud.names[0]]
-                runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
-                            ud.revisions[ud.names[0]]), d, workdir=destdir)
-                runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
-                            branchname), d, workdir=destdir)
+                for idx, name in enumerate(ud.names):
+                    checkoutcmd = 'checkout -B' if not idx else 'branch -f'
+                    branchname =  ud.branches[name]
+                    runfetchcmd("%s %s %s %s" % (ud.basecmd, checkoutcmd, branchname, \
+                                ud.revisions[name]), d, workdir=destdir)
+                    runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
+                                branchname), d, workdir=destdir)
             else:
                 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
 
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 209b13f6..217cd88b 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -647,6 +647,33 @@ class FetcherNetworkTest(FetcherTest):
             self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir))
             self.gitfetcher(dummyurl, dummyurl)
 
+        def test_gitfetch_multi_one_uri(self):
+            rev = "be393f247a08c0a4a50a6a76b8fd57f78295d2a1"
+            rev1 = "85596c9af3bb6407159c6c8de229cbe275aa74ea"
+            rev2 = "28249c42701f9156a0b3153d72d7e46dacab37cb"
+            self.d.setVar("SRCREV_rev1", rev1)
+            self.d.setVar("SRCREV_rev2", rev2)
+            url = "git://git.openembedded.org/bitbake;name=rev1,rev2;branch=1.34,1.36"
+            fetcher = bb.fetch.Fetch([url], self.d);
+            fetcher.download()
+            fetcher.unpack(self.unpackdir)
+            stdout = bb.process.run("git rev-parse master",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev);
+            stdout = bb.process.run("git rev-parse HEAD",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev1);
+            stdout = bb.process.run("git rev-parse 1.34",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev1);
+            stdout = bb.process.run("git rev-parse 1.36",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev2);
+
         def test_git_submodule(self):
             fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d)
             fetcher.download()
-- 
2.14.2

[PATCH v2 2/2] bitbake-user-manual: update git fetcher

[David Vincent]

Add precisions for the multiple branches support in git fetcher. This
support was documented in source code but not in documentation, so
include it.

Signed-off-by: David Vincent <freesilicon@gmail.com>
---
 .../bitbake-user-manual-fetching.xml               | 34 ++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml b/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml
index c721e86e..353e9a3c 100644
--- a/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml
+++ b/doc/bitbake-user-manual/bitbake-user-manual-fetching.xml
@@ -565,8 +565,26 @@
                     <listitem><para><emphasis>"branch":</emphasis>
                         The branch(es) of the Git tree to clone.
                         If unset, this is assumed to be "master".
-                        The number of branch parameters much match the number of
-                        name parameters.
+                        This option also supports multiple branch fetching, with
+                        branches separated by commas.
+                        <note>
+                            <title>Note</title>
+                            <para>
+                                In multiple branches case, the name option must
+                                have the same number of names to match the
+                                branches, which is used to specify the
+                                <link linkend="var-SRCREV"><filename>SRCREV</filename></link>
+                                for the branch.
+                            </para>
+
+                            <para>
+                                After checkout, all branches will be created
+                                with their tip pointing to the corresponding
+                                <filename>SRCREV</filename>. In particular, the
+                                current checked out branch will be the first
+                                provided in the list.
+                            </para>
+                        </note>
                         </para></listitem>
                     <listitem><para><emphasis>"rev":</emphasis>
                         The revision to use for the checkout.
@@ -589,12 +607,24 @@
                         By default, the path is <filename>git/</filename>.
                         </para></listitem>
                 </itemizedlist>
+            </para>
+
+            <para>
                 Here are some example URLs:
                 <literallayout class='monospaced'>
      SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1"
      SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http"
                 </literallayout>
             </para>
+
+            <para>
+                Here is an example URL using multiple branches:
+                <literallayout class='monospaced'>
+     SRC_URI = "git://git.oe.handhelds.org/git/vip.git;name=master,feature;branch=master,features/feature1"
+     SRCREV_master = "ad8b1d659ddd2699ebf7d50ef9de8940b157bfc2"
+     SRCREV_feature = "06c0dbdcba374ca7f92a53d69292d6bb7bc9b0f3"
+                </literallayout>
+            </para>
         </section>
 
         <section id='gitsm-fetcher'>
-- 
2.14.2

[PATCH] fetch/git: fix per-branch unpacking

[David Vincent]

Create branches for each user-supplied name when running checkout and
set start point of branch to the resolved revision.

Signed-off-by: David Vincent <freesilicon@gmail.com>
---
 lib/bb/fetch2/git.py  | 12 +++++++-----
 lib/bb/tests/fetch.py | 27 +++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index 5ef8cd69..e71551ff 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -482,11 +482,13 @@ class Git(FetchMethod):
                             workdir=destdir)
                 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
             elif not ud.nobranch:
-                branchname =  ud.branches[ud.names[0]]
-                runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
-                            ud.revisions[ud.names[0]]), d, workdir=destdir)
-                runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
-                            branchname), d, workdir=destdir)
+                for idx, name in enumerate(ud.names):
+                    checkoutcmd = 'checkout -B' if not idx else 'branch -f'
+                    branchname =  ud.branches[name]
+                    runfetchcmd("%s %s %s %s" % (ud.basecmd, checkoutcmd, branchname, \
+                                ud.revisions[name]), d, workdir=destdir)
+                    runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
+                                branchname), d, workdir=destdir)
             else:
                 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
 
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 209b13f6..217cd88b 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -647,6 +647,33 @@ class FetcherNetworkTest(FetcherTest):
             self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (dummyurl, self.sourcedir))
             self.gitfetcher(dummyurl, dummyurl)
 
+        def test_gitfetch_multi_one_uri(self):
+            rev = "be393f247a08c0a4a50a6a76b8fd57f78295d2a1"
+            rev1 = "85596c9af3bb6407159c6c8de229cbe275aa74ea"
+            rev2 = "28249c42701f9156a0b3153d72d7e46dacab37cb"
+            self.d.setVar("SRCREV_rev1", rev1)
+            self.d.setVar("SRCREV_rev2", rev2)
+            url = "git://git.openembedded.org/bitbake;name=rev1,rev2;branch=1.34,1.36"
+            fetcher = bb.fetch.Fetch([url], self.d);
+            fetcher.download()
+            fetcher.unpack(self.unpackdir)
+            stdout = bb.process.run("git rev-parse master",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev);
+            stdout = bb.process.run("git rev-parse HEAD",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev1);
+            stdout = bb.process.run("git rev-parse 1.34",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev1);
+            stdout = bb.process.run("git rev-parse 1.36",
+                                    cwd=os.path.join(self.unpackdir, "git"))
+            unpack_rev = stdout[0].strip()
+            self.assertEqual(unpack_rev, rev2);
+
         def test_git_submodule(self):
             fetcher = bb.fetch.Fetch(["gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842"], self.d)
             fetcher.download()
-- 
2.14.2