[PATCH] fetch2/git: stop generated tarballs from leaking info

[PATCH] fetch2/git: stop generated tarballs from leaking info

[Olaf Mandel]

When using BB_GENERATE_MIRROR_TARBALLS="1" to generate mirror tarballs
of git repositories, they leaked local information: username, group and
time of the last fetch. Remove all these by setting fixed information:

 * uname = pokybuild
 * gname = users
 * mtime = committer time of newest commit in repo

The username and group value were taken from the archives available on
the downloads.yoctoproject.org mirror. The modification time is chosen
so it still retains some relationship to the contents of the archive.

Signed-off-by: Olaf Mandel <o.mandel@menlosystems.com>
---
 lib/bb/fetch2/git.py  |  5 ++++-
 lib/bb/tests/fetch.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index b3c5e6da..4d06a571 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -462,7 +462,10 @@ class Git(FetchMethod):
 
             logger.info("Creating tarball of git repository")
             with create_atomic(ud.fullmirror) as tfile:
-                runfetchcmd("tar -czf %s ." % tfile, d, workdir=ud.clonedir)
+                mtime = runfetchcmd("git log --all -1 --format=%cD", d,
+                        quiet=True, workdir=ud.clonedir)
+                runfetchcmd("tar -czf %s --owner pokybuild --group users --mtime \"%s\" ."
+                        % (tfile, mtime), d, workdir=ud.clonedir)
             runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
     def clone_shallow_local(self, ud, dest, d):
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index eff12b7c..233ecae7 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -11,6 +11,7 @@ import hashlib
 import tempfile
 import collections
 import os
+import tarfile
 from bb.fetch2 import URI
 from bb.fetch2 import FetchMethod
 import bb
@@ -628,6 +629,35 @@ class GitShallowTarballNamingTest(FetcherTest):
         self.assertIn(self.mirror_tarball, dir)
 
 
+class CleanTarballTest(FetcherTest):
+    def setUp(self):
+        super(CleanTarballTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
+
+        self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+    @skipIfNoNetwork()
+    def test_that_the_tarball_contents_does_not_leak_info(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        fetcher.unpack(self.unpackdir)
+        mtime = bb.process.run('git log --all -1 --format=%ct',
+                cwd=os.path.join(self.unpackdir, 'git'))
+        self.assertEqual(len(mtime), 2)
+        mtime = int(mtime[0])
+
+        archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball))
+        self.assertNotEqual(len(archive.members), 0)
+        for member in archive.members:
+            self.assertEqual(member.uname, 'pokybuild')
+            self.assertEqual(member.gname, 'users')
+            self.assertEqual(member.mtime, mtime)
+
+
 class FetcherLocalTest(FetcherTest):
     def setUp(self):
         def touch(fn):
-- 
2.25.1

[PATCH v2] fetch2/git: stop generated tarballs from leaking info

[Olaf Mandel]

When using BB_GENERATE_MIRROR_TARBALLS="1" to generate mirror tarballs
of git repositories, they leaked local information: username, group and
time of the last fetch. Remove all these by setting fixed information:

 * uname = pokybuild (6000)
 * gname = users (100)
 * mtime = committer time of newest commit in repo

The username and group value were taken from the archives available on
the downloads.yoctoproject.org mirror. The modification time is chosen
so it still retains some relationship to the contents of the archive.

Signed-off-by: Olaf Mandel <o.mandel@menlosystems.com>
---
 lib/bb/fetch2/git.py  |  5 ++++-
 lib/bb/tests/fetch.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index f6f6b63a..ac3fd7ce 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -448,7 +448,10 @@ class Git(FetchMethod):
 
             logger.info("Creating tarball of git repository")
             with create_atomic(ud.fullmirror) as tfile:
-                runfetchcmd("tar -czf %s ." % tfile, d, workdir=ud.clonedir)
+                mtime = runfetchcmd("git log --all -1 --format=%cD", d,
+                        quiet=True, workdir=ud.clonedir)
+                runfetchcmd("tar -czf %s --owner pokybuild --group users --mtime \"%s\" ."
+                        % (tfile, mtime), d, workdir=ud.clonedir)
             runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
     def clone_shallow_local(self, ud, dest, d):
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 301c4683..0f066394 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -11,6 +11,7 @@ import hashlib
 import tempfile
 import collections
 import os
+import tarfile
 from bb.fetch2 import URI
 from bb.fetch2 import FetchMethod
 import bb
@@ -584,6 +585,35 @@ class GitShallowTarballNamingTest(FetcherTest):
         self.assertIn(self.mirror_tarball, dir)
 
 
+class CleanTarballTest(FetcherTest):
+    def setUp(self):
+        super(CleanTarballTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
+
+        self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+    @skipIfNoNetwork()
+    def test_that_the_tarball_contents_does_not_leak_info(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        fetcher.unpack(self.unpackdir)
+        mtime = bb.process.run('git log --all -1 --format=%ct',
+                cwd=os.path.join(self.unpackdir, 'git'))
+        self.assertEqual(len(mtime), 2)
+        mtime = int(mtime[0])
+
+        archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball))
+        self.assertNotEqual(len(archive.members), 0)
+        for member in archive.members:
+            self.assertEqual(member.uname, 'pokybuild')
+            self.assertEqual(member.gname, 'users')
+            self.assertEqual(member.mtime, mtime)
+
+
 class FetcherLocalTest(FetcherTest):
     def setUp(self):
         def touch(fn):
-- 
2.25.1

[PATCH v3] fetch2/git: stop generated tarballs from leaking info

[Olaf Mandel]

When using BB_GENERATE_MIRROR_TARBALLS="1" to generate mirror tarballs
of git repositories, they leaked local information: username, group and
time of the last fetch. Remove all these by setting fixed information:

 * uname = pokybuild (6000)
 * gname = users (100)
 * mtime = committer time of newest commit in repo

The username and group value were taken from the archives available on
the downloads.yoctoproject.org mirror. The modification time is chosen
so it still retains some relationship to the contents of the archive.

Signed-off-by: Olaf Mandel <o.mandel@menlosystems.com>
---
 lib/bb/fetch2/git.py  |  5 ++++-
 lib/bb/tests/fetch.py | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index f6f6b63a..727cebdc 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -448,7 +448,10 @@ class Git(FetchMethod):
 
             logger.info("Creating tarball of git repository")
             with create_atomic(ud.fullmirror) as tfile:
-                runfetchcmd("tar -czf %s ." % tfile, d, workdir=ud.clonedir)
+                mtime = runfetchcmd("git log --all -1 --format=%cD", d,
+                        quiet=True, workdir=ud.clonedir)
+                runfetchcmd("tar -czf %s --owner pokybuild:6000 --group users:100 --mtime \"%s\" ."
+                        % (tfile, mtime), d, workdir=ud.clonedir)
             runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
     def clone_shallow_local(self, ud, dest, d):
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 301c4683..68934e79 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -11,6 +11,7 @@ import hashlib
 import tempfile
 import collections
 import os
+import tarfile
 from bb.fetch2 import URI
 from bb.fetch2 import FetchMethod
 import bb
@@ -584,6 +585,37 @@ class GitShallowTarballNamingTest(FetcherTest):
         self.assertIn(self.mirror_tarball, dir)
 
 
+class CleanTarballTest(FetcherTest):
+    def setUp(self):
+        super(CleanTarballTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
+
+        self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+    @skipIfNoNetwork()
+    def test_that_the_tarball_contents_does_not_leak_info(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        fetcher.unpack(self.unpackdir)
+        mtime = bb.process.run('git log --all -1 --format=%ct',
+                cwd=os.path.join(self.unpackdir, 'git'))
+        self.assertEqual(len(mtime), 2)
+        mtime = int(mtime[0])
+
+        archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball))
+        self.assertNotEqual(len(archive.members), 0)
+        for member in archive.members:
+            self.assertEqual(member.uname, 'pokybuild')
+            self.assertEqual(member.uid, 6000)
+            self.assertEqual(member.gname, 'users')
+            self.assertEqual(member.gid, 100)
+            self.assertEqual(member.mtime, mtime)
+
+
 class FetcherLocalTest(FetcherTest):
     def setUp(self):
         def touch(fn):
-- 
2.25.1

Re: [PATCH v3] fetch2/git: stop generated tarballs from leaking info

[Olaf Mandel]

[-- Attachment #1.1: Type: text/plain, Size: 623 bytes --]

Hello,

Am 28.03.2022 um 19:36 schrieb Olaf Mandel:
> When using BB_GENERATE_MIRROR_TARBALLS="1" to generate mirror tarballs
> of git repositories, they leaked local information: username, group and
> time of the last fetch. Remove all these by setting fixed information:
> 
>   * uname = pokybuild (6000)
>   * gname = users (100)

it was pointed out to me by Marek that instead of using pokybuild:users, 
which seems to be an artifact of the YP autobuilder, I should use the 
"canonical" combination oe:oe. But that raises the question: which 
numerical IDs should I use then?

Best regards,
Olaf Mandel

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 495 bytes --]

Re: [bitbake-devel] [PATCH v3] fetch2/git: stop generated tarballs from leaking info

[Alexandre Belloni]

Hello,

On 05/04/2022 13:38:41+0200, Olaf Mandel wrote:
> Hello,
> 
> Am 28.03.2022 um 19:36 schrieb Olaf Mandel:
> > When using BB_GENERATE_MIRROR_TARBALLS="1" to generate mirror tarballs
> > of git repositories, they leaked local information: username, group and
> > time of the last fetch. Remove all these by setting fixed information:
> > 
> >   * uname = pokybuild (6000)
> >   * gname = users (100)
> 
> it was pointed out to me by Marek that instead of using pokybuild:users,
> which seems to be an artifact of the YP autobuilder, I should use the
> "canonical" combination oe:oe. But that raises the question: which numerical
> IDs should I use then?
> 

Note that patch has already been applied so you'd have to send a patch
on top of master.

> Best regards,
> Olaf Mandel




> 
> -=-=-=-=-=-=-=-=-=-=-=-
> Links: You receive all messages sent to this group.
> View/Reply Online (#13579): https://lists.openembedded.org/g/bitbake-devel/message/13579
> Mute This Topic: https://lists.openembedded.org/mt/90090609/3617179
> Group Owner: bitbake-devel+owner@lists.openembedded.org
> Unsubscribe: https://lists.openembedded.org/g/bitbake-devel/unsub [alexandre.belloni@bootlin.com]
> -=-=-=-=-=-=-=-=-=-=-=-
> 


-- 
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

Re: [bitbake-devel] [PATCH v3] fetch2/git: stop generated tarballs from leaking info

[Richard Purdie]

On Tue, 2022-04-05 at 13:38 +0200, Olaf Mandel wrote:
> Hello,
> 
> Am 28.03.2022 um 19:36 schrieb Olaf Mandel:
> > When using BB_GENERATE_MIRROR_TARBALLS="1" to generate mirror tarballs
> > of git repositories, they leaked local information: username, group and
> > time of the last fetch. Remove all these by setting fixed information:
> > 
> >   * uname = pokybuild (6000)
> >   * gname = users (100)
> 
> it was pointed out to me by Marek that instead of using pokybuild:users, 
> which seems to be an artifact of the YP autobuilder, I should use the 
> "canonical" combination oe:oe. But that raises the question: which 
> numerical IDs should I use then?

I was happy to get a consistent value, I'm less worried about what that value is
but if we want something different I'm ok with that...

Cheers,

Richard

[PATCH] fetch2/git: canonicalize ids in generated tarballs

[Olaf Mandel]

Change the owner information in the mirror tarballs generated using
BB_GENERATE_MIRROR_TARBALLS="1". This is an extension of commit
0178ab83, which used the original pokybuild:user information, but failed
to clean up the numerical user and group ids. Now set the more canonical
values of oe:oe and 0:0.

Signed-off-by: Olaf Mandel <o.mandel@menlosystems.com>
CC: Marek Vasut <marex@denx.de>
---
 lib/bb/fetch2/git.py  | 2 +-
 lib/bb/tests/fetch.py | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/bb/fetch2/git.py b/lib/bb/fetch2/git.py
index 4d06a571..b3eb8248 100644
--- a/lib/bb/fetch2/git.py
+++ b/lib/bb/fetch2/git.py
@@ -464,7 +464,7 @@ class Git(FetchMethod):
             with create_atomic(ud.fullmirror) as tfile:
                 mtime = runfetchcmd("git log --all -1 --format=%cD", d,
                         quiet=True, workdir=ud.clonedir)
-                runfetchcmd("tar -czf %s --owner pokybuild --group users --mtime \"%s\" ."
+                runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
                         % (tfile, mtime), d, workdir=ud.clonedir)
             runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
diff --git a/lib/bb/tests/fetch.py b/lib/bb/tests/fetch.py
index 233ecae7..1152e89c 100644
--- a/lib/bb/tests/fetch.py
+++ b/lib/bb/tests/fetch.py
@@ -653,8 +653,10 @@ class CleanTarballTest(FetcherTest):
         archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball))
         self.assertNotEqual(len(archive.members), 0)
         for member in archive.members:
-            self.assertEqual(member.uname, 'pokybuild')
-            self.assertEqual(member.gname, 'users')
+            self.assertEqual(member.uname, 'oe')
+            self.assertEqual(member.uid, 0)
+            self.assertEqual(member.gname, 'oe')
+            self.assertEqual(member.gid, 0)
             self.assertEqual(member.mtime, mtime)
 
 
-- 
2.25.1