* [bitbake-devel][PATCH] bitbake: Add piping compression library
@ 2021-06-17 17:56 Joshua Watt
2021-06-17 19:38 ` [bitbake-devel][PATCH v2] " Joshua Watt
0 siblings, 1 reply; 6+ messages in thread
From: Joshua Watt @ 2021-06-17 17:56 UTC (permalink / raw)
To: bitbake-devel; +Cc: Joshua Watt
Adds a library that implements file-like objects (similar to
gzip.GzipFile) that can stream to arbitrary compression programs. This
is utilized to implement a LZ4 and zstd compression API.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
bitbake/lib/bb/_pipecompress.py | 170 ++++++++++++++++++++++++++++++++
bitbake/lib/bb/lz4.py | 17 ++++
bitbake/lib/bb/zstd.py | 27 +++++
3 files changed, 214 insertions(+)
create mode 100644 bitbake/lib/bb/_pipecompress.py
create mode 100644 bitbake/lib/bb/lz4.py
create mode 100644 bitbake/lib/bb/zstd.py
diff --git a/bitbake/lib/bb/_pipecompress.py b/bitbake/lib/bb/_pipecompress.py
new file mode 100644
index 0000000000..3404be98d6
--- /dev/null
+++ b/bitbake/lib/bb/_pipecompress.py
@@ -0,0 +1,170 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Helper library to implement streaming compression and decompression using an
+# external process
+#
+# This library should be used directly by end users; a wrapper library for the
+# specific compression tool should be created
+
+import builtins
+import io
+import os
+import subprocess
+
+
+def open_wrap(
+ cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs
+):
+ """
+ Open a compressed file in binary or text mode.
+
+ Users should not call this directly. A specific compression library can use
+ this helper to provide it's own "open" command
+
+ The filename argument can be an actual filename (a str or bytes object), or
+ an existing file object to read from or write to.
+
+ The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
+ binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
+ "rb".
+
+ For binary mode, this function is equivalent to the cls constructor:
+ cls(filename, mode). In this case, the encoding, errors and newline
+ arguments must not be provided.
+
+ For text mode, a cls object is created, and wrapped in an
+ io.TextIOWrapper instance with the specified encoding, error handling
+ behavior, and line ending(s).
+ """
+ if "t" in mode:
+ if "b" in mode:
+ raise ValueError("Invalid mode: %r" % (mode,))
+ else:
+ if encoding is not None:
+ raise ValueError("Argument 'encoding' not supported in binary mode")
+ if errors is not None:
+ raise ValueError("Argument 'errors' not supported in binary mode")
+ if newline is not None:
+ raise ValueError("Argument 'newline' not supported in binary mode")
+
+ file_mode = mode.replace("t", "")
+ if isinstance(filename, (str, bytes, os.PathLike)):
+ binary_file = cls(filename, file_mode, **kwargs)
+ elif hasattr(filename, "read") or hasattr(filename, "write"):
+ binary_file = cls(None, file_mode, fileobj=filename, **kwargs)
+ else:
+ raise TypeError("filename must be a str or bytes object, or a file")
+
+ if "t" in mode:
+ return io.TextIOWrapper(
+ binary_file, encoding, errors, newline, write_through=True
+ )
+ else:
+ return binary_file
+
+
+class PipeFile(io.RawIOBase):
+ """
+ Class that implements generically piping to/from a compression program
+
+ Derived classes should add the function get_compress() and get_decompress()
+ that return the required commands. Input will be piped into stdin and the
+ (de)compressed output should be written to stdout, e.g.:
+
+ class FooFile(PipeCompressionFile):
+ def get_decompress(self):
+ return ["fooc", "--decompress", "--stdout"]
+
+ def get_compress(self):
+ return ["fooc", "--compress", "--stdout"]
+
+ """
+
+ READ = 0
+ WRITE = 1
+
+ def __init__(self, filename=None, mode="rb", *, fileobj=None):
+ if "t" in mode or "U" in mode:
+ raise ValueError("Invalid mode: {!r}".format(mode))
+
+ if not "b" in mode:
+ mode += "b"
+
+ if mode.startswith("r"):
+ self.mode = self.READ
+ elif mode.startswith("w"):
+ self.mode = self.WRITE
+ else:
+ raise ValueError("Invalid mode %r" % mode)
+
+ if fileobj is not None:
+ self.fileobj = fileobj
+ else:
+ self.fileobj = builtins.open(filename, mode or "rb")
+
+ if self.mode == self.READ:
+ self.p = subprocess.Popen(
+ self.get_decompress(),
+ stdin=self.fileobj,
+ stdout=subprocess.PIPE,
+ close_fds=True,
+ )
+ self.pipe = self.p.stdout
+ else:
+ self.p = subprocess.Popen(
+ self.get_compress(),
+ stdin=subprocess.PIPE,
+ stdout=self.fileobj,
+ close_fds=True,
+ )
+ self.pipe = self.p.stdin
+
+ self.__closed = False
+
+ def close(self):
+ if self.closed:
+ return
+
+ self.pipe.close()
+ self.p.wait()
+ self.fileobj.close()
+
+ self.__closed = True
+
+ @property
+ def closed(self):
+ return self.__closed
+
+ def fileno(self):
+ return self.pipe.fileno()
+
+ def flush(self):
+ self.pipe.flush()
+
+ def isatty(self):
+ return self.pipe.isatty()
+
+ def readable(self):
+ return self.mode == self.READ
+
+ def writable(self):
+ return self.mode == self.WRITE
+
+ def readinto(self, b):
+ if self.mode != self.READ:
+ import errno
+
+ raise OSError(
+ errno.EBADF, "read() on write-only %s object" % self.__class__.__name__
+ )
+ return self.pipe.readinto(b)
+
+ def write(self, data):
+ if self.mode != self.WRITE:
+ import errno
+
+ raise OSError(
+ errno.EBADF, "write() on read-only %s object" % self.__class__.__name__
+ )
+ return self.pipe.write(data)
diff --git a/bitbake/lib/bb/lz4.py b/bitbake/lib/bb/lz4.py
new file mode 100644
index 0000000000..c2bda291d3
--- /dev/null
+++ b/bitbake/lib/bb/lz4.py
@@ -0,0 +1,17 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+
+
+def open(*args, **kwargs):
+ return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs)
+
+
+class LZ4File(bb._pipecompress.PipeFile):
+ def get_compress(self):
+ return ["lz4c", "-z", "-c"]
+
+ def get_decompress(self):
+ return ["lz4c", "-d", "-c"]
diff --git a/bitbake/lib/bb/zstd.py b/bitbake/lib/bb/zstd.py
new file mode 100644
index 0000000000..1f46087ec9
--- /dev/null
+++ b/bitbake/lib/bb/zstd.py
@@ -0,0 +1,27 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+import shutil
+
+
+def open(*args, **kwargs):
+ return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
+
+
+class ZstdFile(bb._pipecompress.PipeFile):
+ def __init__(self, *args, num_threads=1, **kwargs):
+ self.num_threads = num_threads
+ super().__init__(*args, **kwargs)
+
+ def _get_zstd(self):
+ if self.num_threads == 1 or not shutil.which("pzstd"):
+ return ["zstd"]
+ return ["pzstd", "-p", "%d" % self.num_threads]
+
+ def get_compress(self):
+ return self._get_zstd() + ["-c"]
+
+ def get_decompress(self):
+ return self._get_zstd() + ["-d", "-c"]
--
2.31.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [bitbake-devel][PATCH v2] bitbake: Add piping compression library
2021-06-17 17:56 [bitbake-devel][PATCH] bitbake: Add piping compression library Joshua Watt
@ 2021-06-17 19:38 ` Joshua Watt
2021-06-23 13:50 ` Alexandre Belloni
2021-07-14 15:01 ` [bitbake-devel][PATCH v3] " Joshua Watt
0 siblings, 2 replies; 6+ messages in thread
From: Joshua Watt @ 2021-06-17 19:38 UTC (permalink / raw)
To: bitbake-devel; +Cc: Joshua Watt
Adds a library that implements file-like objects (similar to
gzip.GzipFile) that can stream to arbitrary compression programs. This
is utilized to implement a LZ4 and zstd compression API.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
V2: Now with tests!
bitbake/bin/bitbake-selftest | 1 +
bitbake/lib/bb/_pipecompress.py | 194 ++++++++++++++++++++++++++++
bitbake/lib/bb/lz4.py | 17 +++
bitbake/lib/bb/tests/compression.py | 82 ++++++++++++
bitbake/lib/bb/zstd.py | 27 ++++
5 files changed, 321 insertions(+)
create mode 100644 bitbake/lib/bb/_pipecompress.py
create mode 100644 bitbake/lib/bb/lz4.py
create mode 100644 bitbake/lib/bb/tests/compression.py
create mode 100644 bitbake/lib/bb/zstd.py
diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
index 757441c754..63762fe794 100755
--- a/bitbake/bin/bitbake-selftest
+++ b/bitbake/bin/bitbake-selftest
@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
"bb.tests.runqueue",
"bb.tests.siggen",
"bb.tests.utils",
+ "bb.tests.compression",
"hashserv.tests",
"layerindexlib.tests.layerindexobj",
"layerindexlib.tests.restapi",
diff --git a/bitbake/lib/bb/_pipecompress.py b/bitbake/lib/bb/_pipecompress.py
new file mode 100644
index 0000000000..4b9f662143
--- /dev/null
+++ b/bitbake/lib/bb/_pipecompress.py
@@ -0,0 +1,194 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Helper library to implement streaming compression and decompression using an
+# external process
+#
+# This library should be used directly by end users; a wrapper library for the
+# specific compression tool should be created
+
+import builtins
+import io
+import os
+import subprocess
+
+
+def open_wrap(
+ cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs
+):
+ """
+ Open a compressed file in binary or text mode.
+
+ Users should not call this directly. A specific compression library can use
+ this helper to provide it's own "open" command
+
+ The filename argument can be an actual filename (a str or bytes object), or
+ an existing file object to read from or write to.
+
+ The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
+ binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
+ "rb".
+
+ For binary mode, this function is equivalent to the cls constructor:
+ cls(filename, mode). In this case, the encoding, errors and newline
+ arguments must not be provided.
+
+ For text mode, a cls object is created, and wrapped in an
+ io.TextIOWrapper instance with the specified encoding, error handling
+ behavior, and line ending(s).
+ """
+ if "t" in mode:
+ if "b" in mode:
+ raise ValueError("Invalid mode: %r" % (mode,))
+ else:
+ if encoding is not None:
+ raise ValueError("Argument 'encoding' not supported in binary mode")
+ if errors is not None:
+ raise ValueError("Argument 'errors' not supported in binary mode")
+ if newline is not None:
+ raise ValueError("Argument 'newline' not supported in binary mode")
+
+ file_mode = mode.replace("t", "")
+ if isinstance(filename, (str, bytes, os.PathLike)):
+ binary_file = cls(filename, file_mode, **kwargs)
+ elif hasattr(filename, "read") or hasattr(filename, "write"):
+ binary_file = cls(None, file_mode, fileobj=filename, **kwargs)
+ else:
+ raise TypeError("filename must be a str or bytes object, or a file")
+
+ if "t" in mode:
+ return io.TextIOWrapper(
+ binary_file, encoding, errors, newline, write_through=True
+ )
+ else:
+ return binary_file
+
+
+class CompressionError(OSError):
+ pass
+
+
+class PipeFile(io.RawIOBase):
+ """
+ Class that implements generically piping to/from a compression program
+
+ Derived classes should add the function get_compress() and get_decompress()
+ that return the required commands. Input will be piped into stdin and the
+ (de)compressed output should be written to stdout, e.g.:
+
+ class FooFile(PipeCompressionFile):
+ def get_decompress(self):
+ return ["fooc", "--decompress", "--stdout"]
+
+ def get_compress(self):
+ return ["fooc", "--compress", "--stdout"]
+
+ """
+
+ READ = 0
+ WRITE = 1
+
+ def __init__(self, filename=None, mode="rb", *, stderr=None, fileobj=None):
+ if "t" in mode or "U" in mode:
+ raise ValueError("Invalid mode: {!r}".format(mode))
+
+ if not "b" in mode:
+ mode += "b"
+
+ if mode.startswith("r"):
+ self.mode = self.READ
+ elif mode.startswith("w"):
+ self.mode = self.WRITE
+ else:
+ raise ValueError("Invalid mode %r" % mode)
+
+ if fileobj is not None:
+ self.fileobj = fileobj
+ else:
+ self.fileobj = builtins.open(filename, mode or "rb")
+
+ if self.mode == self.READ:
+ self.p = subprocess.Popen(
+ self.get_decompress(),
+ stdin=self.fileobj,
+ stdout=subprocess.PIPE,
+ stderr=stderr,
+ close_fds=True,
+ )
+ self.pipe = self.p.stdout
+ else:
+ self.p = subprocess.Popen(
+ self.get_compress(),
+ stdin=subprocess.PIPE,
+ stdout=self.fileobj,
+ stderr=stderr,
+ close_fds=True,
+ )
+ self.pipe = self.p.stdin
+
+ self.__closed = False
+
+ def _check_process(self):
+ if self.p is None:
+ return
+
+ returncode = self.p.wait()
+ if returncode:
+ raise CompressionError("Process died with %d" % returncode)
+ self.p = None
+
+ def close(self):
+ if self.closed:
+ return
+
+ self.pipe.close()
+ if self.p is not None:
+ self._check_process()
+ self.fileobj.close()
+
+ self.__closed = True
+
+ @property
+ def closed(self):
+ return self.__closed
+
+ def fileno(self):
+ return self.pipe.fileno()
+
+ def flush(self):
+ self.pipe.flush()
+
+ def isatty(self):
+ return self.pipe.isatty()
+
+ def readable(self):
+ return self.mode == self.READ
+
+ def writable(self):
+ return self.mode == self.WRITE
+
+ def readinto(self, b):
+ if self.mode != self.READ:
+ import errno
+
+ raise OSError(
+ errno.EBADF, "read() on write-only %s object" % self.__class__.__name__
+ )
+ size = self.pipe.readinto(b)
+ if size == 0:
+ self._check_process()
+ return size
+
+ def write(self, data):
+ if self.mode != self.WRITE:
+ import errno
+
+ raise OSError(
+ errno.EBADF, "write() on read-only %s object" % self.__class__.__name__
+ )
+ data = self.pipe.write(data)
+
+ if not data:
+ self._check_process()
+
+ return data
diff --git a/bitbake/lib/bb/lz4.py b/bitbake/lib/bb/lz4.py
new file mode 100644
index 0000000000..c2bda291d3
--- /dev/null
+++ b/bitbake/lib/bb/lz4.py
@@ -0,0 +1,17 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+
+
+def open(*args, **kwargs):
+ return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs)
+
+
+class LZ4File(bb._pipecompress.PipeFile):
+ def get_compress(self):
+ return ["lz4c", "-z", "-c"]
+
+ def get_decompress(self):
+ return ["lz4c", "-d", "-c"]
diff --git a/bitbake/lib/bb/tests/compression.py b/bitbake/lib/bb/tests/compression.py
new file mode 100644
index 0000000000..f663853901
--- /dev/null
+++ b/bitbake/lib/bb/tests/compression.py
@@ -0,0 +1,82 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from pathlib import Path
+import bb.lz4
+import bb.zstd
+import contextlib
+import os
+import shutil
+import tempfile
+import unittest
+import subprocess
+
+
+class CompressionTests(object):
+ def setUp(self):
+ self._t = tempfile.TemporaryDirectory()
+ self.tmpdir = Path(self._t.name)
+ self.addCleanup(self._t.cleanup)
+
+ def _file_helper(self, mode_suffix, data):
+ tmp_file = self.tmpdir / "compressed"
+
+ with self.do_open(tmp_file, mode="w" + mode_suffix) as f:
+ f.write(data)
+
+ with self.do_open(tmp_file, mode="r" + mode_suffix) as f:
+ read_data = f.read()
+
+ self.assertEqual(read_data, data)
+
+ def test_text_file(self):
+ self._file_helper("t", "Hello")
+
+ def test_binary_file(self):
+ self._file_helper("b", "Hello".encode("utf-8"))
+
+ def _pipe_helper(self, mode_suffix, data):
+ rfd, wfd = os.pipe()
+ with open(rfd, "rb") as r, open(wfd, "wb") as w:
+ with self.do_open(r, mode="r" + mode_suffix) as decompress:
+ with self.do_open(w, mode="w" + mode_suffix) as compress:
+ compress.write(data)
+ read_data = decompress.read()
+
+ self.assertEqual(read_data, data)
+
+ def test_text_pipe(self):
+ self._pipe_helper("t", "Hello")
+
+ def test_binary_pipe(self):
+ self._pipe_helper("b", "Hello".encode("utf-8"))
+
+ def test_bad_decompress(self):
+ tmp_file = self.tmpdir / "compressed"
+ with tmp_file.open("wb") as f:
+ f.write(b"\x00")
+
+ with self.assertRaises(OSError):
+ with self.do_open(tmp_file, mode="rb", stderr=subprocess.DEVNULL) as f:
+ data = f.read()
+
+
+class LZ4Tests(CompressionTests, unittest.TestCase):
+ def get_cls(self):
+ return bb.lz4.LZ4File
+
+ @contextlib.contextmanager
+ def do_open(self, *args, **kwargs):
+ with bb.lz4.open(*args, **kwargs) as f:
+ yield f
+
+
+class ZStdTests(CompressionTests, unittest.TestCase):
+ def get_cls(self):
+ return bb.zstd.ZstdFile
+
+ @contextlib.contextmanager
+ def do_open(self, *args, **kwargs):
+ with bb.zstd.open(*args, **kwargs) as f:
+ yield f
diff --git a/bitbake/lib/bb/zstd.py b/bitbake/lib/bb/zstd.py
new file mode 100644
index 0000000000..1f46087ec9
--- /dev/null
+++ b/bitbake/lib/bb/zstd.py
@@ -0,0 +1,27 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+import shutil
+
+
+def open(*args, **kwargs):
+ return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
+
+
+class ZstdFile(bb._pipecompress.PipeFile):
+ def __init__(self, *args, num_threads=1, **kwargs):
+ self.num_threads = num_threads
+ super().__init__(*args, **kwargs)
+
+ def _get_zstd(self):
+ if self.num_threads == 1 or not shutil.which("pzstd"):
+ return ["zstd"]
+ return ["pzstd", "-p", "%d" % self.num_threads]
+
+ def get_compress(self):
+ return self._get_zstd() + ["-c"]
+
+ def get_decompress(self):
+ return self._get_zstd() + ["-d", "-c"]
--
2.31.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [bitbake-devel][PATCH v2] bitbake: Add piping compression library
2021-06-17 19:38 ` [bitbake-devel][PATCH v2] " Joshua Watt
@ 2021-06-23 13:50 ` Alexandre Belloni
2021-07-14 15:01 ` [bitbake-devel][PATCH v3] " Joshua Watt
1 sibling, 0 replies; 6+ messages in thread
From: Alexandre Belloni @ 2021-06-23 13:50 UTC (permalink / raw)
To: Joshua Watt; +Cc: bitbake-devel
Hello Joshua,
On 17/06/2021 14:38:09-0500, Joshua Watt wrote:
> Adds a library that implements file-like objects (similar to
> gzip.GzipFile) that can stream to arbitrary compression programs. This
> is utilized to implement a LZ4 and zstd compression API.
>
> Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
> ---
> V2: Now with tests!
>
> bitbake/bin/bitbake-selftest | 1 +
> bitbake/lib/bb/_pipecompress.py | 194 ++++++++++++++++++++++++++++
> bitbake/lib/bb/lz4.py | 17 +++
> bitbake/lib/bb/tests/compression.py | 82 ++++++++++++
> bitbake/lib/bb/zstd.py | 27 ++++
Note that this does apply on the poky git repository but not the bitbake
repo unless patch -p2 is used.
> 5 files changed, 321 insertions(+)
> create mode 100644 bitbake/lib/bb/_pipecompress.py
> create mode 100644 bitbake/lib/bb/lz4.py
> create mode 100644 bitbake/lib/bb/tests/compression.py
> create mode 100644 bitbake/lib/bb/zstd.py
>
I believe I have applied the patch correctly and this fails on the
autobuilders with:
FileNotFoundError: [Errno 2] No such file or directory: 'zstd': 'zstd'
https://autobuilder.yoctoproject.org/typhoon/#/builders/79/builds/2234
https://autobuilder.yoctoproject.org/typhoon/#/builders/80/builds/2210
https://autobuilder.yoctoproject.org/typhoon/#/builders/86/builds/2213
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/2248
> diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
> index 757441c754..63762fe794 100755
> --- a/bitbake/bin/bitbake-selftest
> +++ b/bitbake/bin/bitbake-selftest
> @@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
> "bb.tests.runqueue",
> "bb.tests.siggen",
> "bb.tests.utils",
> + "bb.tests.compression",
> "hashserv.tests",
> "layerindexlib.tests.layerindexobj",
> "layerindexlib.tests.restapi",
> diff --git a/bitbake/lib/bb/_pipecompress.py b/bitbake/lib/bb/_pipecompress.py
> new file mode 100644
> index 0000000000..4b9f662143
> --- /dev/null
> +++ b/bitbake/lib/bb/_pipecompress.py
> @@ -0,0 +1,194 @@
> +#
> +# SPDX-License-Identifier: GPL-2.0-only
> +#
> +# Helper library to implement streaming compression and decompression using an
> +# external process
> +#
> +# This library should be used directly by end users; a wrapper library for the
> +# specific compression tool should be created
> +
> +import builtins
> +import io
> +import os
> +import subprocess
> +
> +
> +def open_wrap(
> + cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs
> +):
> + """
> + Open a compressed file in binary or text mode.
> +
> + Users should not call this directly. A specific compression library can use
> + this helper to provide it's own "open" command
> +
> + The filename argument can be an actual filename (a str or bytes object), or
> + an existing file object to read from or write to.
> +
> + The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
> + binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
> + "rb".
> +
> + For binary mode, this function is equivalent to the cls constructor:
> + cls(filename, mode). In this case, the encoding, errors and newline
> + arguments must not be provided.
> +
> + For text mode, a cls object is created, and wrapped in an
> + io.TextIOWrapper instance with the specified encoding, error handling
> + behavior, and line ending(s).
> + """
> + if "t" in mode:
> + if "b" in mode:
> + raise ValueError("Invalid mode: %r" % (mode,))
> + else:
> + if encoding is not None:
> + raise ValueError("Argument 'encoding' not supported in binary mode")
> + if errors is not None:
> + raise ValueError("Argument 'errors' not supported in binary mode")
> + if newline is not None:
> + raise ValueError("Argument 'newline' not supported in binary mode")
> +
> + file_mode = mode.replace("t", "")
> + if isinstance(filename, (str, bytes, os.PathLike)):
> + binary_file = cls(filename, file_mode, **kwargs)
> + elif hasattr(filename, "read") or hasattr(filename, "write"):
> + binary_file = cls(None, file_mode, fileobj=filename, **kwargs)
> + else:
> + raise TypeError("filename must be a str or bytes object, or a file")
> +
> + if "t" in mode:
> + return io.TextIOWrapper(
> + binary_file, encoding, errors, newline, write_through=True
> + )
> + else:
> + return binary_file
> +
> +
> +class CompressionError(OSError):
> + pass
> +
> +
> +class PipeFile(io.RawIOBase):
> + """
> + Class that implements generically piping to/from a compression program
> +
> + Derived classes should add the function get_compress() and get_decompress()
> + that return the required commands. Input will be piped into stdin and the
> + (de)compressed output should be written to stdout, e.g.:
> +
> + class FooFile(PipeCompressionFile):
> + def get_decompress(self):
> + return ["fooc", "--decompress", "--stdout"]
> +
> + def get_compress(self):
> + return ["fooc", "--compress", "--stdout"]
> +
> + """
> +
> + READ = 0
> + WRITE = 1
> +
> + def __init__(self, filename=None, mode="rb", *, stderr=None, fileobj=None):
> + if "t" in mode or "U" in mode:
> + raise ValueError("Invalid mode: {!r}".format(mode))
> +
> + if not "b" in mode:
> + mode += "b"
> +
> + if mode.startswith("r"):
> + self.mode = self.READ
> + elif mode.startswith("w"):
> + self.mode = self.WRITE
> + else:
> + raise ValueError("Invalid mode %r" % mode)
> +
> + if fileobj is not None:
> + self.fileobj = fileobj
> + else:
> + self.fileobj = builtins.open(filename, mode or "rb")
> +
> + if self.mode == self.READ:
> + self.p = subprocess.Popen(
> + self.get_decompress(),
> + stdin=self.fileobj,
> + stdout=subprocess.PIPE,
> + stderr=stderr,
> + close_fds=True,
> + )
> + self.pipe = self.p.stdout
> + else:
> + self.p = subprocess.Popen(
> + self.get_compress(),
> + stdin=subprocess.PIPE,
> + stdout=self.fileobj,
> + stderr=stderr,
> + close_fds=True,
> + )
> + self.pipe = self.p.stdin
> +
> + self.__closed = False
> +
> + def _check_process(self):
> + if self.p is None:
> + return
> +
> + returncode = self.p.wait()
> + if returncode:
> + raise CompressionError("Process died with %d" % returncode)
> + self.p = None
> +
> + def close(self):
> + if self.closed:
> + return
> +
> + self.pipe.close()
> + if self.p is not None:
> + self._check_process()
> + self.fileobj.close()
> +
> + self.__closed = True
> +
> + @property
> + def closed(self):
> + return self.__closed
> +
> + def fileno(self):
> + return self.pipe.fileno()
> +
> + def flush(self):
> + self.pipe.flush()
> +
> + def isatty(self):
> + return self.pipe.isatty()
> +
> + def readable(self):
> + return self.mode == self.READ
> +
> + def writable(self):
> + return self.mode == self.WRITE
> +
> + def readinto(self, b):
> + if self.mode != self.READ:
> + import errno
> +
> + raise OSError(
> + errno.EBADF, "read() on write-only %s object" % self.__class__.__name__
> + )
> + size = self.pipe.readinto(b)
> + if size == 0:
> + self._check_process()
> + return size
> +
> + def write(self, data):
> + if self.mode != self.WRITE:
> + import errno
> +
> + raise OSError(
> + errno.EBADF, "write() on read-only %s object" % self.__class__.__name__
> + )
> + data = self.pipe.write(data)
> +
> + if not data:
> + self._check_process()
> +
> + return data
> diff --git a/bitbake/lib/bb/lz4.py b/bitbake/lib/bb/lz4.py
> new file mode 100644
> index 0000000000..c2bda291d3
> --- /dev/null
> +++ b/bitbake/lib/bb/lz4.py
> @@ -0,0 +1,17 @@
> +#
> +# SPDX-License-Identifier: GPL-2.0-only
> +#
> +
> +import bb._pipecompress
> +
> +
> +def open(*args, **kwargs):
> + return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs)
> +
> +
> +class LZ4File(bb._pipecompress.PipeFile):
> + def get_compress(self):
> + return ["lz4c", "-z", "-c"]
> +
> + def get_decompress(self):
> + return ["lz4c", "-d", "-c"]
> diff --git a/bitbake/lib/bb/tests/compression.py b/bitbake/lib/bb/tests/compression.py
> new file mode 100644
> index 0000000000..f663853901
> --- /dev/null
> +++ b/bitbake/lib/bb/tests/compression.py
> @@ -0,0 +1,82 @@
> +#
> +# SPDX-License-Identifier: GPL-2.0-only
> +#
> +
> +from pathlib import Path
> +import bb.lz4
> +import bb.zstd
> +import contextlib
> +import os
> +import shutil
> +import tempfile
> +import unittest
> +import subprocess
> +
> +
> +class CompressionTests(object):
> + def setUp(self):
> + self._t = tempfile.TemporaryDirectory()
> + self.tmpdir = Path(self._t.name)
> + self.addCleanup(self._t.cleanup)
> +
> + def _file_helper(self, mode_suffix, data):
> + tmp_file = self.tmpdir / "compressed"
> +
> + with self.do_open(tmp_file, mode="w" + mode_suffix) as f:
> + f.write(data)
> +
> + with self.do_open(tmp_file, mode="r" + mode_suffix) as f:
> + read_data = f.read()
> +
> + self.assertEqual(read_data, data)
> +
> + def test_text_file(self):
> + self._file_helper("t", "Hello")
> +
> + def test_binary_file(self):
> + self._file_helper("b", "Hello".encode("utf-8"))
> +
> + def _pipe_helper(self, mode_suffix, data):
> + rfd, wfd = os.pipe()
> + with open(rfd, "rb") as r, open(wfd, "wb") as w:
> + with self.do_open(r, mode="r" + mode_suffix) as decompress:
> + with self.do_open(w, mode="w" + mode_suffix) as compress:
> + compress.write(data)
> + read_data = decompress.read()
> +
> + self.assertEqual(read_data, data)
> +
> + def test_text_pipe(self):
> + self._pipe_helper("t", "Hello")
> +
> + def test_binary_pipe(self):
> + self._pipe_helper("b", "Hello".encode("utf-8"))
> +
> + def test_bad_decompress(self):
> + tmp_file = self.tmpdir / "compressed"
> + with tmp_file.open("wb") as f:
> + f.write(b"\x00")
> +
> + with self.assertRaises(OSError):
> + with self.do_open(tmp_file, mode="rb", stderr=subprocess.DEVNULL) as f:
> + data = f.read()
> +
> +
> +class LZ4Tests(CompressionTests, unittest.TestCase):
> + def get_cls(self):
> + return bb.lz4.LZ4File
> +
> + @contextlib.contextmanager
> + def do_open(self, *args, **kwargs):
> + with bb.lz4.open(*args, **kwargs) as f:
> + yield f
> +
> +
> +class ZStdTests(CompressionTests, unittest.TestCase):
> + def get_cls(self):
> + return bb.zstd.ZstdFile
> +
> + @contextlib.contextmanager
> + def do_open(self, *args, **kwargs):
> + with bb.zstd.open(*args, **kwargs) as f:
> + yield f
> diff --git a/bitbake/lib/bb/zstd.py b/bitbake/lib/bb/zstd.py
> new file mode 100644
> index 0000000000..1f46087ec9
> --- /dev/null
> +++ b/bitbake/lib/bb/zstd.py
> @@ -0,0 +1,27 @@
> +#
> +# SPDX-License-Identifier: GPL-2.0-only
> +#
> +
> +import bb._pipecompress
> +import shutil
> +
> +
> +def open(*args, **kwargs):
> + return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
> +
> +
> +class ZstdFile(bb._pipecompress.PipeFile):
> + def __init__(self, *args, num_threads=1, **kwargs):
> + self.num_threads = num_threads
> + super().__init__(*args, **kwargs)
> +
> + def _get_zstd(self):
> + if self.num_threads == 1 or not shutil.which("pzstd"):
> + return ["zstd"]
> + return ["pzstd", "-p", "%d" % self.num_threads]
> +
> + def get_compress(self):
> + return self._get_zstd() + ["-c"]
> +
> + def get_decompress(self):
> + return self._get_zstd() + ["-d", "-c"]
> --
> 2.31.1
>
--
Alexandre Belloni, co-owner and COO, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com
^ permalink raw reply [flat|nested] 6+ messages in thread
* [bitbake-devel][PATCH v3] bitbake: Add piping compression library
2021-06-17 19:38 ` [bitbake-devel][PATCH v2] " Joshua Watt
2021-06-23 13:50 ` Alexandre Belloni
@ 2021-07-14 15:01 ` Joshua Watt
2021-07-20 8:45 ` Richard Purdie
1 sibling, 1 reply; 6+ messages in thread
From: Joshua Watt @ 2021-07-14 15:01 UTC (permalink / raw)
To: bitbake-devel; +Cc: Joshua Watt
Adds a library that implements file-like objects (similar to
gzip.GzipFile) that can stream to arbitrary compression programs. This
is utilized to implement a LZ4 and zstd compression API.
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
---
V2: Now with tests!
V3: Added compression level for zstd
Tests are skipped if command line programs are missing, which should
prevent needing to update bitbake and oe-core in lock-step.
bitbake/bin/bitbake-selftest | 1 +
bitbake/lib/bb/_pipecompress.py | 194 ++++++++++++++++++++++++++++
bitbake/lib/bb/lz4.py | 17 +++
bitbake/lib/bb/tests/compression.py | 98 ++++++++++++++
bitbake/lib/bb/zstd.py | 28 ++++
5 files changed, 338 insertions(+)
create mode 100644 bitbake/lib/bb/_pipecompress.py
create mode 100644 bitbake/lib/bb/lz4.py
create mode 100644 bitbake/lib/bb/tests/compression.py
create mode 100644 bitbake/lib/bb/zstd.py
diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
index 757441c754..63762fe794 100755
--- a/bitbake/bin/bitbake-selftest
+++ b/bitbake/bin/bitbake-selftest
@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
"bb.tests.runqueue",
"bb.tests.siggen",
"bb.tests.utils",
+ "bb.tests.compression",
"hashserv.tests",
"layerindexlib.tests.layerindexobj",
"layerindexlib.tests.restapi",
diff --git a/bitbake/lib/bb/_pipecompress.py b/bitbake/lib/bb/_pipecompress.py
new file mode 100644
index 0000000000..4b9f662143
--- /dev/null
+++ b/bitbake/lib/bb/_pipecompress.py
@@ -0,0 +1,194 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Helper library to implement streaming compression and decompression using an
+# external process
+#
+# This library should be used directly by end users; a wrapper library for the
+# specific compression tool should be created
+
+import builtins
+import io
+import os
+import subprocess
+
+
+def open_wrap(
+ cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs
+):
+ """
+ Open a compressed file in binary or text mode.
+
+ Users should not call this directly. A specific compression library can use
+ this helper to provide it's own "open" command
+
+ The filename argument can be an actual filename (a str or bytes object), or
+ an existing file object to read from or write to.
+
+ The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
+ binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
+ "rb".
+
+ For binary mode, this function is equivalent to the cls constructor:
+ cls(filename, mode). In this case, the encoding, errors and newline
+ arguments must not be provided.
+
+ For text mode, a cls object is created, and wrapped in an
+ io.TextIOWrapper instance with the specified encoding, error handling
+ behavior, and line ending(s).
+ """
+ if "t" in mode:
+ if "b" in mode:
+ raise ValueError("Invalid mode: %r" % (mode,))
+ else:
+ if encoding is not None:
+ raise ValueError("Argument 'encoding' not supported in binary mode")
+ if errors is not None:
+ raise ValueError("Argument 'errors' not supported in binary mode")
+ if newline is not None:
+ raise ValueError("Argument 'newline' not supported in binary mode")
+
+ file_mode = mode.replace("t", "")
+ if isinstance(filename, (str, bytes, os.PathLike)):
+ binary_file = cls(filename, file_mode, **kwargs)
+ elif hasattr(filename, "read") or hasattr(filename, "write"):
+ binary_file = cls(None, file_mode, fileobj=filename, **kwargs)
+ else:
+ raise TypeError("filename must be a str or bytes object, or a file")
+
+ if "t" in mode:
+ return io.TextIOWrapper(
+ binary_file, encoding, errors, newline, write_through=True
+ )
+ else:
+ return binary_file
+
+
+class CompressionError(OSError):
+ pass
+
+
+class PipeFile(io.RawIOBase):
+ """
+ Class that implements generically piping to/from a compression program
+
+ Derived classes should add the function get_compress() and get_decompress()
+ that return the required commands. Input will be piped into stdin and the
+ (de)compressed output should be written to stdout, e.g.:
+
+ class FooFile(PipeCompressionFile):
+ def get_decompress(self):
+ return ["fooc", "--decompress", "--stdout"]
+
+ def get_compress(self):
+ return ["fooc", "--compress", "--stdout"]
+
+ """
+
+ READ = 0
+ WRITE = 1
+
+ def __init__(self, filename=None, mode="rb", *, stderr=None, fileobj=None):
+ if "t" in mode or "U" in mode:
+ raise ValueError("Invalid mode: {!r}".format(mode))
+
+ if not "b" in mode:
+ mode += "b"
+
+ if mode.startswith("r"):
+ self.mode = self.READ
+ elif mode.startswith("w"):
+ self.mode = self.WRITE
+ else:
+ raise ValueError("Invalid mode %r" % mode)
+
+ if fileobj is not None:
+ self.fileobj = fileobj
+ else:
+ self.fileobj = builtins.open(filename, mode or "rb")
+
+ if self.mode == self.READ:
+ self.p = subprocess.Popen(
+ self.get_decompress(),
+ stdin=self.fileobj,
+ stdout=subprocess.PIPE,
+ stderr=stderr,
+ close_fds=True,
+ )
+ self.pipe = self.p.stdout
+ else:
+ self.p = subprocess.Popen(
+ self.get_compress(),
+ stdin=subprocess.PIPE,
+ stdout=self.fileobj,
+ stderr=stderr,
+ close_fds=True,
+ )
+ self.pipe = self.p.stdin
+
+ self.__closed = False
+
+ def _check_process(self):
+ if self.p is None:
+ return
+
+ returncode = self.p.wait()
+ if returncode:
+ raise CompressionError("Process died with %d" % returncode)
+ self.p = None
+
+ def close(self):
+ if self.closed:
+ return
+
+ self.pipe.close()
+ if self.p is not None:
+ self._check_process()
+ self.fileobj.close()
+
+ self.__closed = True
+
+ @property
+ def closed(self):
+ return self.__closed
+
+ def fileno(self):
+ return self.pipe.fileno()
+
+ def flush(self):
+ self.pipe.flush()
+
+ def isatty(self):
+ return self.pipe.isatty()
+
+ def readable(self):
+ return self.mode == self.READ
+
+ def writable(self):
+ return self.mode == self.WRITE
+
+ def readinto(self, b):
+ if self.mode != self.READ:
+ import errno
+
+ raise OSError(
+ errno.EBADF, "read() on write-only %s object" % self.__class__.__name__
+ )
+ size = self.pipe.readinto(b)
+ if size == 0:
+ self._check_process()
+ return size
+
+ def write(self, data):
+ if self.mode != self.WRITE:
+ import errno
+
+ raise OSError(
+ errno.EBADF, "write() on read-only %s object" % self.__class__.__name__
+ )
+ data = self.pipe.write(data)
+
+ if not data:
+ self._check_process()
+
+ return data
diff --git a/bitbake/lib/bb/lz4.py b/bitbake/lib/bb/lz4.py
new file mode 100644
index 0000000000..c2bda291d3
--- /dev/null
+++ b/bitbake/lib/bb/lz4.py
@@ -0,0 +1,17 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+
+
+def open(*args, **kwargs):
+ return bb._pipecompress.open_wrap(LZ4File, *args, **kwargs)
+
+
+class LZ4File(bb._pipecompress.PipeFile):
+ def get_compress(self):
+ return ["lz4c", "-z", "-c"]
+
+ def get_decompress(self):
+ return ["lz4c", "-d", "-c"]
diff --git a/bitbake/lib/bb/tests/compression.py b/bitbake/lib/bb/tests/compression.py
new file mode 100644
index 0000000000..e47610ccd6
--- /dev/null
+++ b/bitbake/lib/bb/tests/compression.py
@@ -0,0 +1,98 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from pathlib import Path
+import bb.lz4
+import bb.zstd
+import contextlib
+import os
+import shutil
+import tempfile
+import unittest
+import subprocess
+
+
+class CompressionTests(object):
+ def setUp(self):
+ self._t = tempfile.TemporaryDirectory()
+ self.tmpdir = Path(self._t.name)
+ self.addCleanup(self._t.cleanup)
+
+ def _file_helper(self, mode_suffix, data):
+ tmp_file = self.tmpdir / "compressed"
+
+ with self.do_open(tmp_file, mode="w" + mode_suffix) as f:
+ f.write(data)
+
+ with self.do_open(tmp_file, mode="r" + mode_suffix) as f:
+ read_data = f.read()
+
+ self.assertEqual(read_data, data)
+
+ def test_text_file(self):
+ self._file_helper("t", "Hello")
+
+ def test_binary_file(self):
+ self._file_helper("b", "Hello".encode("utf-8"))
+
+ def _pipe_helper(self, mode_suffix, data):
+ rfd, wfd = os.pipe()
+ with open(rfd, "rb") as r, open(wfd, "wb") as w:
+ with self.do_open(r, mode="r" + mode_suffix) as decompress:
+ with self.do_open(w, mode="w" + mode_suffix) as compress:
+ compress.write(data)
+ read_data = decompress.read()
+
+ self.assertEqual(read_data, data)
+
+ def test_text_pipe(self):
+ self._pipe_helper("t", "Hello")
+
+ def test_binary_pipe(self):
+ self._pipe_helper("b", "Hello".encode("utf-8"))
+
+ def test_bad_decompress(self):
+ tmp_file = self.tmpdir / "compressed"
+ with tmp_file.open("wb") as f:
+ f.write(b"\x00")
+
+ with self.assertRaises(OSError):
+ with self.do_open(tmp_file, mode="rb", stderr=subprocess.DEVNULL) as f:
+ data = f.read()
+
+
+class LZ4Tests(CompressionTests, unittest.TestCase):
+ def setUp(self):
+ if shutil.which("lz4c") is None:
+ self.skipTest("'lz4c' not found")
+ super().setUp()
+
+ @contextlib.contextmanager
+ def do_open(self, *args, **kwargs):
+ with bb.lz4.open(*args, **kwargs) as f:
+ yield f
+
+
+class ZStdTests(CompressionTests, unittest.TestCase):
+ def setUp(self):
+ if shutil.which("zstd") is None:
+ self.skipTest("'zstd' not found")
+ super().setUp()
+
+ @contextlib.contextmanager
+ def do_open(self, *args, **kwargs):
+ with bb.zstd.open(*args, **kwargs) as f:
+ yield f
+
+
+class PZStdTests(CompressionTests, unittest.TestCase):
+ def setUp(self):
+ if shutil.which("pzstd") is None:
+ self.skipTest("'pzstd' not found")
+ super().setUp()
+
+ @contextlib.contextmanager
+ def do_open(self, *args, **kwargs):
+ with bb.zstd.open(*args, num_threads=2, **kwargs) as f:
+ yield f
diff --git a/bitbake/lib/bb/zstd.py b/bitbake/lib/bb/zstd.py
new file mode 100644
index 0000000000..858fd062b0
--- /dev/null
+++ b/bitbake/lib/bb/zstd.py
@@ -0,0 +1,28 @@
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb._pipecompress
+import shutil
+
+
+def open(*args, **kwargs):
+ return bb._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
+
+
+class ZstdFile(bb._pipecompress.PipeFile):
+ def __init__(self, *args, num_threads=1, compresslevel=3, **kwargs):
+ self.num_threads = num_threads
+ self.compresslevel = compresslevel
+ super().__init__(*args, **kwargs)
+
+ def _get_zstd(self):
+ if self.num_threads == 1 or not shutil.which("pzstd"):
+ return ["zstd"]
+ return ["pzstd", "-p", "%d" % self.num_threads]
+
+ def get_compress(self):
+ return self._get_zstd() + ["-c", "-%d" % self.compresslevel]
+
+ def get_decompress(self):
+ return self._get_zstd() + ["-d", "-c"]
--
2.32.0
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [bitbake-devel][PATCH v3] bitbake: Add piping compression library
2021-07-14 15:01 ` [bitbake-devel][PATCH v3] " Joshua Watt
@ 2021-07-20 8:45 ` Richard Purdie
2021-07-20 15:03 ` Joshua Watt
0 siblings, 1 reply; 6+ messages in thread
From: Richard Purdie @ 2021-07-20 8:45 UTC (permalink / raw)
To: Joshua Watt, bitbake-devel
On Wed, 2021-07-14 at 10:01 -0500, Joshua Watt wrote:
> Adds a library that implements file-like objects (similar to
> gzip.GzipFile) that can stream to arbitrary compression programs. This
> is utilized to implement a LZ4 and zstd compression API.
>
> Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
> ---
> V2: Now with tests!
> V3: Added compression level for zstd
> Tests are skipped if command line programs are missing, which should
> prevent needing to update bitbake and oe-core in lock-step.
>
>
>
>
>
>
>
>
>
> bitbake/bin/bitbake-selftest | 1 +
> bitbake/lib/bb/_pipecompress.py | 194 ++++++++++++++++++++++++++++
> bitbake/lib/bb/lz4.py | 17 +++
> bitbake/lib/bb/tests/compression.py | 98 ++++++++++++++
> bitbake/lib/bb/zstd.py | 28 ++++
> 5 files changed, 338 insertions(+)
> create mode 100644 bitbake/lib/bb/_pipecompress.py
> create mode 100644 bitbake/lib/bb/lz4.py
> create mode 100644 bitbake/lib/bb/tests/compression.py
> create mode 100644 bitbake/lib/bb/zstd.py
>
> diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
> index 757441c754..63762fe794 100755
> --- a/bitbake/bin/bitbake-selftest
> +++ b/bitbake/bin/bitbake-selftest
> @@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
> "bb.tests.runqueue",
> "bb.tests.siggen",
> "bb.tests.utils",
> + "bb.tests.compression",
> "hashserv.tests",
> "layerindexlib.tests.layerindexobj",
> "layerindexlib.tests.restapi",
> diff --git a/bitbake/lib/bb/_pipecompress.py b/bitbake/lib/bb/_pipecompress.py
> new file mode 100644
> index 0000000000..4b9f662143
> --- /dev/null
> +++ b/bitbake/lib/bb/_pipecompress.py
> @@ -0,0 +1,194 @@
> +#
> +# SPDX-License-Identifier: GPL-2.0-only
> +#
> +# Helper library to implement streaming compression and decompression using an
> +# external process
> +#
> +# This library should be used directly by end users; a wrapper library for the
> +# specific compression tool should be created
Should or should not? :)
I'm fine with adding this but I think it should be under a bb.compress. namespace
rather than directly in bb. I tweaked that patch in master-next to test that.
Cheers,
Richard
^ permalink raw reply [flat|nested] 6+ messages in thread
* Re: [bitbake-devel][PATCH v3] bitbake: Add piping compression library
2021-07-20 8:45 ` Richard Purdie
@ 2021-07-20 15:03 ` Joshua Watt
0 siblings, 0 replies; 6+ messages in thread
From: Joshua Watt @ 2021-07-20 15:03 UTC (permalink / raw)
To: Richard Purdie, bitbake-devel
On 7/20/21 3:45 AM, Richard Purdie wrote:
> On Wed, 2021-07-14 at 10:01 -0500, Joshua Watt wrote:
>> Adds a library that implements file-like objects (similar to
>> gzip.GzipFile) that can stream to arbitrary compression programs. This
>> is utilized to implement a LZ4 and zstd compression API.
>>
>> Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
>> ---
>> V2: Now with tests!
>> V3: Added compression level for zstd
>> Tests are skipped if command line programs are missing, which should
>> prevent needing to update bitbake and oe-core in lock-step.
>>
>>
>>
>>
>>
>>
>>
>>
>>
>> bitbake/bin/bitbake-selftest | 1 +
>> bitbake/lib/bb/_pipecompress.py | 194 ++++++++++++++++++++++++++++
>> bitbake/lib/bb/lz4.py | 17 +++
>> bitbake/lib/bb/tests/compression.py | 98 ++++++++++++++
>> bitbake/lib/bb/zstd.py | 28 ++++
>> 5 files changed, 338 insertions(+)
>> create mode 100644 bitbake/lib/bb/_pipecompress.py
>> create mode 100644 bitbake/lib/bb/lz4.py
>> create mode 100644 bitbake/lib/bb/tests/compression.py
>> create mode 100644 bitbake/lib/bb/zstd.py
>>
>> diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
>> index 757441c754..63762fe794 100755
>> --- a/bitbake/bin/bitbake-selftest
>> +++ b/bitbake/bin/bitbake-selftest
>> @@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
>> "bb.tests.runqueue",
>> "bb.tests.siggen",
>> "bb.tests.utils",
>> + "bb.tests.compression",
>> "hashserv.tests",
>> "layerindexlib.tests.layerindexobj",
>> "layerindexlib.tests.restapi",
>> diff --git a/bitbake/lib/bb/_pipecompress.py b/bitbake/lib/bb/_pipecompress.py
>> new file mode 100644
>> index 0000000000..4b9f662143
>> --- /dev/null
>> +++ b/bitbake/lib/bb/_pipecompress.py
>> @@ -0,0 +1,194 @@
>> +#
>> +# SPDX-License-Identifier: GPL-2.0-only
>> +#
>> +# Helper library to implement streaming compression and decompression using an
>> +# external process
>> +#
>> +# This library should be used directly by end users; a wrapper library for the
>> +# specific compression tool should be created
> Should or should not? :)
>
> I'm fine with adding this but I think it should be under a bb.compress. namespace
> rather than directly in bb. I tweaked that patch in master-next to test that.
That's fine with me. I would hope that having this in the code will
lower the barrier to entry and it start getting used more frequently...
I do have plans to use it once it's available.
>
> Cheers,
>
> Richard
>
>
^ permalink raw reply [flat|nested] 6+ messages in thread
end of thread, other threads:[~2021-07-20 15:03 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-06-17 17:56 [bitbake-devel][PATCH] bitbake: Add piping compression library Joshua Watt
2021-06-17 19:38 ` [bitbake-devel][PATCH v2] " Joshua Watt
2021-06-23 13:50 ` Alexandre Belloni
2021-07-14 15:01 ` [bitbake-devel][PATCH v3] " Joshua Watt
2021-07-20 8:45 ` Richard Purdie
2021-07-20 15:03 ` Joshua Watt
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.