* [Buildroot] [PATCH 1/1] utils/update-rust: add new util for updating rust/rust-bin
@ 2022-09-26 20:45 James Hilliard
2023-08-26 21:51 ` Thomas Petazzoni via buildroot
0 siblings, 1 reply; 3+ messages in thread
From: James Hilliard @ 2022-09-26 20:45 UTC (permalink / raw)
To: buildroot; +Cc: James Hilliard
Manually updating the rust package is tedious and slow as we have to
update and validate hashes for all supported rust-bin arch specific
toolchains.
To simplify this process add a python script which will update and
validate hashes and signatures for the new desired rust version.
This script is additionally capable of resuming an update if
interrupted which may be useful on slower network connections
as validating gpg signatures requires fully downloading each rust
toolchain distribution file.
This script has no external dependencies other than the optional
python-gnupg library which is needed for gpg signature validation.
Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
utils/update-rust | 226 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 226 insertions(+)
create mode 100755 utils/update-rust
diff --git a/utils/update-rust b/utils/update-rust
new file mode 100755
index 0000000000..2aad3fffa3
--- /dev/null
+++ b/utils/update-rust
@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+"""
+
+Utility for updating rust
+
+"""
+import argparse
+import errno
+import urllib.request
+import tempfile
+import shutil
+import io
+import os
+import hashlib
+from pathlib import Path
+from os import fdopen
+
+try:
+ # Requires the python-gnupg library
+ from gnupg import GPG
+except ImportError:
+ print("Unable to verify signatures, python-gnupg required")
+ GPG = None
+
+RUST_KEY = None
+RUST_KEY_URL = "https://static.rust-lang.org/rust-key.gpg.ascii"
+
+TOPDIR = Path(__file__).resolve().parent.parent
+PACKAGE_DIR = TOPDIR.joinpath("package")
+RUST_DIR = PACKAGE_DIR.joinpath("rust")
+RUST_BIN_DIR = PACKAGE_DIR.joinpath("rust-bin")
+RUST_MK_FILE = RUST_DIR.joinpath("rust.mk")
+RUST_HASH_FILE = RUST_DIR.joinpath("rust.hash")
+RUST_BIN_MK_FILE = RUST_BIN_DIR.joinpath("rust-bin.mk")
+RUST_BIN_HASH_FILE = RUST_BIN_DIR.joinpath("rust-bin.hash")
+
+
+def url_data(url):
+ with urllib.request.urlopen(url) as f:
+ return f.read()
+
+
+def rust_verify(sha256, sig_url):
+ global RUST_KEY
+ file_data = url_data(sig_url.rpartition(".")[0])
+ file_data_sha256 = hashlib.sha256(file_data).hexdigest()
+ valid_sig = True
+ if file_data_sha256 != sha256:
+ valid_sig = False
+ if GPG is not None:
+ try:
+ tmp_dir = tempfile.mkdtemp()
+ gpg = GPG(gnupghome=tmp_dir)
+ if RUST_KEY is None:
+ RUST_KEY = url_data(RUST_KEY_URL)
+ import_result = gpg.import_keys(RUST_KEY)
+ gpg.trust_keys(import_result.fingerprints, "TRUST_ULTIMATE")
+ sig_data = url_data(sig_url)
+ with tempfile.NamedTemporaryFile() as tmp_file:
+ tmp_file.write(file_data)
+ verified = gpg.verify_file(io.BytesIO(sig_data), tmp_file.name)
+ if verified.trust_level is None:
+ print("Signature validation for %s failed." % sig_url)
+ valid_sig = False
+ if verified.trust_level < verified.TRUST_ULTIMATE:
+ print(
+ "Signature validation for %s failed: %s"
+ % (sig_url, verified.trust_text)
+ )
+ valid_sig = False
+ finally:
+ try:
+ shutil.rmtree(tmp_dir)
+ except OSError as exc:
+ if exc.errno != errno.ENOENT:
+ valid_sig = False
+ raise exc
+ else:
+ print("Skipping validation for %s" % sig_url)
+ return valid_sig
+
+
+def update_hash_file_entry(hash_file, old_version, new_version):
+ tmpfd, tmpfpath = tempfile.mkstemp()
+ updated = False
+ with fdopen(tmpfd, "w") as new_file:
+ with hash_file.open("r") as old_file:
+ line = old_file.readline()
+ while line:
+ words = line.split()
+ sha256_url_line = None
+ sig_url_line = None
+ sha256_hash_line = None
+ has_section_start = (
+ len(words) == 3 and words[0] == "#" and words[1] == "From"
+ )
+ is_old_version = old_version in words[2]
+ is_new_version = new_version in words[2]
+ needs_version_update = is_old_version and not is_new_version
+ if updated is False and has_section_start and needs_version_update:
+ if is_old_version:
+ old_sha256_url = words[2]
+ new_sha256_url = old_sha256_url.replace(
+ old_version, new_version
+ )
+ sha256_url_line = line.replace(old_version, new_version)
+ new_sha256_data = url_data(new_sha256_url)
+ print(
+ "processing: %s" % new_sha256_data.strip().decode("utf-8")
+ )
+ line = old_file.readline()
+ words = line.split()
+ if (
+ len(words) == 4
+ and words[0] == "#"
+ and words[1] == "Verified"
+ and words[2] == "using"
+ ):
+ old_sig_url = words[3]
+ if old_version in old_sig_url:
+ new_sig_url = old_sig_url.replace(
+ old_version, new_version
+ )
+ sig_url_line = line.replace(old_version, new_version)
+ new_sha256_hash = new_sha256_data.split()[0].decode(
+ "ascii"
+ )
+ new_sha256_archive = new_sha256_data.split()[1].decode(
+ "ascii"
+ )
+ if (
+ rust_verify(new_sha256_hash, new_sig_url)
+ is not True
+ ):
+ raise Exception("signature verification failed")
+ line = old_file.readline()
+ words = line.split()
+ if len(words) == 3 and words[0] == "sha256":
+ old_sha256_hash = words[1]
+ old_archive_name = words[2]
+ if old_version in old_archive_name:
+ sha256_hash_line = line.replace(
+ old_version, new_version
+ ).replace(old_sha256_hash, new_sha256_hash)
+ if (
+ new_sha256_archive
+ == sha256_hash_line.split()[2]
+ ):
+ updated = True
+ else:
+ raise Exception(
+ "archive name mismatch in sha256 file"
+ )
+ if updated is True:
+ new_file.write(sha256_url_line)
+ new_file.write(sig_url_line)
+ new_file.write(sha256_hash_line)
+ else:
+ new_file.write(line)
+ line = old_file.readline()
+ if updated:
+ shutil.copymode(hash_file, tmpfpath)
+ shutil.move(tmpfpath, hash_file)
+ else:
+ os.remove(tmpfpath)
+
+ return updated
+
+
+def update_hash_file(hash_file, old_version, new_version):
+ updated = True
+ while updated:
+ updated = update_hash_file_entry(hash_file, old_version, new_version)
+
+
+def update_mk_file(mk_file, old_version, new_version):
+ tmpfd, tmpfpath = tempfile.mkstemp()
+ updated = False
+ with fdopen(tmpfd, "w") as new_file:
+ with mk_file.open() as old_file:
+ version_var = mk_file.stem.upper().replace("-", "_") + "_VERSION"
+ for line in old_file.readlines():
+ words = line.split()
+ if (
+ len(words) != 0
+ and words[0] == version_var
+ and words[1] == "="
+ and words[2] == old_version
+ ):
+ updated = True
+ new_file.write(line.replace(old_version, new_version))
+ else:
+ new_file.write(line)
+
+ if updated:
+ shutil.copymode(mk_file, tmpfpath)
+ shutil.move(tmpfpath, mk_file)
+ else:
+ os.remove(tmpfpath)
+
+ return updated
+
+
+def get_old_version(mk_file):
+ with mk_file.open() as f:
+ version_var = mk_file.stem.upper().replace("-", "_") + "_VERSION"
+ for line in f.readlines():
+ words = line.split()
+ if len(words) != 0 and words[0] == version_var:
+ return words[-1]
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Update rust")
+ parser.add_argument("version", help="Rust version to update to", type=str)
+
+ args = parser.parse_args()
+ new_version = args.version
+ update_hash_file(RUST_HASH_FILE, get_old_version(RUST_MK_FILE), new_version)
+ update_mk_file(RUST_MK_FILE, get_old_version(RUST_MK_FILE), new_version)
+ update_hash_file(RUST_BIN_HASH_FILE, get_old_version(RUST_BIN_MK_FILE), new_version)
+ update_mk_file(RUST_BIN_MK_FILE, get_old_version(RUST_BIN_MK_FILE), new_version)
+
+
+if __name__ == "__main__":
+ main()
--
2.34.1
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [Buildroot] [PATCH 1/1] utils/update-rust: add new util for updating rust/rust-bin
2022-09-26 20:45 [Buildroot] [PATCH 1/1] utils/update-rust: add new util for updating rust/rust-bin James Hilliard
@ 2023-08-26 21:51 ` Thomas Petazzoni via buildroot
2023-08-27 6:56 ` James Hilliard
0 siblings, 1 reply; 3+ messages in thread
From: Thomas Petazzoni via buildroot @ 2023-08-26 21:51 UTC (permalink / raw)
To: James Hilliard; +Cc: buildroot
Hello James,
On Mon, 26 Sep 2022 14:45:05 -0600
James Hilliard <james.hilliard1@gmail.com> wrote:
> Manually updating the rust package is tedious and slow as we have to
> update and validate hashes for all supported rust-bin arch specific
> toolchains.
>
> To simplify this process add a python script which will update and
> validate hashes and signatures for the new desired rust version.
>
> This script is additionally capable of resuming an update if
> interrupted which may be useful on slower network connections
> as validating gpg signatures requires fully downloading each rust
> toolchain distribution file.
>
> This script has no external dependencies other than the optional
> python-gnupg library which is needed for gpg signature validation.
>
> Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
Sorry for the long delay in getting back to you on this. Some comments
below.
> diff --git a/utils/update-rust b/utils/update-rust
> new file mode 100755
> index 0000000000..2aad3fffa3
> --- /dev/null
> +++ b/utils/update-rust
> @@ -0,0 +1,226 @@
> +#!/usr/bin/env python3
> +"""
> +
> +Utility for updating rust
> +
> +"""
> +import argparse
> +import errno
> +import urllib.request
> +import tempfile
> +import shutil
> +import io
> +import os
> +import hashlib
> +from pathlib import Path
> +from os import fdopen
> +
> +try:
> + # Requires the python-gnupg library
> + from gnupg import GPG
> +except ImportError:
> + print("Unable to verify signatures, python-gnupg required")
> + GPG = None
I don't like the fact that GnuPG is optional. Indeed, in the .hash
file, we indicate that the tarballs have been verified against their
signature. This script updates the hash file, keeping those comments
that the hashes have been verified against the GnuPG signature... but
in fact if one doesn't have python-gnupg installed, it's not the case.
> +def update_hash_file_entry(hash_file, old_version, new_version):
> + tmpfd, tmpfpath = tempfile.mkstemp()
> + updated = False
> + with fdopen(tmpfd, "w") as new_file:
> + with hash_file.open("r") as old_file:
This function looks extremely complicated. I would suggest to try one
of those approaches:
- Entirely regenerate the hash file
- Use regexps to modify the hashes + version
Another nit:
+ if (
+ len(words) == 4
+ and words[0] == "#"
+ and words[1] == "Verified"
+ and words[2] == "using"
+ ):
It is quite uncommon in Python to use parenthesis around conditions.
> +def update_mk_file(mk_file, old_version, new_version):
> + tmpfd, tmpfpath = tempfile.mkstemp()
> + updated = False
> + with fdopen(tmpfd, "w") as new_file:
> + with mk_file.open() as old_file:
> + version_var = mk_file.stem.upper().replace("-", "_") + "_VERSION"
> + for line in old_file.readlines():
> + words = line.split()
> + if (
> + len(words) != 0
> + and words[0] == version_var
> + and words[1] == "="
> + and words[2] == old_version
> + ):
> + updated = True
> + new_file.write(line.replace(old_version, new_version))
> + else:
> + new_file.write(line)
Same: use a regexp to modify the version.
Could you rework your script based on those suggestions?
Thanks a lot!
Thomas
--
Thomas Petazzoni, co-owner and CEO, Bootlin
Embedded Linux and Kernel engineering and training
https://bootlin.com
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [Buildroot] [PATCH 1/1] utils/update-rust: add new util for updating rust/rust-bin
2023-08-26 21:51 ` Thomas Petazzoni via buildroot
@ 2023-08-27 6:56 ` James Hilliard
0 siblings, 0 replies; 3+ messages in thread
From: James Hilliard @ 2023-08-27 6:56 UTC (permalink / raw)
To: Thomas Petazzoni; +Cc: buildroot
On Sat, Aug 26, 2023 at 5:51 PM Thomas Petazzoni
<thomas.petazzoni@bootlin.com> wrote:
>
> Hello James,
>
> On Mon, 26 Sep 2022 14:45:05 -0600
> James Hilliard <james.hilliard1@gmail.com> wrote:
>
> > Manually updating the rust package is tedious and slow as we have to
> > update and validate hashes for all supported rust-bin arch specific
> > toolchains.
> >
> > To simplify this process add a python script which will update and
> > validate hashes and signatures for the new desired rust version.
> >
> > This script is additionally capable of resuming an update if
> > interrupted which may be useful on slower network connections
> > as validating gpg signatures requires fully downloading each rust
> > toolchain distribution file.
> >
> > This script has no external dependencies other than the optional
> > python-gnupg library which is needed for gpg signature validation.
> >
> > Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
>
> Sorry for the long delay in getting back to you on this. Some comments
> below.
>
> > diff --git a/utils/update-rust b/utils/update-rust
> > new file mode 100755
> > index 0000000000..2aad3fffa3
> > --- /dev/null
> > +++ b/utils/update-rust
> > @@ -0,0 +1,226 @@
> > +#!/usr/bin/env python3
> > +"""
> > +
> > +Utility for updating rust
> > +
> > +"""
> > +import argparse
> > +import errno
> > +import urllib.request
> > +import tempfile
> > +import shutil
> > +import io
> > +import os
> > +import hashlib
> > +from pathlib import Path
> > +from os import fdopen
> > +
> > +try:
> > + # Requires the python-gnupg library
> > + from gnupg import GPG
> > +except ImportError:
> > + print("Unable to verify signatures, python-gnupg required")
> > + GPG = None
>
> I don't like the fact that GnuPG is optional. Indeed, in the .hash
> file, we indicate that the tarballs have been verified against their
> signature. This script updates the hash file, keeping those comments
> that the hashes have been verified against the GnuPG signature... but
> in fact if one doesn't have python-gnupg installed, it's not the case.
I didn't want to make python-gnupg mandatory here in case someone needed
to change rust versions on a build host that didn't have python-gnupg installed
for whatever reason.
>
>
> > +def update_hash_file_entry(hash_file, old_version, new_version):
> > + tmpfd, tmpfpath = tempfile.mkstemp()
> > + updated = False
> > + with fdopen(tmpfd, "w") as new_file:
> > + with hash_file.open("r") as old_file:
>
> This function looks extremely complicated. I would suggest to try one
> of those approaches:
Yeah, this function is super ugly
>
> - Entirely regenerate the hash file
>
> - Use regexps to modify the hashes + version
Hmm, yeah that might work better, I'm trying to remember if there was
a reason I was avoiding the regexps approach but it's been a while and
I kind of forgot why I did things this way. I think I was trying to ensure that
the signature and archive files were processed as a group for whatever
reason but I'm not entirely sure.
>
> Another nit:
>
> + if (
> + len(words) == 4
> + and words[0] == "#"
> + and words[1] == "Verified"
> + and words[2] == "using"
> + ):
>
> It is quite uncommon in Python to use parenthesis around conditions.
I think this style was generated by the python black code formatting tool(I
was trying to clean up the style as this was quite ugly before as well).
>
> > +def update_mk_file(mk_file, old_version, new_version):
> > + tmpfd, tmpfpath = tempfile.mkstemp()
> > + updated = False
> > + with fdopen(tmpfd, "w") as new_file:
> > + with mk_file.open() as old_file:
> > + version_var = mk_file.stem.upper().replace("-", "_") + "_VERSION"
> > + for line in old_file.readlines():
> > + words = line.split()
> > + if (
> > + len(words) != 0
> > + and words[0] == version_var
> > + and words[1] == "="
> > + and words[2] == old_version
> > + ):
> > + updated = True
> > + new_file.write(line.replace(old_version, new_version))
> > + else:
> > + new_file.write(line)
>
> Same: use a regexp to modify the version.
>
> Could you rework your script based on those suggestions?
Yeah, I'll take a look at reworking this when I get some time, if I
can figure out
or remember how this code worked. This update_mk_file function is super
ugly/terrible but it at least hasn't ever broken since I wrote it from
what I can
tell when doing rust version updates.
>
> Thanks a lot!
>
> Thomas
> --
> Thomas Petazzoni, co-owner and CEO, Bootlin
> Embedded Linux and Kernel engineering and training
> https://bootlin.com
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2023-08-27 6:56 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-09-26 20:45 [Buildroot] [PATCH 1/1] utils/update-rust: add new util for updating rust/rust-bin James Hilliard
2023-08-26 21:51 ` Thomas Petazzoni via buildroot
2023-08-27 6:56 ` James Hilliard
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).