* [Buildroot] [PATCH v2 1/1] utils/genrandconfig: migrate to asyncio subprocess calls
@ 2022-04-13 22:40 James Hilliard
2023-02-06 16:39 ` Arnout Vandecappelle
0 siblings, 1 reply; 3+ messages in thread
From: James Hilliard @ 2022-04-13 22:40 UTC (permalink / raw)
To: buildroot; +Cc: James Hilliard
Since genrandconfig no longer appears to support python2 we can
migrate the subprocess calls to use asyncio variants.
This has the advantage of allowing for runners like autobuild-run to
integrate directly into genrandconfig by calling the asyncio
gen_config using importlib instead of having to run genrandconfig as
a subprocess.
Using asyncio is advantageous here as it eliminates the requirement
for the runner to deal with blocking subprocess calls(by having to
use threading for example).
Also cleanup some unused functions/python2 compatibility shims.
Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
---
Changes v1 -> v2:
- compatibility shims for python 3.4 and newer
---
utils/genrandconfig | 94 +++++++++++++++++++++++++++++----------------
1 file changed, 61 insertions(+), 33 deletions(-)
diff --git a/utils/genrandconfig b/utils/genrandconfig
index 59fe34e58d..aa82a41d5c 100755
--- a/utils/genrandconfig
+++ b/utils/genrandconfig
@@ -18,23 +18,18 @@
# This script generates a random configuration for testing Buildroot.
-import contextlib
+import asyncio
import csv
import os
from random import randint
-import subprocess
import sys
from distutils.version import StrictVersion
import platform
-if sys.hexversion >= 0x3000000:
- import urllib.request as _urllib
+if sys.version_info < (3, 8):
+ from asyncio import coroutine
else:
- import urllib2 as _urllib
-
-
-def urlopen_closing(uri):
- return contextlib.closing(_urllib.urlopen(uri))
+ from types import coroutine
class SystemInfo:
@@ -63,6 +58,7 @@ class SystemInfo:
# --
return None
+ @coroutine
def has(self, prog):
"""Checks whether a program is available.
Lazily evaluates missing entries.
@@ -78,11 +74,13 @@ class SystemInfo:
have_it = self.find_prog(prog)
# java[c] needs special care
if have_it and prog in ('java', 'javac'):
- with open(os.devnull, "w") as devnull:
- if subprocess.call("%s -version | grep gcj" % prog,
- shell=True,
- stdout=devnull, stderr=devnull) != 1:
- have_it = False
+ proc = yield from asyncio.create_subprocess_shell(
+ "%s -version | grep gcj" % prog,
+ stdout=asyncio.subprocess.DEVNULL,
+ stderr=asyncio.subprocess.DEVNULL)
+ ret = yield from proc.wait()
+ if ret != 1:
+ have_it = False
# --
self.progs[prog] = have_it
return have_it
@@ -159,6 +157,7 @@ def get_toolchain_configs(toolchains_csv, buildrootdir):
return configs
+@coroutine
def is_toolchain_usable(configfile, config):
"""Check if the toolchain is actually usable."""
@@ -179,8 +178,12 @@ def is_toolchain_usable(configfile, config):
'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \
'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64_BE=y\n' in configlines or \
'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines:
- ldd_version_output = subprocess.check_output(['ldd', '--version'])
- glibc_version = ldd_version_output.splitlines()[0].split()[-1]
+ proc = yield from asyncio.create_subprocess_exec(
+ 'ldd', '--version', stdout=asyncio.subprocess.PIPE)
+ ldd_version_output, _ = yield from proc.communicate()
+ if proc.returncode:
+ return False
+ glibc_version = ldd_version_output.splitlines()[0].split()[-1].decode('utf-8')
if StrictVersion('2.14') > StrictVersion(glibc_version):
print("WARN: ignoring the Linaro ARM toolchains because too old host glibc", file=sys.stderr)
return False
@@ -188,6 +191,7 @@ def is_toolchain_usable(configfile, config):
return True
+@coroutine
def fixup_config(sysinfo, configfile):
"""Finalize the configuration and reject any problematic combinations
@@ -202,7 +206,8 @@ def fixup_config(sysinfo, configfile):
BR2_TOOLCHAIN_EXTERNAL_URL = 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/'
- if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"):
+ has_java = yield from sysinfo.has("java")
+ if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not has_java:
return False
# The ctng toolchain is affected by PR58854
if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
@@ -507,6 +512,7 @@ def fixup_config(sysinfo, configfile):
return True
+@coroutine
def gen_config(args):
"""Generate a new random configuration
@@ -565,7 +571,8 @@ def gen_config(args):
# Randomly enable BR2_REPRODUCIBLE 10% of times
# also enable tar filesystem images for testing
- if sysinfo.has("diffoscope") and randint(0, 10) == 0:
+ has_diffoscope = yield from sysinfo.has("diffoscope")
+ if has_diffoscope and randint(0, 10) == 0:
configlines.append("BR2_REPRODUCIBLE=y\n")
configlines.append("BR2_TARGET_ROOTFS_TAR=y\n")
@@ -579,10 +586,14 @@ def gen_config(args):
with open(configfile, "w+") as configf:
configf.writelines(configlines)
- subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
- "olddefconfig"])
+ proc = yield from asyncio.create_subprocess_exec(
+ "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
+ ret = yield from proc.wait()
+ if ret:
+ return ret
- if not is_toolchain_usable(configfile, toolchainconfig):
+ toolchain_usable = yield from is_toolchain_usable(configfile, toolchainconfig)
+ if not toolchain_usable:
return 2
# Now, generate the random selection of packages, and fixup
@@ -596,21 +607,34 @@ def gen_config(args):
file=sys.stderr)
return 1
bounded_loop -= 1
- subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
- "KCONFIG_PROBABILITY=%d" % randint(1, 20),
- "randpackageconfig" if args.toolchains_csv else "randconfig"])
-
- if fixup_config(sysinfo, configfile):
+ proc = yield from asyncio.create_subprocess_exec(
+ "make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
+ "KCONFIG_PROBABILITY=%d" % randint(1, 20),
+ "randpackageconfig" if args.toolchains_csv else "randconfig")
+ ret = yield from proc.wait()
+ if ret:
+ return ret
+
+ ret = yield from fixup_config(sysinfo, configfile)
+ if ret:
break
- subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
- "olddefconfig"])
+ proc = yield from asyncio.create_subprocess_exec(
+ "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
+ ret = yield from proc.wait()
+ if ret:
+ return ret
- subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
- "savedefconfig"])
+ proc = yield from asyncio.create_subprocess_exec(
+ "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "savedefconfig")
+ ret = yield from proc.wait()
+ if ret:
+ return ret
- return subprocess.call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
- "dependencies"])
+ proc = yield from asyncio.create_subprocess_exec(
+ "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "dependencies")
+ ret = yield from proc.wait()
+ return ret
if __name__ == '__main__':
@@ -642,7 +666,11 @@ if __name__ == '__main__':
args.outputdir = os.path.abspath(args.outputdir)
try:
- ret = gen_config(args)
+ if sys.version_info < (3, 7):
+ loop = asyncio.get_event_loop()
+ ret = loop.run_until_complete(gen_config(args))
+ else:
+ ret = asyncio.run(gen_config(args))
except Exception as e:
print(str(e), file=sys.stderr)
parser.exit(1)
--
2.25.1
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [Buildroot] [PATCH v2 1/1] utils/genrandconfig: migrate to asyncio subprocess calls
2022-04-13 22:40 [Buildroot] [PATCH v2 1/1] utils/genrandconfig: migrate to asyncio subprocess calls James Hilliard
@ 2023-02-06 16:39 ` Arnout Vandecappelle
2023-02-06 16:45 ` James Hilliard
0 siblings, 1 reply; 3+ messages in thread
From: Arnout Vandecappelle @ 2023-02-06 16:39 UTC (permalink / raw)
To: James Hilliard, buildroot
Hi James,
On 14/04/2022 00:40, James Hilliard wrote:
> Since genrandconfig no longer appears to support python2 we can
> migrate the subprocess calls to use asyncio variants.
>
> This has the advantage of allowing for runners like autobuild-run to
> integrate directly into genrandconfig by calling the asyncio
> gen_config using importlib instead of having to run genrandconfig as
> a subprocess.
>
> Using asyncio is advantageous here as it eliminates the requirement
> for the runner to deal with blocking subprocess calls(by having to
> use threading for example).
>
> Also cleanup some unused functions/python2 compatibility shims.
>
> Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
I finally applied to master. Needed to resolve some conflicts, but otherwise
no changes. I still do have a few comments though.
> ---
> Changes v1 -> v2:
> - compatibility shims for python 3.4 and newer
> ---
> utils/genrandconfig | 94 +++++++++++++++++++++++++++++----------------
> 1 file changed, 61 insertions(+), 33 deletions(-)
>
> diff --git a/utils/genrandconfig b/utils/genrandconfig
> index 59fe34e58d..aa82a41d5c 100755
> --- a/utils/genrandconfig
> +++ b/utils/genrandconfig
> @@ -18,23 +18,18 @@
>
> # This script generates a random configuration for testing Buildroot.
>
> -import contextlib
> +import asyncio
> import csv
> import os
> from random import randint
> -import subprocess
> import sys
> from distutils.version import StrictVersion
> import platform
>
> -if sys.hexversion >= 0x3000000:
> - import urllib.request as _urllib
> +if sys.version_info < (3, 8):
> + from asyncio import coroutine
> else:
> - import urllib2 as _urllib
> -
> -
> -def urlopen_closing(uri):
> - return contextlib.closing(_urllib.urlopen(uri))
> + from types import coroutine
>
>
> class SystemInfo:
> @@ -63,6 +58,7 @@ class SystemInfo:
> # --
> return None
>
> + @coroutine
> def has(self, prog):
Isn't the usual way to use
async def has(self, prog):
? Or does that mean something different?
> """Checks whether a program is available.
> Lazily evaluates missing entries.
> @@ -78,11 +74,13 @@ class SystemInfo:
> have_it = self.find_prog(prog)
> # java[c] needs special care
> if have_it and prog in ('java', 'javac'):
> - with open(os.devnull, "w") as devnull:
> - if subprocess.call("%s -version | grep gcj" % prog,
> - shell=True,
> - stdout=devnull, stderr=devnull) != 1:
> - have_it = False
> + proc = yield from asyncio.create_subprocess_shell(
> + "%s -version | grep gcj" % prog,
Perhaps we should also start using f-strings. That bumps the requirement to
python 3.6 but I don't think it's unreasonable for genrandconfig.
> + stdout=asyncio.subprocess.DEVNULL,
> + stderr=asyncio.subprocess.DEVNULL)
> + ret = yield from proc.wait()
> + if ret != 1:
> + have_it = False
> # --
> self.progs[prog] = have_it
> return have_it
> @@ -159,6 +157,7 @@ def get_toolchain_configs(toolchains_csv, buildrootdir):
> return configs
>
>
> +@coroutine
> def is_toolchain_usable(configfile, config):
> """Check if the toolchain is actually usable."""
>
> @@ -179,8 +178,12 @@ def is_toolchain_usable(configfile, config):
> 'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \
> 'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64_BE=y\n' in configlines or \
> 'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines:
> - ldd_version_output = subprocess.check_output(['ldd', '--version'])
> - glibc_version = ldd_version_output.splitlines()[0].split()[-1]
> + proc = yield from asyncio.create_subprocess_exec(
> + 'ldd', '--version', stdout=asyncio.subprocess.PIPE)
> + ldd_version_output, _ = yield from proc.communicate()
> + if proc.returncode:
> + return False
> + glibc_version = ldd_version_output.splitlines()[0].split()[-1].decode('utf-8')
The decode was already added in commit
12e4f7c5c43fb7b4db6d6548b9dbabb9c1b5f875 so I didn't modify this line.
> if StrictVersion('2.14') > StrictVersion(glibc_version):
> print("WARN: ignoring the Linaro ARM toolchains because too old host glibc", file=sys.stderr)
> return False
> @@ -188,6 +191,7 @@ def is_toolchain_usable(configfile, config):
> return True
>
>
> +@coroutine
> def fixup_config(sysinfo, configfile):
> """Finalize the configuration and reject any problematic combinations
>
> @@ -202,7 +206,8 @@ def fixup_config(sysinfo, configfile):
>
> BR2_TOOLCHAIN_EXTERNAL_URL = 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/'
>
> - if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"):
> + has_java = yield from sysinfo.has("java")
> + if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not has_java:
> return False
> # The ctng toolchain is affected by PR58854
> if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
> @@ -507,6 +512,7 @@ def fixup_config(sysinfo, configfile):
> return True
>
>
> +@coroutine
> def gen_config(args):
> """Generate a new random configuration
>
> @@ -565,7 +571,8 @@ def gen_config(args):
>
> # Randomly enable BR2_REPRODUCIBLE 10% of times
> # also enable tar filesystem images for testing
> - if sysinfo.has("diffoscope") and randint(0, 10) == 0:
> + has_diffoscope = yield from sysinfo.has("diffoscope")
> + if has_diffoscope and randint(0, 10) == 0:
> configlines.append("BR2_REPRODUCIBLE=y\n")
> configlines.append("BR2_TARGET_ROOTFS_TAR=y\n")
>
> @@ -579,10 +586,14 @@ def gen_config(args):
> with open(configfile, "w+") as configf:
> configf.writelines(configlines)
>
> - subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> - "olddefconfig"])
> + proc = yield from asyncio.create_subprocess_exec(
> + "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
> + ret = yield from proc.wait()
> + if ret:
> + return ret
>
> - if not is_toolchain_usable(configfile, toolchainconfig):
> + toolchain_usable = yield from is_toolchain_usable(configfile, toolchainconfig)
> + if not toolchain_usable:
> return 2
>
> # Now, generate the random selection of packages, and fixup
> @@ -596,21 +607,34 @@ def gen_config(args):
> file=sys.stderr)
> return 1
> bounded_loop -= 1
> - subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> - "KCONFIG_PROBABILITY=%d" % randint(1, 20),
> - "randpackageconfig" if args.toolchains_csv else "randconfig"])
> -
> - if fixup_config(sysinfo, configfile):
> + proc = yield from asyncio.create_subprocess_exec(
> + "make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> + "KCONFIG_PROBABILITY=%d" % randint(1, 20),
> + "randpackageconfig" if args.toolchains_csv else "randconfig")
This was refactored a bit, can you check if I fixed it up correctly?
Regards,
Arnout
> + ret = yield from proc.wait()
> + if ret:
> + return ret
> +
> + ret = yield from fixup_config(sysinfo, configfile)
> + if ret:
> break
>
> - subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> - "olddefconfig"])
> + proc = yield from asyncio.create_subprocess_exec(
> + "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
> + ret = yield from proc.wait()
> + if ret:
> + return ret
>
> - subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> - "savedefconfig"])
> + proc = yield from asyncio.create_subprocess_exec(
> + "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "savedefconfig")
> + ret = yield from proc.wait()
> + if ret:
> + return ret
>
> - return subprocess.call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> - "dependencies"])
> + proc = yield from asyncio.create_subprocess_exec(
> + "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "dependencies")
> + ret = yield from proc.wait()
> + return ret
>
>
> if __name__ == '__main__':
> @@ -642,7 +666,11 @@ if __name__ == '__main__':
> args.outputdir = os.path.abspath(args.outputdir)
>
> try:
> - ret = gen_config(args)
> + if sys.version_info < (3, 7):
> + loop = asyncio.get_event_loop()
> + ret = loop.run_until_complete(gen_config(args))
> + else:
> + ret = asyncio.run(gen_config(args))
> except Exception as e:
> print(str(e), file=sys.stderr)
> parser.exit(1)
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply [flat|nested] 3+ messages in thread
* Re: [Buildroot] [PATCH v2 1/1] utils/genrandconfig: migrate to asyncio subprocess calls
2023-02-06 16:39 ` Arnout Vandecappelle
@ 2023-02-06 16:45 ` James Hilliard
0 siblings, 0 replies; 3+ messages in thread
From: James Hilliard @ 2023-02-06 16:45 UTC (permalink / raw)
To: Arnout Vandecappelle; +Cc: buildroot
On Mon, Feb 6, 2023 at 9:39 AM Arnout Vandecappelle <arnout@mind.be> wrote:
>
> Hi James,
>
> On 14/04/2022 00:40, James Hilliard wrote:
> > Since genrandconfig no longer appears to support python2 we can
> > migrate the subprocess calls to use asyncio variants.
> >
> > This has the advantage of allowing for runners like autobuild-run to
> > integrate directly into genrandconfig by calling the asyncio
> > gen_config using importlib instead of having to run genrandconfig as
> > a subprocess.
> >
> > Using asyncio is advantageous here as it eliminates the requirement
> > for the runner to deal with blocking subprocess calls(by having to
> > use threading for example).
> >
> > Also cleanup some unused functions/python2 compatibility shims.
> >
> > Signed-off-by: James Hilliard <james.hilliard1@gmail.com>
>
> I finally applied to master. Needed to resolve some conflicts, but otherwise
> no changes. I still do have a few comments though.
>
> > ---
> > Changes v1 -> v2:
> > - compatibility shims for python 3.4 and newer
> > ---
> > utils/genrandconfig | 94 +++++++++++++++++++++++++++++----------------
> > 1 file changed, 61 insertions(+), 33 deletions(-)
> >
> > diff --git a/utils/genrandconfig b/utils/genrandconfig
> > index 59fe34e58d..aa82a41d5c 100755
> > --- a/utils/genrandconfig
> > +++ b/utils/genrandconfig
> > @@ -18,23 +18,18 @@
> >
> > # This script generates a random configuration for testing Buildroot.
> >
> > -import contextlib
> > +import asyncio
> > import csv
> > import os
> > from random import randint
> > -import subprocess
> > import sys
> > from distutils.version import StrictVersion
> > import platform
> >
> > -if sys.hexversion >= 0x3000000:
> > - import urllib.request as _urllib
> > +if sys.version_info < (3, 8):
> > + from asyncio import coroutine
> > else:
> > - import urllib2 as _urllib
> > -
> > -
> > -def urlopen_closing(uri):
> > - return contextlib.closing(_urllib.urlopen(uri))
> > + from types import coroutine
> >
> >
> > class SystemInfo:
> > @@ -63,6 +58,7 @@ class SystemInfo:
> > # --
> > return None
> >
> > + @coroutine
> > def has(self, prog):
>
> Isn't the usual way to use
>
> async def has(self, prog):
>
> ? Or does that mean something different?
Yeah, the weird style here was for python 3.4 compat, but if 3.6 is fine
this should work:
https://patchwork.ozlabs.org/project/buildroot/patch/20230206150847.3621475-1-james.hilliard1@gmail.com/
>
>
> > """Checks whether a program is available.
> > Lazily evaluates missing entries.
> > @@ -78,11 +74,13 @@ class SystemInfo:
> > have_it = self.find_prog(prog)
> > # java[c] needs special care
> > if have_it and prog in ('java', 'javac'):
> > - with open(os.devnull, "w") as devnull:
> > - if subprocess.call("%s -version | grep gcj" % prog,
> > - shell=True,
> > - stdout=devnull, stderr=devnull) != 1:
> > - have_it = False
> > + proc = yield from asyncio.create_subprocess_shell(
> > + "%s -version | grep gcj" % prog,
>
> Perhaps we should also start using f-strings. That bumps the requirement to
> python 3.6 but I don't think it's unreasonable for genrandconfig.
Yeah, older than 3.6 starts getting pretty annoying.
>
> > + stdout=asyncio.subprocess.DEVNULL,
> > + stderr=asyncio.subprocess.DEVNULL)
> > + ret = yield from proc.wait()
> > + if ret != 1:
> > + have_it = False
> > # --
> > self.progs[prog] = have_it
> > return have_it
> > @@ -159,6 +157,7 @@ def get_toolchain_configs(toolchains_csv, buildrootdir):
> > return configs
> >
> >
> > +@coroutine
> > def is_toolchain_usable(configfile, config):
> > """Check if the toolchain is actually usable."""
> >
> > @@ -179,8 +178,12 @@ def is_toolchain_usable(configfile, config):
> > 'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64=y\n' in configlines or \
> > 'BR2_TOOLCHAIN_EXTERNAL_LINARO_AARCH64_BE=y\n' in configlines or \
> > 'BR2_TOOLCHAIN_EXTERNAL_LINARO_ARMEB=y\n' in configlines:
> > - ldd_version_output = subprocess.check_output(['ldd', '--version'])
> > - glibc_version = ldd_version_output.splitlines()[0].split()[-1]
> > + proc = yield from asyncio.create_subprocess_exec(
> > + 'ldd', '--version', stdout=asyncio.subprocess.PIPE)
> > + ldd_version_output, _ = yield from proc.communicate()
> > + if proc.returncode:
> > + return False
> > + glibc_version = ldd_version_output.splitlines()[0].split()[-1].decode('utf-8')
>
> The decode was already added in commit
> 12e4f7c5c43fb7b4db6d6548b9dbabb9c1b5f875 so I didn't modify this line.
>
> > if StrictVersion('2.14') > StrictVersion(glibc_version):
> > print("WARN: ignoring the Linaro ARM toolchains because too old host glibc", file=sys.stderr)
> > return False
> > @@ -188,6 +191,7 @@ def is_toolchain_usable(configfile, config):
> > return True
> >
> >
> > +@coroutine
> > def fixup_config(sysinfo, configfile):
> > """Finalize the configuration and reject any problematic combinations
> >
> > @@ -202,7 +206,8 @@ def fixup_config(sysinfo, configfile):
> >
> > BR2_TOOLCHAIN_EXTERNAL_URL = 'BR2_TOOLCHAIN_EXTERNAL_URL="http://autobuild.buildroot.org/toolchains/tarballs/'
> >
> > - if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not sysinfo.has("java"):
> > + has_java = yield from sysinfo.has("java")
> > + if "BR2_NEEDS_HOST_JAVA=y\n" in configlines and not has_java:
> > return False
> > # The ctng toolchain is affected by PR58854
> > if 'BR2_PACKAGE_LTTNG_TOOLS=y\n' in configlines and \
> > @@ -507,6 +512,7 @@ def fixup_config(sysinfo, configfile):
> > return True
> >
> >
> > +@coroutine
> > def gen_config(args):
> > """Generate a new random configuration
> >
> > @@ -565,7 +571,8 @@ def gen_config(args):
> >
> > # Randomly enable BR2_REPRODUCIBLE 10% of times
> > # also enable tar filesystem images for testing
> > - if sysinfo.has("diffoscope") and randint(0, 10) == 0:
> > + has_diffoscope = yield from sysinfo.has("diffoscope")
> > + if has_diffoscope and randint(0, 10) == 0:
> > configlines.append("BR2_REPRODUCIBLE=y\n")
> > configlines.append("BR2_TARGET_ROOTFS_TAR=y\n")
> >
> > @@ -579,10 +586,14 @@ def gen_config(args):
> > with open(configfile, "w+") as configf:
> > configf.writelines(configlines)
> >
> > - subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> > - "olddefconfig"])
> > + proc = yield from asyncio.create_subprocess_exec(
> > + "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
> > + ret = yield from proc.wait()
> > + if ret:
> > + return ret
> >
> > - if not is_toolchain_usable(configfile, toolchainconfig):
> > + toolchain_usable = yield from is_toolchain_usable(configfile, toolchainconfig)
> > + if not toolchain_usable:
> > return 2
> >
> > # Now, generate the random selection of packages, and fixup
> > @@ -596,21 +607,34 @@ def gen_config(args):
> > file=sys.stderr)
> > return 1
> > bounded_loop -= 1
> > - subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> > - "KCONFIG_PROBABILITY=%d" % randint(1, 20),
> > - "randpackageconfig" if args.toolchains_csv else "randconfig"])
> > -
> > - if fixup_config(sysinfo, configfile):
> > + proc = yield from asyncio.create_subprocess_exec(
> > + "make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> > + "KCONFIG_PROBABILITY=%d" % randint(1, 20),
> > + "randpackageconfig" if args.toolchains_csv else "randconfig")
>
> This was refactored a bit, can you check if I fixed it up correctly?
Yeah, looks good from what I can tell.
>
> Regards,
> Arnout
>
> > + ret = yield from proc.wait()
> > + if ret:
> > + return ret
> > +
> > + ret = yield from fixup_config(sysinfo, configfile)
> > + if ret:
> > break
> >
> > - subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> > - "olddefconfig"])
> > + proc = yield from asyncio.create_subprocess_exec(
> > + "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "olddefconfig")
> > + ret = yield from proc.wait()
> > + if ret:
> > + return ret
> >
> > - subprocess.check_call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> > - "savedefconfig"])
> > + proc = yield from asyncio.create_subprocess_exec(
> > + "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "savedefconfig")
> > + ret = yield from proc.wait()
> > + if ret:
> > + return ret
> >
> > - return subprocess.call(["make", "O=%s" % args.outputdir, "-C", args.buildrootdir,
> > - "dependencies"])
> > + proc = yield from asyncio.create_subprocess_exec(
> > + "make", "O=%s" % args.outputdir, "-C", args.buildrootdir, "dependencies")
> > + ret = yield from proc.wait()
> > + return ret
> >
> >
> > if __name__ == '__main__':
> > @@ -642,7 +666,11 @@ if __name__ == '__main__':
> > args.outputdir = os.path.abspath(args.outputdir)
> >
> > try:
> > - ret = gen_config(args)
> > + if sys.version_info < (3, 7):
> > + loop = asyncio.get_event_loop()
> > + ret = loop.run_until_complete(gen_config(args))
> > + else:
> > + ret = asyncio.run(gen_config(args))
> > except Exception as e:
> > print(str(e), file=sys.stderr)
> > parser.exit(1)
_______________________________________________
buildroot mailing list
buildroot@buildroot.org
https://lists.buildroot.org/mailman/listinfo/buildroot
^ permalink raw reply [flat|nested] 3+ messages in thread
end of thread, other threads:[~2023-02-06 16:46 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-04-13 22:40 [Buildroot] [PATCH v2 1/1] utils/genrandconfig: migrate to asyncio subprocess calls James Hilliard
2023-02-06 16:39 ` Arnout Vandecappelle
2023-02-06 16:45 ` James Hilliard
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.