Multi-threaded npm build

Multi-threaded npm build

[michal.frynas]

[-- Attachment #1: Type: text/plain, Size: 2643 bytes --]

Task do_configure delivered by npm.bbclass takes a lot of time, the longer shrinkwrap dependency list the longer time needed for do_configure to execute.
Following improvement allows to execute do_configure on any available CPI threads.

> 
> diff --git a/bitbake/lib/bb/fetch2/npmsw.py
> b/bitbake/lib/bb/fetch2/npmsw.py
> index 0c3511d8ab..0e6075859c 100644
> --- a/bitbake/lib/bb/fetch2/npmsw.py
> +++ b/bitbake/lib/bb/fetch2/npmsw.py
> @@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity
> from bb.fetch2.npm import npm_localfile
> from bb.fetch2.npm import npm_unpack
> from bb.utils import is_semver
> +from concurrent.futures import ThreadPoolExecutor
> 
> -def foreach_dependencies(shrinkwrap, callback=None, dev=False):
> +def foreach_dependencies(shrinkwrap, callback=None, dev=False,
> executor=None):
> """
> Run a callback for each dependencies of a shrinkwrap file.
> The callback is using the format:
> @@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap, callback=None,
> dev=False):
> continue
> elif deps[name].get("bundled", False):
> continue
> -                callback(name, deps[name], subtree)
> +                if executor:
> +                    executor.submit(callback, name=name,
> params=deps[name], deptree=subtree)
> +                else:
> +                    callback(name, deps[name], subtree)
> 
> _walk_deps(shrinkwrap.get("dependencies", {}), [])
> 
> diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
> index 068032a1e5..1869b63cdb 100644
> --- a/meta/classes/npm.bbclass
> +++ b/meta/classes/npm.bbclass
> @@ -80,6 +80,7 @@ python npm_do_configure() {
> from bb.fetch2.npm import npm_unpack
> from bb.fetch2.npmsw import foreach_dependencies
> from bb.progress import OutOfProgressHandler
> +    from concurrent.futures import ThreadPoolExecutor
> 
> bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
> bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
> @@ -164,9 +165,11 @@ python npm_do_configure() {
> progress_done += 1
> progress.write("%d/%d" % (progress_done, progress_total))
> 
> +    executor = ThreadPoolExecutor(max_workers = os.cpu_count())
> dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
> foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
> -    foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
> +    foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev,
> executor)
> +    executor.shutdown(True)
> 
> # Configure the main package
> with tempfile.TemporaryDirectory() as tmpdir:
> 
>

[-- Attachment #2: Type: text/html, Size: 15362 bytes --]

Re: [OE-core] Multi-threaded npm build

[Randy MacLeod]

[-- Attachment #1: Type: text/plain, Size: 3194 bytes --]

On 2020-12-02 11:25 a.m., michal.frynas@sigma.se wrote:
> Task do_configure delivered by npm.bbclass takes a lot of time, the 
> longer shrinkwrap dependency list the longer time needed for 
> do_configure to execute.
> Following improvement allows to execute do_configure on any available 
> CPI threads.
Thanks for the patch Michal.

Can you resend to:
    bitbake-devel@lists.openembedded.org

using git send-email as per:
https://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded

Thanks,

../Randy


>     diff --git a/bitbake/lib/bb/fetch2/npmsw.py
>     b/bitbake/lib/bb/fetch2/npmsw.py
>     index 0c3511d8ab..0e6075859c 100644
>     --- a/bitbake/lib/bb/fetch2/npmsw.py
>     +++ b/bitbake/lib/bb/fetch2/npmsw.py
>     @@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity
>      from bb.fetch2.npm import npm_localfile
>      from bb.fetch2.npm import npm_unpack
>      from bb.utils import is_semver
>     +from concurrent.futures import ThreadPoolExecutor
>     -def foreach_dependencies(shrinkwrap, callback=None, dev=False):
>     +def foreach_dependencies(shrinkwrap, callback=None, dev=False,
>     executor=None):
>          """
>      Run a callback for each dependencies of a shrinkwrap file.
>      The callback is using the format:
>     @@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap,
>     callback=None, dev=False):
>              continue
>          elif deps[name].get("bundled", False):
>              continue
>     -       callback(name, deps[name], subtree)
>     +       if executor:
>     +           executor.submit(callback, name=name,
>     params=deps[name], deptree=subtree)
>     +       else:
>     +           callback(name, deps[name], subtree)
>      _walk_deps(shrinkwrap.get("dependencies", {}), [])
>     diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
>     index 068032a1e5..1869b63cdb 100644
>     --- a/meta/classes/npm.bbclass
>     +++ b/meta/classes/npm.bbclass
>     @@ -80,6 +80,7 @@ python npm_do_configure() {
>          from bb.fetch2.npm import npm_unpack
>          from bb.fetch2.npmsw import foreach_dependencies
>          from bb.progress import OutOfProgressHandler
>     +    from concurrent.futures import ThreadPoolExecutor
>      bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
>      bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
>     @@ -164,9 +165,11 @@ python npm_do_configure() {
>      progress_done += 1
>      progress.write("%d/%d" % (progress_done, progress_total))
>     + executor = ThreadPoolExecutor(max_workers = os.cpu_count())
>          dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
>      foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
>     - foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
>     + foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev,
>     executor)
>     + executor.shutdown(True)
>          # Configure the main package
>          with tempfile.TemporaryDirectory() as tmpdir:
>
>
> 
>

-- 
# Randy MacLeod
# Wind River Linux


[-- Attachment #2: Type: text/html, Size: 20526 bytes --]

Re: [OE-core] Multi-threaded npm build

[Marek Belisko]

Hi Michal,

On Wed, Dec 2, 2020 at 5:25 PM <michal.frynas@sigma.se> wrote:
>
> Task do_configure delivered by npm.bbclass takes a lot of time, the longer shrinkwrap dependency list the longer time needed for do_configure to execute.
> Following improvement allows to execute do_configure on any available CPI threads.
Do you have some number how fast it is using with and without your
patch? I'm currently hitting the same issue with the slow npm
configure phase.
>
> diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py
> index 0c3511d8ab..0e6075859c 100644
> --- a/bitbake/lib/bb/fetch2/npmsw.py
> +++ b/bitbake/lib/bb/fetch2/npmsw.py
> @@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity
>  from bb.fetch2.npm import npm_localfile
>  from bb.fetch2.npm import npm_unpack
>  from bb.utils import is_semver
> +from concurrent.futures import ThreadPoolExecutor
>
> -def foreach_dependencies(shrinkwrap, callback=None, dev=False):
> +def foreach_dependencies(shrinkwrap, callback=None, dev=False, executor=None):
>      """
>          Run a callback for each dependencies of a shrinkwrap file.
>          The callback is using the format:
> @@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
>                      continue
>                  elif deps[name].get("bundled", False):
>                      continue
> -                callback(name, deps[name], subtree)
> +                if executor:
> +                    executor.submit(callback, name=name, params=deps[name], deptree=subtree)
> +                else:
> +                    callback(name, deps[name], subtree)
>
>      _walk_deps(shrinkwrap.get("dependencies", {}), [])
>
> diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
> index 068032a1e5..1869b63cdb 100644
> --- a/meta/classes/npm.bbclass
> +++ b/meta/classes/npm.bbclass
> @@ -80,6 +80,7 @@ python npm_do_configure() {
>      from bb.fetch2.npm import npm_unpack
>      from bb.fetch2.npmsw import foreach_dependencies
>      from bb.progress import OutOfProgressHandler
> +    from concurrent.futures import ThreadPoolExecutor
>
>      bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
>      bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
> @@ -164,9 +165,11 @@ python npm_do_configure() {
>              progress_done += 1
>              progress.write("%d/%d" % (progress_done, progress_total))
>
> +    executor = ThreadPoolExecutor(max_workers = os.cpu_count())
>      dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
>      foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
> -    foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
> +    foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev, executor)
> +    executor.shutdown(True)
>
>      # Configure the main package
>      with tempfile.TemporaryDirectory() as tmpdir:
>
>
>
>
> 
>

BR,

marek

Re: [OE-core] Multi-threaded npm build

[Jean-Marie Lemetayer]

[-- Attachment #1: Type: text/plain, Size: 3237 bytes --]

Hi Michal,

Thanks, this is definitely helpful.

Have you checked if your improvement is tested. I think it should be tested
by the test_devtool_add_npm under meta/lib/oeqa/selftest/cases/devtool.py

See:
 - https://wiki.yoctoproject.org/wiki/Oe-selftest
 -
https://git.yoctoproject.org/cgit/cgit.cgi/poky/commit/?id=22dd46cc34629d0750177fddff2e1c178c854340

Best Regards,
Jean-Marie


On Wed, Dec 2, 2020 at 5:26 PM <michal.frynas@sigma.se> wrote:

> Task do_configure delivered by npm.bbclass takes a lot of time, the longer
> shrinkwrap dependency list the longer time needed for do_configure to
> execute.
> Following improvement allows to execute do_configure on any available CPI
> threads.
>
> diff --git a/bitbake/lib/bb/fetch2/npmsw.py
> b/bitbake/lib/bb/fetch2/npmsw.py
> index 0c3511d8ab..0e6075859c 100644
> --- a/bitbake/lib/bb/fetch2/npmsw.py
> +++ b/bitbake/lib/bb/fetch2/npmsw.py
> @@ -29,8 +29,9 @@ from bb.fetch2.npm import npm_integrity
>  from bb.fetch2.npm import npm_localfile
>  from bb.fetch2.npm import npm_unpack
>  from bb.utils import is_semver
> +from concurrent.futures import ThreadPoolExecutor
>
> -def foreach_dependencies(shrinkwrap, callback=None, dev=False):
> +def foreach_dependencies(shrinkwrap, callback=None, dev=False,
> executor=None):
>      """
>          Run a callback for each dependencies of a shrinkwrap file.
>          The callback is using the format:
> @@ -49,7 +50,10 @@ def foreach_dependencies(shrinkwrap, callback=None,
> dev=False):
>                      continue
>                  elif deps[name].get("bundled", False):
>                      continue
> -                callback(name, deps[name], subtree)
> +                if executor:
> +                    executor.submit(callback, name=name,
> params=deps[name], deptree=subtree)
> +                else:
> +                    callback(name, deps[name], subtree)
>
>      _walk_deps(shrinkwrap.get("dependencies", {}), [])
>
> diff --git a/meta/classes/npm.bbclass b/meta/classes/npm.bbclass
> index 068032a1e5..1869b63cdb 100644
> --- a/meta/classes/npm.bbclass
> +++ b/meta/classes/npm.bbclass
> @@ -80,6 +80,7 @@ python npm_do_configure() {
>      from bb.fetch2.npm import npm_unpack
>      from bb.fetch2.npmsw import foreach_dependencies
>      from bb.progress import OutOfProgressHandler
> +    from concurrent.futures import ThreadPoolExecutor
>
>      bb.utils.remove(d.getVar("NPM_CACHE"), recurse=True)
>      bb.utils.remove(d.getVar("NPM_PACKAGE"), recurse=True)
> @@ -164,9 +165,11 @@ python npm_do_configure() {
>              progress_done += 1
>              progress.write("%d/%d" % (progress_done, progress_total))
>
> +    executor = ThreadPoolExecutor(max_workers = os.cpu_count())
>      dev = bb.utils.to_boolean(d.getVar("NPM_INSTALL_DEV"), False)
>      foreach_dependencies(orig_shrinkwrap, _count_dependency, dev)
> -    foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev)
> +    foreach_dependencies(orig_shrinkwrap, _cache_dependency, dev,
> executor)
> +    executor.shutdown(True)
>
>      # Configure the main package
>      with tempfile.TemporaryDirectory() as tmpdir:
>
>
>
>
> 
>
>

[-- Attachment #2: Type: text/html, Size: 14689 bytes --]