[PATCH v3 1/4] sstate.bbclass: track found files on mirrors with a counter

[PATCH v3 1/4] sstate.bbclass: track found files on mirrors with a counter

[Jose Quaresma]

We don't need extra python collections to count the found files
on the sstate cache and sstate mirrors.
The main found collections provides all the files that were found,
then we only need to count the files on sstate mirror

Signed-off-by: Jose Quaresma <quaresma.jose@gmail.com>
---
 meta/classes/sstate.bbclass | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 2175ace4c4..2575750247 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -871,8 +871,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
 
 def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
     found = set()
-    foundLocal = set()
-    foundNet = set()
     missed = set()
 
     def gethash(task):
@@ -905,12 +903,11 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         if os.path.exists(sstatefile):
             bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
             found.add(tid)
-            foundLocal.add(tid)
-            continue
         else:
-            missed.add(tid)
             bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
+            missed.add(tid)
 
+    foundMirrors = 0
     mirrors = d.getVar("SSTATE_MIRRORS")
     if mirrors:
         # Copy the data object and override DL_DIR and SRC_URI
@@ -950,8 +947,9 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                             connection_cache=thread_worker.connection_cache)
                 fetcher.checkstatus()
                 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
+                foundMirrors += 1
                 found.add(tid)
-                foundNet.add(tid)
+
                 if tid in missed:
                     missed.remove(tid)
             except:
@@ -1013,7 +1011,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         match = 0
         if total:
             match = len(found) / total * 100
-        bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
+        bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" %
+            (total, len(found)-foundMirrors, foundMirrors, len(missed), currentcount, match, complete))
 
     if hasattr(bb.parse.siggen, "checkhashes"):
         bb.parse.siggen.checkhashes(sq_data, missed, found, d)
-- 
2.32.0

[PATCH v3 2/4] sstate.bbclass: only search on the mirrors for the missing files

[Jose Quaresma]

On the first search we found some files on the local sstate cache.
The missing files are know as well when this step finish.
When we have sstate mirrors we don't need to iterate all files again
because we already know what's missing.

Signed-off-by: Jose Quaresma <quaresma.jose@gmail.com>
---
 meta/classes/sstate.bbclass | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 2575750247..c3c145e7f3 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -949,11 +949,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
                 foundMirrors += 1
                 found.add(tid)
-
-                if tid in missed:
-                    missed.remove(tid)
+                missed.remove(tid)
             except:
-                missed.add(tid)
                 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
                 pass
             if len(tasklist) >= min_tasks:
@@ -961,9 +958,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
 
         tasklist = []
         min_tasks = 100
-        for tid in sq_data['hash']:
-            if tid in found:
-                continue
+        for tid in missed:
             spec, extrapath, tname = getpathcomponents(tid, d)
             sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
             tasklist.append((tid, sstatefile))
-- 
2.32.0

[PATCH v3 3/4] sstate.bbclass: get the number of threads from BB_NUMBER_THREADS

[Jose Quaresma]

- bitbake BB_NUMBER_THREADS uses cpu_count from oe utils that uses
the python os.sched_getaffinity and it is more acurrate.

 grep -nH ^BB_NUMBER_THREADS meta/conf/bitbake.conf
 meta/conf/bitbake.conf:806:BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()}"

- multiprocessing.cpu_count() returns the number of CPUs on the host,
not the number of usable CPUs on the host. If the user is using
scheduler affinity then the number of usable CPUs may be less,
so when determining how many cores we can use check the affinity instead.

Signed-off-by: Jose Quaresma <quaresma.jose@gmail.com>
---
 meta/classes/sstate.bbclass | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index c3c145e7f3..bc14b5e264 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -964,13 +964,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
             tasklist.append((tid, sstatefile))
 
         if tasklist:
+            nproc = min(d.getVar("BB_NUMBER_THREADS"), len(tasklist))
+
             if len(tasklist) >= min_tasks:
                 msg = "Checking sstate mirror object availability"
                 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
 
-            import multiprocessing
-            nproc = min(multiprocessing.cpu_count(), len(tasklist))
-
             bb.event.enable_threadlock()
             pool = oe.utils.ThreadedPool(nproc, len(tasklist),
                     worker_init=checkstatus_init, worker_end=checkstatus_end)
-- 
2.32.0

[PATCH v3 4/4] sstate.bbclass: sstate mirror progress bar cleanup

[Jose Quaresma]

We only has the progress bar when we have more than 100 objects.
So check for this and store the result to show the progress bar.

Signed-off-by: Jose Quaresma <quaresma.jose@gmail.com>
---
 meta/classes/sstate.bbclass | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index bc14b5e264..9cf57466ae 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -953,20 +953,24 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
             except:
                 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
                 pass
-            if len(tasklist) >= min_tasks:
+
+            if progress:
                 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
 
         tasklist = []
-        min_tasks = 100
         for tid in missed:
             spec, extrapath, tname = getpathcomponents(tid, d)
             sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
             tasklist.append((tid, sstatefile))
 
+        progress = False
+        if len(tasklist) >= 100:
+            progress = True
+
         if tasklist:
             nproc = min(d.getVar("BB_NUMBER_THREADS"), len(tasklist))
 
-            if len(tasklist) >= min_tasks:
+            if progress:
                 msg = "Checking sstate mirror object availability"
                 bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
 
@@ -979,7 +983,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
             pool.wait_completion()
             bb.event.disable_threadlock()
 
-            if len(tasklist) >= min_tasks:
+            if progress:
                 bb.event.fire(bb.event.ProcessFinished(msg), d)
 
     inheritlist = d.getVar("INHERIT")
-- 
2.32.0

Re: [OE-core] [PATCH v3 3/4] sstate.bbclass: get the number of threads from BB_NUMBER_THREADS

[Richard Purdie]

On Wed, 2021-08-18 at 17:24 +0100, Jose Quaresma wrote:
> - bitbake BB_NUMBER_THREADS uses cpu_count from oe utils that uses
> the python os.sched_getaffinity and it is more acurrate.
> 
>  grep -nH ^BB_NUMBER_THREADS meta/conf/bitbake.conf
>  meta/conf/bitbake.conf:806:BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()}"
> 
> - multiprocessing.cpu_count() returns the number of CPUs on the host,
> not the number of usable CPUs on the host. If the user is using
> scheduler affinity then the number of usable CPUs may be less,
> so when determining how many cores we can use check the affinity instead.
> 
> Signed-off-by: Jose Quaresma <quaresma.jose@gmail.com>
> ---
>  meta/classes/sstate.bbclass | 5 ++---
>  1 file changed, 2 insertions(+), 3 deletions(-)
> 
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index c3c145e7f3..bc14b5e264 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -964,13 +964,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
>              tasklist.append((tid, sstatefile))
>  
> 
> 
> 
>          if tasklist:
> +            nproc = min(d.getVar("BB_NUMBER_THREADS"), len(tasklist))
> +
>              if len(tasklist) >= min_tasks:
>                  msg = "Checking sstate mirror object availability"
>                  bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
>  
> 
> 
> 
> -            import multiprocessing
> -            nproc = min(multiprocessing.cpu_count(), len(tasklist))
> -
>              bb.event.enable_threadlock()
>              pool = oe.utils.ThreadedPool(nproc, len(tasklist),
>                      worker_init=checkstatus_init, worker_end=checkstatus_end)


https://autobuilder.yoctoproject.org/typhoon/#/builders/69/builds/3825/steps/17/logs/stdio

Cheers,

Richard

Re: [OE-core] [PATCH v3 3/4] sstate.bbclass: get the number of threads from BB_NUMBER_THREADS

[Jose Quaresma]

Hi,

d.getVar("BB_NUMBER_THREADS") returns a string and it needs to be
converted to int.

-nproc = min(d.getVar("BB_NUMBER_THREADS"), len(tasklist))
+nproc = min(int(d.getVar("BB_NUMBER_THREADS")), len(tasklist))

fixed in V4

Richard Purdie <richard.purdie@linuxfoundation.org> escreveu no dia
quarta, 18/08/2021 à(s) 20:34:
>
> On Wed, 2021-08-18 at 17:24 +0100, Jose Quaresma wrote:
> > - bitbake BB_NUMBER_THREADS uses cpu_count from oe utils that uses
> > the python os.sched_getaffinity and it is more acurrate.
> >
> >  grep -nH ^BB_NUMBER_THREADS meta/conf/bitbake.conf
> >  meta/conf/bitbake.conf:806:BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()}"
> >
> > - multiprocessing.cpu_count() returns the number of CPUs on the host,
> > not the number of usable CPUs on the host. If the user is using
> > scheduler affinity then the number of usable CPUs may be less,
> > so when determining how many cores we can use check the affinity instead.
> >
> > Signed-off-by: Jose Quaresma <quaresma.jose@gmail.com>
> > ---
> >  meta/classes/sstate.bbclass | 5 ++---
> >  1 file changed, 2 insertions(+), 3 deletions(-)
> >
> > diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> > index c3c145e7f3..bc14b5e264 100644
> > --- a/meta/classes/sstate.bbclass
> > +++ b/meta/classes/sstate.bbclass
> > @@ -964,13 +964,12 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
> >              tasklist.append((tid, sstatefile))
> >
> >
> >
> >
> >          if tasklist:
> > +            nproc = min(d.getVar("BB_NUMBER_THREADS"), len(tasklist))
> > +
> >              if len(tasklist) >= min_tasks:
> >                  msg = "Checking sstate mirror object availability"
> >                  bb.event.fire(bb.event.ProcessStarted(msg, len(tasklist)), d)
> >
> >
> >
> >
> > -            import multiprocessing
> > -            nproc = min(multiprocessing.cpu_count(), len(tasklist))
> > -
> >              bb.event.enable_threadlock()
> >              pool = oe.utils.ThreadedPool(nproc, len(tasklist),
> >                      worker_init=checkstatus_init, worker_end=checkstatus_end)
>
>
> https://autobuilder.yoctoproject.org/typhoon/#/builders/69/builds/3825/steps/17/logs/stdio
>
> Cheers,
>
> Richard
>
>


-- 
Best regards,

José Quaresma