[PATCH] sstate.bbclass: count the files on mirrors using the pre local files

[PATCH] sstate.bbclass: count the files on mirrors using the pre local files

[Jose Quaresma]

We don't need extra python collections to count the found files
on the sstate cache and sstate mirrors.

The main found collections provides all the files that were found.
Then we only need to store the nunber of files that found on the
local sstate cache and the files that found on the sstate cache
mirror is derived from that.

Signed-off-by: Jose Quaresma <quaresma.jose@gmail.com>
---
 meta/classes/sstate.bbclass | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 92a73114bb..f06778a3ee 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
 
 def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, **kwargs):
     found = set()
-    foundLocal = set()
-    foundNet = set()
     missed = set()
 
     def gethash(task):
@@ -925,14 +923,13 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
 
         if os.path.exists(sstatefile):
-            bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
             found.add(tid)
-            foundLocal.add(tid)
-            continue
+            bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
         else:
             missed.add(tid)
             bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
 
+    foundLocal = len(found)
     mirrors = d.getVar("SSTATE_MIRRORS")
     if mirrors:
         # Copy the data object and override DL_DIR and SRC_URI
@@ -973,7 +970,6 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
                 fetcher.checkstatus()
                 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
                 found.add(tid)
-                foundNet.add(tid)
                 if tid in missed:
                     missed.remove(tid)
             except:
@@ -1034,7 +1030,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
         match = 0
         if total:
             match = len(found) / total * 100
-        bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal), len(foundNet),len(missed), currentcount, match, complete))
+        bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d Current %d (%d%% match, %d%% complete)" %
+            (total, foundLocal, len(found)-foundLocal, len(missed), currentcount, match, complete))
 
     if hasattr(bb.parse.siggen, "checkhashes"):
         bb.parse.siggen.checkhashes(sq_data, missed, found, d)
-- 
2.33.0

Re: [PATCH] sstate.bbclass: count the files on mirrors using the pre local files

[Jose Quaresma]

[-- Attachment #1: Type: text/plain, Size: 3177 bytes --]

ping

Jose Quaresma <quaresma.jose@gmail.com> escreveu no dia domingo, 3/10/2021
à(s) 22:54:

> We don't need extra python collections to count the found files
> on the sstate cache and sstate mirrors.
>
> The main found collections provides all the files that were found.
> Then we only need to store the nunber of files that found on the
> local sstate cache and the files that found on the sstate cache
> mirror is derived from that.
>
> Signed-off-by: Jose Quaresma <quaresma.jose@gmail.com>
> ---
>  meta/classes/sstate.bbclass | 11 ++++-------
>  1 file changed, 4 insertions(+), 7 deletions(-)
>
> diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
> index 92a73114bb..f06778a3ee 100644
> --- a/meta/classes/sstate.bbclass
> +++ b/meta/classes/sstate.bbclass
> @@ -893,8 +893,6 @@ BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
>
>  def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0,
> summary=True, **kwargs):
>      found = set()
> -    foundLocal = set()
> -    foundNet = set()
>      missed = set()
>
>      def gethash(task):
> @@ -925,14 +923,13 @@ def sstate_checkhashes(sq_data, d, siginfo=False,
> currentcount=0, summary=True,
>          sstatefile = d.expand("${SSTATE_DIR}/" + extrapath +
> generate_sstatefn(spec, gethash(tid), tname, siginfo, d))
>
>          if os.path.exists(sstatefile):
> -            bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
>              found.add(tid)
> -            foundLocal.add(tid)
> -            continue
> +            bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
>          else:
>              missed.add(tid)
>              bb.debug(2, "SState: Looked for but didn't find file %s" %
> sstatefile)
>
> +    foundLocal = len(found)
>      mirrors = d.getVar("SSTATE_MIRRORS")
>      if mirrors:
>          # Copy the data object and override DL_DIR and SRC_URI
> @@ -973,7 +970,6 @@ def sstate_checkhashes(sq_data, d, siginfo=False,
> currentcount=0, summary=True,
>                  fetcher.checkstatus()
>                  bb.debug(2, "SState: Successful fetch test for %s" %
> srcuri)
>                  found.add(tid)
> -                foundNet.add(tid)
>                  if tid in missed:
>                      missed.remove(tid)
>              except:
> @@ -1034,7 +1030,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False,
> currentcount=0, summary=True,
>          match = 0
>          if total:
>              match = len(found) / total * 100
> -        bb.plain("Sstate summary: Wanted %d Local %d Network %d Missed %d
> Current %d (%d%% match, %d%% complete)" % (total, len(foundLocal),
> len(foundNet),len(missed), currentcount, match, complete))
> +        bb.plain("Sstate summary: Wanted %d Local %d Mirrors %d Missed %d
> Current %d (%d%% match, %d%% complete)" %
> +            (total, foundLocal, len(found)-foundLocal, len(missed),
> currentcount, match, complete))
>
>      if hasattr(bb.parse.siggen, "checkhashes"):
>          bb.parse.siggen.checkhashes(sq_data, missed, found, d)
> --
> 2.33.0
>
>

-- 
Best regards,

José Quaresma

[-- Attachment #2: Type: text/html, Size: 4024 bytes --]