[PATCH][autobuilder-helper][warrior 01/41] config.json: Disable PRSERV

[PATCH][autobuilder-helper][warrior 01/41] config.json: Disable PRSERV

[Richard Purdie]

This was copied from the old autobuilder configuration without much thought.
It would only be effective if we had a common PRSERV or saved the database
but we don't. It therefore makes sense to disable it.

One problem it was causing was inconsistency in the buildhistory output as
PKGR would change "r0" to "r0.0" and vice versa. The issue depended on whether
the build has coming from sstate or not and the settings the sstate had been
built with.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/config.json b/config.json
index 573082a..90b78f3 100644
--- a/config.json
+++ b/config.json
@@ -24,7 +24,6 @@
         "DISTRO" : "poky",
         "SDKMACHINE" : "i686",
         "PACKAGE_CLASSES" : "package_rpm package_deb package_ipk",
-        "PRSERV" : "PRSERV_HOST = 'localhost:0'",
         "DLDIR" : "DL_DIR = '${BASE_SHAREDDIR}/current_sources'",
         "SSTATEDIR" : ["SSTATE_DIR ?= '${BASE_SHAREDDIR}/pub/sstate-warrior'"],
         "SSTATEDIR_RELEASE" : ["SSTATE_MIRRORS += 'file://.* file://${BASE_SHAREDDIR}/pub/sstate/PATH'", "SSTATE_DIR ?= '${BASE_PUBLISHDIR}/sstate/@RELEASENUM@'"],
@@ -67,7 +66,6 @@
                 "SANITYTARGETS" : "core-image-sato:do_testsdk core-image-minimal:do_testsdkext core-image-sato:do_testsdkext"
             },
             "step3" : {
-                "PRSERV" : false,
                 "BUILDHISTORY" : false,
                 "EXTRACMDS" : ["${SCRIPTSDIR}/checkvnc; DISPLAY=:1 oe-selftest -r runqemu meta_ide -j 15"],
                 "ADDLAYER" : ["${BUILDDIR}/../meta-selftest"]
@@ -124,7 +122,6 @@
             "MACHINE" : "qemux86-64",
             "SDKMACHINE" : "x86_64",
             "PACKAGE_CLASSES" : "package_rpm",
-            "PRSERV" : false,
             "extravars" : [
                 "RPM_GPG_SIGN_CHUNK = '1'"
             ],
@@ -142,7 +139,6 @@
         "trigger-build" : {
             "SDKMACHINE" : "x86_64",
             "MACHINE" : "qemux86",
-            "PRSERV" : false,
             "step1" : {
                 "BBTARGETS" : "universe -c fetch -k",
                 "extravars" : [
@@ -235,7 +231,6 @@
                 "MACHINE" : "qemux86-64",
                 "SDKMACHINE" : "x86_64",
                 "PACKAGE_CLASSES" : "package_rpm",
-                "PRSERV" : false,
                 "extravars" : [
                     "RPM_GPG_SIGN_CHUNK = '1'"
                 ],
@@ -596,7 +591,6 @@
             "MACHINE" : "qemux86",
             "SDKMACHINE" : "x86_64",
             "BBTARGETS" : "universe:do_checkuri",
-            "PRSERV" : false,
             "extravars" : [
                 "SOURCE_MIRROR_FETCH = '1'",
                 "BB_NUMBER_THREADS = '1'",
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 02/41] config.json: Hide WARNINGS we don't care about for meta-gplv2 testing

[Richard Purdie]

We're not interested in these warnings, the license incompatibility is
expected. By hiding these, we'll notice when warnings we do care about
appear.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config.json b/config.json
index 90b78f3..43526de 100644
--- a/config.json
+++ b/config.json
@@ -553,7 +553,8 @@
             "MACHINE" : "qemux86",
             "BBTARGETS" : "core-image-minimal core-image-full-cmdline",
             "extravars" : [
-                "INCOMPATIBLE_LICENSE = '*GPLv3'"
+                "INCOMPATIBLE_LICENSE = '*GPLv3'",
+                "WARN_QA_remove = 'incompatible-license'"
             ],
             "EXTRACMDS" : [
                 "../../yocto-autobuilder-helper/scripts/check-gplv3"
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 03/41] README: Add pointer to the mailing list for patches

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 README | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README b/README
index a6c4f1d..2747510 100644
--- a/README
+++ b/README
@@ -25,3 +25,8 @@ would also allow customisation.
 Authors: 
   Richard Purdie <richard.purdie@linuxfoundation.org>
   Joshua Lock <joshua.g.lock@intel.com>
+
+Contributions:
+
+Patches for this code should be sent to the yocto@yoctoproject.org mailing list
+with [yocto-autobuilder-helper] in the subject.
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 04/41] layer-config: fixing silent failures from always exiting '0'

[Richard Purdie]

From: Thomas Goodwin <btgoodwin@geontech.com>

The return value from bitbakecmd was not being returned when
errors occurred which allowed shared-repo-unpack to succeed
despite the failure.  This fix changes to check_call and a
try-catch when attempting to add repos that fail for whatever
reason during add-layer, like a missing conf/layer.conf at
the top level or a previously-added layer breaks parsing
because of missing dependencies.

Signed-off-by: Thomas Goodwin <btgoodwin@geontech.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/layer-config | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/scripts/layer-config b/scripts/layer-config
index bfc62da..286451a 100755
--- a/scripts/layer-config
+++ b/scripts/layer-config
@@ -24,9 +24,7 @@ args = parser.parse_args()
 ourconfig = utils.loadconfig()
 
 def bitbakecmd(targetdir, cmd):
-    ret = subprocess.call(". ./oe-init-build-env; %s" % cmd, shell=True, cwd=targetdir)
-    if ret:
-        utils.printheader("ERROR: Command %s failed with exit code %d, see errors above." % (cmd, ret))
+    subprocess.check_call(". ./oe-init-build-env; %s" % cmd, shell=True, cwd=targetdir)
 
 needrepos = utils.getconfigvar("NEEDREPOS", ourconfig, args.target, None)
 
@@ -51,5 +49,8 @@ if callinit:
 for repo in needrepos:
     if repo in repos and "no-layer-add" in repos[repo] and repos[repo]["no-layer-add"]:
         continue
-    bitbakecmd(args.abworkdir, "bitbake-layers add-layer %s" % (args.abworkdir + "/" + repo))
-
+    try:
+        bitbakecmd(args.abworkdir, "bitbake-layers add-layer %s" % (args.abworkdir + "/" + repo))
+    except subprocess.CalledProcessError as e:
+        utils.printheader("ERROR: Command %s failed with exit code %d, see errors above." % (e.cmd, e.returncode))
+        sys.exit(e.returncode)
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 05/41] layer-config, shared-repo-unpack: Sub-repos in NEEDREPOS

[Richard Purdie]

From: Thomas Goodwin <btgoodwin@geontech.com>

The previous fixes requires the user to set "no-layer-add"
for a repo and then use ADDLAYER to insert the sub-repos
(e.g., meta-openmbedded/meta-oe) as a two-part process.
This means that you would also have to specify that flag
if a repo that is a layer with dependencies is in the
list so that it can be inserted in the correct order later
via ADDLAYER to avoid parsing problems.  This fix allows
for specifying a NEEDREPOS with the subdirectory of the
target layer (e.g., meta-openembedded/meta-oe) so that
there is no need for the "no-layer-add" followed by
ADDLAYER combination.  The entire meta-openembedded
repo would be moved into place, and the sublayer added
to bblayers.conf.

Signed-off-by: Thomas Goodwin <btgoodwin@geontech.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/layer-config       | 24 +++++++++++++++---------
 scripts/shared-repo-unpack |  4 ++--
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/scripts/layer-config b/scripts/layer-config
index 286451a..bb1b681 100755
--- a/scripts/layer-config
+++ b/scripts/layer-config
@@ -33,21 +33,27 @@ callinit = False
 repos = utils.getconfig("repo-defaults", ourconfig)
 
 for repo in needrepos:
-    checkdir = repo
-    if repo in repos:
-        if "call-init" in repos[repo] and repos[repo]["call-init"]:
+    repo_basename = repo.split('/')[0]
+    checkdir = repo_basename
+    if repo_basename in repos:
+        if "call-init" in repos[repo_basename] and repos[repo_basename]["call-init"]:
             callinit = True
-        if "checkout-dirname" in repos[repo]:
-            checkdir = repos[repo]["checkout-dirname"]
-    utils.mkdir(args.abworkdir + "/" + checkdir)
-    for f in os.listdir(args.abworkdir + "/repos/" + repo):
-        subprocess.check_call(['mv', args.abworkdir + "/repos/" + repo + "/" + f, args.abworkdir + "/" + checkdir + "/"])
+        if "checkout-dirname" in repos[repo_basename]:
+            checkdir = repos[repo_basename]["checkout-dirname"]
+
+    source = args.abworkdir + "/repos/" + repo_basename
+    destination = args.abworkdir + "/" + checkdir
+    if not os.path.isdir(destination) or callinit:
+        utils.mkdir(destination)
+        for f in os.listdir(source):
+            subprocess.check_call(['mv', source + "/" + f, destination + "/"])
 
 if callinit:
     subprocess.check_call(". ./oe-init-build-env", shell=True, cwd=args.abworkdir)
 
 for repo in needrepos:
-    if repo in repos and "no-layer-add" in repos[repo] and repos[repo]["no-layer-add"]:
+    repo_basename = repo.split('/')[0]
+    if repo_basename in repos and "no-layer-add" in repos[repo_basename] and repos[repo_basename]["no-layer-add"]:
         continue
     try:
         bitbakecmd(args.abworkdir, "bitbake-layers add-layer %s" % (args.abworkdir + "/" + repo))
diff --git a/scripts/shared-repo-unpack b/scripts/shared-repo-unpack
index a281897..92f0ccf 100755
--- a/scripts/shared-repo-unpack
+++ b/scripts/shared-repo-unpack
@@ -46,9 +46,9 @@ with open(args.repojson) as f:
     repos = json.load(f)
 
 targetsubdir = args.abworkdir + "/repos"
-
+needrepos_baseddirs = [r.split('/')[0] for r in needrepos]
 for repo in sorted(repos.keys()):
-    if repo not in needrepos:
+    if repo not in needrepos_baseddirs:
         continue
     targetrepodir = "%s/%s" % (targetsubdir, repo)
     if args.cache_dir:
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 06/41] prepare-shared-repos: Use temporary non-NFS directory for layer clones for speed

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/prepare-shared-repos | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/scripts/prepare-shared-repos b/scripts/prepare-shared-repos
index eb89f5b..9ed0772 100755
--- a/scripts/prepare-shared-repos
+++ b/scripts/prepare-shared-repos
@@ -8,6 +8,7 @@ import os
 import sys
 import subprocess
 import errno
+import tempfile
 
 import utils
 
@@ -31,8 +32,11 @@ with open(args.repojson) as f:
 
 stashdir = utils.getconfig("REPO_STASH_DIR", ourconfig)
 
-for repo in sorted(repos.keys()):
-    utils.printheader("Intially fetching repo %s" % repo)
-    utils.fetchgitrepo(args.sharedsrcdir, repo, repos[repo], stashdir)
-    if args.publish_dir:
-        utils.publishrepo(args.sharedsrcdir, repo, args.publish_dir)
+with tempfile.TemporaryDirectory(prefix="shared-repo-temp-", dir="/tmp") as tempdir:
+    for repo in sorted(repos.keys()):
+        utils.printheader("Intially fetching repo %s" % repo)
+        utils.fetchgitrepo(tempdir, repo, repos[repo], stashdir)
+        if args.publish_dir:
+            utils.publishrepo(tempdir, repo, args.publish_dir)
+
+    subprocess.check_call(["rsync", "-a", tempdir + "/*", args.sharedsrcdir])
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 07/41] prepare-shared-repos: UAllow shell globing to fix failure

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/prepare-shared-repos | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/prepare-shared-repos b/scripts/prepare-shared-repos
index 9ed0772..2a2b343 100755
--- a/scripts/prepare-shared-repos
+++ b/scripts/prepare-shared-repos
@@ -39,4 +39,4 @@ with tempfile.TemporaryDirectory(prefix="shared-repo-temp-", dir="/tmp") as temp
         if args.publish_dir:
             utils.publishrepo(tempdir, repo, args.publish_dir)
 
-    subprocess.check_call(["rsync", "-a", tempdir + "/*", args.sharedsrcdir])
+    subprocess.check_call(["rsync", "-a", tempdir + "/*", args.sharedsrcdir], shell=True)
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 08/41] prepare-shared-repos: Really fix shell globbing

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/prepare-shared-repos | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/prepare-shared-repos b/scripts/prepare-shared-repos
index 2a2b343..8400d09 100755
--- a/scripts/prepare-shared-repos
+++ b/scripts/prepare-shared-repos
@@ -39,4 +39,4 @@ with tempfile.TemporaryDirectory(prefix="shared-repo-temp-", dir="/tmp") as temp
         if args.publish_dir:
             utils.publishrepo(tempdir, repo, args.publish_dir)
 
-    subprocess.check_call(["rsync", "-a", tempdir + "/*", args.sharedsrcdir], shell=True)
+    subprocess.check_call("rsync -a " + tempdir + "/* " + args.sharedsrcdir, shell=True)
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 09/41] config.json: Replaced occurrencies of /home/pokybuild with ${BASE_HOMEDIR}

[Richard Purdie]

From: Marco Cavallini <m.cavallini@koansoftware.com>

Signed-off-by: Marco Cavallini <m.cavallini@koansoftware.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/config.json b/config.json
index 43526de..e7c7b55 100644
--- a/config.json
+++ b/config.json
@@ -16,8 +16,8 @@
     "WEBPUBLISH_DIR" : "${BASE_SHAREDDIR}/",
     "WEBPUBLISH_URL" : "https://autobuilder.yocto.io/",
 
-    "BUILDPERF_STATEDIR" : "/home/pokybuild/buildperf",
-    "BUILDPERF_RESULTSDIR" : "/home/pokybuild/buildperf-results",
+    "BUILDPERF_STATEDIR" : "${BASE_HOMEDIR}/buildperf",
+    "BUILDPERF_RESULTSDIR" : "${BASE_HOMEDIR}/buildperf-results",
 
     "defaults" : {
         "NEEDREPOS" : ["poky"],
@@ -109,7 +109,7 @@
             "SSTATEDIR_RELEASE" : ["SSTATE_DIR ?= '${HELPERBUILDDIR}/sstate'"],
             "PACKAGE_CLASSES" : "package_rpm",
             "EXTRAPLAINCMDS" : [
-                "${SCRIPTSDIR}/build-perf-test-wrapper -r ${BUILDPERF_RESULTSDIR} -E yocto-perf@lists.yoctoproject.org -d ${BUILDPERF_STATEDIR}/downloads -w /home/pokybuild/build-perf-test -p ${HELPERRESULTSDIR}/${HELPERTARGET} -R ${HELPERREPONAME} -b ${HELPERBRANCHNAME} --push git@push.yoctoproject.org:yocto-buildstats"
+                "${SCRIPTSDIR}/build-perf-test-wrapper -r ${BUILDPERF_RESULTSDIR} -E yocto-perf@lists.yoctoproject.org -d ${BUILDPERF_STATEDIR}/downloads -w ${BASE_HOMEDIR}/build-perf-test -p ${HELPERRESULTSDIR}/${HELPERTARGET} -R ${HELPERREPONAME} -b ${HELPERBRANCHNAME} --push git@push.yoctoproject.org:yocto-buildstats"
             ],
             "extravars" : [
                 "BB_NUMBER_THREADS = '24'",
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 10/41] shared-repo-unpack: suppressing exception, returning exit code

[Richard Purdie]

From: Thomas Goodwin <btgoodwin@geontech.com>

The preceeding errors that would be shown should be enough to
identify the problem since it would have been a problem
running bitbake-layers add-layer, indicating a misconfiguration
of config.json.

Signed-off-by: Thomas Goodwin <btgoodwin@geontech.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/shared-repo-unpack | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/shared-repo-unpack b/scripts/shared-repo-unpack
index 92f0ccf..4a2b1c9 100755
--- a/scripts/shared-repo-unpack
+++ b/scripts/shared-repo-unpack
@@ -61,4 +61,7 @@ for repo in sorted(repos.keys()):
         if args.publish_dir:
             utils.publishrepo(targetsubdir, repo, args.publish_dir)
 
-subprocess.check_call([scriptsdir + "/layer-config", args.abworkdir, args.target])
+try:
+    subprocess.check_call([scriptsdir + "/layer-config", args.abworkdir, args.target])
+except subprocess.CalledProcessError as e:
+    sys.exit(e.returncode)
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 11/41] run-config: Preface log output with auto.conf settings to aid debugging

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/run-config | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scripts/run-config b/scripts/run-config
index 0097f97..b7540a5 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -121,6 +121,14 @@ def bitbakecmd(builddir, cmd, report, stepnum, logsuffix, oeenv=True):
     print("Running '%s' with output to %s" % (cmd, log))
     flush()
 
+    autoconf = builddir + "/conf/auto.conf"
+    if os.path.exists(autoconf):
+        with open(autoconf, "r") as inf, open(log, "a") as outf:
+            outf.write("auto.conf settings:\n")
+            for line in inf.readlines():
+                outf.write(line)
+            outf.write("\n")
+
     with subprocess.Popen(cmd, shell=True, cwd=builddir + "/..", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1) as p, open(log, 'ab') as f:
         for line in p.stdout:
             if not args.quietlogging:
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 12/41] run-config: Add support for wrapping builds in a buildtools tarball

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/run-config | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/scripts/run-config b/scripts/run-config
index b7540a5..e946e90 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -8,6 +8,8 @@ import os
 import sys
 import subprocess
 import errno
+import fnmatch
+import glob
 
 import utils
 
@@ -80,6 +82,34 @@ if args.target in ourconfig['overrides']:
 
 hp.printheader("Target task %s has %d steps" % (args.target, maxsteps))
 
+bttarball = None
+if "buildtools" in ourconfig and args.workername:
+    btcfg = utils.getconfig("buildtools", ourconfig)
+    for entry in btcfg:
+        if fnmatch.fnmatch(args.workername, entry):
+            bttarball = btcfg[entry]
+            break
+
+btenv = None
+if bttarball:
+    btdir = args.builddir + "../buildtools"
+    if not os.path.exists(btdir):
+        btdlpath = utils.getconfig("BASE_SHAREDDIR", ourconfig) + "/buildtools/" + os.path.basename(bttarball)
+        if not os.path.exists(btdlpath):
+            os.makedirs(os.path.dirname(btdlpath), exist_ok=True)
+            subprocess.check_call(["wget", "-O", btdlpath, bttarball])
+            os.chmod(btdlpath, 0o775)
+        subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
+    btenv = glob.glob(btdir + "/environment-setup*")
+    # We either parse or wrap all our execution calls, rock and a hard place :(
+    with open(btenv[0], "r") as f:
+        for line in f.readlines():
+            if line.startswith("export "):
+                line = line.split(" ", 1)[1].split("=", 1)
+                if "$PATH" in line[1]:
+                    line[1] = line[1].replace("$PATH", os.environ["PATH"])
+                os.environ[line[0]] = line[1]
+
 finalret = 0
 
 def flush():
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 13/41] config.json: Enable buildtools for centos7 workers (as a test for now)

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/config.json b/config.json
index e7c7b55..3e69544 100644
--- a/config.json
+++ b/config.json
@@ -8,6 +8,8 @@
     "BUILD_HISTORY_DIRECTPUSH" : ["poky:morty", "poky:pyro", "poky:rocko", "poky:sumo", "poky:thud", "poky:warrior", "poky:zeus", "poky:master"],
     "BUILD_HISTORY_FORKPUSH" : {"poky-contrib:ross/mut" : "poky:master", "poky:master-next" : "poky:master"},
 
+    "BUILDTOOLS_URL" : "http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M2/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0%2Bsnapshot-20200122.sh",
+
     "REPO_STASH_DIR" : "${BASE_HOMEDIR}/git/mirror",
     "TRASH_DIR" : "${BASE_HOMEDIR}/git/trash",
 
@@ -868,5 +870,8 @@
             "branch" : "master",
             "revision" : "HEAD"
         }
+    },
+    "buildtools" : {
+        "centos7*" : "${BUILDTOOLS_URL}"
     }
 }
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 14/41] config.json: Disable buildtools tarball on centos7 as its missing crypt.h

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config.json b/config.json
index 3e69544..49a26e7 100644
--- a/config.json
+++ b/config.json
@@ -872,6 +872,6 @@
         }
     },
     "buildtools" : {
-        "centos7*" : "${BUILDTOOLS_URL}"
+        "disabled-centos7*" : "${BUILDTOOLS_URL}"
     }
 }
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 15/41] scripts/run-config: Fix buildtools path

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/run-config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/run-config b/scripts/run-config
index e946e90..cbfe073 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -92,7 +92,7 @@ if "buildtools" in ourconfig and args.workername:
 
 btenv = None
 if bttarball:
-    btdir = args.builddir + "../buildtools"
+    btdir = args.builddir + "/../buildtools"
     if not os.path.exists(btdir):
         btdlpath = utils.getconfig("BASE_SHAREDDIR", ourconfig) + "/buildtools/" + os.path.basename(bttarball)
         if not os.path.exists(btdlpath):
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 16/41] send-qa-email: Limit results clone to shallow depth to avoid timeouts

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/send-qa-email | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/send-qa-email b/scripts/send-qa-email
index d2f4574..7dbc290 100755
--- a/scripts/send-qa-email
+++ b/scripts/send-qa-email
@@ -61,7 +61,7 @@ if 'poky' in repos and os.path.exists(resulttool) and args.results_dir:
 
     tempdir = tempfile.mkdtemp(prefix='sendqaemail.')
     try:
-        subprocess.check_call(["git", "clone", "git@git.yoctoproject.org:yocto-testresults", tempdir])
+        subprocess.check_call(["git", "clone", "git@git.yoctoproject.org:yocto-testresults", tempdir, "--depth", "5"])
 
         # If the base comparision branch isn't present regression comparision won't work
         # at least until we can tell the tool to ignore internal branch information
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 17/41] scripts/run-config: Handle multiple builds racing with lockfile for buildtools

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/run-config | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/scripts/run-config b/scripts/run-config
index cbfe073..ce1f36f 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -10,6 +10,7 @@ import subprocess
 import errno
 import fnmatch
 import glob
+import fcntl
 
 import utils
 
@@ -95,10 +96,15 @@ if bttarball:
     btdir = args.builddir + "/../buildtools"
     if not os.path.exists(btdir):
         btdlpath = utils.getconfig("BASE_SHAREDDIR", ourconfig) + "/buildtools/" + os.path.basename(bttarball)
-        if not os.path.exists(btdlpath):
+        btlock = btdlpath + ".lock"
+        if not os.path.exists(os.path.dirname(btdlpath)):
             os.makedirs(os.path.dirname(btdlpath), exist_ok=True)
-            subprocess.check_call(["wget", "-O", btdlpath, bttarball])
-            os.chmod(btdlpath, 0o775)
+        with open(btlock, 'a+') as lf:
+            fileno = lf.fileno()
+            fcntl.flock(fileno, fcntl.LOCK_EX)
+            if not os.path.exists(btdlpath):
+                subprocess.check_call(["wget", "-O", btdlpath, bttarball])
+                os.chmod(btdlpath, 0o775)
         subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
     btenv = glob.glob(btdir + "/environment-setup*")
     # We either parse or wrap all our execution calls, rock and a hard place :(
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 18/41] scripts/run-config: Handle quoting issues in buildtools environment files

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/run-config | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/run-config b/scripts/run-config
index ce1f36f..b4f1d58 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -111,9 +111,11 @@ if bttarball:
     with open(btenv[0], "r") as f:
         for line in f.readlines():
             if line.startswith("export "):
-                line = line.split(" ", 1)[1].split("=", 1)
+                line = line.strip().split(" ", 1)[1].split("=", 1)
                 if "$PATH" in line[1]:
                     line[1] = line[1].replace("$PATH", os.environ["PATH"])
+                if line[1].startswith(("'", '"')):
+                    line[1] = line[1][1:-1]
                 os.environ[line[0]] = line[1]
 
 finalret = 0
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 19/41] scripts/run-config: Add support for local filesystem buildtools urls

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/run-config | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/run-config b/scripts/run-config
index b4f1d58..d4ff6b3 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -103,7 +103,10 @@ if bttarball:
             fileno = lf.fileno()
             fcntl.flock(fileno, fcntl.LOCK_EX)
             if not os.path.exists(btdlpath):
-                subprocess.check_call(["wget", "-O", btdlpath, bttarball])
+                if bttarball.startswith("/"):
+                    subprocess.check_call(["cp", bttarball, btdlpath])
+                else:
+                    subprocess.check_call(["wget", "-O", btdlpath, bttarball])
                 os.chmod(btdlpath, 0o775)
         subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
     btenv = glob.glob(btdir + "/environment-setup*")
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 20/41] scripts/run-config: Ensure racing for the lockfile doesn't traceback

[Richard Purdie]

Traceback (most recent call last):
  File "yocto-autobuilder-helper/scripts/run-config", line 116, in <module>
    fcntl.flock(fileno, fcntl.LOCK_EX)
OSError: [Errno 22] Invalid argument

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/run-config | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/scripts/run-config b/scripts/run-config
index d4ff6b3..55287c3 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -99,15 +99,20 @@ if bttarball:
         btlock = btdlpath + ".lock"
         if not os.path.exists(os.path.dirname(btdlpath)):
             os.makedirs(os.path.dirname(btdlpath), exist_ok=True)
-        with open(btlock, 'a+') as lf:
-            fileno = lf.fileno()
-            fcntl.flock(fileno, fcntl.LOCK_EX)
-            if not os.path.exists(btdlpath):
-                if bttarball.startswith("/"):
-                    subprocess.check_call(["cp", bttarball, btdlpath])
-                else:
-                    subprocess.check_call(["wget", "-O", btdlpath, bttarball])
-                os.chmod(btdlpath, 0o775)
+        while True:
+            try:
+                with open(btlock, 'a+') as lf:
+                    fileno = lf.fileno()
+                    fcntl.flock(fileno, fcntl.LOCK_EX)
+                    if not os.path.exists(btdlpath):
+                        if bttarball.startswith("/"):
+                            subprocess.check_call(["cp", bttarball, btdlpath])
+                        else:
+                            subprocess.check_call(["wget", "-O", btdlpath, bttarball])
+                        os.chmod(btdlpath, 0o775)
+            except OSError:
+                # We raced with someone else, try again
+                pass
         subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
     btenv = glob.glob(btdir + "/environment-setup*")
     # We either parse or wrap all our execution calls, rock and a hard place :(
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 21/41] run-config: Fix lockfile loop handling

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/run-config | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/run-config b/scripts/run-config
index 55287c3..4d795b9 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -110,6 +110,7 @@ if bttarball:
                         else:
                             subprocess.check_call(["wget", "-O", btdlpath, bttarball])
                         os.chmod(btdlpath, 0o775)
+                break
             except OSError:
                 # We raced with someone else, try again
                 pass
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 22/41] config.json: Update to 3.1M3 for buildtools tarball and apply to centos7/debian8

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/config.json b/config.json
index 49a26e7..ea5947e 100644
--- a/config.json
+++ b/config.json
@@ -8,7 +8,8 @@
     "BUILD_HISTORY_DIRECTPUSH" : ["poky:morty", "poky:pyro", "poky:rocko", "poky:sumo", "poky:thud", "poky:warrior", "poky:zeus", "poky:master"],
     "BUILD_HISTORY_FORKPUSH" : {"poky-contrib:ross/mut" : "poky:master", "poky:master-next" : "poky:master"},
 
-    "BUILDTOOLS_URL" : "http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M2/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0%2Bsnapshot-20200122.sh",
+    "BUILDTOOLS_URL_TEMPLOCAL" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200309-15/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200309.sh",
+    "BUILDTOOLS_URL" : "http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh",
 
     "REPO_STASH_DIR" : "${BASE_HOMEDIR}/git/mirror",
     "TRASH_DIR" : "${BASE_HOMEDIR}/git/trash",
@@ -872,6 +873,7 @@
         }
     },
     "buildtools" : {
-        "disabled-centos7*" : "${BUILDTOOLS_URL}"
+        "debian8*" : "${BUILDTOOLS_URL}",
+        "centos7*" : "${BUILDTOOLS_URL}"
     }
 }
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 23/41] config.json: Apply buildtools tarball to centos7 perf worker too

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config.json b/config.json
index ea5947e..b5d8264 100644
--- a/config.json
+++ b/config.json
@@ -874,6 +874,7 @@
     },
     "buildtools" : {
         "debian8*" : "${BUILDTOOLS_URL}",
-        "centos7*" : "${BUILDTOOLS_URL}"
+        "centos7*" : "${BUILDTOOLS_URL}",
+        "perf-centos7*" : "${BUILDTOOLS_URL}"
     }
 }
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 24/41] scripts/utils.py: Fix confusing naming of getcomparisonbranch() return value

[Richard Purdie]

From: Steve Sakoman <steve@sakoman.com>

This code was copy/pasted but in this case the variables should
be called "compare", not base.

Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/utils.py b/scripts/utils.py
index 68652d9..5542fe2 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -358,9 +358,9 @@ def getcomparisonbranch(ourconfig, reponame, branchname):
     if (reponame + ":" + branchname) in getconfig("BUILD_HISTORY_FORKPUSH", ourconfig):
         base = getconfig("BUILD_HISTORY_FORKPUSH", ourconfig)[reponame + ":" + branchname]
         if base:
-            baserepo, basebranch = base.split(":")
-            print("Comparing to %s\n" % (basebranch))
-            return branchname, basebranch
+            comparerepo, comparebranch = base.split(":")
+            print("Comparing to %s\n" % (comparebranch))
+            return branchname, comparebranch
     if (reponame + ":" + branchname) in getconfig("BUILD_HISTORY_DIRECTPUSH", ourconfig):
         return branchname, None
     return None, None
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 25/41] scripts/send-qa-email: fix bug in git push logic for yocto-testresults

[Richard Purdie]

From: Steve Sakoman <steve@sakoman.com>

We were mistakenly doing a force push if the branch was in either
BUILD_HISTORY_FORKPUSH or BUILD_HISTORY_DIRECTPUSH.

Now we force push for branches in BUILD_HISTORY_FORKPUSH, regular push
for branches in BUILD_HISTORY_DIRECTPUSH, and no push if the branch is
in neither list.

Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/send-qa-email | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/send-qa-email b/scripts/send-qa-email
index 7dbc290..a265a7c 100755
--- a/scripts/send-qa-email
+++ b/scripts/send-qa-email
@@ -77,10 +77,10 @@ if 'poky' in repos and os.path.exists(resulttool) and args.results_dir:
                 extraopts = None
 
         subprocess.check_call([resulttool, "store", args.results_dir, tempdir])
-        if basebranch:
+        if comparebranch:
             subprocess.check_call(["git", "push", "--all", "--force"], cwd=tempdir)
             subprocess.check_call(["git", "push", "--tags", "--force"], cwd=tempdir)
-        else:
+        elif basebranch:
             subprocess.check_call(["git", "push", "--all"], cwd=tempdir)
             subprocess.check_call(["git", "push", "--tags"], cwd=tempdir)
 
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 26/41] scripts/run-config: Add extra debug about buildtools to show when active

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/run-config | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/run-config b/scripts/run-config
index 4d795b9..54d74dc 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -96,6 +96,7 @@ if bttarball:
     btdir = args.builddir + "/../buildtools"
     if not os.path.exists(btdir):
         btdlpath = utils.getconfig("BASE_SHAREDDIR", ourconfig) + "/buildtools/" + os.path.basename(bttarball)
+        print("Extracting buildtools %s" % bttarball)
         btlock = btdlpath + ".lock"
         if not os.path.exists(os.path.dirname(btdlpath)):
             os.makedirs(os.path.dirname(btdlpath), exist_ok=True)
@@ -116,6 +117,7 @@ if bttarball:
                 pass
         subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
     btenv = glob.glob(btdir + "/environment-setup*")
+    print("Using buildtools %s" % btenv)
     # We either parse or wrap all our execution calls, rock and a hard place :(
     with open(btenv[0], "r") as f:
         for line in f.readlines():
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 27/41] scripts/run-config: Share buildtools setup code with shared-repo-unpack

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/run-config         | 49 +------------------------------------
 scripts/shared-repo-unpack |  2 ++
 scripts/utils.py           | 50 ++++++++++++++++++++++++++++++++++++++
 3 files changed, 53 insertions(+), 48 deletions(-)

diff --git a/scripts/run-config b/scripts/run-config
index 54d74dc..814f0cc 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -8,9 +8,6 @@ import os
 import sys
 import subprocess
 import errno
-import fnmatch
-import glob
-import fcntl
 
 import utils
 
@@ -83,51 +80,7 @@ if args.target in ourconfig['overrides']:
 
 hp.printheader("Target task %s has %d steps" % (args.target, maxsteps))
 
-bttarball = None
-if "buildtools" in ourconfig and args.workername:
-    btcfg = utils.getconfig("buildtools", ourconfig)
-    for entry in btcfg:
-        if fnmatch.fnmatch(args.workername, entry):
-            bttarball = btcfg[entry]
-            break
-
-btenv = None
-if bttarball:
-    btdir = args.builddir + "/../buildtools"
-    if not os.path.exists(btdir):
-        btdlpath = utils.getconfig("BASE_SHAREDDIR", ourconfig) + "/buildtools/" + os.path.basename(bttarball)
-        print("Extracting buildtools %s" % bttarball)
-        btlock = btdlpath + ".lock"
-        if not os.path.exists(os.path.dirname(btdlpath)):
-            os.makedirs(os.path.dirname(btdlpath), exist_ok=True)
-        while True:
-            try:
-                with open(btlock, 'a+') as lf:
-                    fileno = lf.fileno()
-                    fcntl.flock(fileno, fcntl.LOCK_EX)
-                    if not os.path.exists(btdlpath):
-                        if bttarball.startswith("/"):
-                            subprocess.check_call(["cp", bttarball, btdlpath])
-                        else:
-                            subprocess.check_call(["wget", "-O", btdlpath, bttarball])
-                        os.chmod(btdlpath, 0o775)
-                break
-            except OSError:
-                # We raced with someone else, try again
-                pass
-        subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
-    btenv = glob.glob(btdir + "/environment-setup*")
-    print("Using buildtools %s" % btenv)
-    # We either parse or wrap all our execution calls, rock and a hard place :(
-    with open(btenv[0], "r") as f:
-        for line in f.readlines():
-            if line.startswith("export "):
-                line = line.strip().split(" ", 1)[1].split("=", 1)
-                if "$PATH" in line[1]:
-                    line[1] = line[1].replace("$PATH", os.environ["PATH"])
-                if line[1].startswith(("'", '"')):
-                    line[1] = line[1][1:-1]
-                os.environ[line[0]] = line[1]
+utils.setup_buildtools_tarball(ourconfig, args.workername, args.builddir + "/../buildtools")
 
 finalret = 0
 
diff --git a/scripts/shared-repo-unpack b/scripts/shared-repo-unpack
index 4a2b1c9..7dc250c 100755
--- a/scripts/shared-repo-unpack
+++ b/scripts/shared-repo-unpack
@@ -61,6 +61,8 @@ for repo in sorted(repos.keys()):
         if args.publish_dir:
             utils.publishrepo(targetsubdir, repo, args.publish_dir)
 
+utils.setup_buildtools_tarball(ourconfig, args.workername, args.abworkdir + "/buildtools")
+
 try:
     subprocess.check_call([scriptsdir + "/layer-config", args.abworkdir, args.target])
 except subprocess.CalledProcessError as e:
diff --git a/scripts/utils.py b/scripts/utils.py
index 5542fe2..d7b9ae7 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -8,6 +8,9 @@ import codecs
 import sys
 import re
 import argparse
+import fnmatch
+import glob
+import fcntl
 
 #
 # Check if config contains all the listed params
@@ -364,3 +367,50 @@ def getcomparisonbranch(ourconfig, reponame, branchname):
     if (reponame + ":" + branchname) in getconfig("BUILD_HISTORY_DIRECTPUSH", ourconfig):
         return branchname, None
     return None, None
+
+def setup_buildtools_tarball(ourconfig, workername, btdir):
+    bttarball = None
+    if "buildtools" in ourconfig and workername:
+        btcfg = getconfig("buildtools", ourconfig)
+        for entry in btcfg:
+            if fnmatch.fnmatch(workername, entry):
+                bttarball = btcfg[entry]
+                break
+
+    btenv = None
+    if bttarball:
+        btdir = os.path.abspath(btdir)
+        if not os.path.exists(btdir):
+            btdlpath = getconfig("BASE_SHAREDDIR", ourconfig) + "/buildtools/" + os.path.basename(bttarball)
+            print("Extracting buildtools %s" % bttarball)
+            btlock = btdlpath + ".lock"
+            if not os.path.exists(os.path.dirname(btdlpath)):
+                os.makedirs(os.path.dirname(btdlpath), exist_ok=True)
+            while True:
+                try:
+                    with open(btlock, 'a+') as lf:
+                        fileno = lf.fileno()
+                        fcntl.flock(fileno, fcntl.LOCK_EX)
+                        if not os.path.exists(btdlpath):
+                            if bttarball.startswith("/"):
+                                subprocess.check_call(["cp", bttarball, btdlpath])
+                            else:
+                                subprocess.check_call(["wget", "-O", btdlpath, bttarball])
+                            os.chmod(btdlpath, 0o775)
+                    break
+                except OSError:
+                    # We raced with someone else, try again
+                    pass
+            subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
+        btenv = glob.glob(btdir + "/environment-setup*")
+        print("Using buildtools %s" % btenv)
+        # We either parse or wrap all our execution calls, rock and a hard place :(
+        with open(btenv[0], "r") as f:
+            for line in f.readlines():
+                if line.startswith("export "):
+                    line = line.strip().split(" ", 1)[1].split("=", 1)
+                    if "$PATH" in line[1]:
+                        line[1] = line[1].replace("$PATH", os.environ["PATH"])
+                    if line[1].startswith(("'", '"')):
+                        line[1] = line[1][1:-1]
+                    os.environ[line[0]] = line[1]
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 28/41] config.json: Use new buildtools tarball

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config.json b/config.json
index b5d8264..c94605b 100644
--- a/config.json
+++ b/config.json
@@ -9,7 +9,8 @@
     "BUILD_HISTORY_FORKPUSH" : {"poky-contrib:ross/mut" : "poky:master", "poky:master-next" : "poky:master"},
 
     "BUILDTOOLS_URL_TEMPLOCAL" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200309-15/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200309.sh",
-    "BUILDTOOLS_URL" : "http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh",
+    "BUILDTOOLS_URL_TEMPLOCAL2" : "http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh",
+    "BUILDTOOLS_URL" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200427-22/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.1.sh",
 
     "REPO_STASH_DIR" : "${BASE_HOMEDIR}/git/mirror",
     "TRASH_DIR" : "${BASE_HOMEDIR}/git/trash",
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 29/41] config.json: Update to new buildtools tarball

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config.json b/config.json
index c94605b..70cee81 100644
--- a/config.json
+++ b/config.json
@@ -10,7 +10,7 @@
 
     "BUILDTOOLS_URL_TEMPLOCAL" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200309-15/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200309.sh",
     "BUILDTOOLS_URL_TEMPLOCAL2" : "http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh",
-    "BUILDTOOLS_URL" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200427-22/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.1.sh",
+    "BUILDTOOLS_URL" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200430-14/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.1.sh",
 
     "REPO_STASH_DIR" : "${BASE_HOMEDIR}/git/mirror",
     "TRASH_DIR" : "${BASE_HOMEDIR}/git/trash",
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 30/41] config.json: Use buildtools from 3.2 M1 to fix bugs in openssl reporting

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 385fdf2295d514d0b0883957a110825235f9647f)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
---
 config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config.json b/config.json
index 70cee81..fbf262b 100644
--- a/config.json
+++ b/config.json
@@ -10,7 +10,7 @@
 
     "BUILDTOOLS_URL_TEMPLOCAL" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200309-15/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200309.sh",
     "BUILDTOOLS_URL_TEMPLOCAL2" : "http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh",
-    "BUILDTOOLS_URL" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200430-14/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.1.sh",
+    "BUILDTOOLS_URL" : "http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.2_M1/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.1+snapshot-20200617.sh",
 
     "REPO_STASH_DIR" : "${BASE_HOMEDIR}/git/mirror",
     "TRASH_DIR" : "${BASE_HOMEDIR}/git/trash",
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 31/41] scripts/publish-artefacts: Convert md5sum to sha256sum

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 80c53d03ea32e3b4831e8945eaa2e1d643d0ce89)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
---
 scripts/publish-artefacts | 88 +++++++++++++++++++--------------------
 1 file changed, 44 insertions(+), 44 deletions(-)

diff --git a/scripts/publish-artefacts b/scripts/publish-artefacts
index c263a34..2611cc8 100755
--- a/scripts/publish-artefacts
+++ b/scripts/publish-artefacts
@@ -3,11 +3,11 @@ WORKDIR=$1
 DEST=$2
 target=$3
 
-md5sums(){
+sha256sums(){
     dest=$1
-    for x in `find $dest -maxdepth 5 -type f`; do 
-        if [ ${x##*.} != md5sum ]; then 
-            md5sum $x >> $x.md5sum
+    for x in `find $dest -maxdepth 5 -type f`; do
+        if [ ${x##*.} != sha256sum ]; then
+            sha256sum $x >> $x.sha256sum
         fi
     done
 }
@@ -18,7 +18,7 @@ case "$target" in
     "qemuarm"|"qemuarm64"|"qemumips64"|"qemumips"|"qemuppc"|"qemux86"|"qemux86-64"|"edgerouter"|"mpc8315e-rdb"|"genericx86"|"genericx86-64"|"beaglebone")
         mkdir -p $DEST/toolchain/i686
         mkdir -p $DEST/toolchain/x86_64
-        md5sums $TMPDIR/deploy/sdk
+        sha256sums $TMPDIR/deploy/sdk
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/sdk/poky-*i686-core-image* $DEST/toolchain/i686
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/sdk/poky-*x86_64-core-image* $DEST/toolchain/x86_64
         ;;
@@ -34,7 +34,7 @@ case "$target" in
         mkdir -p $DEST/toolchain/i686
         mkdir -p $DEST/toolchain/x86_64
         mkdir -p $DEST/toolchain/aarch64
-        md5sums $TMPDIR/deploy/sdk
+        sha256sums $TMPDIR/deploy/sdk
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/sdk/*buildtools* $DEST/buildtools
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/sdk/i686-nativesdk-libc* $DEST/toolchain/i686
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/sdk/x86_64-nativesdk-libc* $DEST/toolchain/x86_64
@@ -42,165 +42,165 @@ case "$target" in
         ;;
     "qemuarm64")
         mkdir -p $DEST/machines/qemu/qemuarm64
-        md5sums $TMPDIR/deploy/images/qemuarm64
+        sha256sums $TMPDIR/deploy/images/qemuarm64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemuarm64/*qemuarm64* $DEST/machines/qemu/qemuarm64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemuarm64/*Image* $DEST/machines/qemu/qemuarm64
         ;;
     "qemuarm")
         mkdir -p $DEST/machines/qemu/qemuarm
-        md5sums $TMPDIR/deploy/images/qemuarm
+        sha256sums $TMPDIR/deploy/images/qemuarm
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemuarm/*qemuarm* $DEST/machines/qemu/qemuarm
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemuarm/*Image* $DEST/machines/qemu/qemuarm
         ;;
     "beaglebone")
         mkdir -p $DEST/machines/beaglebone-yocto
-        md5sums $TMPDIR/deploy/images/beaglebone-yocto
+        sha256sums $TMPDIR/deploy/images/beaglebone-yocto
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/beaglebone-yocto/*Image* $DEST/machines/beaglebone-yocto
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/beaglebone-yocto/*u-boot* $DEST/machines/beaglebone-yocto
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/beaglebone-yocto/*beaglebone-yocto* $DEST/machines/beaglebone-yocto
         ;;
     "qemuarm-alt")
         mkdir -p $DEST/machines/qemu/qemuarm-lsb
-        md5sums $TMPDIR/deploy/images/qemuarm
+        sha256sums $TMPDIR/deploy/images/qemuarm
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemuarm/*qemuarm* $DEST/machines/qemu/qemuarm-lsb
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemuarm/*Image* $DEST/machines/qemu/qemuarm-lsb
         ;;
     "beaglebone-alt")
         mkdir -p $DEST/machines/beaglebone-yocto-lsb
-        md5sums $TMPDIR/deploy/images/beaglebone-yocto
+        sha256sums $TMPDIR/deploy/images/beaglebone-yocto
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/beaglebone-yocto/*Image* $DEST/machines/beaglebone-yocto-lsb
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/beaglebone-yocto/*u-boot* $DEST/machines/beaglebone-yocto-lsb
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/beaglebone-yocto/*beaglebone-yocto* $DEST/machines/beaglebone-yocto-lsb
         ;;
     "qemumips64")
         mkdir -p $DEST/machines/qemu/qemumips64
-        md5sums $TMPDIR/deploy/images/qemumips64
+        sha256sums $TMPDIR/deploy/images/qemumips64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemumips64/*qemumips64* $DEST/machines/qemu/qemumips64
         ;;
     "qemumips")
         mkdir -p $DEST/machines/qemu/qemumips
-        md5sums $TMPDIR/deploy/images/qemumips
+        sha256sums $TMPDIR/deploy/images/qemumips
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemumips/*qemumips* $DEST/machines/qemu/qemumips
         ;;
     "edgerouter")
         mkdir -p $DEST/machines/edgerouter
-        md5sums $TMPDIR/deploy/images/edgerouter
+        sha256sums $TMPDIR/deploy/images/edgerouter
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/edgerouter/*edgerouter* $DEST/machines/edgerouter
         ;;
     "qemumips-alt")
         mkdir -p $DEST/machines/qemu/qemumips-lsb
-        md5sums $TMPDIR/deploy/images/qemumips
+        sha256sums $TMPDIR/deploy/images/qemumips
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemumips/*qemumips* $DEST/machines/qemu/qemumips-lsb
         ;;
     "edgerouter-alt")
         mkdir -p $DEST/machines/edgerouter-lsb
-        md5sums $TMPDIR/deploy/images/edgerouter
+        sha256sums $TMPDIR/deploy/images/edgerouter
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/edgerouter/*edgerouter* $DEST/machines/edgerouter-lsb
         ;;
     "qemuppc")
         mkdir -p $DEST/machines/qemu/qemuppc
-        md5sums $TMPDIR/deploy/images/qemuppc
+        sha256sums $TMPDIR/deploy/images/qemuppc
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemuppc/*qemuppc* $DEST/machines/qemu/qemuppc
         ;;
     "mpc8315e-rdb")
         mkdir -p $DEST/machines/mpc8315e-rdb
-        md5sums $TMPDIR/deploy/images/mpc8315e-rdb
+        sha256sums $TMPDIR/deploy/images/mpc8315e-rdb
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/mpc8315e-rdb/*mpc8315* $DEST/machines/mpc8315e-rdb
         ;;
     "qemuppc-alt")
         mkdir -p $DEST/machines/qemu/qemuppc-lsb
-        md5sums $TMPDIR/deploy/images/qemuppc
+        sha256sums $TMPDIR/deploy/images/qemuppc
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemuppc/*qemuppc* $DEST/machines/qemu/qemuppc-lsb
         ;;
     "mpc8315e-rdb-alt")
         mkdir -p $DEST/machines/mpc8315e-rdb-lsb
-        md5sums $TMPDIR/deploy/images/mpc8315e-rdb
+        sha256sums $TMPDIR/deploy/images/mpc8315e-rdb
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/mpc8315e-rdb/*mpc8315* $DEST/machines/mpc8315e-rdb-lsb
         ;;
     "wic")
         mkdir -p $DEST/machines/qemu/qemux86
-        md5sums $TMPDIR/deploy/wic_images/qemux86
+        sha256sums $TMPDIR/deploy/wic_images/qemux86
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/qemux86/*/*/*.direct $DEST/machines/qemu/qemux86
-        cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/qemux86/*/*/*.direct.md5sum $DEST/machines/qemu/qemux86
+        cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/qemux86/*/*/*.direct.sha256sum $DEST/machines/qemu/qemux86
         mkdir -p $DEST/machines/genericx86
-        md5sums $TMPDIR/deploy/wic_images/genericx86
+        sha256sums $TMPDIR/deploy/wic_images/genericx86
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/genericx86/*/*/*.direct $DEST/machines/genericx86
-        cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/genericx86/*/*/*.direct.md5sum $DEST/machines/genericx86
+        cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/genericx86/*/*/*.direct.sha256sum $DEST/machines/genericx86
         mkdir -p $DEST/machines/qemu/qemux86-64
-        md5sums $TMPDIR/deploy/wic_images/qemux86-64
+        sha256sums $TMPDIR/deploy/wic_images/qemux86-64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/qemux86-64/*/*/*.direct $DEST/machines/qemu/qemux86-64
-        cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/qemux86-64/*/*/*.direct.md5sum $DEST/machines/qemu/qemux86-64
+        cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/qemux86-64/*/*/*.direct.sha256sum $DEST/machines/qemu/qemux86-64
         mkdir -p $DEST/machines/genericx86-64
-        md5sums $TMPDIR/deploy/wic_images/genericx86-64
+        sha256sums $TMPDIR/deploy/wic_images/genericx86-64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/genericx86-64/*/*/*.direct $DEST/machines/genericx86-64
-        cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/genericx86-64/*/*/*.direct.md5sum $DEST/machines/genericx86-64
+        cp -R --no-dereference --preserve=links $TMPDIR/deploy/wic_images/genericx86-64/*/*/*.direct.sha256sum $DEST/machines/genericx86-64
         ;;
     "qemux86-64")
         mkdir -p $DEST/machines/qemu/qemux86-64
-        md5sums $TMPDIR/deploy/images/qemux86-64
+        sha256sums $TMPDIR/deploy/images/qemux86-64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemux86-64/*qemux86-64* $DEST/machines/qemu/qemux86-64
         ;;
     "genericx86-64")
         mkdir -p $DEST/machines/genericx86-64
-        md5sums $TMPDIR/deploy/images/genericx86-64
+        sha256sums $TMPDIR/deploy/images/genericx86-64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/genericx86-64/*genericx86-64* $DEST/machines/genericx86-64
         ;;
     "qemux86-64-alt")
         mkdir -p $DEST/machines/qemu/qemux86-64-lsb
-        md5sums $TMPDIR/deploy/images/qemux86-64
+        sha256sums $TMPDIR/deploy/images/qemux86-64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemux86-64/*qemux86-64* $DEST/machines/qemu/qemux86-64-lsb
         ;;
     "genericx86-64-alt")
         mkdir -p $DEST/machines/genericx86-64-lsb
-        md5sums $TMPDIR/deploy/images/genericx86-64
+        sha256sums $TMPDIR/deploy/images/genericx86-64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/genericx86-64/*genericx86-64* $DEST/machines/genericx86-64-lsb
         ;;
     "nightly-x86-64-bsp")
         rm -rf $DEST/$target/images/intel-corei7-64/*
         mkdir -p $DEST/$target/images/intel-corei7-64
-        md5sums $TMPDIR/deploy/images/intel-corei7-64
+        sha256sums $TMPDIR/deploy/images/intel-corei7-64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/intel-corei7-64/bzImage* $DEST/$target/images/intel-corei7-64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/intel-corei7-64/*core-image-sato-sdk-intel-corei7-64*tar* $DEST/$target/images/intel-corei7-64
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/intel-corei7-64/*modules-* $DEST/$target/images/intel-corei7-64
         ;;
     "qemux86")
         mkdir -p $DEST/machines/qemu/qemux86
-        md5sums $TMPDIR/deploy/images/qemux86
+        sha256sums $TMPDIR/deploy/images/qemux86
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemux86/*qemux86* $DEST/machines/qemu/qemux86
         ;;
     "genericx86")
         mkdir -p $DEST/machines/genericx86
-        md5sums $TMPDIR/deploy/images/genericx86
+        sha256sums $TMPDIR/deploy/images/genericx86
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/genericx86/*genericx86* $DEST/machines/genericx86
         ;;
     "qemux86-alt")
         mkdir -p $DEST/machines/qemu/qemux86-lsb
-        md5sums $TMPDIR/deploy/images/qemux86
+        sha256sums $TMPDIR/deploy/images/qemux86
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemux86/*qemux86* $DEST/machines/qemu/qemux86-lsb
         ;;
     "genericx86-alt")
         mkdir -p $DEST/machines/genericx86-lsb
-        md5sums $TMPDIR/deploy/images/genericx86
+        sha256sums $TMPDIR/deploy/images/genericx86
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/genericx86/*genericx86* $DEST/machines/genericx86-lsb
         ;;
     "poky-tiny")
         mkdir -p $DEST/machines/qemu/qemu-tiny
-        md5sums $TMPDIR/deploy/images/qemux86
+        sha256sums $TMPDIR/deploy/images/qemux86
         cp -R --no-dereference --preserve=links $TMPDIR/deploy/images/qemux86/*qemux86* $DEST/machines/qemu/qemu-tiny
         ;;
     "eclipse-plugin-neon")
         DIR=$WORKDIR/../scripts
-        # create md5sums only for the zip files
-        for x in `ls $DIR/*.zip`; do md5sum $x >> $x.md5sum; done
+        # create sha256sums only for the zip files
+        for x in `ls $DIR/*.zip`; do sha256sum $x >> $x.sha256sum; done
         mkdir -p $DEST/eclipse-plugin/neon
-        cp --no-dereference --preserve=links $DIR/org.*.zip $DIR/org.*.md5sum $DEST/eclipse-plugin/neon
+        cp --no-dereference --preserve=links $DIR/org.*.zip $DIR/org.*.sha256sum $DEST/eclipse-plugin/neon
         ;;
     "eclipse-plugin-oxygen")
         DIR=$WORKDIR/../scripts
-        # create md5sums only for the zip files
-        for x in `ls $DIR/*.zip`; do md5sum $x >> $x.md5sum; done
+        # create sha256sums only for the zip files
+        for x in `ls $DIR/*.zip`; do sha256sum $x >> $x.sha256sum; done
         mkdir -p $DEST/eclipse-plugin/oxygen
-        cp --no-dereference --preserve=links $DIR/org.*.zip $DIR/org.*.md5sum $DEST/eclipse-plugin/oxygen
+        cp --no-dereference --preserve=links $DIR/org.*.zip $DIR/org.*.sha256sum $DEST/eclipse-plugin/oxygen
         ;;
 esac
 
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 32/41] publish-artefacts: Improve sha256 checksum handling

[Richard Purdie]

From: "Tummalapalli, Vineela" <vineela.tummalapalli@intel.com>

The existing checksum files were encoding full paths. This patch changes
this to use just the filename which allows relocation of the directories
and is more appropriate for sharing over http.

Signed-off-by: Vineela Tummalapalli <vineela.tummalapalli@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 835ac325a4ed3e421a5a9bfd7c75f552ae2103cf)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
---
 scripts/publish-artefacts | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scripts/publish-artefacts b/scripts/publish-artefacts
index 2611cc8..d5cf86c 100755
--- a/scripts/publish-artefacts
+++ b/scripts/publish-artefacts
@@ -5,10 +5,14 @@ target=$3
 
 sha256sums(){
     dest=$1
-    for x in `find $dest -maxdepth 5 -type f`; do
-        if [ ${x##*.} != sha256sum ]; then
-            sha256sum $x >> $x.sha256sum
-        fi
+    for x in `find -L $dest -maxdepth 5 -type f`; do
+        for w in $x;do
+            if [ ${w##*.} != sha256sum ]; then
+                shasum=`sha256sum $w | awk '{print $1}'`
+                filename=${w##*/}
+                echo $shasum $filename >> $w.sha256sum
+            fi
+        done
     done
 }
 
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 33/41] scripts/utils.py: Convert md5sum to sha256sum for layer tarballs

[Richard Purdie]

From: Vineela <vineela.tummalapalli@intel.com>

Signed-off-by: Vineela Tummalapalli <vineela.tummalapalli@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 3aefa5a763125c760813a8f8434c574ecd3ca99b)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
---
 scripts/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/utils.py b/scripts/utils.py
index d7b9ae7..ec43832 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -251,7 +251,7 @@ def publishrepo(clonedir, repo, publishdir):
     revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=sharedrepo).decode('utf-8').strip()
     archive_name = repo + "-" + revision + ".tar.bz2"
     subprocess.check_call("git archive --format=tar HEAD --prefix=" + repo + "/ | bzip2 -c > " + archive_name, shell=True, cwd=sharedrepo)
-    subprocess.check_call("md5sum " + archive_name + " >> " + archive_name + ".md5sum", shell=True, cwd=sharedrepo)
+    subprocess.check_call("sha256sum " + archive_name + " >> " + archive_name + ".sha256sum", shell=True, cwd=sharedrepo)
     mkdir(publishdir)
     subprocess.check_call("rsync -av " + archive_name + "* " + publishdir, shell=True, cwd=sharedrepo)
 
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 34/41] build-perf/send-qa-email: Fallback to master as a comparision branch

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/build-perf-test-wrapper | 3 +++
 scripts/send-qa-email           | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/scripts/build-perf-test-wrapper b/scripts/build-perf-test-wrapper
index e7b24ad..660ef82 100755
--- a/scripts/build-perf-test-wrapper
+++ b/scripts/build-perf-test-wrapper
@@ -185,6 +185,9 @@ if git_repo:
         basebranch, comparebranch = utils.getcomparisonbranch(ourconfig, args.repo, args.branch)
         if comparebranch:
             extraopts = extraopts + " --branch2 %s" % (comparebranch)
+        else:
+            print("No comparision branch found, comparing to master")
+            extraopts = extraopts + " --branch2 master"
 
     print("\nGenerating test report")
     open(report_txt, "w").close()
diff --git a/scripts/send-qa-email b/scripts/send-qa-email
index a265a7c..b4d4cec 100755
--- a/scripts/send-qa-email
+++ b/scripts/send-qa-email
@@ -54,6 +54,9 @@ if 'poky' in repos and os.path.exists(resulttool) and args.results_dir:
         extraopts = " --branch %s --commit %s" % (branch, revision)
     if comparebranch:
         extraopts = extraopts + " --branch2 %s" % (comparebranch)
+    elif basebranch:
+        print("No comparision branch found, comparing to master")
+        extraopts = extraopts + " --branch2 master"
 
     report = subprocess.check_output([resulttool, "report", args.results_dir])
     with open(args.results_dir + "/testresult-report.txt", "wb") as f:
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 35/41] send-qa-email: Fix stable branch test result history issues

[Richard Purdie]

On stable release branch builds, it will clone "master" since comparebranch
is None. It will then fail to push since there is already a dunfell
branch upstream with changes on which it needs to rebase on top of.
The change to use shallow clones significantly increases this problem.

To fix this, fall back to cloning basebranch by name, then master
if it doesn't exist.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/send-qa-email | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/scripts/send-qa-email b/scripts/send-qa-email
index b4d4cec..abc17e2 100755
--- a/scripts/send-qa-email
+++ b/scripts/send-qa-email
@@ -55,8 +55,8 @@ if 'poky' in repos and os.path.exists(resulttool) and args.results_dir:
     if comparebranch:
         extraopts = extraopts + " --branch2 %s" % (comparebranch)
     elif basebranch:
-        print("No comparision branch found, comparing to master")
-        extraopts = extraopts + " --branch2 master"
+        print("No comparision branch found, comparing to %s" % basebranch)
+        extraopts = extraopts + " --branch2 %s" % basebranch
 
     report = subprocess.check_output([resulttool, "report", args.results_dir])
     with open(args.results_dir + "/testresult-report.txt", "wb") as f:
@@ -64,7 +64,16 @@ if 'poky' in repos and os.path.exists(resulttool) and args.results_dir:
 
     tempdir = tempfile.mkdtemp(prefix='sendqaemail.')
     try:
-        subprocess.check_call(["git", "clone", "git@git.yoctoproject.org:yocto-testresults", tempdir, "--depth", "5"])
+        cloneopts = []
+        if comparebranch:
+            cloneopts = ["--branch", comparebranch]
+        elif basebranch:
+            cloneopts = ["--branch", basebranch]
+        try:
+            subprocess.check_call(["git", "clone", "git@git.yoctoproject.org:yocto-testresults", tempdir, "--depth", "5"] + cloneopts)
+        except subprocess.CalledProcessError:
+            print("No comparision branch found, falling back to master")
+            subprocess.check_call(["git", "clone", "git@git.yoctoproject.org:yocto-testresults", tempdir, "--depth", "5"])
 
         # If the base comparision branch isn't present regression comparision won't work
         # at least until we can tell the tool to ignore internal branch information
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 36/41] scripts/utils: Add more logging for fetchgitrepo

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scripts/utils.py b/scripts/utils.py
index ec43832..1dfc9a2 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -232,13 +232,17 @@ def fetchgitrepo(clonedir, repo, params, stashdir):
     sharedrepo = "%s/%s" % (clonedir, repo)
     branch = params["branch"]
     revision = params["revision"]
+    print("Checking for stash at: " + stashdir + "/" + repo)
     if os.path.exists(stashdir + "/" + repo):
+        print("Cloning from stash to %s..." % sharedrepo)
         subprocess.check_call(["git", "clone", "file://%s/%s" % (stashdir, repo), "%s/%s" % (clonedir, repo)])
         subprocess.check_call(["git", "remote", "rm", "origin"], cwd=sharedrepo)
         subprocess.check_call(["git", "remote", "add", "origin", params["url"]], cwd=sharedrepo)
+        print("Updating from origin...")
         subprocess.check_call(["git", "fetch", "origin"], cwd=sharedrepo)
         subprocess.check_call(["git", "fetch", "origin", "-t"], cwd=sharedrepo)
     else:
+        print("Cloning from origin to %s..." % sharedrepo)
         subprocess.check_call(["git", "clone", params["url"], sharedrepo])
 
     subprocess.check_call(["git", "checkout", branch], cwd=sharedrepo)
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 37/41] scripts/utils: Ensuring new logging is flushed from buffers

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/utils.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/utils.py b/scripts/utils.py
index 1dfc9a2..10e3cc1 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -233,18 +233,24 @@ def fetchgitrepo(clonedir, repo, params, stashdir):
     branch = params["branch"]
     revision = params["revision"]
     print("Checking for stash at: " + stashdir + "/" + repo)
+    flush()
     if os.path.exists(stashdir + "/" + repo):
         print("Cloning from stash to %s..." % sharedrepo)
+        flush()
         subprocess.check_call(["git", "clone", "file://%s/%s" % (stashdir, repo), "%s/%s" % (clonedir, repo)])
         subprocess.check_call(["git", "remote", "rm", "origin"], cwd=sharedrepo)
         subprocess.check_call(["git", "remote", "add", "origin", params["url"]], cwd=sharedrepo)
         print("Updating from origin...")
+        flush()
         subprocess.check_call(["git", "fetch", "origin"], cwd=sharedrepo)
         subprocess.check_call(["git", "fetch", "origin", "-t"], cwd=sharedrepo)
     else:
         print("Cloning from origin to %s..." % sharedrepo)
+        flush()
         subprocess.check_call(["git", "clone", params["url"], sharedrepo])
 
+    print("Updating checkout...")
+    flush()
     subprocess.check_call(["git", "checkout", branch], cwd=sharedrepo)
     # git reset revision==HEAD won't help, we need to reset onto the potentially fetched origin branch
     subprocess.check_call(["git", "reset", "origin/" + branch, "--hard"], cwd=sharedrepo)
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 38/41] scripts/utils: Ensure buildtools tarball hash matches any supplied value

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/utils.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/scripts/utils.py b/scripts/utils.py
index 10e3cc1..46346fb 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -378,6 +378,25 @@ def getcomparisonbranch(ourconfig, reponame, branchname):
         return branchname, None
     return None, None
 
+def sha256_file(filename):
+    """
+    Return the hex string representation of the 256-bit SHA checksum of
+    filename.
+    """
+    import hashlib
+    import mmap
+
+    method = hashlib.sha256()
+    with open(filename, "rb") as f:
+        try:
+            with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
+                for chunk in iter(lambda: mm.read(8192), b''):
+                    method.update(chunk)
+        except ValueError:
+            # You can't mmap() an empty file so silence this exception
+            pass
+    return method.hexdigest()
+
 def setup_buildtools_tarball(ourconfig, workername, btdir):
     bttarball = None
     if "buildtools" in ourconfig and workername:
@@ -389,6 +408,9 @@ def setup_buildtools_tarball(ourconfig, workername, btdir):
 
     btenv = None
     if bttarball:
+        sha256 = None
+        if ";" in bttarball:
+            bttarball, sha256 = bttarball.split(";")
         btdir = os.path.abspath(btdir)
         if not os.path.exists(btdir):
             btdlpath = getconfig("BASE_SHAREDDIR", ourconfig) + "/buildtools/" + os.path.basename(bttarball)
@@ -401,6 +423,10 @@ def setup_buildtools_tarball(ourconfig, workername, btdir):
                     with open(btlock, 'a+') as lf:
                         fileno = lf.fileno()
                         fcntl.flock(fileno, fcntl.LOCK_EX)
+                        if sha256 and os.path.exists(btdlpath):
+                            dl_sha256 = sha256_file(btdlpath)
+                            if dl_sha256 != sha256:
+                                os.unlink(btdlpath)
                         if not os.path.exists(btdlpath):
                             if bttarball.startswith("/"):
                                 subprocess.check_call(["cp", bttarball, btdlpath])
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 39/41] config.json: Build buildtools-extended-tarball for x86_64 and aarch64

[Richard Purdie]

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config.json b/config.json
index fbf262b..c0cf55b 100644
--- a/config.json
+++ b/config.json
@@ -524,7 +524,7 @@
             "MACHINE" : "qemux86-64",
             "step1" : {
                 "SDKMACHINE" : "x86_64",
-                "BBTARGETS" : "buildtools-tarball uninative-tarball"
+                "BBTARGETS" : "buildtools-tarball buildtools-extended-tarball uninative-tarball"
             },
             "step2" : {
                 "SDKMACHINE" : "i686",
@@ -532,7 +532,7 @@
             },
             "step3" : {
                 "SDKMACHINE" : "aarch64",
-                "BBTARGETS" : "uninative-tarball"
+                "BBTARGETS" : "buildtools-tarball buildtools-extended-tarball uninative-tarball"
             }
         },
         "build-appliance" : {
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 40/41] scripts/send-qa-email: Update to dedicated push capable domain

[Richard Purdie]

From: Michael Halstead <mhalstead@linuxfoundation.org>

Signed-off-by: Michael Halstead <mhalstead@linuxfoundation.org>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 scripts/send-qa-email | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/send-qa-email b/scripts/send-qa-email
index abc17e2..f38c876 100755
--- a/scripts/send-qa-email
+++ b/scripts/send-qa-email
@@ -70,10 +70,10 @@ if 'poky' in repos and os.path.exists(resulttool) and args.results_dir:
         elif basebranch:
             cloneopts = ["--branch", basebranch]
         try:
-            subprocess.check_call(["git", "clone", "git@git.yoctoproject.org:yocto-testresults", tempdir, "--depth", "5"] + cloneopts)
+            subprocess.check_call(["git", "clone", "git@push.yoctoproject.org:yocto-testresults", tempdir, "--depth", "5"] + cloneopts)
         except subprocess.CalledProcessError:
             print("No comparision branch found, falling back to master")
-            subprocess.check_call(["git", "clone", "git@git.yoctoproject.org:yocto-testresults", tempdir, "--depth", "5"])
+            subprocess.check_call(["git", "clone", "git@push.yoctoproject.org:yocto-testresults", tempdir, "--depth", "5"])
 
         # If the base comparision branch isn't present regression comparision won't work
         # at least until we can tell the tool to ignore internal branch information
-- 
2.25.1

[PATCH][autobuilder-helper][warrior 41/41] config.json: Update buildtools config

[Richard Purdie]

Need to use buildtools on centos8 and fedora32 as they lack makeinfo
as part of the default install now.

Also use buildtools to standard and extended as appropriate.

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
---
 config.json | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/config.json b/config.json
index c0cf55b..dac5374 100644
--- a/config.json
+++ b/config.json
@@ -8,9 +8,8 @@
     "BUILD_HISTORY_DIRECTPUSH" : ["poky:morty", "poky:pyro", "poky:rocko", "poky:sumo", "poky:thud", "poky:warrior", "poky:zeus", "poky:master"],
     "BUILD_HISTORY_FORKPUSH" : {"poky-contrib:ross/mut" : "poky:master", "poky:master-next" : "poky:master"},
 
-    "BUILDTOOLS_URL_TEMPLOCAL" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200309-15/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200309.sh",
-    "BUILDTOOLS_URL_TEMPLOCAL2" : "http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.1_M3/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0+snapshot-20200315.sh",
-    "BUILDTOOLS_URL" : "http://downloads.yoctoproject.org/releases/yocto/milestones/yocto-3.2_M1/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.1+snapshot-20200617.sh",
+    "BUILDTOOLS_URL" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200907-11/buildtools/x86_64-buildtools-nativesdk-standalone-3.0.4.sh;f0c06b2cfd2f5d92ef5fb592f4d726f9ad270d83d7846f817a08abfd017d159d",
+    "BUILDTOOLS_URL_EXT" : "/srv/autobuilder/autobuilder.yoctoproject.org/pub/non-release/20200907-11/buildtools/x86_64-buildtools-extended-nativesdk-standalone-3.0.4.sh;a5fea6b64109a604f3718dfb2a7b5ad32402d13ed5472d9d791e7bf9900621bd",
 
     "REPO_STASH_DIR" : "${BASE_HOMEDIR}/git/mirror",
     "TRASH_DIR" : "${BASE_HOMEDIR}/git/trash",
@@ -876,6 +875,11 @@
     "buildtools" : {
         "debian8*" : "${BUILDTOOLS_URL}",
         "centos7*" : "${BUILDTOOLS_URL}",
+        "centos8*" : "${BUILDTOOLS_URL_EXT}",
+        "debian10*" : "${BUILDTOOLS_URL_EXT}",
+        "tumbleweed*" : "${BUILDTOOLS_URL_EXT}",
+        "fedora31*" : "${BUILDTOOLS_URL_EXT}",
+        "fedora32*" : "${BUILDTOOLS_URL_EXT}",
         "perf-centos7*" : "${BUILDTOOLS_URL}"
     }
 }
-- 
2.25.1