[OE-core][PATCH v2 REPOST 1/2] bitbake: fetch2/wget: checkstatus: cleanup URLError exception

[OE-core][PATCH v2 REPOST 1/2] bitbake: fetch2/wget: checkstatus: cleanup URLError exception

[Nicola Lunghi]

The exception for urllib.error.URLError can only happen on the

  with opener.open(r) as response:

line.
Limit the try except span to this line and move the retry logic of
commit 312f1a5e741 outside of the try/except catch

see also commit:
  312f1a5e741: bitbake: fetchption: attempt checkstatus again if it fails

Signed-off-by: Nicola Lunghi <nick83ola@gmail.com>
---
 bitbake/lib/bb/fetch2/wget.py | 60 +++++++++++++++++------------------
 1 file changed, 29 insertions(+), 31 deletions(-)

diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index 725586d2b5..5aef55a9c6 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -295,42 +295,40 @@ class Wget(FetchMethod):
             handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context()))
         opener = urllib.request.build_opener(*handlers)
 
-        try:
-            uri = ud.url.split(";")[0]
-            r = urllib.request.Request(uri)
-            r.get_method = lambda: "HEAD"
-            # Some servers (FusionForge, as used on Alioth) require that the
-            # optional Accept header is set.
-            r.add_header("Accept", "*/*")
-            def add_basic_auth(login_str, request):
-                '''Adds Basic auth to http request, pass in login:password as string'''
-                import base64
-                encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
-                authheader = "Basic %s" % encodeuser
-                r.add_header("Authorization", authheader)
-
-            if ud.user and ud.pswd:
-                add_basic_auth(ud.user + ':' + ud.pswd, r)
+        uri = ud.url.split(";")[0]
+        r = urllib.request.Request(uri)
+        r.get_method = lambda: "HEAD"
+        # Some servers (FusionForge, as used on Alioth) require that the
+        # optional Accept header is set.
+        r.add_header("Accept", "*/*")
+        def add_basic_auth(login_str, request):
+            '''Adds Basic auth to http request, pass in login:password as string'''
+            import base64
+            encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
+            authheader = "Basic %s" % encodeuser
+            r.add_header("Authorization", authheader)
 
-            try:
-                import netrc
-                n = netrc.netrc()
-                login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
-                add_basic_auth("%s:%s" % (login, password), r)
-            except (TypeError, ImportError, IOError, netrc.NetrcParseError):
-                pass
+        if ud.user and ud.pswd:
+            add_basic_auth(ud.user + ':' + ud.pswd, r)
 
+        try:
+            import netrc
+            n = netrc.netrc()
+            login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
+            add_basic_auth("%s:%s" % (login, password), r)
+        except (TypeError, ImportError, IOError, netrc.NetrcParseError):
+            pass
+
+        try:
             with opener.open(r) as response:
                 pass
+            return True
         except urllib.error.URLError as e:
-            if try_again:
-                logger.debug(2, "checkstatus: trying again")
-                return self.checkstatus(fetch, ud, d, False)
-            else:
-                # debug for now to avoid spamming the logs in e.g. remote sstate searches
-                logger.debug(2, "checkstatus() urlopen failed: %s" % e)
-                return False
-        return True
+            logger.debug(2, "checkstatus() urlopen failed: %s" % e)
+        if try_again:
+            logger.debug(2, "checkstatus: trying again")
+            return self.checkstatus(fetch, ud, d, False)
+        return False
 
     def _parse_path(self, regex, s):
         """
-- 
2.20.1

[OE-core][PATCH v2 REPOST 2/2] bitbake: fetch2/wget: checkstatus: use the same arguments as parent

[Nicola Lunghi]

the commit:
  312f1a5e741: bitbake: fetchption: attempt checkstatus again if it fails

was introducing an extra parameter to the checkstatus function, overriding
the parent.

Avoid that and remove recursion as is not needed.
Also introduce a small delay between the two requests.

Signed-off-by: Nicola Lunghi <nick83ola@gmail.com>
---
 bitbake/lib/bb/fetch2/wget.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index 5aef55a9c6..67b4d58924 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -121,7 +121,7 @@ class Wget(FetchMethod):
 
         return True
 
-    def checkstatus(self, fetch, ud, d, try_again=True):
+    def checkstatus(self, fetch, ud, d):
         class HTTPConnectionCache(http.client.HTTPConnection):
             if fetch.connection_cache:
                 def connect(self):
@@ -324,10 +324,21 @@ class Wget(FetchMethod):
                 pass
             return True
         except urllib.error.URLError as e:
-            logger.debug(2, "checkstatus() urlopen failed: %s" % e)
-        if try_again:
-            logger.debug(2, "checkstatus: trying again")
-            return self.checkstatus(fetch, ud, d, False)
+            logger.debug(2, "checkstatus() urlopen failed: %s", e)
+
+        # Some services such as SourceForge seem to struggle to keep up under
+        # load, with the result that over half of the autobuilder checkuri
+        # runs fail with sourceforge.net "connection timed out". Attempt to
+        # mitigate this by re-attempting once the network operation on failure.
+        import time
+        time.sleep(0.2)
+        logger.debug(2, "checkstatus: trying again")
+        try:
+            with opener.open(r) as response:
+                pass
+            return True
+        except urllib.error.URLError as e:
+            logger.debug(2, "checkstatus() urlopen failed for the second time: %s", e)
         return False
 
     def _parse_path(self, regex, s):
-- 
2.20.1