Re: [RFC PATCH] toaster: support authentication required proxy connection

Re: [RFC PATCH] toaster: support authentication required proxy connection

[Damian, Alexandru]

[-- Attachment #1: Type: text/plain, Size: 4526 bytes --]

Hello,

Thank you for sending this patch. There are several aspects that, I think,
need clarification.

* Can you please expand the commit message, explaining a bit better the
feature that you add, and how it can be tested ?

* Can you please look into using urllib2 to handle the connections, as this
library has better support for authenticated proxies, probably better than
Httplib2 ?

* I am not sure that reading the nameserver list is needed - the proxy_rdns
needs to be either True or False, not a nameserver value.


Thank you for taking an interest in Toaster !

Cheers,
Alex

On Thu, Aug 20, 2015 at 11:18 AM, Bian Naimeng <biannm@cn.fujitsu.com>
wrote:

> Make sure layersource can be downloaded from authentication required proxy
> connection.
>
> Signed-off-by: Bian Naimeng <biannm@cn.fujitsu.com>
> ---
>  lib/toaster/orm/models.py | 58
> ++++++++++++++++++++++++++---------------------
>  1 file changed, 32 insertions(+), 26 deletions(-)
>
> diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py
> index 92fcaa7..8371d9b 100644
> --- a/lib/toaster/orm/models.py
> +++ b/lib/toaster/orm/models.py
> @@ -817,7 +817,7 @@ class LayerIndexLayerSource(LayerSource):
>          assert self.apiurl is not None
>          from django.db import transaction, connection
>
> -        import httplib, urlparse, json
> +        import httplib2, urlparse, json
>          import os
>          proxy_settings = os.environ.get("http_proxy", None)
>
> @@ -825,34 +825,40 @@ class LayerIndexLayerSource(LayerSource):
>              conn = None
>              _parsedurl = urlparse.urlparse(apiurl)
>              path = _parsedurl.path
> -            query = _parsedurl.query
> -            def parse_url(url):
> -                parsedurl = urlparse.urlparse(url)
> +
> +            def get_nameserver_ip():
> +                from os import path
> +                # Get Nameserver from /etc/resolv.conf
> +                RESOLV_CONF="/etc/resolv.conf"
> +                if not path.isfile(RESOLV_CONF):
> +                    logger.info("%s does not exist" % RESOLV_CONF)
> +                    return None
> +
> +                fh = None
>                  try:
> -                    (host, port) = parsedurl.netloc.split(":")
> -                except ValueError:
> -                    host = parsedurl.netloc
> -                    port = None
> +                    fh = open(RESOLV_CONF, 'r')
> +                    for line in fh:
> +                        if not line.startswith('nameserver'):
> +                            continue
> +                        return line.strip().split()[-1]
> +                finally:
> +                    if fh is not None:
> +                        fh.close()
> +                return None
> +
> +            pi = None
> +            if proxy_settings is not None:
> +                pi = httplib2.proxy_info_from_environment()
> +                rdns = get_nameserver_ip()
> +                if rdns is not None:
> +                    pi.proxy_rdns = rdns
>
> -                if port is None:
> -                    port = 80
> -                else:
> -                    port = int(port)
> -                return (host, port)
> +            httpcon = httplib2.Http(proxy_info=pi)
> +            res, content = httpcon.request(apiurl, 'GET')
>
> -            if proxy_settings is None:
> -                host, port = parse_url(apiurl)
> -                conn = httplib.HTTPConnection(host, port)
> -                conn.request("GET", path + "?" + query)
> -            else:
> -                host, port = parse_url(proxy_settings)
> -                conn = httplib.HTTPConnection(host, port)
> -                conn.request("GET", apiurl)
> -
> -            r = conn.getresponse()
> -            if r.status != 200:
> -                raise Exception("Failed to read " + path + ": %d %s" %
> (r.status, r.reason))
> -            return json.loads(r.read())
> +            if res.status != 200:
> +                raise Exception("Failed to read " + path + ": %d %s" %
> (res.status, res.reason))
> +            return json.loads(content)
>
>          # verify we can get the basic api
>          try:
> --
> 1.9.1
>
> --
> _______________________________________________
> toaster mailing list
> toaster@yoctoproject.org
> https://lists.yoctoproject.org/listinfo/toaster
>



-- 
Alex Damian
Yocto Project
SSG / OTC

[-- Attachment #2: Type: text/html, Size: 6563 bytes --]

[RFC PATCH] toaster: support authentication required proxy connection

[Bian Naimeng]

Make sure layersource can be downloaded from authentication required proxy connection.

Signed-off-by: Bian Naimeng <biannm@cn.fujitsu.com>
---
 lib/toaster/orm/models.py | 58 ++++++++++++++++++++++++++---------------------
 1 file changed, 32 insertions(+), 26 deletions(-)

diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py
index 92fcaa7..8371d9b 100644
--- a/lib/toaster/orm/models.py
+++ b/lib/toaster/orm/models.py
@@ -817,7 +817,7 @@ class LayerIndexLayerSource(LayerSource):
         assert self.apiurl is not None
         from django.db import transaction, connection
 
-        import httplib, urlparse, json
+        import httplib2, urlparse, json
         import os
         proxy_settings = os.environ.get("http_proxy", None)
 
@@ -825,34 +825,40 @@ class LayerIndexLayerSource(LayerSource):
             conn = None
             _parsedurl = urlparse.urlparse(apiurl)
             path = _parsedurl.path
-            query = _parsedurl.query
-            def parse_url(url):
-                parsedurl = urlparse.urlparse(url)
+
+            def get_nameserver_ip():
+                from os import path
+                # Get Nameserver from /etc/resolv.conf
+                RESOLV_CONF="/etc/resolv.conf"
+                if not path.isfile(RESOLV_CONF):
+                    logger.info("%s does not exist" % RESOLV_CONF)
+                    return None
+
+                fh = None
                 try:
-                    (host, port) = parsedurl.netloc.split(":")
-                except ValueError:
-                    host = parsedurl.netloc
-                    port = None
+                    fh = open(RESOLV_CONF, 'r')
+                    for line in fh:
+                        if not line.startswith('nameserver'):
+                            continue
+                        return line.strip().split()[-1]
+                finally:
+                    if fh is not None:
+                        fh.close()
+                return None
+
+            pi = None
+            if proxy_settings is not None:
+                pi = httplib2.proxy_info_from_environment()
+                rdns = get_nameserver_ip()
+                if rdns is not None:
+                    pi.proxy_rdns = rdns
 
-                if port is None:
-                    port = 80
-                else:
-                    port = int(port)
-                return (host, port)
+            httpcon = httplib2.Http(proxy_info=pi)
+            res, content = httpcon.request(apiurl, 'GET')
 
-            if proxy_settings is None:
-                host, port = parse_url(apiurl)
-                conn = httplib.HTTPConnection(host, port)
-                conn.request("GET", path + "?" + query)
-            else:
-                host, port = parse_url(proxy_settings)
-                conn = httplib.HTTPConnection(host, port)
-                conn.request("GET", apiurl)
-
-            r = conn.getresponse()
-            if r.status != 200:
-                raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
-            return json.loads(r.read())
+            if res.status != 200:
+                raise Exception("Failed to read " + path + ": %d %s" % (res.status, res.reason))
+            return json.loads(content)
 
         # verify we can get the basic api
         try:
-- 
1.9.1

Re: [RFC PATCH] toaster: support authentication required proxy connection

[Bian, Naimeng]

Thanks for your comments.
I will send V2 path.

Thanks
 Bian

From: Damian, Alexandru [mailto:alexandru.damian@intel.com] 
Sent: Thursday, August 20, 2015 6:04 PM
Cc: toaster@yoctoproject.org
Subject: Re: [Toaster] [RFC PATCH] toaster: support authentication required proxy connection

Hello,
Thank you for sending this patch. There are several aspects that, I think, need clarification.
* Can you please expand the commit message, explaining a bit better the feature that you add, and how it can be tested ?

* Can you please look into using urllib2 to handle the connections, as this library has better support for authenticated proxies, probably better than Httplib2 ?
* I am not sure that reading the nameserver list is needed - the proxy_rdns needs to be either True or False, not a nameserver value.


Thank you for taking an interest in Toaster !
Cheers,
Alex

On Thu, Aug 20, 2015 at 11:18 AM, Bian Naimeng <biannm@cn.fujitsu.com> wrote:
Make sure layersource can be downloaded from authentication required proxy connection.

Signed-off-by: Bian Naimeng <biannm@cn.fujitsu.com>
---
 lib/toaster/orm/models.py | 58 ++++++++++++++++++++++++++---------------------
 1 file changed, 32 insertions(+), 26 deletions(-)

diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py
index 92fcaa7..8371d9b 100644
--- a/lib/toaster/orm/models.py
+++ b/lib/toaster/orm/models.py
@@ -817,7 +817,7 @@ class LayerIndexLayerSource(LayerSource):
         assert self.apiurl is not None
         from django.db import transaction, connection

-        import httplib, urlparse, json
+        import httplib2, urlparse, json
         import os
         proxy_settings = os.environ.get("http_proxy", None)

@@ -825,34 +825,40 @@ class LayerIndexLayerSource(LayerSource):
             conn = None
             _parsedurl = urlparse.urlparse(apiurl)
             path = _parsedurl.path
-            query = _parsedurl.query
-            def parse_url(url):
-                parsedurl = urlparse.urlparse(url)
+
+            def get_nameserver_ip():
+                from os import path
+                # Get Nameserver from /etc/resolv.conf
+                RESOLV_CONF="/etc/resolv.conf"
+                if not path.isfile(RESOLV_CONF):
+                    logger.info("%s does not exist" % RESOLV_CONF)
+                    return None
+
+                fh = None
                 try:
-                    (host, port) = parsedurl.netloc.split(":")
-                except ValueError:
-                    host = parsedurl.netloc
-                    port = None
+                    fh = open(RESOLV_CONF, 'r')
+                    for line in fh:
+                        if not line.startswith('nameserver'):
+                            continue
+                        return line.strip().split()[-1]
+                finally:
+                    if fh is not None:
+                        fh.close()
+                return None
+
+            pi = None
+            if proxy_settings is not None:
+                pi = httplib2.proxy_info_from_environment()
+                rdns = get_nameserver_ip()
+                if rdns is not None:
+                    pi.proxy_rdns = rdns

-                if port is None:
-                    port = 80
-                else:
-                    port = int(port)
-                return (host, port)
+            httpcon = httplib2.Http(proxy_info=pi)
+            res, content = httpcon.request(apiurl, 'GET')

-            if proxy_settings is None:
-                host, port = parse_url(apiurl)
-                conn = httplib.HTTPConnection(host, port)
-                conn.request("GET", path + "?" + query)
-            else:
-                host, port = parse_url(proxy_settings)
-                conn = httplib.HTTPConnection(host, port)
-                conn.request("GET", apiurl)
-
-            r = conn.getresponse()
-            if r.status != 200:
-                raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
-            return json.loads(r.read())
+            if res.status != 200:
+                raise Exception("Failed to read " + path + ": %d %s" % (res.status, res.reason))
+            return json.loads(content)

         # verify we can get the basic api
         try:
--
1.9.1

--
_______________________________________________
toaster mailing list
toaster@yoctoproject.org
https://lists.yoctoproject.org/listinfo/toaster



-- 
Alex Damian
Yocto Project
SSG / OTC