All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH] distrodata: Take account proxies on distrodata tasks
@ 2015-09-02  8:59 leonardo.sandoval.gonzalez
  2015-09-02 18:09 ` Aníbal Limón
  2015-09-30 15:16 ` Leonardo Sandoval
  0 siblings, 2 replies; 4+ messages in thread
From: leonardo.sandoval.gonzalez @ 2015-09-02  8:59 UTC (permalink / raw)
  To: openembedded-core

From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>

Proxies defined in the enviroment where not taken into account
on the distrodata tasks. This commit implied passing the datastore
into the distro_check library and context manager for the
urllib.urlopen function.

One way to run distrodata tasks is using 'universe' as target and the
'all' distrodata task:

      $ bitbake universe -c distrodataall
      $ bitbake universe -c distro_checkall
      $ bitbake universe -c checklicenseall

Logs are located under TMPDIR/log

[YOCTO #7567]

Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
---
 meta/classes/distrodata.bbclass |   6 +--
 meta/lib/oe/distro_check.py     | 113 +++++++++++++++++++++++-----------------
 2 files changed, 69 insertions(+), 50 deletions(-)

diff --git a/meta/classes/distrodata.bbclass b/meta/classes/distrodata.bbclass
index 4168e43..0a706ef 100644
--- a/meta/classes/distrodata.bbclass
+++ b/meta/classes/distrodata.bbclass
@@ -33,7 +33,7 @@ python do_distrodata_np() {
         tmpdir = d.getVar('TMPDIR', True)
         distro_check_dir = os.path.join(tmpdir, "distro_check")
         datetime = localdata.getVar('DATETIME', True)
-        dist_check.update_distro_data(distro_check_dir, datetime)
+        dist_check.update_distro_data(distro_check_dir, datetime, localdata)
 
         if pn.find("-native") != -1:
             pnstripped = pn.split("-native")
@@ -118,7 +118,7 @@ python do_distrodata() {
         tmpdir = d.getVar('TMPDIR', True)
         distro_check_dir = os.path.join(tmpdir, "distro_check")
         datetime = localdata.getVar('DATETIME', True)
-        dist_check.update_distro_data(distro_check_dir, datetime)
+        dist_check.update_distro_data(distro_check_dir, datetime, localdata)
 
         pn = d.getVar("PN", True)
         bb.note("Package Name: %s" % pn)
@@ -402,7 +402,7 @@ python do_distro_check() {
     bb.utils.mkdirhier(logpath)
     result_file = os.path.join(logpath, "distrocheck.csv")
     datetime = localdata.getVar('DATETIME', True)
-    dc.update_distro_data(distro_check_dir, datetime)
+    dc.update_distro_data(distro_check_dir, datetime, localdata)
 
     # do the comparison
     result = dc.compare_in_distro_packages_list(distro_check_dir, d)
diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py
index 8ed5b0e..b3419ce 100644
--- a/meta/lib/oe/distro_check.py
+++ b/meta/lib/oe/distro_check.py
@@ -1,7 +1,28 @@
-def get_links_from_url(url):
+from contextlib import contextmanager
+@contextmanager
+def create_socket(url, d):
+    import urllib
+    socket = urllib.urlopen(url, proxies=get_proxies(d))
+    try:
+        yield socket
+    finally:
+        socket.close()
+
+def get_proxies(d):
+    import os
+    proxykeys = ['HTTP_PROXY', 'http_proxy',
+                 'HTTPS_PROXY', 'https_proxy',
+                 'FTP_PROXY', 'ftp_proxy',
+                 'FTPS_PROXY', 'ftps_proxy',
+                 'NO_PROXY', 'no_proxy',
+                 'ALL_PROXY', 'all_proxy']
+    proxyvalues = map(lambda key: d.getVar(key, True), proxykeys)
+    return dict(zip(proxykeys, proxyvalues))
+
+def get_links_from_url(url, d):
     "Return all the href links found on the web location"
 
-    import urllib, sgmllib
+    import sgmllib
     
     class LinksParser(sgmllib.SGMLParser):
         def parse(self, s):
@@ -24,19 +45,18 @@ def get_links_from_url(url):
             "Return the list of hyperlinks."
             return self.hyperlinks
 
-    sock = urllib.urlopen(url)
-    webpage = sock.read()
-    sock.close()
+    with create_socket(url,d) as sock:
+        webpage = sock.read()
 
     linksparser = LinksParser()
     linksparser.parse(webpage)
     return linksparser.get_hyperlinks()
 
-def find_latest_numeric_release(url):
+def find_latest_numeric_release(url, d):
     "Find the latest listed numeric release on the given url"
     max=0
     maxstr=""
-    for link in get_links_from_url(url):
+    for link in get_links_from_url(url, d):
         try:
             release = float(link)
         except:
@@ -70,7 +90,7 @@ def clean_package_list(package_list):
     return set.keys()
 
 
-def get_latest_released_meego_source_package_list():
+def get_latest_released_meego_source_package_list(d):
     "Returns list of all the name os packages in the latest meego distro"
 
     package_names = []
@@ -82,11 +102,11 @@ def get_latest_released_meego_source_package_list():
     package_list=clean_package_list(package_names)
     return "1.0", package_list
 
-def get_source_package_list_from_url(url, section):
+def get_source_package_list_from_url(url, section, d):
     "Return a sectioned list of package names from a URL list"
 
     bb.note("Reading %s: %s" % (url, section))
-    links = get_links_from_url(url)
+    links = get_links_from_url(url, d)
     srpms = filter(is_src_rpm, links)
     names_list = map(package_name_from_srpm, srpms)
 
@@ -96,44 +116,44 @@ def get_source_package_list_from_url(url, section):
 
     return new_pkgs
 
-def get_latest_released_fedora_source_package_list():
+def get_latest_released_fedora_source_package_list(d):
     "Returns list of all the name os packages in the latest fedora distro"
-    latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/")
+    latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d)
 
-    package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main")
+    package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main", d)
 
 #    package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything")
-    package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates")
+    package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d)
 
     package_list=clean_package_list(package_names)
         
     return latest, package_list
 
-def get_latest_released_opensuse_source_package_list():
+def get_latest_released_opensuse_source_package_list(d):
     "Returns list of all the name os packages in the latest opensuse distro"
-    latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/")
+    latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d)
 
-    package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main")
-    package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates")
+    package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d)
+    package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates", d)
 
     package_list=clean_package_list(package_names)
     return latest, package_list
 
-def get_latest_released_mandriva_source_package_list():
+def get_latest_released_mandriva_source_package_list(d):
     "Returns list of all the name os packages in the latest mandriva distro"
-    latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/")
-    package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main")
+    latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d)
+    package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d)
 #    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib")
-    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates")
+    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d)
 
     package_list=clean_package_list(package_names)
     return latest, package_list
 
-def find_latest_debian_release(url):
+def find_latest_debian_release(url, d):
     "Find the latest listed debian release on the given url"
 
     releases = []
-    for link in get_links_from_url(url):
+    for link in get_links_from_url(url, d):
         if link[:6] == "Debian":
             if ';' not in link:
                 releases.append(link)
@@ -143,16 +163,15 @@ def find_latest_debian_release(url):
     except:
         return "_NotFound_"
 
-def get_debian_style_source_package_list(url, section):
+def get_debian_style_source_package_list(url, section, d):
     "Return the list of package-names stored in the debian style Sources.gz file"
-    import urllib
-    sock = urllib.urlopen(url)
-    import tempfile
-    tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
-    tmpfilename=tmpfile.name
-    tmpfile.write(sock.read())
-    sock.close()
-    tmpfile.close()
+    with create_socket(url,d) as sock:
+        webpage = sock.read()
+        import tempfile
+        tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
+        tmpfilename=tmpfile.name
+        tmpfile.write(sock.read())
+        tmpfile.close()
     import gzip
     bb.note("Reading %s: %s" % (url, section))
 
@@ -165,41 +184,41 @@ def get_debian_style_source_package_list(url, section):
 
     return package_names
 
-def get_latest_released_debian_source_package_list():
+def get_latest_released_debian_source_package_list(d):
     "Returns list of all the name os packages in the latest debian distro"
-    latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/")
+    latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d)
     url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz" 
-    package_names = get_debian_style_source_package_list(url, "main")
+    package_names = get_debian_style_source_package_list(url, "main", d)
 #    url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz" 
 #    package_names += get_debian_style_source_package_list(url, "contrib")
     url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz" 
-    package_names += get_debian_style_source_package_list(url, "updates")
+    package_names += get_debian_style_source_package_list(url, "updates", d)
     package_list=clean_package_list(package_names)
     return latest, package_list
 
-def find_latest_ubuntu_release(url):
+def find_latest_ubuntu_release(url, d):
     "Find the latest listed ubuntu release on the given url"
     url += "?C=M;O=D" # Descending Sort by Last Modified
-    for link in get_links_from_url(url):
+    for link in get_links_from_url(url, d):
         if link[-8:] == "-updates":
             return link[:-8]
     return "_NotFound_"
 
-def get_latest_released_ubuntu_source_package_list():
+def get_latest_released_ubuntu_source_package_list(d):
     "Returns list of all the name os packages in the latest ubuntu distro"
-    latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/")
+    latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d)
     url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
-    package_names = get_debian_style_source_package_list(url, "main")
+    package_names = get_debian_style_source_package_list(url, "main", d)
 #    url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest
 #    package_names += get_debian_style_source_package_list(url, "multiverse")
 #    url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest
 #    package_names += get_debian_style_source_package_list(url, "universe")
     url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
-    package_names += get_debian_style_source_package_list(url, "updates")
+    package_names += get_debian_style_source_package_list(url, "updates", d)
     package_list=clean_package_list(package_names)
     return latest, package_list
 
-def create_distro_packages_list(distro_check_dir):
+def create_distro_packages_list(distro_check_dir, d):
     pkglst_dir = os.path.join(distro_check_dir, "package_lists")
     if not os.path.isdir (pkglst_dir):
         os.makedirs(pkglst_dir)
@@ -220,7 +239,7 @@ def create_distro_packages_list(distro_check_dir):
     begin = datetime.now()
     for distro in per_distro_functions:
         name = distro[0]
-        release, package_list = distro[1]()
+        release, package_list = distro[1](d)
         bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
         package_list_file = os.path.join(pkglst_dir, name + "-" + release)
         f = open(package_list_file, "w+b")
@@ -231,7 +250,7 @@ def create_distro_packages_list(distro_check_dir):
     delta = end - begin
     bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds)
 
-def update_distro_data(distro_check_dir, datetime):
+def update_distro_data(distro_check_dir, datetime, d):
     """
         If distro packages list data is old then rebuild it.
         The operations has to be protected by a lock so that
@@ -258,7 +277,7 @@ def update_distro_data(distro_check_dir, datetime):
         if saved_datetime[0:8] != datetime[0:8]:
             bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
             bb.note("Regenerating distro package lists")
-            create_distro_packages_list(distro_check_dir)
+            create_distro_packages_list(distro_check_dir, d)
             f.seek(0)
             f.write(datetime)
 
-- 
1.8.4.5



^ permalink raw reply related	[flat|nested] 4+ messages in thread
* Re: [PATCH] distrodata: Take account proxies on distrodata tasks
  2015-09-02  8:59 [PATCH] distrodata: Take account proxies on distrodata tasks leonardo.sandoval.gonzalez
@ 2015-09-02 18:09 ` Aníbal Limón
  2015-09-02 18:57   ` Leonardo Sandoval
  2015-09-30 15:16 ` Leonardo Sandoval
  1 sibling, 1 reply; 4+ messages in thread
From: Aníbal Limón @ 2015-09-02 18:09 UTC (permalink / raw)
  To: leonardo.sandoval.gonzalez, openembedded-core

HI Leo,

This is a way to do it but implies that functions doing distro_data 
needs to know about the data store
and ii can be improved only exporting the proxies in the interface 
function update_data,

If you don't specify proxies into urllib.urlopen it search into environment.

For export proxies you can use as an example [1], see below.

Regards,
     alimon

[1] 
http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/bitbake/lib/bb/fetch2/wget.py#n221


On 02/09/15 03:59, leonardo.sandoval.gonzalez@linux.intel.com wrote:
> From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
>
> Proxies defined in the enviroment where not taken into account
> on the distrodata tasks. This commit implied passing the datastore
> into the distro_check library and context manager for the
> urllib.urlopen function.
>
> One way to run distrodata tasks is using 'universe' as target and the
> 'all' distrodata task:
>
>        $ bitbake universe -c distrodataall
>        $ bitbake universe -c distro_checkall
>        $ bitbake universe -c checklicenseall
>
> Logs are located under TMPDIR/log
>
> [YOCTO #7567]
>
> Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
> ---
>   meta/classes/distrodata.bbclass |   6 +--
>   meta/lib/oe/distro_check.py     | 113 +++++++++++++++++++++++-----------------
>   2 files changed, 69 insertions(+), 50 deletions(-)
>
> diff --git a/meta/classes/distrodata.bbclass b/meta/classes/distrodata.bbclass
> index 4168e43..0a706ef 100644
> --- a/meta/classes/distrodata.bbclass
> +++ b/meta/classes/distrodata.bbclass
> @@ -33,7 +33,7 @@ python do_distrodata_np() {
>           tmpdir = d.getVar('TMPDIR', True)
>           distro_check_dir = os.path.join(tmpdir, "distro_check")
>           datetime = localdata.getVar('DATETIME', True)
> -        dist_check.update_distro_data(distro_check_dir, datetime)
> +        dist_check.update_distro_data(distro_check_dir, datetime, localdata)
>   
>           if pn.find("-native") != -1:
>               pnstripped = pn.split("-native")
> @@ -118,7 +118,7 @@ python do_distrodata() {
>           tmpdir = d.getVar('TMPDIR', True)
>           distro_check_dir = os.path.join(tmpdir, "distro_check")
>           datetime = localdata.getVar('DATETIME', True)
> -        dist_check.update_distro_data(distro_check_dir, datetime)
> +        dist_check.update_distro_data(distro_check_dir, datetime, localdata)
>   
>           pn = d.getVar("PN", True)
>           bb.note("Package Name: %s" % pn)
> @@ -402,7 +402,7 @@ python do_distro_check() {
>       bb.utils.mkdirhier(logpath)
>       result_file = os.path.join(logpath, "distrocheck.csv")
>       datetime = localdata.getVar('DATETIME', True)
> -    dc.update_distro_data(distro_check_dir, datetime)
> +    dc.update_distro_data(distro_check_dir, datetime, localdata)
>   
>       # do the comparison
>       result = dc.compare_in_distro_packages_list(distro_check_dir, d)
> diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py
> index 8ed5b0e..b3419ce 100644
> --- a/meta/lib/oe/distro_check.py
> +++ b/meta/lib/oe/distro_check.py
> @@ -1,7 +1,28 @@
> -def get_links_from_url(url):
> +from contextlib import contextmanager
> +@contextmanager
> +def create_socket(url, d):
> +    import urllib
> +    socket = urllib.urlopen(url, proxies=get_proxies(d))
> +    try:
> +        yield socket
> +    finally:
> +        socket.close()
> +
> +def get_proxies(d):
> +    import os
> +    proxykeys = ['HTTP_PROXY', 'http_proxy',
> +                 'HTTPS_PROXY', 'https_proxy',
> +                 'FTP_PROXY', 'ftp_proxy',
> +                 'FTPS_PROXY', 'ftps_proxy',
> +                 'NO_PROXY', 'no_proxy',
> +                 'ALL_PROXY', 'all_proxy']
> +    proxyvalues = map(lambda key: d.getVar(key, True), proxykeys)
> +    return dict(zip(proxykeys, proxyvalues))
> +
> +def get_links_from_url(url, d):
>       "Return all the href links found on the web location"
>   
> -    import urllib, sgmllib
> +    import sgmllib
>       
>       class LinksParser(sgmllib.SGMLParser):
>           def parse(self, s):
> @@ -24,19 +45,18 @@ def get_links_from_url(url):
>               "Return the list of hyperlinks."
>               return self.hyperlinks
>   
> -    sock = urllib.urlopen(url)
> -    webpage = sock.read()
> -    sock.close()
> +    with create_socket(url,d) as sock:
> +        webpage = sock.read()
>   
>       linksparser = LinksParser()
>       linksparser.parse(webpage)
>       return linksparser.get_hyperlinks()
>   
> -def find_latest_numeric_release(url):
> +def find_latest_numeric_release(url, d):
>       "Find the latest listed numeric release on the given url"
>       max=0
>       maxstr=""
> -    for link in get_links_from_url(url):
> +    for link in get_links_from_url(url, d):
>           try:
>               release = float(link)
>           except:
> @@ -70,7 +90,7 @@ def clean_package_list(package_list):
>       return set.keys()
>   
>   
> -def get_latest_released_meego_source_package_list():
> +def get_latest_released_meego_source_package_list(d):
>       "Returns list of all the name os packages in the latest meego distro"
>   
>       package_names = []
> @@ -82,11 +102,11 @@ def get_latest_released_meego_source_package_list():
>       package_list=clean_package_list(package_names)
>       return "1.0", package_list
>   
> -def get_source_package_list_from_url(url, section):
> +def get_source_package_list_from_url(url, section, d):
>       "Return a sectioned list of package names from a URL list"
>   
>       bb.note("Reading %s: %s" % (url, section))
> -    links = get_links_from_url(url)
> +    links = get_links_from_url(url, d)
>       srpms = filter(is_src_rpm, links)
>       names_list = map(package_name_from_srpm, srpms)
>   
> @@ -96,44 +116,44 @@ def get_source_package_list_from_url(url, section):
>   
>       return new_pkgs
>   
> -def get_latest_released_fedora_source_package_list():
> +def get_latest_released_fedora_source_package_list(d):
>       "Returns list of all the name os packages in the latest fedora distro"
> -    latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/")
> +    latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d)
>   
> -    package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main")
> +    package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main", d)
>   
>   #    package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything")
> -    package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates")
> +    package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d)
>   
>       package_list=clean_package_list(package_names)
>           
>       return latest, package_list
>   
> -def get_latest_released_opensuse_source_package_list():
> +def get_latest_released_opensuse_source_package_list(d):
>       "Returns list of all the name os packages in the latest opensuse distro"
> -    latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/")
> +    latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d)
>   
> -    package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main")
> -    package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates")
> +    package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d)
> +    package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates", d)
>   
>       package_list=clean_package_list(package_names)
>       return latest, package_list
>   
> -def get_latest_released_mandriva_source_package_list():
> +def get_latest_released_mandriva_source_package_list(d):
>       "Returns list of all the name os packages in the latest mandriva distro"
> -    latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/")
> -    package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main")
> +    latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d)
> +    package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d)
>   #    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib")
> -    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates")
> +    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d)
>   
>       package_list=clean_package_list(package_names)
>       return latest, package_list
>   
> -def find_latest_debian_release(url):
> +def find_latest_debian_release(url, d):
>       "Find the latest listed debian release on the given url"
>   
>       releases = []
> -    for link in get_links_from_url(url):
> +    for link in get_links_from_url(url, d):
>           if link[:6] == "Debian":
>               if ';' not in link:
>                   releases.append(link)
> @@ -143,16 +163,15 @@ def find_latest_debian_release(url):
>       except:
>           return "_NotFound_"
>   
> -def get_debian_style_source_package_list(url, section):
> +def get_debian_style_source_package_list(url, section, d):
>       "Return the list of package-names stored in the debian style Sources.gz file"
> -    import urllib
> -    sock = urllib.urlopen(url)
> -    import tempfile
> -    tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
> -    tmpfilename=tmpfile.name
> -    tmpfile.write(sock.read())
> -    sock.close()
> -    tmpfile.close()
> +    with create_socket(url,d) as sock:
> +        webpage = sock.read()
> +        import tempfile
> +        tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
> +        tmpfilename=tmpfile.name
> +        tmpfile.write(sock.read())
> +        tmpfile.close()
>       import gzip
>       bb.note("Reading %s: %s" % (url, section))
>   
> @@ -165,41 +184,41 @@ def get_debian_style_source_package_list(url, section):
>   
>       return package_names
>   
> -def get_latest_released_debian_source_package_list():
> +def get_latest_released_debian_source_package_list(d):
>       "Returns list of all the name os packages in the latest debian distro"
> -    latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/")
> +    latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d)
>       url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
> -    package_names = get_debian_style_source_package_list(url, "main")
> +    package_names = get_debian_style_source_package_list(url, "main", d)
>   #    url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz"
>   #    package_names += get_debian_style_source_package_list(url, "contrib")
>       url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
> -    package_names += get_debian_style_source_package_list(url, "updates")
> +    package_names += get_debian_style_source_package_list(url, "updates", d)
>       package_list=clean_package_list(package_names)
>       return latest, package_list
>   
> -def find_latest_ubuntu_release(url):
> +def find_latest_ubuntu_release(url, d):
>       "Find the latest listed ubuntu release on the given url"
>       url += "?C=M;O=D" # Descending Sort by Last Modified
> -    for link in get_links_from_url(url):
> +    for link in get_links_from_url(url, d):
>           if link[-8:] == "-updates":
>               return link[:-8]
>       return "_NotFound_"
>   
> -def get_latest_released_ubuntu_source_package_list():
> +def get_latest_released_ubuntu_source_package_list(d):
>       "Returns list of all the name os packages in the latest ubuntu distro"
> -    latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/")
> +    latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d)
>       url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
> -    package_names = get_debian_style_source_package_list(url, "main")
> +    package_names = get_debian_style_source_package_list(url, "main", d)
>   #    url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest
>   #    package_names += get_debian_style_source_package_list(url, "multiverse")
>   #    url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest
>   #    package_names += get_debian_style_source_package_list(url, "universe")
>       url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
> -    package_names += get_debian_style_source_package_list(url, "updates")
> +    package_names += get_debian_style_source_package_list(url, "updates", d)
>       package_list=clean_package_list(package_names)
>       return latest, package_list
>   
> -def create_distro_packages_list(distro_check_dir):
> +def create_distro_packages_list(distro_check_dir, d):
>       pkglst_dir = os.path.join(distro_check_dir, "package_lists")
>       if not os.path.isdir (pkglst_dir):
>           os.makedirs(pkglst_dir)
> @@ -220,7 +239,7 @@ def create_distro_packages_list(distro_check_dir):
>       begin = datetime.now()
>       for distro in per_distro_functions:
>           name = distro[0]
> -        release, package_list = distro[1]()
> +        release, package_list = distro[1](d)
>           bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
>           package_list_file = os.path.join(pkglst_dir, name + "-" + release)
>           f = open(package_list_file, "w+b")
> @@ -231,7 +250,7 @@ def create_distro_packages_list(distro_check_dir):
>       delta = end - begin
>       bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds)
>   
> -def update_distro_data(distro_check_dir, datetime):
> +def update_distro_data(distro_check_dir, datetime, d):
>       """
>           If distro packages list data is old then rebuild it.
>           The operations has to be protected by a lock so that
> @@ -258,7 +277,7 @@ def update_distro_data(distro_check_dir, datetime):
>           if saved_datetime[0:8] != datetime[0:8]:
>               bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
>               bb.note("Regenerating distro package lists")
> -            create_distro_packages_list(distro_check_dir)

Use export proxies,

export_proxies(d)

> +            create_distro_packages_list(distro_check_dir, d)
>               f.seek(0)
>               f.write(datetime)
>   



^ permalink raw reply	[flat|nested] 4+ messages in thread
* Re: [PATCH] distrodata: Take account proxies on distrodata tasks
  2015-09-02 18:09 ` Aníbal Limón
@ 2015-09-02 18:57   ` Leonardo Sandoval
  0 siblings, 0 replies; 4+ messages in thread
From: Leonardo Sandoval @ 2015-09-02 18:57 UTC (permalink / raw)
  To: Aníbal Limón, openembedded-core

I am passing the proxies directly into the urlopen function, this way I 
do not pollute the environment. I saw also [1] but I found less 
intrusive the way I implemented it.


On 09/02/2015 01:09 PM, Aníbal Limón wrote:
> HI Leo,
>
> This is a way to do it but implies that functions doing distro_data
> needs to know about the data store
> and ii can be improved only exporting the proxies in the interface
> function update_data,
>
> If you don't specify proxies into urllib.urlopen it search into
> environment.
>
> For export proxies you can use as an example [1], see below.
>
> Regards,
>      alimon
>
> [1]
> http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/bitbake/lib/bb/fetch2/wget.py#n221
>
>
>
> On 02/09/15 03:59, leonardo.sandoval.gonzalez@linux.intel.com wrote:
>> From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
>>
>> Proxies defined in the enviroment where not taken into account
>> on the distrodata tasks. This commit implied passing the datastore
>> into the distro_check library and context manager for the
>> urllib.urlopen function.
>>
>> One way to run distrodata tasks is using 'universe' as target and the
>> 'all' distrodata task:
>>
>>        $ bitbake universe -c distrodataall
>>        $ bitbake universe -c distro_checkall
>>        $ bitbake universe -c checklicenseall
>>
>> Logs are located under TMPDIR/log
>>
>> [YOCTO #7567]
>>
>> Signed-off-by: Leonardo Sandoval
>> <leonardo.sandoval.gonzalez@linux.intel.com>
>> ---
>>   meta/classes/distrodata.bbclass |   6 +--
>>   meta/lib/oe/distro_check.py     | 113
>> +++++++++++++++++++++++-----------------
>>   2 files changed, 69 insertions(+), 50 deletions(-)
>>
>> diff --git a/meta/classes/distrodata.bbclass
>> b/meta/classes/distrodata.bbclass
>> index 4168e43..0a706ef 100644
>> --- a/meta/classes/distrodata.bbclass
>> +++ b/meta/classes/distrodata.bbclass
>> @@ -33,7 +33,7 @@ python do_distrodata_np() {
>>           tmpdir = d.getVar('TMPDIR', True)
>>           distro_check_dir = os.path.join(tmpdir, "distro_check")
>>           datetime = localdata.getVar('DATETIME', True)
>> -        dist_check.update_distro_data(distro_check_dir, datetime)
>> +        dist_check.update_distro_data(distro_check_dir, datetime,
>> localdata)
>>           if pn.find("-native") != -1:
>>               pnstripped = pn.split("-native")
>> @@ -118,7 +118,7 @@ python do_distrodata() {
>>           tmpdir = d.getVar('TMPDIR', True)
>>           distro_check_dir = os.path.join(tmpdir, "distro_check")
>>           datetime = localdata.getVar('DATETIME', True)
>> -        dist_check.update_distro_data(distro_check_dir, datetime)
>> +        dist_check.update_distro_data(distro_check_dir, datetime,
>> localdata)
>>           pn = d.getVar("PN", True)
>>           bb.note("Package Name: %s" % pn)
>> @@ -402,7 +402,7 @@ python do_distro_check() {
>>       bb.utils.mkdirhier(logpath)
>>       result_file = os.path.join(logpath, "distrocheck.csv")
>>       datetime = localdata.getVar('DATETIME', True)
>> -    dc.update_distro_data(distro_check_dir, datetime)
>> +    dc.update_distro_data(distro_check_dir, datetime, localdata)
>>       # do the comparison
>>       result = dc.compare_in_distro_packages_list(distro_check_dir, d)
>> diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py
>> index 8ed5b0e..b3419ce 100644
>> --- a/meta/lib/oe/distro_check.py
>> +++ b/meta/lib/oe/distro_check.py
>> @@ -1,7 +1,28 @@
>> -def get_links_from_url(url):
>> +from contextlib import contextmanager
>> +@contextmanager
>> +def create_socket(url, d):
>> +    import urllib
>> +    socket = urllib.urlopen(url, proxies=get_proxies(d))
>> +    try:
>> +        yield socket
>> +    finally:
>> +        socket.close()
>> +
>> +def get_proxies(d):
>> +    import os
>> +    proxykeys = ['HTTP_PROXY', 'http_proxy',
>> +                 'HTTPS_PROXY', 'https_proxy',
>> +                 'FTP_PROXY', 'ftp_proxy',
>> +                 'FTPS_PROXY', 'ftps_proxy',
>> +                 'NO_PROXY', 'no_proxy',
>> +                 'ALL_PROXY', 'all_proxy']
>> +    proxyvalues = map(lambda key: d.getVar(key, True), proxykeys)
>> +    return dict(zip(proxykeys, proxyvalues))
>> +
>> +def get_links_from_url(url, d):
>>       "Return all the href links found on the web location"
>> -    import urllib, sgmllib
>> +    import sgmllib
>>       class LinksParser(sgmllib.SGMLParser):
>>           def parse(self, s):
>> @@ -24,19 +45,18 @@ def get_links_from_url(url):
>>               "Return the list of hyperlinks."
>>               return self.hyperlinks
>> -    sock = urllib.urlopen(url)
>> -    webpage = sock.read()
>> -    sock.close()
>> +    with create_socket(url,d) as sock:
>> +        webpage = sock.read()
>>       linksparser = LinksParser()
>>       linksparser.parse(webpage)
>>       return linksparser.get_hyperlinks()
>> -def find_latest_numeric_release(url):
>> +def find_latest_numeric_release(url, d):
>>       "Find the latest listed numeric release on the given url"
>>       max=0
>>       maxstr=""
>> -    for link in get_links_from_url(url):
>> +    for link in get_links_from_url(url, d):
>>           try:
>>               release = float(link)
>>           except:
>> @@ -70,7 +90,7 @@ def clean_package_list(package_list):
>>       return set.keys()
>> -def get_latest_released_meego_source_package_list():
>> +def get_latest_released_meego_source_package_list(d):
>>       "Returns list of all the name os packages in the latest meego
>> distro"
>>       package_names = []
>> @@ -82,11 +102,11 @@ def get_latest_released_meego_source_package_list():
>>       package_list=clean_package_list(package_names)
>>       return "1.0", package_list
>> -def get_source_package_list_from_url(url, section):
>> +def get_source_package_list_from_url(url, section, d):
>>       "Return a sectioned list of package names from a URL list"
>>       bb.note("Reading %s: %s" % (url, section))
>> -    links = get_links_from_url(url)
>> +    links = get_links_from_url(url, d)
>>       srpms = filter(is_src_rpm, links)
>>       names_list = map(package_name_from_srpm, srpms)
>> @@ -96,44 +116,44 @@ def get_source_package_list_from_url(url, section):
>>       return new_pkgs
>> -def get_latest_released_fedora_source_package_list():
>> +def get_latest_released_fedora_source_package_list(d):
>>       "Returns list of all the name os packages in the latest fedora
>> distro"
>> -    latest =
>> find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/")
>>
>> +    latest =
>> find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/",
>> d)
>> -    package_names =
>> get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/"
>> % latest, "main")
>> +    package_names =
>> get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/"
>> % latest, "main", d)
>>   #    package_names +=
>> get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/"
>> % latest, "everything")
>> -    package_names +=
>> get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/"
>> % latest, "updates")
>> +    package_names +=
>> get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/"
>> % latest, "updates", d)
>>       package_list=clean_package_list(package_names)
>>       return latest, package_list
>> -def get_latest_released_opensuse_source_package_list():
>> +def get_latest_released_opensuse_source_package_list(d):
>>       "Returns list of all the name os packages in the latest opensuse
>> distro"
>> -    latest =
>> find_latest_numeric_release("http://download.opensuse.org/source/distribution/")
>>
>> +    latest =
>> find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d)
>>
>> -    package_names =
>> get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/"
>> % latest, "main")
>> -    package_names +=
>> get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/"
>> % latest, "updates")
>> +    package_names =
>> get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/"
>> % latest, "main", d)
>> +    package_names +=
>> get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/"
>> % latest, "updates", d)
>>       package_list=clean_package_list(package_names)
>>       return latest, package_list
>> -def get_latest_released_mandriva_source_package_list():
>> +def get_latest_released_mandriva_source_package_list(d):
>>       "Returns list of all the name os packages in the latest mandriva
>> distro"
>> -    latest =
>> find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/")
>>
>> -    package_names =
>> get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/"
>> % latest, "main")
>> +    latest =
>> find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/",
>> d)
>> +    package_names =
>> get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/"
>> % latest, "main", d)
>>   #    package_names +=
>> get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/"
>> % latest, "contrib")
>> -    package_names +=
>> get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/"
>> % latest, "updates")
>> +    package_names +=
>> get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/"
>> % latest, "updates", d)
>>       package_list=clean_package_list(package_names)
>>       return latest, package_list
>> -def find_latest_debian_release(url):
>> +def find_latest_debian_release(url, d):
>>       "Find the latest listed debian release on the given url"
>>       releases = []
>> -    for link in get_links_from_url(url):
>> +    for link in get_links_from_url(url, d):
>>           if link[:6] == "Debian":
>>               if ';' not in link:
>>                   releases.append(link)
>> @@ -143,16 +163,15 @@ def find_latest_debian_release(url):
>>       except:
>>           return "_NotFound_"
>> -def get_debian_style_source_package_list(url, section):
>> +def get_debian_style_source_package_list(url, section, d):
>>       "Return the list of package-names stored in the debian style
>> Sources.gz file"
>> -    import urllib
>> -    sock = urllib.urlopen(url)
>> -    import tempfile
>> -    tmpfile = tempfile.NamedTemporaryFile(mode='wb',
>> prefix='oecore.', suffix='.tmp', delete=False)
>> -    tmpfilename=tmpfile.name
>> -    tmpfile.write(sock.read())
>> -    sock.close()
>> -    tmpfile.close()
>> +    with create_socket(url,d) as sock:
>> +        webpage = sock.read()
>> +        import tempfile
>> +        tmpfile = tempfile.NamedTemporaryFile(mode='wb',
>> prefix='oecore.', suffix='.tmp', delete=False)
>> +        tmpfilename=tmpfile.name
>> +        tmpfile.write(sock.read())
>> +        tmpfile.close()
>>       import gzip
>>       bb.note("Reading %s: %s" % (url, section))
>> @@ -165,41 +184,41 @@ def get_debian_style_source_package_list(url,
>> section):
>>       return package_names
>> -def get_latest_released_debian_source_package_list():
>> +def get_latest_released_debian_source_package_list(d):
>>       "Returns list of all the name os packages in the latest debian
>> distro"
>> -    latest =
>> find_latest_debian_release("http://ftp.debian.org/debian/dists/")
>> +    latest =
>> find_latest_debian_release("http://ftp.debian.org/debian/dists/", d)
>>       url =
>> "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
>> -    package_names = get_debian_style_source_package_list(url, "main")
>> +    package_names = get_debian_style_source_package_list(url, "main", d)
>>   #    url =
>> "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz"
>>   #    package_names += get_debian_style_source_package_list(url,
>> "contrib")
>>       url =
>> "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
>>
>> -    package_names += get_debian_style_source_package_list(url,
>> "updates")
>> +    package_names += get_debian_style_source_package_list(url,
>> "updates", d)
>>       package_list=clean_package_list(package_names)
>>       return latest, package_list
>> -def find_latest_ubuntu_release(url):
>> +def find_latest_ubuntu_release(url, d):
>>       "Find the latest listed ubuntu release on the given url"
>>       url += "?C=M;O=D" # Descending Sort by Last Modified
>> -    for link in get_links_from_url(url):
>> +    for link in get_links_from_url(url, d):
>>           if link[-8:] == "-updates":
>>               return link[:-8]
>>       return "_NotFound_"
>> -def get_latest_released_ubuntu_source_package_list():
>> +def get_latest_released_ubuntu_source_package_list(d):
>>       "Returns list of all the name os packages in the latest ubuntu
>> distro"
>> -    latest =
>> find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/")
>> +    latest =
>> find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d)
>>       url =
>> "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" %
>> latest
>> -    package_names = get_debian_style_source_package_list(url, "main")
>> +    package_names = get_debian_style_source_package_list(url, "main", d)
>>   #    url =
>> "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz"
>> % latest
>>   #    package_names += get_debian_style_source_package_list(url,
>> "multiverse")
>>   #    url =
>> "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz"
>> % latest
>>   #    package_names += get_debian_style_source_package_list(url,
>> "universe")
>>       url =
>> "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz"
>> % latest
>> -    package_names += get_debian_style_source_package_list(url,
>> "updates")
>> +    package_names += get_debian_style_source_package_list(url,
>> "updates", d)
>>       package_list=clean_package_list(package_names)
>>       return latest, package_list
>> -def create_distro_packages_list(distro_check_dir):
>> +def create_distro_packages_list(distro_check_dir, d):
>>       pkglst_dir = os.path.join(distro_check_dir, "package_lists")
>>       if not os.path.isdir (pkglst_dir):
>>           os.makedirs(pkglst_dir)
>> @@ -220,7 +239,7 @@ def create_distro_packages_list(distro_check_dir):
>>       begin = datetime.now()
>>       for distro in per_distro_functions:
>>           name = distro[0]
>> -        release, package_list = distro[1]()
>> +        release, package_list = distro[1](d)
>>           bb.note("Distro: %s, Latest Release: %s, # src packages: %d"
>> % (name, release, len(package_list)))
>>           package_list_file = os.path.join(pkglst_dir, name + "-" +
>> release)
>>           f = open(package_list_file, "w+b")
>> @@ -231,7 +250,7 @@ def create_distro_packages_list(distro_check_dir):
>>       delta = end - begin
>>       bb.note("package_list generatiosn took this much time: %d
>> seconds" % delta.seconds)
>> -def update_distro_data(distro_check_dir, datetime):
>> +def update_distro_data(distro_check_dir, datetime, d):
>>       """
>>           If distro packages list data is old then rebuild it.
>>           The operations has to be protected by a lock so that
>> @@ -258,7 +277,7 @@ def update_distro_data(distro_check_dir, datetime):
>>           if saved_datetime[0:8] != datetime[0:8]:
>>               bb.note("The build datetime did not match: saved:%s
>> current:%s" % (saved_datetime, datetime))
>>               bb.note("Regenerating distro package lists")
>> -            create_distro_packages_list(distro_check_dir)
>
> Use export proxies,
>
> export_proxies(d)
>
>> +            create_distro_packages_list(distro_check_dir, d)
>>               f.seek(0)
>>               f.write(datetime)
>


^ permalink raw reply	[flat|nested] 4+ messages in thread
* Re: [PATCH] distrodata: Take account proxies on distrodata tasks
  2015-09-02  8:59 [PATCH] distrodata: Take account proxies on distrodata tasks leonardo.sandoval.gonzalez
  2015-09-02 18:09 ` Aníbal Limón
@ 2015-09-30 15:16 ` Leonardo Sandoval
  1 sibling, 0 replies; 4+ messages in thread
From: Leonardo Sandoval @ 2015-09-30 15:16 UTC (permalink / raw)
  To: openembedded-core

ping..

This patch is pending for merge.


On 09/02/2015 03:59 AM, leonardo.sandoval.gonzalez@linux.intel.com wrote:
> From: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
>
> Proxies defined in the enviroment where not taken into account
> on the distrodata tasks. This commit implied passing the datastore
> into the distro_check library and context manager for the
> urllib.urlopen function.
>
> One way to run distrodata tasks is using 'universe' as target and the
> 'all' distrodata task:
>
>        $ bitbake universe -c distrodataall
>        $ bitbake universe -c distro_checkall
>        $ bitbake universe -c checklicenseall
>
> Logs are located under TMPDIR/log
>
> [YOCTO #7567]
>
> Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>
> ---
>   meta/classes/distrodata.bbclass |   6 +--
>   meta/lib/oe/distro_check.py     | 113 +++++++++++++++++++++++-----------------
>   2 files changed, 69 insertions(+), 50 deletions(-)
>
> diff --git a/meta/classes/distrodata.bbclass b/meta/classes/distrodata.bbclass
> index 4168e43..0a706ef 100644
> --- a/meta/classes/distrodata.bbclass
> +++ b/meta/classes/distrodata.bbclass
> @@ -33,7 +33,7 @@ python do_distrodata_np() {
>           tmpdir = d.getVar('TMPDIR', True)
>           distro_check_dir = os.path.join(tmpdir, "distro_check")
>           datetime = localdata.getVar('DATETIME', True)
> -        dist_check.update_distro_data(distro_check_dir, datetime)
> +        dist_check.update_distro_data(distro_check_dir, datetime, localdata)
>
>           if pn.find("-native") != -1:
>               pnstripped = pn.split("-native")
> @@ -118,7 +118,7 @@ python do_distrodata() {
>           tmpdir = d.getVar('TMPDIR', True)
>           distro_check_dir = os.path.join(tmpdir, "distro_check")
>           datetime = localdata.getVar('DATETIME', True)
> -        dist_check.update_distro_data(distro_check_dir, datetime)
> +        dist_check.update_distro_data(distro_check_dir, datetime, localdata)
>
>           pn = d.getVar("PN", True)
>           bb.note("Package Name: %s" % pn)
> @@ -402,7 +402,7 @@ python do_distro_check() {
>       bb.utils.mkdirhier(logpath)
>       result_file = os.path.join(logpath, "distrocheck.csv")
>       datetime = localdata.getVar('DATETIME', True)
> -    dc.update_distro_data(distro_check_dir, datetime)
> +    dc.update_distro_data(distro_check_dir, datetime, localdata)
>
>       # do the comparison
>       result = dc.compare_in_distro_packages_list(distro_check_dir, d)
> diff --git a/meta/lib/oe/distro_check.py b/meta/lib/oe/distro_check.py
> index 8ed5b0e..b3419ce 100644
> --- a/meta/lib/oe/distro_check.py
> +++ b/meta/lib/oe/distro_check.py
> @@ -1,7 +1,28 @@
> -def get_links_from_url(url):
> +from contextlib import contextmanager
> +@contextmanager
> +def create_socket(url, d):
> +    import urllib
> +    socket = urllib.urlopen(url, proxies=get_proxies(d))
> +    try:
> +        yield socket
> +    finally:
> +        socket.close()
> +
> +def get_proxies(d):
> +    import os
> +    proxykeys = ['HTTP_PROXY', 'http_proxy',
> +                 'HTTPS_PROXY', 'https_proxy',
> +                 'FTP_PROXY', 'ftp_proxy',
> +                 'FTPS_PROXY', 'ftps_proxy',
> +                 'NO_PROXY', 'no_proxy',
> +                 'ALL_PROXY', 'all_proxy']
> +    proxyvalues = map(lambda key: d.getVar(key, True), proxykeys)
> +    return dict(zip(proxykeys, proxyvalues))
> +
> +def get_links_from_url(url, d):
>       "Return all the href links found on the web location"
>
> -    import urllib, sgmllib
> +    import sgmllib
>
>       class LinksParser(sgmllib.SGMLParser):
>           def parse(self, s):
> @@ -24,19 +45,18 @@ def get_links_from_url(url):
>               "Return the list of hyperlinks."
>               return self.hyperlinks
>
> -    sock = urllib.urlopen(url)
> -    webpage = sock.read()
> -    sock.close()
> +    with create_socket(url,d) as sock:
> +        webpage = sock.read()
>
>       linksparser = LinksParser()
>       linksparser.parse(webpage)
>       return linksparser.get_hyperlinks()
>
> -def find_latest_numeric_release(url):
> +def find_latest_numeric_release(url, d):
>       "Find the latest listed numeric release on the given url"
>       max=0
>       maxstr=""
> -    for link in get_links_from_url(url):
> +    for link in get_links_from_url(url, d):
>           try:
>               release = float(link)
>           except:
> @@ -70,7 +90,7 @@ def clean_package_list(package_list):
>       return set.keys()
>
>
> -def get_latest_released_meego_source_package_list():
> +def get_latest_released_meego_source_package_list(d):
>       "Returns list of all the name os packages in the latest meego distro"
>
>       package_names = []
> @@ -82,11 +102,11 @@ def get_latest_released_meego_source_package_list():
>       package_list=clean_package_list(package_names)
>       return "1.0", package_list
>
> -def get_source_package_list_from_url(url, section):
> +def get_source_package_list_from_url(url, section, d):
>       "Return a sectioned list of package names from a URL list"
>
>       bb.note("Reading %s: %s" % (url, section))
> -    links = get_links_from_url(url)
> +    links = get_links_from_url(url, d)
>       srpms = filter(is_src_rpm, links)
>       names_list = map(package_name_from_srpm, srpms)
>
> @@ -96,44 +116,44 @@ def get_source_package_list_from_url(url, section):
>
>       return new_pkgs
>
> -def get_latest_released_fedora_source_package_list():
> +def get_latest_released_fedora_source_package_list(d):
>       "Returns list of all the name os packages in the latest fedora distro"
> -    latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/")
> +    latest = find_latest_numeric_release("http://archive.fedoraproject.org/pub/fedora/linux/releases/", d)
>
> -    package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main")
> +    package_names = get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/releases/%s/Fedora/source/SRPMS/" % latest, "main", d)
>
>   #    package_names += get_source_package_list_from_url("http://download.fedora.redhat.com/pub/fedora/linux/releases/%s/Everything/source/SPRMS/" % latest, "everything")
> -    package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates")
> +    package_names += get_source_package_list_from_url("http://archive.fedoraproject.org/pub/fedora/linux/updates/%s/SRPMS/" % latest, "updates", d)
>
>       package_list=clean_package_list(package_names)
>
>       return latest, package_list
>
> -def get_latest_released_opensuse_source_package_list():
> +def get_latest_released_opensuse_source_package_list(d):
>       "Returns list of all the name os packages in the latest opensuse distro"
> -    latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/")
> +    latest = find_latest_numeric_release("http://download.opensuse.org/source/distribution/",d)
>
> -    package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main")
> -    package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates")
> +    package_names = get_source_package_list_from_url("http://download.opensuse.org/source/distribution/%s/repo/oss/suse/src/" % latest, "main", d)
> +    package_names += get_source_package_list_from_url("http://download.opensuse.org/update/%s/rpm/src/" % latest, "updates", d)
>
>       package_list=clean_package_list(package_names)
>       return latest, package_list
>
> -def get_latest_released_mandriva_source_package_list():
> +def get_latest_released_mandriva_source_package_list(d):
>       "Returns list of all the name os packages in the latest mandriva distro"
> -    latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/")
> -    package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main")
> +    latest = find_latest_numeric_release("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/", d)
> +    package_names = get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/release/" % latest, "main", d)
>   #    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/contrib/release/" % latest, "contrib")
> -    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates")
> +    package_names += get_source_package_list_from_url("http://distrib-coffee.ipsl.jussieu.fr/pub/linux/MandrivaLinux/official/%s/SRPMS/main/updates/" % latest, "updates", d)
>
>       package_list=clean_package_list(package_names)
>       return latest, package_list
>
> -def find_latest_debian_release(url):
> +def find_latest_debian_release(url, d):
>       "Find the latest listed debian release on the given url"
>
>       releases = []
> -    for link in get_links_from_url(url):
> +    for link in get_links_from_url(url, d):
>           if link[:6] == "Debian":
>               if ';' not in link:
>                   releases.append(link)
> @@ -143,16 +163,15 @@ def find_latest_debian_release(url):
>       except:
>           return "_NotFound_"
>
> -def get_debian_style_source_package_list(url, section):
> +def get_debian_style_source_package_list(url, section, d):
>       "Return the list of package-names stored in the debian style Sources.gz file"
> -    import urllib
> -    sock = urllib.urlopen(url)
> -    import tempfile
> -    tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
> -    tmpfilename=tmpfile.name
> -    tmpfile.write(sock.read())
> -    sock.close()
> -    tmpfile.close()
> +    with create_socket(url,d) as sock:
> +        webpage = sock.read()
> +        import tempfile
> +        tmpfile = tempfile.NamedTemporaryFile(mode='wb', prefix='oecore.', suffix='.tmp', delete=False)
> +        tmpfilename=tmpfile.name
> +        tmpfile.write(sock.read())
> +        tmpfile.close()
>       import gzip
>       bb.note("Reading %s: %s" % (url, section))
>
> @@ -165,41 +184,41 @@ def get_debian_style_source_package_list(url, section):
>
>       return package_names
>
> -def get_latest_released_debian_source_package_list():
> +def get_latest_released_debian_source_package_list(d):
>       "Returns list of all the name os packages in the latest debian distro"
> -    latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/")
> +    latest = find_latest_debian_release("http://ftp.debian.org/debian/dists/", d)
>       url = "http://ftp.debian.org/debian/dists/stable/main/source/Sources.gz"
> -    package_names = get_debian_style_source_package_list(url, "main")
> +    package_names = get_debian_style_source_package_list(url, "main", d)
>   #    url = "http://ftp.debian.org/debian/dists/stable/contrib/source/Sources.gz"
>   #    package_names += get_debian_style_source_package_list(url, "contrib")
>       url = "http://ftp.debian.org/debian/dists/stable-proposed-updates/main/source/Sources.gz"
> -    package_names += get_debian_style_source_package_list(url, "updates")
> +    package_names += get_debian_style_source_package_list(url, "updates", d)
>       package_list=clean_package_list(package_names)
>       return latest, package_list
>
> -def find_latest_ubuntu_release(url):
> +def find_latest_ubuntu_release(url, d):
>       "Find the latest listed ubuntu release on the given url"
>       url += "?C=M;O=D" # Descending Sort by Last Modified
> -    for link in get_links_from_url(url):
> +    for link in get_links_from_url(url, d):
>           if link[-8:] == "-updates":
>               return link[:-8]
>       return "_NotFound_"
>
> -def get_latest_released_ubuntu_source_package_list():
> +def get_latest_released_ubuntu_source_package_list(d):
>       "Returns list of all the name os packages in the latest ubuntu distro"
> -    latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/")
> +    latest = find_latest_ubuntu_release("http://archive.ubuntu.com/ubuntu/dists/", d)
>       url = "http://archive.ubuntu.com/ubuntu/dists/%s/main/source/Sources.gz" % latest
> -    package_names = get_debian_style_source_package_list(url, "main")
> +    package_names = get_debian_style_source_package_list(url, "main", d)
>   #    url = "http://archive.ubuntu.com/ubuntu/dists/%s/multiverse/source/Sources.gz" % latest
>   #    package_names += get_debian_style_source_package_list(url, "multiverse")
>   #    url = "http://archive.ubuntu.com/ubuntu/dists/%s/universe/source/Sources.gz" % latest
>   #    package_names += get_debian_style_source_package_list(url, "universe")
>       url = "http://archive.ubuntu.com/ubuntu/dists/%s-updates/main/source/Sources.gz" % latest
> -    package_names += get_debian_style_source_package_list(url, "updates")
> +    package_names += get_debian_style_source_package_list(url, "updates", d)
>       package_list=clean_package_list(package_names)
>       return latest, package_list
>
> -def create_distro_packages_list(distro_check_dir):
> +def create_distro_packages_list(distro_check_dir, d):
>       pkglst_dir = os.path.join(distro_check_dir, "package_lists")
>       if not os.path.isdir (pkglst_dir):
>           os.makedirs(pkglst_dir)
> @@ -220,7 +239,7 @@ def create_distro_packages_list(distro_check_dir):
>       begin = datetime.now()
>       for distro in per_distro_functions:
>           name = distro[0]
> -        release, package_list = distro[1]()
> +        release, package_list = distro[1](d)
>           bb.note("Distro: %s, Latest Release: %s, # src packages: %d" % (name, release, len(package_list)))
>           package_list_file = os.path.join(pkglst_dir, name + "-" + release)
>           f = open(package_list_file, "w+b")
> @@ -231,7 +250,7 @@ def create_distro_packages_list(distro_check_dir):
>       delta = end - begin
>       bb.note("package_list generatiosn took this much time: %d seconds" % delta.seconds)
>
> -def update_distro_data(distro_check_dir, datetime):
> +def update_distro_data(distro_check_dir, datetime, d):
>       """
>           If distro packages list data is old then rebuild it.
>           The operations has to be protected by a lock so that
> @@ -258,7 +277,7 @@ def update_distro_data(distro_check_dir, datetime):
>           if saved_datetime[0:8] != datetime[0:8]:
>               bb.note("The build datetime did not match: saved:%s current:%s" % (saved_datetime, datetime))
>               bb.note("Regenerating distro package lists")
> -            create_distro_packages_list(distro_check_dir)
> +            create_distro_packages_list(distro_check_dir, d)
>               f.seek(0)
>               f.write(datetime)
>
>


^ permalink raw reply	[flat|nested] 4+ messages in thread
end of thread, other threads:[~2015-09-30 15:14 UTC | newest]

Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2015-09-02  8:59 [PATCH] distrodata: Take account proxies on distrodata tasks leonardo.sandoval.gonzalez
2015-09-02 18:09 ` Aníbal Limón
2015-09-02 18:57   ` Leonardo Sandoval
2015-09-30 15:16 ` Leonardo Sandoval

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.