[Documentation][PATCH] scripts: Migrate code to run with Python 3 and Bitbake 1.31.1

* [Documentation][PATCH] scripts: Migrate code to run with Python 3 and Bitbake 1.31.1
@ 2016-09-27 18:21 Fabio Berton
  2016-09-27 22:01 ` Otavio Salvador
  0 siblings, 1 reply; 3+ messages in thread
From: Fabio Berton @ 2016-09-27 18:21 UTC (permalink / raw)
  To: meta-freescale

- The urllib2 module has been split across several modules in Python
  3.0 named urllib.request and urllib.error. Adapt code to use
  urllib.request.

- Remove deprecated Python 2 function appy(). Use 2to3 script to
  generate new code.

- Adapt code to use bitbake with multi-config support.
  This feature was add on bitbake commit:
    5287991691578825c847bac2368e9b51c0ede3f0

- Adapt pickle to use Python 3.

Signed-off-by: Fabio Berton <fabio.berton@ossystems.com.br>
---
 scripts/bitbake-metadata2doc.py     |  4 +--
 scripts/bitbake-metadata2doc.sh     |  2 +-
 scripts/doc_utils.py                |  3 +-
 scripts/extract-bitbake-metadata.py | 59 +++++++++++++++++--------------------
 scripts/format_machine_list.py      |  2 +-
 scripts/generate-bugs-table.py      |  8 ++---
 scripts/test-sheet-parser.py        |  2 +-
 7 files changed, 37 insertions(+), 43 deletions(-)

diff --git a/scripts/bitbake-metadata2doc.py b/scripts/bitbake-metadata2doc.py
index 0149dd4..98af3f8 100644
--- a/scripts/bitbake-metadata2doc.py
+++ b/scripts/bitbake-metadata2doc.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 # -*- encoding: utf-8 -*-
 
 import os
@@ -496,7 +496,7 @@ gitdm_dir = sys.argv[4]
 start_commit = sys.argv[5]
 end_commit = sys.argv[6]
 
-data_fd = open(data_file, 'r')
+data_fd = open(data_file, 'rb')
 data = pickle.load(data_fd)
 data_fd.close()
 
diff --git a/scripts/bitbake-metadata2doc.sh b/scripts/bitbake-metadata2doc.sh
index a6191d1..3616fe1 100755
--- a/scripts/bitbake-metadata2doc.sh
+++ b/scripts/bitbake-metadata2doc.sh
@@ -82,7 +82,7 @@ for machine in $machines; do
     echo "Using $build_dir as build directory"
     MACHINE=$machine . ./setup-environment `basename $build_dir`
 
-    MACHINE=$machine python $anchor/extract-bitbake-metadata.py \
+    MACHINE=$machine python3 $anchor/extract-bitbake-metadata.py \
         $anchor/$marshalled_data_file \
         apptrk \
         barebox \
diff --git a/scripts/doc_utils.py b/scripts/doc_utils.py
index 9d11bd8..6125f13 100644
--- a/scripts/doc_utils.py
+++ b/scripts/doc_utils.py
@@ -19,8 +19,7 @@ def tabularize(lines, spacing=2):
 
     spc = ' ' * spacing
     if lines:
-        col_widths = map(lambda col: apply(max, map(len, col) + [0]),
-                         apply(zip, lines))
+        col_widths = [max(*list(map(len, col)) + [0]) for col in zip(*lines)]
         return '\n'.join([format_header(lines[0], col_widths, spc),
                           format_body(lines[1:], col_widths, spc),
                           format_border(col_widths)]) + \
diff --git a/scripts/extract-bitbake-metadata.py b/scripts/extract-bitbake-metadata.py
index d4b1634..8b97366 100644
--- a/scripts/extract-bitbake-metadata.py
+++ b/scripts/extract-bitbake-metadata.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """ From https://github.com/kergoth/bb/blob/master/libexec/bbcmd.py """
 
@@ -47,7 +47,7 @@ class Tinfoil(bb.tinfoil.Tinfoil):
         bb.providers.logger.setLevel(logging.ERROR)
         bb.taskdata.logger.setLevel(logging.CRITICAL)
         self.cooker_data = None
-        self.taskdata = None
+        self.taskdata = {}
 
         self.localdata = bb.data.createCopy(self.config_data)
         self.localdata.finalize()
@@ -56,9 +56,8 @@ class Tinfoil(bb.tinfoil.Tinfoil):
 
 
     def prepare_taskdata(self, provided=None, rprovided=None):
-        self.cache_data = self.cooker.recipecache
-        if self.taskdata is None:
-            self.taskdata = bb.taskdata.TaskData(abort=False)
+        self.cache_data = self.cooker.recipecaches['']
+        self.taskdata[''] = self.taskdata.get('', bb.taskdata.TaskData(abort=False))
 
         if provided:
             self.add_provided(provided)
@@ -68,9 +67,9 @@ class Tinfoil(bb.tinfoil.Tinfoil):
 
     def add_rprovided(self, rprovided):
         for item in rprovided:
-            self.taskdata.add_rprovider(self.localdata, self.cache_data, item)
+            self.taskdata[''].add_rprovider(self.localdata, self.cache_data, item)
 
-        self.taskdata.add_unresolved(self.localdata, self.cache_data)
+        self.taskdata[''].add_unresolved(self.localdata, self.cache_data)
 
     def add_provided(self, provided):
         if 'world' in provided:
@@ -84,9 +83,9 @@ class Tinfoil(bb.tinfoil.Tinfoil):
             provided.extend(self.cache_data.universe_target)
 
         for item in provided:
-            self.taskdata.add_provider(self.localdata, self.cache_data, item)
+            self.taskdata[''].add_provider(self.localdata, self.cache_data, item)
 
-        self.taskdata.add_unresolved(self.localdata, self.cache_data)
+        self.taskdata[''].add_unresolved(self.localdata, self.cache_data)
 
     def rec_get_dependees(self, targetid, depth=0, seen=None):
         if seen is None:
@@ -99,41 +98,41 @@ class Tinfoil(bb.tinfoil.Tinfoil):
                 yield _id, _depth
 
     def get_dependees(self, targetid, seen):
-        dep_fnids = self.taskdata.get_dependees(targetid)
+        dep_fnids = self.taskdata[''].get_dependees(targetid)
         for dep_fnid in dep_fnids:
             if dep_fnid in seen:
                 continue
             seen.add(dep_fnid)
-            for target in self.taskdata.build_targets:
-                if dep_fnid in self.taskdata.build_targets[target]:
+            for target in self.taskdata[''].build_targets:
+                if dep_fnid in self.taskdata[''].build_targets[target]:
                     yield dep_fnid, target
 
     def get_buildid(self, target):
-        if not self.taskdata.have_build_target(target):
+        if not self.taskdata[''].have_build_target(target):
             if target in self.cooker.recipecache.ignored_dependencies:
                 return
 
-            reasons = self.taskdata.get_reasons(target)
+            reasons = self.taskdata[''].get_reasons(target)
             if reasons:
                 self.logger.error("No buildable '%s' recipe found:\n%s", target, "\n".join(reasons))
             else:
                 self.logger.error("No '%s' recipe found", target)
             return
         else:
-            return self.taskdata.getbuild_id(target)
+            return self.taskdata[''].getbuild_id(target)
 
     def target_filenames(self):
         """Return the filenames of all of taskdata's targets"""
         filenames = set()
 
-        for targetid in self.taskdata.build_targets:
-            fnid = self.taskdata.build_targets[targetid][0]
-            fn = self.taskdata.fn_index[fnid]
+        for targetid in self.taskdata[''].build_targets:
+            fnid = self.taskdata[''].build_targets[targetid][0]
+            fn = self.taskdata[''].fn_index[fnid]
             filenames.add(fn)
 
-        for targetid in self.taskdata.run_targets:
-            fnid = self.taskdata.run_targets[targetid][0]
-            fn = self.taskdata.fn_index[fnid]
+        for targetid in self.taskdata[''].run_targets:
+            fnid = self.taskdata[''].run_targets[targetid][0]
+            fn = self.taskdata[''].fn_index[fnid]
             filenames.add(fn)
 
         return filenames
@@ -173,20 +172,16 @@ class Tinfoil(bb.tinfoil.Tinfoil):
     def build_target_to_fn(self, target):
         """Given a target, prepare taskdata and return a filename"""
         self.prepare_taskdata([target])
-        targetid = self.get_buildid(target)
-        if targetid is None:
-            return
-        fnid = self.taskdata.build_targets[targetid][0]
-        fn = self.taskdata.fn_index[fnid]
+        if target in self.taskdata[''].build_targets and self.taskdata[''].build_targets[target]:
+            fn = self.taskdata[''].build_targets[target][0]
         return fn
 
     def parse_recipe_file(self, recipe_filename):
         """Given a recipe filename, do a full parse of it"""
+        bb_cache = bb.cache.NoCache(self.cooker.databuilder)
         appends = self.cooker.collection.get_file_appends(recipe_filename)
         try:
-            recipe_data = bb.cache.Cache.loadDataFull(recipe_filename,
-                                                      appends,
-                                                      self.config_data)
+            recipe_data = bb_cache.loadDataFull(recipe_filename, appends)
         except Exception:
             raise
         return recipe_data
@@ -267,7 +262,7 @@ import pickle
 
 def load_data(data_file):
     try:
-        fd = open(data_file, 'r')
+        fd = open(data_file, 'rb')
         data = pickle.load(fd)
         fd.close()
         return data
@@ -275,8 +270,8 @@ def load_data(data_file):
         return {}
 
 def dump_data(data, data_file):
-    fd = open(data_file, 'w')
-    pickle.dump(data, fd)
+    fd = open(data_file, 'wb')
+    pickle.dump(data, fd, protocol=2)
     fd.close()
 
 def extract_bitbake_metadata(recipes):
diff --git a/scripts/format_machine_list.py b/scripts/format_machine_list.py
index 778d4fc..2980e86 100755
--- a/scripts/format_machine_list.py
+++ b/scripts/format_machine_list.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 from doc_utils import tabularize
 import csv
 import os
diff --git a/scripts/generate-bugs-table.py b/scripts/generate-bugs-table.py
index e9829bd..13390a3 100755
--- a/scripts/generate-bugs-table.py
+++ b/scripts/generate-bugs-table.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 # Call this file on the command line with "-h" as argument to get all the
 # available options.
 
@@ -8,7 +8,7 @@ import os
 import re
 import sys
 import time
-import urllib2
+import urllib.request
 from doc_utils import tabularize
 
 BASE_DIRECTORY = os.path.dirname(os.path.realpath(__file__ + "/../"))
@@ -20,8 +20,8 @@ OPEN_BUGS_URL = "https://bugzilla.yoctoproject.org/buglist.cgi?quicksearch=meta-
 OPEN_BUGS_OUTPUT_FILE_PATH = BASE_DIRECTORY + "/release-notes/source/open_bugs.inc"
 
 def request_bug_list(url):
-    buffer = urllib2.urlopen(url)
-    csv_bug_list = buffer.read()
+    buffer = urllib.request.urlopen(url)
+    csv_bug_list = buffer.read().decode()
     buffer.close()
     return csv_bug_list
 
diff --git a/scripts/test-sheet-parser.py b/scripts/test-sheet-parser.py
index bb6f5da..967f476 100755
--- a/scripts/test-sheet-parser.py
+++ b/scripts/test-sheet-parser.py
@@ -1,4 +1,4 @@
-#! /usr/bin/env python
+#! /usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import os
-- 
2.1.4



^ permalink raw reply related	[flat|nested] 3+ messages in thread