[PATCH 1/5] python3: generalise make commands

[PATCH 1/5] python3: generalise make commands

[Ross Burton]

We're repeating the same make invocation over and over, twice without setting
OPT=${CFLAGS} which doesn't seem right.

Centralise the make invocation to clean up the tasks.

Signed-off-by: Ross Burton <ross.burton@intel.com>
---
 meta/recipes-devtools/python/python3_3.5.5.bb | 48 +++++++++------------------
 1 file changed, 16 insertions(+), 32 deletions(-)

diff --git a/meta/recipes-devtools/python/python3_3.5.5.bb b/meta/recipes-devtools/python/python3_3.5.5.bb
index 9e31fa6f299..8e94e38a2d5 100644
--- a/meta/recipes-devtools/python/python3_3.5.5.bb
+++ b/meta/recipes-devtools/python/python3_3.5.5.bb
@@ -95,6 +95,18 @@ do_configure_append() {
 	autoreconf -Wcross --verbose --install --force --exclude=autopoint ../Python-${PV}/Modules/_ctypes/libffi
 }
 
+run_make() {
+	oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \
+		HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \
+		STAGING_LIBDIR=${STAGING_LIBDIR} \
+		STAGING_INCDIR=${STAGING_INCDIR} \
+		STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \
+		LIB=${baselib} \
+		ARCH=${TARGET_ARCH} \
+		OPT="${CFLAGS}" \
+		"$@"
+}
+
 do_compile() {
         # regenerate platform specific files, because they depend on system headers
         cd ${S}/Lib/plat-linux*
@@ -125,14 +137,7 @@ do_compile() {
 	# then call do_install twice we get Makefile.orig == Makefile.sysroot
 	install -m 0644 Makefile Makefile.sysroot
 
-        oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \
-                HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \
-                STAGING_LIBDIR=${STAGING_LIBDIR} \
-                STAGING_INCDIR=${STAGING_INCDIR} \
-                STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \
-                LIB=${baselib} \
-                ARCH=${TARGET_ARCH} \
-                OPT="${CFLAGS}" profile-opt
+    run_make profile-opt
 
         if ${@bb.utils.contains('MACHINE_FEATURES', 'qemu-usermode', 'true', 'false', d)}; then
                 qemu_binary="${@qemu_wrapper_cmdline(d, '${STAGING_DIR_TARGET}', ['${B}', '${STAGING_DIR_TARGET}/${base_libdir}'])}"
@@ -145,14 +150,7 @@ EOF
                 ./pgo-image-qemuwrapper ${B}/python ${PYTHON3_PROFILE_TASK} || true
 	fi
 
-        oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \
-                HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \
-                STAGING_LIBDIR=${STAGING_LIBDIR} \
-                STAGING_INCDIR=${STAGING_INCDIR} \
-                STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \
-                LIB=${baselib} \
-                ARCH=${TARGET_ARCH} \
-                OPT="${CFLAGS}" clean_and_use_profile
+    run_make clean_and_use_profile
 }
 
 do_install() {
@@ -165,23 +163,9 @@ do_install() {
 
 	# rerun the build once again with original makefile this time
 	# run install in a separate step to avoid compile/install race
-	oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \
-		HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \
-		STAGING_LIBDIR=${STAGING_LIBDIR} \
-		STAGING_INCDIR=${STAGING_INCDIR} \
-		STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \
-		LIB=${baselib} \
-		ARCH=${TARGET_ARCH} \
-		DESTDIR=${D} LIBDIR=${libdir} build_all_use_profile
+	run_make DESTDIR=${D} LIBDIR=${libdir} build_all_use_profile
 	
-	oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \
-		HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \
-		STAGING_LIBDIR=${STAGING_LIBDIR} \
-		STAGING_INCDIR=${STAGING_INCDIR} \
-		STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \
-		LIB=${baselib} \
-		ARCH=${TARGET_ARCH} \
-		DESTDIR=${D} LIBDIR=${libdir} install
+	run_make DESTDIR=${D} LIBDIR=${libdir} install
 
 	# avoid conflict with 2to3 from Python 2
 	rm -f ${D}/${bindir}/2to3
-- 
2.11.0

[PATCH 2/5] python3: add toggle for profile-guided-optimisation

[Ross Burton]

Add a PACKAGECONFIG for profile-guided-optimisation, and default to whether
qemu-usermode is available.

Move --enable-optimizations to the pgo optimisation as all this currently does
is use the PGO rules, causing a performance hit if PGO isn't actually used.

Signed-off-by: Ross Burton <ross.burton@intel.com>
---
 meta/recipes-devtools/python/python3_3.5.5.bb | 54 +++++++++++++++------------
 1 file changed, 31 insertions(+), 23 deletions(-)

diff --git a/meta/recipes-devtools/python/python3_3.5.5.bb b/meta/recipes-devtools/python/python3_3.5.5.bb
index 8e94e38a2d5..aad2e3abe60 100644
--- a/meta/recipes-devtools/python/python3_3.5.5.bb
+++ b/meta/recipes-devtools/python/python3_3.5.5.bb
@@ -73,7 +73,8 @@ CACHED_CONFIGUREVARS = "ac_cv_have_chflags=no \
 TARGET_CC_ARCH += "-DNDEBUG -fno-inline"
 SDK_CC_ARCH += "-DNDEBUG -fno-inline"
 EXTRA_OEMAKE += "CROSS_COMPILE=yes"
-EXTRA_OECONF += "CROSSPYTHONPATH=${STAGING_LIBDIR_NATIVE}/python${PYTHON_MAJMIN}/lib-dynload/ --without-ensurepip --enable-optimizations"
+EXTRA_OECONF += "CROSSPYTHONPATH=${STAGING_LIBDIR_NATIVE}/python${PYTHON_MAJMIN}/lib-dynload/ --without-ensurepip"
+
 PYTHON3_PROFILE_TASK ?= "${S}/Tools/pybench/pybench.py -n 1"
 
 export CROSS_COMPILE = "${TARGET_PREFIX}"
@@ -87,8 +88,10 @@ export CROSSPYTHONPATH = "${STAGING_LIBDIR_NATIVE}/python${PYTHON_MAJMIN}/lib-dy
 # No ctypes option for python 3
 PYTHONLSBOPTS = ""
 
-PACKAGECONFIG ??= "readline"
+PACKAGECONFIG ??= "readline ${@bb.utils.contains('MACHINE_FEATURES', 'qemu-usermode', 'pgo', '', d)}"
 PACKAGECONFIG[readline] = ",,readline"
+# Use profile guided optimisation by running PyBench inside qemu-user
+PACKAGECONFIG[pgo] = "--enable-optimizations"
 
 do_configure_append() {
 	rm -f ${S}/Makefile.orig
@@ -108,16 +111,16 @@ run_make() {
 }
 
 do_compile() {
-        # regenerate platform specific files, because they depend on system headers
-        cd ${S}/Lib/plat-linux*
-        include=${STAGING_INCDIR} ${STAGING_BINDIR_NATIVE}/python3-native/python3 \
-                ${S}/Tools/scripts/h2py.py -i '(u_long)' \
-                ${STAGING_INCDIR}/dlfcn.h \
-                ${STAGING_INCDIR}/linux/cdrom.h \
-                ${STAGING_INCDIR}/netinet/in.h \
-                ${STAGING_INCDIR}/sys/types.h
-        sed -e 's,${STAGING_DIR_HOST},,g' -i *.py
-        cd -
+	# regenerate platform specific files, because they depend on system headers
+	cd ${S}/Lib/plat-linux*
+	include=${STAGING_INCDIR} ${STAGING_BINDIR_NATIVE}/python3-native/python3 \
+		${S}/Tools/scripts/h2py.py -i '(u_long)' \
+		${STAGING_INCDIR}/dlfcn.h \
+		${STAGING_INCDIR}/linux/cdrom.h \
+		${STAGING_INCDIR}/netinet/in.h \
+		${STAGING_INCDIR}/sys/types.h
+	sed -e 's,${STAGING_DIR_HOST},,g' -i *.py
+	cd -
 
 	# remove any bogus LD_LIBRARY_PATH
 	sed -i -e s,RUNSHARED=.*,RUNSHARED=, Makefile
@@ -137,20 +140,21 @@ do_compile() {
 	# then call do_install twice we get Makefile.orig == Makefile.sysroot
 	install -m 0644 Makefile Makefile.sysroot
 
-    run_make profile-opt
-
-        if ${@bb.utils.contains('MACHINE_FEATURES', 'qemu-usermode', 'true', 'false', d)}; then
-                qemu_binary="${@qemu_wrapper_cmdline(d, '${STAGING_DIR_TARGET}', ['${B}', '${STAGING_DIR_TARGET}/${base_libdir}'])}"
-                cat > pgo-image-qemuwrapper << EOF
+	if ${@bb.utils.contains('PACKAGECONFIG', 'pgo', 'true', 'false', d)}; then
+		run_make profile-opt
+		qemu_binary="${@qemu_wrapper_cmdline(d, '${STAGING_DIR_TARGET}', ['${B}', '${STAGING_DIR_TARGET}/${base_libdir}'])}"
+		cat > pgo-image-qemuwrapper << EOF
 #!/bin/sh
 set -x
 $qemu_binary "\$@"
 EOF
-                chmod +x pgo-image-qemuwrapper
-                ./pgo-image-qemuwrapper ${B}/python ${PYTHON3_PROFILE_TASK} || true
+		chmod +x pgo-image-qemuwrapper
+		./pgo-image-qemuwrapper ${B}/python ${PYTHON3_PROFILE_TASK} || true
+		run_make clean_and_use_profile
+	else
+		run_make libpython3.so
+		run_make
 	fi
-
-    run_make clean_and_use_profile
 }
 
 do_install() {
@@ -163,8 +167,12 @@ do_install() {
 
 	# rerun the build once again with original makefile this time
 	# run install in a separate step to avoid compile/install race
-	run_make DESTDIR=${D} LIBDIR=${libdir} build_all_use_profile
-	
+	if ${@bb.utils.contains('PACKAGECONFIG', 'pgo', 'true', 'false', d)}; then
+		run_make DESTDIR=${D} LIBDIR=${libdir} build_all_use_profile
+	else
+		run_make DESTDIR=${D} LIBDIR=${libdir}
+	fi
+
 	run_make DESTDIR=${D} LIBDIR=${libdir} install
 
 	# avoid conflict with 2to3 from Python 2
-- 
2.11.0

[PATCH 3/5] python3: trim PGO patch

[Ross Burton]

There's no need to delete the line that removes the profile data, as we're not
using it after the build.  This reduces the size of the patch, making it easier
to maintain.

Signed-off-by: Ross Burton <ross.burton@intel.com>
---
 .../0002-Makefile-add-target-to-split-profile-generation.patch        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/meta/recipes-devtools/python/python3/0002-Makefile-add-target-to-split-profile-generation.patch b/meta/recipes-devtools/python/python3/0002-Makefile-add-target-to-split-profile-generation.patch
index bb01c5bcb46..2b4ba316e45 100644
--- a/meta/recipes-devtools/python/python3/0002-Makefile-add-target-to-split-profile-generation.patch
+++ b/meta/recipes-devtools/python/python3/0002-Makefile-add-target-to-split-profile-generation.patch
@@ -19,7 +19,7 @@ diff --git a/Makefile.pre.in b/Makefile.pre.in
 index 84bc3ff..017a2c4 100644
 --- a/Makefile.pre.in
 +++ b/Makefile.pre.in
-@@ -469,13 +469,11 @@ profile-opt:
+@@ -469,13 +469,12 @@ profile-opt:
  	$(MAKE) profile-removal
  	$(MAKE) build_all_generate_profile
  	$(MAKE) profile-removal
@@ -31,7 +31,7 @@ index 84bc3ff..017a2c4 100644
  	@echo "Rebuilding with profile guided optimizations:"
  	$(MAKE) clean
  	$(MAKE) build_all_use_profile
--	$(MAKE) profile-removal
+ 	$(MAKE) profile-removal
  
  build_all_generate_profile:
  	$(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(EXTRA_CFLAGS) $(PGO_PROF_GEN_FLAG) @LTOFLAGS@" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG) @LTOFLAGS@" LIBS="$(LIBS)"
-- 
2.11.0

[PATCH 4/5] python3: don't use runtime checks to identify float endianism

[Ross Burton]

Python uses AC_RUN_IFELSE to determine the byte order for floats and doubles,
and falls back onto "I don't know" if it can't run code.  This results in
crippled floating point numbers in Python, and the regression tests fail.

Instead of running code, take a macro from autoconf-archive which compiles C
with a special double in which has an ASCII representation, and then greps the
binary to identify the format.

Signed-off-by: Ross Burton <ross.burton@intel.com>
---
 .../python/python3/float-endian.patch              | 212 +++++++++++++++++++++
 meta/recipes-devtools/python/python3_3.5.5.bb      |   1 +
 2 files changed, 213 insertions(+)
 create mode 100644 meta/recipes-devtools/python/python3/float-endian.patch

diff --git a/meta/recipes-devtools/python/python3/float-endian.patch b/meta/recipes-devtools/python/python3/float-endian.patch
new file mode 100644
index 00000000000..6ba3f5c252d
--- /dev/null
+++ b/meta/recipes-devtools/python/python3/float-endian.patch
@@ -0,0 +1,212 @@
+Python uses AC_RUN_IFELSE to determine the byte order for floats and doubles,
+and falls back onto "I don't know" if it can't run code.  This results in
+crippled floating point numbers in Python, and the regression tests fail.
+
+Instead of running code, take a macro from autoconf-archive which compiles C
+with a special double in which has an ASCII representation, and then greps the
+binary to identify the format.
+
+Upstream-Status: Submitted [https://bugs.python.org/issue34585]
+Signed-off-by: Ross Burton <ross.burton@intel.com>
+
+From 50df2a4c3a65ed06322be7c26d42b06ce81730c1 Mon Sep 17 00:00:00 2001
+From: Ross Burton <ross.burton@intel.com>
+Date: Wed, 5 Sep 2018 11:45:52 +0100
+Subject: [PATCH] Don't do runtime test to get float byte order
+
+---
+ configure.ac                     | 74 +++++------------------------------
+ m4/ax_c_float_words_bigendian.m4 | 83 ++++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 92 insertions(+), 65 deletions(-)
+ create mode 100644 m4/ax_c_float_words_bigendian.m4
+
+diff --git a/configure.ac b/configure.ac
+index c9b755f0f4..1215969871 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -9,6 +9,8 @@ AC_PREREQ(2.65)
+ 
+ AC_INIT(python, PYTHON_VERSION, https://bugs.python.org/)
+ 
++AC_CONFIG_MACRO_DIR(m4)
++
+ AC_SUBST(BASECPPFLAGS)
+ if test "$srcdir" != . -a "$srcdir" != "$(pwd)"; then
+     # If we're building out-of-tree, we need to make sure the following
+@@ -4128,77 +4130,19 @@ fi
+ # * Check for various properties of floating point *
+ # **************************************************
+ 
+-AC_MSG_CHECKING(whether C doubles are little-endian IEEE 754 binary64)
+-AC_CACHE_VAL(ac_cv_little_endian_double, [
+-AC_RUN_IFELSE([AC_LANG_SOURCE([[
+-#include <string.h>
+-int main() {
+-    double x = 9006104071832581.0;
+-    if (memcmp(&x, "\x05\x04\x03\x02\x01\xff\x3f\x43", 8) == 0)
+-        return 0;
+-    else
+-        return 1;
+-}
+-]])],
+-[ac_cv_little_endian_double=yes],
+-[ac_cv_little_endian_double=no],
+-[ac_cv_little_endian_double=no])])
+-AC_MSG_RESULT($ac_cv_little_endian_double)
+-if test "$ac_cv_little_endian_double" = yes
+-then
+-  AC_DEFINE(DOUBLE_IS_LITTLE_ENDIAN_IEEE754, 1,
+-  [Define if C doubles are 64-bit IEEE 754 binary format, stored
+-   with the least significant byte first])
+-fi
+-
+-AC_MSG_CHECKING(whether C doubles are big-endian IEEE 754 binary64)
+-AC_CACHE_VAL(ac_cv_big_endian_double, [
+-AC_RUN_IFELSE([AC_LANG_SOURCE([[
+-#include <string.h>
+-int main() {
+-    double x = 9006104071832581.0;
+-    if (memcmp(&x, "\x43\x3f\xff\x01\x02\x03\x04\x05", 8) == 0)
+-        return 0;
+-    else
+-        return 1;
+-}
+-]])],
+-[ac_cv_big_endian_double=yes],
+-[ac_cv_big_endian_double=no],
+-[ac_cv_big_endian_double=no])])
+-AC_MSG_RESULT($ac_cv_big_endian_double)
+-if test "$ac_cv_big_endian_double" = yes
++AX_C_FLOAT_WORDS_BIGENDIAN
++if test "$ax_cv_c_float_words_bigendian" = "yes"
+ then
+   AC_DEFINE(DOUBLE_IS_BIG_ENDIAN_IEEE754, 1,
+   [Define if C doubles are 64-bit IEEE 754 binary format, stored
+    with the most significant byte first])
+-fi
+-
+-# Some ARM platforms use a mixed-endian representation for doubles.
+-# While Python doesn't currently have full support for these platforms
+-# (see e.g., issue 1762561), we can at least make sure that float <-> string
+-# conversions work.
+-AC_MSG_CHECKING(whether C doubles are ARM mixed-endian IEEE 754 binary64)
+-AC_CACHE_VAL(ac_cv_mixed_endian_double, [
+-AC_RUN_IFELSE([AC_LANG_SOURCE([[
+-#include <string.h>
+-int main() {
+-    double x = 9006104071832581.0;
+-    if (memcmp(&x, "\x01\xff\x3f\x43\x05\x04\x03\x02", 8) == 0)
+-        return 0;
+-    else
+-        return 1;
+-}
+-]])],
+-[ac_cv_mixed_endian_double=yes],
+-[ac_cv_mixed_endian_double=no],
+-[ac_cv_mixed_endian_double=no])])
+-AC_MSG_RESULT($ac_cv_mixed_endian_double)
+-if test "$ac_cv_mixed_endian_double" = yes
++elif test "$ax_cv_c_float_words_bigendian" = "no"
+ then
+-  AC_DEFINE(DOUBLE_IS_ARM_MIXED_ENDIAN_IEEE754, 1,
++  AC_DEFINE(DOUBLE_IS_LITTLE_ENDIAN_IEEE754, 1,
+   [Define if C doubles are 64-bit IEEE 754 binary format, stored
+-   in ARM mixed-endian order (byte order 45670123)])
++   with the least significant byte first])
++else
++  AC_MSG_ERROR([Cannot identify floating point byte order])
+ fi
+ 
+ # The short float repr introduced in Python 3.1 requires the
+diff --git a/m4/ax_c_float_words_bigendian.m4 b/m4/ax_c_float_words_bigendian.m4
+new file mode 100644
+index 0000000000..216b90d803
+--- /dev/null
++++ b/m4/ax_c_float_words_bigendian.m4
+@@ -0,0 +1,83 @@
++# ===============================================================================
++#  https://www.gnu.org/software/autoconf-archive/ax_c_float_words_bigendian.html
++# ===============================================================================
++#
++# SYNOPSIS
++#
++#   AX_C_FLOAT_WORDS_BIGENDIAN([ACTION-IF-TRUE], [ACTION-IF-FALSE], [ACTION-IF-UNKNOWN])
++#
++# DESCRIPTION
++#
++#   Checks the ordering of words within a multi-word float. This check is
++#   necessary because on some systems (e.g. certain ARM systems), the float
++#   word ordering can be different from the byte ordering. In a multi-word
++#   float context, "big-endian" implies that the word containing the sign
++#   bit is found in the memory location with the lowest address. This
++#   implementation was inspired by the AC_C_BIGENDIAN macro in autoconf.
++#
++#   The endianness is detected by first compiling C code that contains a
++#   special double float value, then grepping the resulting object file for
++#   certain strings of ASCII values. The double is specially crafted to have
++#   a binary representation that corresponds with a simple string. In this
++#   implementation, the string "noonsees" was selected because the
++#   individual word values ("noon" and "sees") are palindromes, thus making
++#   this test byte-order agnostic. If grep finds the string "noonsees" in
++#   the object file, the target platform stores float words in big-endian
++#   order. If grep finds "seesnoon", float words are in little-endian order.
++#   If neither value is found, the user is instructed to specify the
++#   ordering.
++#
++# LICENSE
++#
++#   Copyright (c) 2008 Daniel Amelang <dan@amelang.net>
++#
++#   Copying and distribution of this file, with or without modification, are
++#   permitted in any medium without royalty provided the copyright notice
++#   and this notice are preserved. This file is offered as-is, without any
++#   warranty.
++
++#serial 11
++
++AC_DEFUN([AX_C_FLOAT_WORDS_BIGENDIAN],
++  [AC_CACHE_CHECK(whether float word ordering is bigendian,
++                  ax_cv_c_float_words_bigendian, [
++
++ax_cv_c_float_words_bigendian=unknown
++AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
++
++double d = 90904234967036810337470478905505011476211692735615632014797120844053488865816695273723469097858056257517020191247487429516932130503560650002327564517570778480236724525140520121371739201496540132640109977779420565776568942592.0;
++
++]])], [
++
++if grep noonsees conftest.$ac_objext >/dev/null ; then
++  ax_cv_c_float_words_bigendian=yes
++fi
++if grep seesnoon conftest.$ac_objext >/dev/null ; then
++  if test "$ax_cv_c_float_words_bigendian" = unknown; then
++    ax_cv_c_float_words_bigendian=no
++  else
++    ax_cv_c_float_words_bigendian=unknown
++  fi
++fi
++
++])])
++
++case $ax_cv_c_float_words_bigendian in
++  yes)
++    m4_default([$1],
++      [AC_DEFINE([FLOAT_WORDS_BIGENDIAN], 1,
++                 [Define to 1 if your system stores words within floats
++                  with the most significant word first])]) ;;
++  no)
++    $2 ;;
++  *)
++    m4_default([$3],
++      [AC_MSG_ERROR([
++
++Unknown float word ordering. You need to manually preset
++ax_cv_c_float_words_bigendian=no (or yes) according to your system.
++
++    ])]) ;;
++esac
++
++])# AX_C_FLOAT_WORDS_BIGENDIAN
+-- 
+2.11.0
+
diff --git a/meta/recipes-devtools/python/python3_3.5.5.bb b/meta/recipes-devtools/python/python3_3.5.5.bb
index aad2e3abe60..b89e3b04c86 100644
--- a/meta/recipes-devtools/python/python3_3.5.5.bb
+++ b/meta/recipes-devtools/python/python3_3.5.5.bb
@@ -41,6 +41,7 @@ SRC_URI += "\
             file://pass-missing-libraries-to-Extension-for-mul.patch \
             file://Use-correct-CFLAGS-for-extensions-when-cross-compili.patch \
             file://0002-Makefile-add-target-to-split-profile-generation.patch \
+			file://float-endian.patch \
            "
 SRC_URI[md5sum] = "f3763edf9824d5d3a15f5f646083b6e0"
 SRC_URI[sha256sum] = "063d2c3b0402d6191b90731e0f735c64830e7522348aeb7ed382a83165d45009"
-- 
2.11.0

[PATCH 5/5] python3: use regrtest instead of PyBench for profile-guided-optimisation

[Ross Burton]

PyBench takes a long time to run, also upstream have removed it from Python and
instead use test.regrtest —pgo to profile the interpreter.

The results are good: not only does Python compile faster (~300s vs ~600s on my
machine) but Phoronix’s PyBench test runs in 2130ms compared to 2229ms when
using PyBench to train (and 2345ms with PGO disabled).

Signed-off-by: Ross Burton <ross.burton@intel.com>
---
 meta/recipes-devtools/python/python3_3.5.5.bb | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/meta/recipes-devtools/python/python3_3.5.5.bb b/meta/recipes-devtools/python/python3_3.5.5.bb
index b89e3b04c86..386b46a763d 100644
--- a/meta/recipes-devtools/python/python3_3.5.5.bb
+++ b/meta/recipes-devtools/python/python3_3.5.5.bb
@@ -76,7 +76,7 @@ SDK_CC_ARCH += "-DNDEBUG -fno-inline"
 EXTRA_OEMAKE += "CROSS_COMPILE=yes"
 EXTRA_OECONF += "CROSSPYTHONPATH=${STAGING_LIBDIR_NATIVE}/python${PYTHON_MAJMIN}/lib-dynload/ --without-ensurepip"
 
-PYTHON3_PROFILE_TASK ?= "${S}/Tools/pybench/pybench.py -n 1"
+PYTHON3_PROFILE_TASK ?= "./python -m test.regrtest --pgo test_grammar test_opcodes test_dict test_builtin test_exceptions test_types test_support || true"
 
 export CROSS_COMPILE = "${TARGET_PREFIX}"
 export _PYTHON_PROJECT_BASE = "${B}"
@@ -144,13 +144,15 @@ do_compile() {
 	if ${@bb.utils.contains('PACKAGECONFIG', 'pgo', 'true', 'false', d)}; then
 		run_make profile-opt
 		qemu_binary="${@qemu_wrapper_cmdline(d, '${STAGING_DIR_TARGET}', ['${B}', '${STAGING_DIR_TARGET}/${base_libdir}'])}"
-		cat > pgo-image-qemuwrapper << EOF
+		cat >pgo-wrapper <<EOF
 #!/bin/sh
-set -x
+cd ${B}
 $qemu_binary "\$@"
 EOF
-		chmod +x pgo-image-qemuwrapper
-		./pgo-image-qemuwrapper ${B}/python ${PYTHON3_PROFILE_TASK} || true
+		chmod +x pgo-wrapper
+		bbnote Gathering profiling data
+		./pgo-wrapper ${PYTHON3_PROFILE_TASK}
+		bbnote Profiling data gathered, rebuilding
 		run_make clean_and_use_profile
 	else
 		run_make libpython3.so
-- 
2.11.0