[U-Boot] [PATCHv3 12/13] gitlab/travis: Rework how and when we use virtualenv in order to use python3

From: Tom Rini <trini@konsulko.com>
To: u-boot@lists.denx.de
Subject: [U-Boot] [PATCHv3 12/13] gitlab/travis: Rework how and when we use virtualenv in order to use python3
Date: Thu, 24 Oct 2019 11:59:27 -0400	[thread overview]
Message-ID: <20191024155928.28616-13-trini@konsulko.com> (raw)
In-Reply-To: <20191024155928.28616-1-trini@konsulko.com>

As things stand today, we have tools that CI requires where "python"
must be "python2".  We need to use a virtualenv and pip in order to
ensure that our pytest tests can be run.  Rework things slightly so
that:
- On Travis-CI, we install python-pyelftools for the platforms that
  require pyelftools to be installed.
- On GitLab-CI, we move to a newer base image that includes python3-pip
  and continue to use a virtualenv per job that needs it, for the
  correct set of packages.

Reviewed-by: Simon Glass <sjg@chromium.org>
Tested-by: Stephen Warren <swarren@nvidia.com>
Tested-by: Simon Glass <sjg@chromium.org> [on sandbox]
Signed-off-by: Tom Rini <trini@konsulko.com>
---
Changes in v2:
- Add Travis-CI support here, and stop calling it a work-around.  This
  is just the path forward until we have everything using python3 and at
  that point evaluate what the right changes, if any, are.
---
 .gitlab-ci.yml | 10 ++++------
 .travis.yml    | 30 +++++++++++++++---------------
 2 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5a34321570ce..9b295ac710fe 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,7 +2,7 @@
 
 # Grab our configured image.  The source for this is found at:
 # https://gitlab.denx.de/u-boot/gitlab-ci-runner
-image: trini/u-boot-gitlab-ci-runner:bionic-20190912.1-03Oct2019
+image: trini/u-boot-gitlab-ci-runner:bionic-20191010-20Oct2019
 
 # We run some tests in different order, to catch some failures quicker.
 stages:
@@ -18,11 +18,6 @@ stages:
     - git clone --depth=1 git://github.com/swarren/uboot-test-hooks.git /tmp/uboot-test-hooks
     - ln -s travis-ci /tmp/uboot-test-hooks/bin/`hostname`
     - ln -s travis-ci /tmp/uboot-test-hooks/py/`hostname`
-    - virtualenv /tmp/venv
-    - . /tmp/venv/bin/activate
-    - pip install pytest
-    - pip install python-subunit
-    - pip install coverage
     - grub-mkimage --prefix="" -o ~/grub_x86.efi -O i386-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
     - grub-mkimage --prefix="" -o ~/grub_x64.efi -O x86_64-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
     - mkdir ~/grub2-arm
@@ -47,6 +42,9 @@ stages:
     # never prevent any test from running. That way, we can always pass
     # "-k something" even when $TEST_PY_TEST_SPEC doesnt need a custom
     # value.
+    - virtualenv -p /usr/bin/python3 /tmp/venv
+    - . /tmp/venv/bin/activate
+    - pip install -r test/py/requirements.txt
     - export UBOOT_TRAVIS_BUILD_DIR=/tmp/.bm-work/${TEST_PY_BD};
       export PATH=/opt/qemu/bin:/tmp/uboot-test-hooks/bin:${PATH};
       export PYTHONPATH=/tmp/uboot-test-hooks/py/travis-ci;
diff --git a/.travis.yml b/.travis.yml
index f61278f9656a..d15b3f742639 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,7 +21,9 @@ addons:
     - build-essential
     - libsdl1.2-dev
     - python
-    - python-virtualenv
+    - python-pyelftools
+    - python3-virtualenv
+    - python3-pip
     - swig
     - libpython-dev
     - iasl
@@ -47,11 +49,6 @@ install:
  - echo -e "arc = /tmp/arc_gnu_2018.09_prebuilt_uclibc_le_archs_linux_install" >> ~/.buildman
  - echo -e "\n[toolchain-alias]\nsh = sh2\n" >> ~/.buildman
  - cat ~/.buildman
- - virtualenv /tmp/venv
- - . /tmp/venv/bin/activate
- - pip install pytest
- - pip install python-subunit
- - pip install pyelftools
  - grub-mkimage --prefix="" -o ~/grub_x86.efi -O i386-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
  - grub-mkimage --prefix="" -o ~/grub_x64.efi -O x86_64-efi normal  echo lsefimmap lsefi lsefisystab efinet tftp minicmd
  - mkdir ~/grub2-arm
@@ -136,15 +133,6 @@ script:
    cp ~/grub_x64.efi $UBOOT_TRAVIS_BUILD_DIR/;
    cp ~/grub2-arm/usr/lib/grub2/arm-efi/grub.efi $UBOOT_TRAVIS_BUILD_DIR/grub_arm.efi;
    cp ~/grub2-arm64/usr/lib/grub2/arm64-efi/grub.efi $UBOOT_TRAVIS_BUILD_DIR/grub_arm64.efi;
-   if [[ "${TEST_PY_BD}" != "" ]]; then
-     ./test/py/test.py --bd ${TEST_PY_BD} ${TEST_PY_ID}
-       -k "${TEST_PY_TEST_SPEC:-not a_test_which_does_not_exist}"
-       --build-dir "$UBOOT_TRAVIS_BUILD_DIR";
-     ret=$?;
-     if [[ $ret -ne 0 ]]; then
-       exit $ret;
-     fi;
-   fi;
    if [[ -n "${TEST_PY_TOOLS}" ]]; then
      PYTHONPATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt"
      PATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH}"
@@ -154,6 +142,18 @@ script:
      PYTHONPATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt"
      PATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH}"
      ./tools/dtoc/dtoc -t;
+   fi;
+   if [[ "${TEST_PY_BD}" != "" ]]; then
+     virtualenv -p /usr/bin/python3 /tmp/venv;
+     . /tmp/venv/bin/activate;
+     pip install -r test/py/requirements.txt;
+     ./test/py/test.py --bd ${TEST_PY_BD} ${TEST_PY_ID}
+       -k "${TEST_PY_TEST_SPEC:-not a_test_which_does_not_exist}"
+       --build-dir "$UBOOT_TRAVIS_BUILD_DIR";
+     ret=$?;
+     if [[ $ret -ne 0 ]]; then
+       exit $ret;
+     fi;
    fi
 
 matrix:
-- 
2.17.1
