[PATCH 0/2] Toaster test improvements for autobuilder

[PATCH 0/2] Toaster test improvements for autobuilder

[Alexander Lussier-Cullen]

Changes to toaster tests to allow for faster builds and to avoid
usage of the root level /tmp directory. Intended to improve test
execution on the autobuilder.

Alexander Lussier-Cullen (2):
  Toaster: tests: add passthroughs for relevant build environment
    variables
  Toaster: make django temp directory configurable

 bitbake/lib/toaster/tests/builds/buildtest.py    | 9 +++++++++
 bitbake/lib/toaster/toastermain/settings.py      | 7 ++++---
 bitbake/lib/toaster/toastermain/settings_test.py | 4 ++--
 bitbake/lib/toaster/tox.ini                      | 4 ++++
 4 files changed, 19 insertions(+), 5 deletions(-)

-- 
2.34.1

[PATCH 1/2] Toaster: tests: add passthroughs for relevant build environment variables

[Alexander Lussier-Cullen]

Toaster build tests create new build environments which do not utlize
any existing build environment variables.
In particular, DL_DIR and SSTATE_DIR from an existing environment can
be passed in to allow for faster builds.

Adding these as passthroughs specifically resolves slow builds
related to the autobuilder integrations of the toaster test suite.

Signed-off-by: Alexander Lussier-Cullen <alexander.lussier-cullen@savoirfairelinux.com>
---
 bitbake/lib/toaster/tests/builds/buildtest.py | 9 +++++++++
 bitbake/lib/toaster/tox.ini                   | 3 +++
 2 files changed, 12 insertions(+)

diff --git a/bitbake/lib/toaster/tests/builds/buildtest.py b/bitbake/lib/toaster/tests/builds/buildtest.py
index 13b51fb0d8..53cd7a9ffa 100644
--- a/bitbake/lib/toaster/tests/builds/buildtest.py
+++ b/bitbake/lib/toaster/tests/builds/buildtest.py
@@ -116,6 +116,15 @@ class BuildTest(unittest.TestCase):
         project = Project.objects.create_project(name=BuildTest.PROJECT_NAME,
                                                  release=release)
 
+        passthrough_variable_names = ["SSTATE_DIR", "DL_DIR"]
+        for variable_name in passthrough_variable_names:
+            current_variable = os.environ.get(variable_name)
+            if current_variable:
+                ProjectVariable.objects.get_or_create(
+                    name=variable_name,
+                    value=current_variable,
+                    project=project)
+
         if os.environ.get("TOASTER_TEST_USE_SSTATE_MIRROR"):
             ProjectVariable.objects.get_or_create(
                 name="SSTATE_MIRRORS",
diff --git a/bitbake/lib/toaster/tox.ini b/bitbake/lib/toaster/tox.ini
index d058558b05..9928657106 100644
--- a/bitbake/lib/toaster/tox.ini
+++ b/bitbake/lib/toaster/tox.ini
@@ -5,6 +5,9 @@ toxworkdir = {env:TOX_WORKDIR:.tox}
 passenv = *
 
 [testenv]
+passenv =
+    SSTATE_DIR
+    DL_DIR
 setenv =
     DJANGO_SETTINGS_MODULE=toastermain.settings_test
     TOASTER_BUILDSERVER=1
-- 
2.34.1

[PATCH 2/2] Toaster: make django temp directory configurable

[Alexander Lussier-Cullen]

Modify the django settings to make cache, logs and databases save
to a configurable directory with TOASTER_DJANGO_TMPDIR instead of
the machine root level /tmp directory.

In addition, add this variable to the passthroughs for tox.

This change is made to facilitate cleaning test files for runs on
the autobuilder and avoid errors involving the persisted database.

Signed-off-by: Alexander Lussier-Cullen <alexander.lussier-cullen@savoirfairelinux.com>
---
 bitbake/lib/toaster/toastermain/settings.py      | 7 ++++---
 bitbake/lib/toaster/toastermain/settings_test.py | 4 ++--
 bitbake/lib/toaster/tox.ini                      | 1 +
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/bitbake/lib/toaster/toastermain/settings.py b/bitbake/lib/toaster/toastermain/settings.py
index 69aa5a10c0..3c12359366 100644
--- a/bitbake/lib/toaster/toastermain/settings.py
+++ b/bitbake/lib/toaster/toastermain/settings.py
@@ -148,6 +148,8 @@ STATICFILES_FINDERS = (
 # Make this unique, and don't share it with anybody.
 SECRET_KEY = 'NOT_SUITABLE_FOR_HOSTED_DEPLOYMENT'
 
+TMPDIR = os.environ.get('TOASTER_DJANGO_TMPDIR', '/tmp')
+
 class InvalidString(str):
     def __mod__(self, other):
         from django.template.base import TemplateSyntaxError
@@ -214,7 +216,7 @@ CACHES = {
     #        },
            'default': {
                'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-               'LOCATION': '/tmp/toaster_cache_%d' % os.getuid(),
+               'LOCATION': '%s/toaster_cache_%d' % (TMPDIR, os.getuid()),
                'TIMEOUT': 1,
             }
           }
@@ -312,7 +314,7 @@ for t in os.walk(os.path.dirname(currentdir)):
 LOGGING = LOGGING_SETTINGS
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
-BUILDDIR = os.environ.get("BUILDDIR", "/tmp")
+BUILDDIR = os.environ.get("BUILDDIR", TMPDIR)
 
 # LOG VIEWER
 # https://pypi.org/project/django-log-viewer/
@@ -325,7 +327,6 @@ LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL']
 # Optionally you can set the next variables in order to customize the admin:
 LOG_VIEWER_FILE_LIST_TITLE = "Logs list"
 
-
 if DEBUG and SQL_DEBUG:
     LOGGING['loggers']['django.db.backends'] = {
             'level': 'DEBUG',
diff --git a/bitbake/lib/toaster/toastermain/settings_test.py b/bitbake/lib/toaster/toastermain/settings_test.py
index 6538d9e453..74def2d240 100644
--- a/bitbake/lib/toaster/toastermain/settings_test.py
+++ b/bitbake/lib/toaster/toastermain/settings_test.py
@@ -19,10 +19,10 @@ TEMPLATE_DEBUG = DEBUG
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': '/tmp/toaster-test-db.sqlite',
+        'NAME': '%s/toaster-test-db.sqlite' % TMPDIR,
         'TEST': {
             'ENGINE': 'django.db.backends.sqlite3',
-            'NAME': '/tmp/toaster-test-db.sqlite',
+            'NAME': '%s/toaster-test-db.sqlite' % TMPDIR,
         }
     }
 }
diff --git a/bitbake/lib/toaster/tox.ini b/bitbake/lib/toaster/tox.ini
index 9928657106..1516a527ae 100644
--- a/bitbake/lib/toaster/tox.ini
+++ b/bitbake/lib/toaster/tox.ini
@@ -8,6 +8,7 @@ passenv = *
 passenv =
     SSTATE_DIR
     DL_DIR
+    TOASTER_DJANGO_TMPDIR
 setenv =
     DJANGO_SETTINGS_MODULE=toastermain.settings_test
     TOASTER_BUILDSERVER=1
-- 
2.34.1