[PATCH 1/6] KVM test: Introducing test_setup library

[PATCH 1/6] KVM test: Introducing test_setup library

[Lucas Meneghel Rodrigues]

In order to concentrate setup classes for the KVM autotest
tests, create test_setup.py. This library will contain
code used to perform actions before to the actual test
execution, putting some hooks on the test postprocessing
code. The first class in there is the UnattendedInstallConfig
class, that prepares the environment for unattended installs.

Advantages with doing this in framework code:

- Setup errors are easier to figure out than having a
'pre command failed' error reason.
- We can use test.tmpdir to store temp dirs, which makes
things even cleaner and less intrusive in the system.
- Less code duplication.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/test_setup.py |  494 ++++++++++++++++++++++++++++++++++++++++
 1 files changed, 494 insertions(+), 0 deletions(-)
 create mode 100644 client/tests/kvm/test_setup.py

diff --git a/client/tests/kvm/test_setup.py b/client/tests/kvm/test_setup.py
new file mode 100644
index 0000000..7b7ef14
--- /dev/null
+++ b/client/tests/kvm/test_setup.py
@@ -0,0 +1,494 @@
+"""
+Library to perform pre/post test setup for KVM autotest.
+"""
+import os, sys, shutil, tempfile, re, ConfigParser, glob, inspect, commands
+import logging
+from autotest_lib.client.common_lib import error
+from autotest_lib.client.bin import utils
+
+
+@error.context_aware
+def cleanup(dir):
+    """
+    If dir is a mountpoint, do what is possible to unmount it. Afterwards,
+    try to remove it.
+
+    @param dir: Directory to be cleaned up.
+    """
+    error.context("cleaning up unattended install directory %s" % dir)
+    if os.path.ismount(dir):
+        utils.run('fuser -k %s' % dir, ignore_status=True)
+        utils.run('umount %s' % dir)
+    if os.path.isdir(dir):
+        shutil.rmtree(dir)
+
+
+@error.context_aware
+def clean_old_image(image):
+    """
+    Clean a leftover image file from previous processes. If it contains a
+    mounted file system, do the proper cleanup procedures.
+
+    @param image: Path to image to be cleaned up.
+    """
+    error.context("cleaning up old leftover image %s" % image)
+    if os.path.exists(image):
+        mtab = open('/etc/mtab', 'r')
+        mtab_contents = mtab.read()
+        mtab.close()
+        if image in mtab_contents:
+            utils.run('fuser -k %s' % image, ignore_status=True)
+            utils.run('umount %s' % image)
+        os.remove(image)
+
+
+class Disk(object):
+    """
+    Abstract class for Disk objects, with the common methods implemented.
+    """
+    def __init__(self):
+        self.path = None
+
+
+    def setup_answer_file(self, filename, contents):
+        utils.open_write_close(os.path.join(self.mount, filename), contents)
+
+
+    def copy_to(self, src):
+        dst = os.path.join(self.mount, os.path.basename(src))
+        if os.path.isdir(src):
+            shutil.copytree(src, dst)
+        elif os.path.isfile(src):
+            shutil.copyfile(src, dst)
+
+
+    def close(self):
+        os.chmod(self.path, 0755)
+        cleanup(self.mount)
+        logging.debug("Disk %s successfuly set", self.path)
+
+
+class FloppyDisk(Disk):
+    """
+    Represents a 1.44 MB floppy disk. We can copy files to it, and setup it in
+    convenient ways.
+    """
+    @error.context_aware
+    def __init__(self, path, qemu_img_binary, tmpdir):
+        error.context("Creating unattended install floppy image %s" % path)
+        self.tmpdir = tmpdir
+        self.mount = tempfile.mkdtemp(prefix='floppy_', dir=self.tmpdir)
+        self.virtio_mount = None
+        self.path = path
+        clean_old_image(path)
+        if not os.path.isdir(os.path.dirname(path)):
+            os.makedirs(os.path.dirname(path))
+
+        try:
+            c_cmd = '%s create -f raw %s 1440k' % (qemu_img_binary, path)
+            utils.run(c_cmd)
+            f_cmd = 'mkfs.msdos -s 1 %s' % path
+            utils.run(f_cmd)
+            m_cmd = 'mount -o loop,rw %s %s' % (path, self.mount)
+            utils.run(m_cmd)
+        except error.CmdError, e:
+            cleanup(self.mount)
+            raise
+
+
+    def _copy_virtio_drivers(self, virtio_floppy):
+        """
+        Copy the virtio drivers on the virtio floppy to the install floppy.
+
+        1) Mount the floppy containing the viostor drivers
+        2) Copy its contents to the root of the install floppy
+        """
+        virtio_mount = tempfile.mkdtemp(prefix='virtio_floppy_',
+                                        dir=self.tmpdir)
+
+        pwd = os.getcwd()
+        try:
+            m_cmd = 'mount -o loop %s %s' % (virtio_floppy, virtio_mount)
+            utils.run(m_cmd)
+            os.chdir(virtio_mount)
+            path_list = glob.glob('*')
+            for path in path_list:
+                self.copy_to(path)
+        finally:
+            os.chdir(pwd)
+            cleanup(virtio_mount)
+
+
+    def setup_virtio_win2003(self, virtio_floppy, virtio_oemsetup_id):
+        """
+        Setup the install floppy with the virtio storage drivers, win2003 style.
+
+        Win2003 and WinXP depend on the file txtsetup.oem file to install
+        the virtio drivers from the floppy, which is a .ini file.
+        Process:
+
+        1) Copy the virtio drivers on the virtio floppy to the install floppy
+        2) Parse the ini file with config parser
+        3) Modify the identifier of the default session that is going to be
+           executed on the config parser object
+        4) Re-write the config file to the disk
+        """
+        self._copy_virtio_drivers(virtio_floppy)
+        txtsetup_oem = os.path.join(self.mount, 'txtsetup.oem')
+        if not os.path.isfile(txtsetup_oem):
+            raise IOError('File txtsetup.oem not found on the install '
+                          'floppy. Please verify if your floppy virtio '
+                          'driver image has this file')
+        parser = ConfigParser.ConfigParser()
+        parser.read(txtsetup_oem)
+        if not parser.has_section('Defaults'):
+            raise ValueError('File txtsetup.oem does not have the session '
+                             '"Defaults". Please check txtsetup.oem')
+        default_driver = parser.get('Defaults', 'SCSI')
+        if default_driver != virtio_oemsetup_id:
+            parser.set('Defaults', 'SCSI', virtio_oemsetup_id)
+            fp = open(txtsetup_oem, 'w')
+            parser.write(fp)
+            fp.close()
+
+
+    def setup_virtio_win2008(self, virtio_floppy):
+        """
+        Setup the install floppy with the virtio storage drivers, win2008 style.
+
+        Win2008, Vista and 7 require people to point out the path to the drivers
+        on the unattended file, so we just need to copy the drivers to the
+        driver floppy disk.
+        Process:
+
+        1) Copy the virtio drivers on the virtio floppy to the install floppy
+        """
+        self._copy_virtio_drivers(virtio_floppy)
+
+
+class CdromDisk(Disk):
+    """
+    Represents a CDROM disk that we can master according to our needs.
+    """
+    def __init__(self, path, tmpdir):
+        self.mount = tempfile.mkdtemp(prefix='cdrom_unattended_', dir=tmpdir)
+        self.path = path
+        clean_old_image(path)
+        if not os.path.isdir(os.path.dirname(path)):
+            os.makedirs(os.path.dirname(path))
+
+
+    @error.context_aware
+    def close(self):
+        error.context("Creating unattended install CD image %s" % self.path)
+        g_cmd = ('mkisofs -o %s -max-iso9660-filenames '
+                 '-relaxed-filenames -D --input-charset iso8859-1 '
+                 '%s' % (self.path, self.mount))
+        utils.run(g_cmd)
+
+        os.chmod(self.path, 0755)
+        cleanup(self.mount)
+        logging.debug("unattended install CD image %s successfuly created",
+                      self.path)
+
+
+class UnattendedInstallConfig(object):
+    """
+    Creates a floppy disk image that will contain a config file for unattended
+    OS install. The parameters to the script are retrieved from environment
+    variables.
+    """
+    def __init__(self, test, params):
+        """
+        Sets class atributes from test parameters.
+
+        @param test: KVM test object.
+        @param params: Dictionary with test parameters.
+        """
+        root_dir = test.bindir
+        images_dir = os.path.join(root_dir, 'images')
+        self.deps_dir = os.path.join(root_dir, 'deps')
+        self.unattended_dir = os.path.join(root_dir, 'unattended')
+
+        attributes = ['kernel_args', 'finish_program', 'cdrom_cd1',
+                      'unattended_file', 'medium', 'url', 'kernel', 'initrd',
+                      'nfs_server', 'nfs_dir', 'install_virtio', 'floppy',
+                      'cdrom_unattended', 'boot_path', 'extra_params',
+                      'qemu_img_binary']
+
+        for a in attributes:
+            setattr(self, a, params.get(a, ''))
+
+        if self.install_virtio == 'yes':
+            v_attributes = ['virtio_floppy', 'virtio_storage_path',
+                            'virtio_network_path', 'virtio_oemsetup_id',
+                            'virtio_network_installer']
+            for va in v_attributes:
+                setattr(self, va, params.get(va, ''))
+
+        self.tmpdir = test.tmpdir
+
+        if getattr(self, 'unattended_file'):
+            self.unattended_file = os.path.join(root_dir, self.unattended_file)
+
+        if getattr(self, 'qemu_img_binary'):
+            if not os.path.isfile(getattr(self, 'qemu_img_binary')):
+                self.qemu_img_binary = os.path.join(root_dir,
+                                                    self.qemu_img_binary)
+
+        if getattr(self, 'cdrom_cd1'):
+            self.cdrom_cd1 = os.path.join(root_dir, self.cdrom_cd1)
+        self.cdrom_cd1_mount = tempfile.mkdtemp(prefix='cdrom_cd1_',
+                                                dir=self.tmpdir)
+        if self.medium == 'nfs':
+            self.nfs_mount = tempfile.mkdtemp(prefix='nfs_',
+                                              dir=self.tmpdir)
+
+        if getattr(self, 'floppy'):
+            self.floppy = os.path.join(root_dir, self.floppy)
+            if not os.path.isdir(os.path.dirname(self.floppy)):
+                os.makedirs(os.path.dirname(self.floppy))
+
+        self.image_path = os.path.dirname(self.kernel)
+
+
+    @error.context_aware
+    def render_answer_file(self):
+        """
+        Replace KVM_TEST_CDKEY (in the unattended file) with the cdkey
+        provided for this test and replace the KVM_TEST_MEDIUM with
+        the tree url or nfs address provided for this test.
+
+        @return: Answer file contents
+        """
+        error.base_context('Rendering final answer file')
+        error.context('Reading answer file %s' % self.unattended_file)
+        unattended_contents = open(self.unattended_file).read()
+        dummy_cdkey_re = r'\bKVM_TEST_CDKEY\b'
+        real_cdkey = os.environ.get('KVM_TEST_cdkey')
+        if re.search(dummy_cdkey_re, unattended_contents):
+            if real_cdkey:
+                unattended_contents = re.sub(dummy_cdkey_re, real_cdkey,
+                                             unattended_contents)
+            else:
+                print ("WARNING: 'cdkey' required but not specified for "
+                       "this unattended installation")
+
+        dummy_medium_re = r'\bKVM_TEST_MEDIUM\b'
+        if self.medium == "cdrom":
+            content = "cdrom"
+        elif self.medium == "url":
+            content = "url --url %s" % self.url
+        elif self.medium == "nfs":
+            content = "nfs --server=%s --dir=%s" % (self.nfs_server,
+                                                    self.nfs_dir)
+        else:
+            raise ValueError("Unexpected installation medium %s" % self.url)
+
+        unattended_contents = re.sub(dummy_medium_re, content,
+                                     unattended_contents)
+
+        def replace_virtio_key(contents, dummy_re, env):
+            """
+            Replace a virtio dummy string with contents.
+
+            If install_virtio is not set, replace it with a dummy string.
+
+            @param contents: Contents of the unattended file
+            @param dummy_re: Regular expression used to search on the.
+                    unattended file contents.
+            @param env: Name of the environment variable.
+            """
+            dummy_path = "C:"
+            driver = os.environ.get(env, '')
+
+            if re.search(dummy_re, contents):
+                if self.install_virtio == "yes":
+                    if driver.endswith("msi"):
+                        driver = 'msiexec /passive /package ' + driver
+                    else:
+                        try:
+                            # Let's escape windows style paths properly
+                            drive, path = driver.split(":")
+                            driver = drive + ":" + re.escape(path)
+                        except:
+                            pass
+                    contents = re.sub(dummy_re, driver, contents)
+                else:
+                    contents = re.sub(dummy_re, dummy_path, contents)
+            return contents
+
+        vdict = {r'\bKVM_TEST_STORAGE_DRIVER_PATH\b':
+                 'KVM_TEST_virtio_storage_path',
+                 r'\bKVM_TEST_NETWORK_DRIVER_PATH\b':
+                 'KVM_TEST_virtio_network_path',
+                 r'\bKVM_TEST_VIRTIO_NETWORK_INSTALLER\b':
+                 'KVM_TEST_virtio_network_installer_path'}
+
+        for vkey in vdict:
+            unattended_contents = replace_virtio_key(unattended_contents,
+                                                     vkey, vdict[vkey])
+
+        logging.debug("Unattended install contents:")
+        for line in unattended_contents.splitlines():
+            logging.debug(line)
+        return unattended_contents
+
+
+    def setup_boot_disk(self):
+        answer_contents = self.render_answer_file()
+
+        if self.unattended_file.endswith('.sif'):
+            dest_fname = 'winnt.sif'
+            setup_file = 'winnt.bat'
+            boot_disk = FloppyDisk(self.floppy, self.qemu_img_binary,
+                                   self.tmpdir)
+            boot_disk.setup_answer_file(dest_fname, answer_contents)
+            setup_file_path = os.path.join(self.unattended_dir, setup_file)
+            boot_disk.copy_to(setup_file_path)
+            if self.install_virtio == "yes":
+                boot_disk.setup_virtio_win2003(self.virtio_floppy,
+                                               self.virtio_oemsetup_id)
+            boot_disk.copy_to(self.finish_program)
+
+        elif self.unattended_file.endswith('.ks'):
+            # Red Hat kickstart install
+            dest_fname = 'ks.cfg'
+            if self.cdrom_unattended:
+                boot_disk = CdromDisk(self.cdrom_unattended, self.tmpdir)
+            elif self.floppy:
+                boot_disk = FloppyDisk(self.floppy, self.qemu_img_binary,
+                                       self.tmpdir)
+            else:
+                raise ValueError("Neither cdrom_unattended nor floppy set "
+                                 "on the config file, please verify")
+            boot_disk.setup_answer_file(dest_fname, answer_contents)
+
+        elif self.unattended_file.endswith('.xml'):
+            if "autoyast" in self.extra_params:
+                # SUSE autoyast install
+                dest_fname = "autoinst.xml"
+                if self.cdrom_unattended:
+                    boot_disk = CdromDisk(self.cdrom_unattended)
+                elif self.floppy:
+                    boot_disk = FloppyDisk(self.floppy, self.qemu_img_binary,
+                                           self.tmpdir)
+                else:
+                    raise ValueError("Neither cdrom_unattended nor floppy set "
+                                     "on the config file, please verify")
+                boot_disk.setup_answer_file(dest_fname, answer_contents)
+
+            else:
+                # Windows unattended install
+                dest_fname = "autounattend.xml"
+                boot_disk = FloppyDisk(self.floppy, self.qemu_img_binary,
+                                       self.tmpdir)
+                boot_disk.setup_answer_file(dest_fname, answer_contents)
+                if self.install_virtio == "yes":
+                    boot_disk.setup_virtio_win2008(self.virtio_floppy)
+                boot_disk.copy_to(self.finish_program)
+
+        else:
+            raise ValueError('Unknown answer file type: %s' %
+                             self.unattended_file)
+
+        boot_disk.close()
+
+
+    @error.context_aware
+    def setup_cdrom(self):
+        """
+        Mount cdrom and copy vmlinuz and initrd.img.
+        """
+        error.context("Copying vmlinuz and initrd.img from install cdrom %s" %
+                      self.cdrom_cd1)
+        m_cmd = ('mount -t iso9660 -v -o loop,ro %s %s' %
+                 (self.cdrom_cd1, self.cdrom_cd1_mount))
+        utils.run(m_cmd)
+
+        try:
+            if not os.path.isdir(self.image_path):
+                os.makedirs(self.image_path)
+            kernel_fetch_cmd = ("cp %s/%s/%s %s" %
+                                (self.cdrom_cd1_mount, self.boot_path,
+                                 os.path.basename(self.kernel), self.kernel))
+            utils.run(kernel_fetch_cmd)
+            initrd_fetch_cmd = ("cp %s/%s/%s %s" %
+                                (self.cdrom_cd1_mount, self.boot_path,
+                                 os.path.basename(self.initrd), self.initrd))
+            utils.run(initrd_fetch_cmd)
+        finally:
+            cleanup(self.cdrom_cd1_mount)
+
+
+    @error.context_aware
+    def setup_url(self):
+        """
+        Download the vmlinuz and initrd.img from URL.
+        """
+        error.context("downloading vmlinuz and initrd.img from %s" % self.url)
+        os.chdir(self.image_path)
+        kernel_fetch_cmd = "wget -q %s/%s/%s" % (self.url, self.boot_path,
+                                                 os.path.basename(self.kernel))
+        initrd_fetch_cmd = "wget -q %s/%s/%s" % (self.url, self.boot_path,
+                                                 os.path.basename(self.initrd))
+
+        if os.path.exists(self.kernel):
+            os.remove(self.kernel)
+        if os.path.exists(self.initrd):
+            os.remove(self.initrd)
+
+        utils.run(kernel_fetch_cmd)
+        utils.run(initrd_fetch_cmd)
+
+
+    def setup_nfs(self):
+        """
+        Copy the vmlinuz and initrd.img from nfs.
+        """
+        error.context("copying the vmlinuz and initrd.img from NFS share")
+
+        m_cmd = ("mount %s:%s %s -o ro" %
+                 (self.nfs_server, self.nfs_dir, self.nfs_mount))
+        utils.run(m_cmd)
+
+        try:
+            kernel_fetch_cmd = ("cp %s/%s/%s %s" %
+                                (self.nfs_mount, self.boot_path,
+                                os.path.basename(self.kernel), self.image_path))
+            utils.run(kernel_fetch_cmd)
+            initrd_fetch_cmd = ("cp %s/%s/%s %s" %
+                                (self.nfs_mount, self.boot_path,
+                                os.path.basename(self.initrd), self.image_path))
+            utils.run(initrd_fetch_cmd)
+        finally:
+            cleanup(self.nfs_mount)
+
+
+    def setup(self):
+        """
+        Configure the environment for unattended install.
+
+        Uses an appropriate strategy according to each install model.
+        """
+        logging.info("Starting unattended install setup")
+
+        logging.debug("Variables set:")
+        for member in inspect.getmembers(self):
+            name, value = member
+            attribute = getattr(self, name)
+            if not (name.startswith("__") or callable(attribute) or not value):
+                logging.debug("    %s: %s", name, value)
+
+        if self.unattended_file and (self.floppy or self.cdrom_unattended):
+            self.setup_boot_disk()
+        if self.medium == "cdrom":
+            if self.kernel and self.initrd:
+                self.setup_cdrom()
+        elif self.medium == "url":
+            self.setup_url()
+        elif self.medium == "nfs":
+            self.setup_nfs()
+        else:
+            raise ValueError("Unexpected installation method %s" %
+                             self.medium)
-- 
1.7.3.4

[PATCH 2/6] KVM test: Make unattended _install use the new pre script

[Lucas Meneghel Rodrigues]

Also, get rid of references to the old unattended install
script.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/kvm_preprocessing.py  |    6 +++++-
 client/tests/kvm/tests_base.cfg.sample |    2 --
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/client/tests/kvm/kvm_preprocessing.py b/client/tests/kvm/kvm_preprocessing.py
index 41455cf..12adb6a 100644
--- a/client/tests/kvm/kvm_preprocessing.py
+++ b/client/tests/kvm/kvm_preprocessing.py
@@ -1,7 +1,7 @@
 import sys, os, time, commands, re, logging, signal, glob, threading, shutil
 from autotest_lib.client.bin import test, utils
 from autotest_lib.client.common_lib import error
-import kvm_vm, kvm_utils, kvm_subprocess, kvm_monitor, ppm_utils
+import kvm_vm, kvm_utils, kvm_subprocess, kvm_monitor, ppm_utils, test_setup
 try:
     import PIL.Image
 except ImportError:
@@ -258,6 +258,10 @@ def preprocess(test, params, env):
         h = kvm_utils.HugePageConfig(params)
         h.setup()
 
+    if params.get("type") == "unattended_install":
+        u = test_setup.UnattendedInstallConfig(test, params)
+        u.setup()
+
     # Execute any pre_commands
     if params.get("pre_command"):
         process_command(test, params, env, params.get("pre_command"),
diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
index 184a582..c727c32 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -97,7 +97,6 @@ variants:
         kill_vm_gracefully = yes
         kill_vm_on_error = yes
         force_create_image = yes
-        pre_command += " scripts/unattended.py;"
         extra_params += " -boot d"
         guest_port_unattended_install = 12323
         kernel = vmlinuz
@@ -381,7 +380,6 @@ variants:
                 # The support VM is identical to the tested VM in every way
                 # except for the image name which ends with '-supportvm'.
                 type = unattended_install
-                pre_command += " scripts/unattended.py;"
                 extra_params += " -boot d"
                 force_create_image = yes
                 kill_vm = yes
-- 
1.7.3.4

[PATCH 3/6] KVM test: Removing scripts/unattended.py

[Lucas Meneghel Rodrigues]

Now that its functionality was implemented as part of
the framework.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/scripts/unattended.py |  543 --------------------------------
 client/tests/kvm/tests_base.cfg.sample |    2 -
 2 files changed, 0 insertions(+), 545 deletions(-)
 delete mode 100755 client/tests/kvm/scripts/unattended.py

diff --git a/client/tests/kvm/scripts/unattended.py b/client/tests/kvm/scripts/unattended.py
deleted file mode 100755
index e65fe46..0000000
--- a/client/tests/kvm/scripts/unattended.py
+++ /dev/null
@@ -1,543 +0,0 @@
-#!/usr/bin/python
-"""
-Simple script to setup unattended installs on KVM guests.
-"""
-# -*- coding: utf-8 -*-
-import os, sys, shutil, tempfile, re, ConfigParser, glob, inspect, commands
-import common
-
-
-SCRIPT_DIR = os.path.dirname(sys.modules[__name__].__file__)
-KVM_TEST_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
-
-
-class SetupError(Exception):
-    """
-    Simple wrapper for the builtin Exception class.
-    """
-    pass
-
-
-def find_command(cmd):
-    """
-    Searches for a command on common paths, error if it can't find it.
-
-    @param cmd: Command to be found.
-    """
-    if os.path.exists(cmd):
-        return cmd
-    for dir in ["/usr/local/sbin", "/usr/local/bin",
-                "/usr/sbin", "/usr/bin", "/sbin", "/bin"]:
-        file = os.path.join(dir, cmd)
-        if os.path.exists(file):
-            return file
-    raise ValueError('Missing command: %s' % cmd)
-
-
-def run(cmd, info=None):
-    """
-    Run a command and throw an exception if it fails.
-    Optionally, you can provide additional contextual info.
-
-    @param cmd: Command string.
-    @param reason: Optional string that explains the context of the failure.
-
-    @raise: SetupError if command fails.
-    """
-    print "Running '%s'" % cmd
-    cmd_name = cmd.split(' ')[0]
-    find_command(cmd_name)
-    status, output = commands.getstatusoutput(cmd)
-    if status:
-        e_msg = ('Command %s failed.\nStatus:%s\nOutput:%s' %
-                 (cmd, status, output))
-        if info is not None:
-            e_msg += '\nAdditional Info:%s' % info
-        raise SetupError(e_msg)
-
-    return (status, output.strip())
-
-
-def cleanup(dir):
-    """
-    If dir is a mountpoint, do what is possible to unmount it. Afterwards,
-    try to remove it.
-
-    @param dir: Directory to be cleaned up.
-    """
-    print "Cleaning up directory %s" % dir
-    if os.path.ismount(dir):
-        os.system('fuser -k %s' % dir)
-        run('umount %s' % dir, info='Could not unmount %s' % dir)
-    if os.path.isdir(dir):
-        shutil.rmtree(dir)
-
-
-def clean_old_image(image):
-    """
-    Clean a leftover image file from previous processes. If it contains a
-    mounted file system, do the proper cleanup procedures.
-
-    @param image: Path to image to be cleaned up.
-    """
-    if os.path.exists(image):
-        mtab = open('/etc/mtab', 'r')
-        mtab_contents = mtab.read()
-        mtab.close()
-        if image in mtab_contents:
-            os.system('fuser -k %s' % image)
-            os.system('umount %s' % image)
-        os.remove(image)
-
-
-class Disk(object):
-    """
-    Abstract class for Disk objects, with the common methods implemented.
-    """
-    def __init__(self):
-        self.path = None
-
-
-    def setup_answer_file(self, filename, contents):
-        answer_file = open(os.path.join(self.mount, filename), 'w')
-        answer_file.write(contents)
-        answer_file.close()
-
-
-    def copy_to(self, src):
-        dst = os.path.join(self.mount, os.path.basename(src))
-        if os.path.isdir(src):
-            shutil.copytree(src, dst)
-        elif os.path.isfile(src):
-            shutil.copyfile(src, dst)
-
-
-    def close(self):
-        os.chmod(self.path, 0755)
-        cleanup(self.mount)
-        print "Disk %s successfuly set" % self.path
-
-
-class FloppyDisk(Disk):
-    """
-    Represents a 1.44 MB floppy disk. We can copy files to it, and setup it in
-    convenient ways.
-    """
-    def __init__(self, path):
-        print "Creating floppy unattended image %s" % path
-        qemu_img_binary = os.environ['KVM_TEST_qemu_img_binary']
-        if not os.path.isabs(qemu_img_binary):
-            qemu_img_binary = os.path.join(KVM_TEST_DIR, qemu_img_binary)
-        if not os.path.exists(qemu_img_binary):
-            raise SetupError('The qemu-img binary that is supposed to be used '
-                             '(%s) does not exist. Please verify your '
-                             'configuration' % qemu_img_binary)
-
-        self.mount = tempfile.mkdtemp(prefix='floppy_', dir='/tmp')
-        self.virtio_mount = None
-        self.path = path
-        clean_old_image(path)
-        if not os.path.isdir(os.path.dirname(path)):
-            os.makedirs(os.path.dirname(path))
-
-        try:
-            c_cmd = '%s create -f raw %s 1440k' % (qemu_img_binary, path)
-            run(c_cmd, info='Could not create floppy image')
-            f_cmd = 'mkfs.msdos -s 1 %s' % path
-            run(f_cmd, info='Error formatting floppy image')
-            m_cmd = 'mount -o loop,rw %s %s' % (path, self.mount)
-            run(m_cmd, info='Could not mount floppy image')
-        except:
-            cleanup(self.mount)
-            raise
-
-
-    def _copy_virtio_drivers(self, virtio_floppy):
-        """
-        Copy the virtio drivers on the virtio floppy to the install floppy.
-
-        1) Mount the floppy containing the viostor drivers
-        2) Copy its contents to the root of the install floppy
-        """
-        virtio_mount = tempfile.mkdtemp(prefix='virtio_floppy_', dir='/tmp')
-
-        pwd = os.getcwd()
-        try:
-            m_cmd = 'mount -o loop %s %s' % (virtio_floppy, virtio_mount)
-            run(m_cmd, info='Could not mount virtio floppy driver')
-            os.chdir(virtio_mount)
-            path_list = glob.glob('*')
-            for path in path_list:
-                self.copy_to(path)
-        finally:
-            os.chdir(pwd)
-            cleanup(virtio_mount)
-
-
-    def setup_virtio_win2003(self, virtio_floppy, virtio_oemsetup_id):
-        """
-        Setup the install floppy with the virtio storage drivers, win2003 style.
-
-        Win2003 and WinXP depend on the file txtsetup.oem file to install
-        the virtio drivers from the floppy, which is a .ini file.
-        Process:
-
-        1) Copy the virtio drivers on the virtio floppy to the install floppy
-        2) Parse the ini file with config parser
-        3) Modify the identifier of the default session that is going to be
-           executed on the config parser object
-        4) Re-write the config file to the disk
-        """
-        self._copy_virtio_drivers(virtio_floppy)
-        txtsetup_oem = os.path.join(self.mount, 'txtsetup.oem')
-        if not os.path.isfile(txtsetup_oem):
-            raise SetupError('File txtsetup.oem not found on the install '
-                             'floppy. Please verify if your floppy virtio '
-                             'driver image has this file')
-        parser = ConfigParser.ConfigParser()
-        parser.read(txtsetup_oem)
-        if not parser.has_section('Defaults'):
-            raise SetupError('File txtsetup.oem does not have the session '
-                             '"Defaults". Please check txtsetup.oem')
-        default_driver = parser.get('Defaults', 'SCSI')
-        if default_driver != virtio_oemsetup_id:
-            parser.set('Defaults', 'SCSI', virtio_oemsetup_id)
-            fp = open(txtsetup_oem, 'w')
-            parser.write(fp)
-            fp.close()
-
-
-    def setup_virtio_win2008(self, virtio_floppy):
-        """
-        Setup the install floppy with the virtio storage drivers, win2008 style.
-
-        Win2008, Vista and 7 require people to point out the path to the drivers
-        on the unattended file, so we just need to copy the drivers to the
-        driver floppy disk.
-        Process:
-
-        1) Copy the virtio drivers on the virtio floppy to the install floppy
-        """
-        self._copy_virtio_drivers(virtio_floppy)
-
-
-class CdromDisk(Disk):
-    """
-    Represents a CDROM disk that we can master according to our needs.
-    """
-    def __init__(self, path):
-        print "Creating ISO unattended image %s" % path
-        self.mount = tempfile.mkdtemp(prefix='cdrom_unattended_', dir='/tmp')
-        self.path = path
-        clean_old_image(path)
-        if not os.path.isdir(os.path.dirname(path)):
-            os.makedirs(os.path.dirname(path))
-
-
-    def close(self):
-        g_cmd = ('mkisofs -o %s -max-iso9660-filenames '
-                 '-relaxed-filenames -D --input-charset iso8859-1 '
-                 '%s' % (self.path, self.mount))
-        run(g_cmd, info='Could not generate iso with answer file')
-
-        os.chmod(self.path, 0755)
-        cleanup(self.mount)
-        print "Disk %s successfuly set" % self.path
-
-
-class UnattendedInstall(object):
-    """
-    Creates a floppy disk image that will contain a config file for unattended
-    OS install. The parameters to the script are retrieved from environment
-    variables.
-    """
-    def __init__(self):
-        """
-        Gets params from environment variables and sets class attributes.
-        """
-        images_dir = os.path.join(KVM_TEST_DIR, 'images')
-        self.deps_dir = os.path.join(KVM_TEST_DIR, 'deps')
-        self.unattended_dir = os.path.join(KVM_TEST_DIR, 'unattended')
-
-        attributes = ['kernel_args', 'finish_program', 'cdrom_cd1',
-                      'unattended_file', 'medium', 'url', 'kernel', 'initrd',
-                      'nfs_server', 'nfs_dir', 'install_virtio', 'floppy',
-                      'cdrom_unattended', 'boot_path', 'extra_params']
-
-        for a in attributes:
-            self._setattr(a)
-
-        if self.install_virtio == 'yes':
-            v_attributes = ['virtio_floppy', 'virtio_storage_path',
-                            'virtio_network_path', 'virtio_oemsetup_id',
-                            'virtio_network_installer']
-            for va in v_attributes:
-                self._setattr(va)
-
-        if self.cdrom_cd1:
-            self.cdrom_cd1 = os.path.join(KVM_TEST_DIR, self.cdrom_cd1)
-        self.cdrom_cd1_mount = tempfile.mkdtemp(prefix='cdrom_cd1_', dir='/tmp')
-        if self.medium == 'nfs':
-            self.nfs_mount = tempfile.mkdtemp(prefix='nfs_', dir='/tmp')
-
-        if self.floppy:
-            self.floppy = os.path.join(KVM_TEST_DIR, self.floppy)
-            if not os.path.isdir(os.path.dirname(self.floppy)):
-                os.makedirs(os.path.dirname(self.floppy))
-
-        self.image_path = os.path.dirname(self.kernel)
-
-
-    def _setattr(self, key):
-        """
-        Populate class attributes with contents of environment variables.
-
-        Example: KVM_TEST_medium will populate self.medium.
-
-        @param key: Name of the class attribute we desire to have.
-        """
-        env_name = 'KVM_TEST_%s' % key
-        value = os.environ.get(env_name, '')
-        setattr(self, key, value)
-
-
-    def render_answer_file(self):
-        # Replace KVM_TEST_CDKEY (in the unattended file) with the cdkey
-        # provided for this test and replace the KVM_TEST_MEDIUM with
-        # the tree url or nfs address provided for this test.
-        unattended_contents = open(self.unattended_file).read()
-        dummy_cdkey_re = r'\bKVM_TEST_CDKEY\b'
-        real_cdkey = os.environ.get('KVM_TEST_cdkey')
-        if re.search(dummy_cdkey_re, unattended_contents):
-            if real_cdkey:
-                unattended_contents = re.sub(dummy_cdkey_re, real_cdkey,
-                                             unattended_contents)
-            else:
-                print ("WARNING: 'cdkey' required but not specified for "
-                       "this unattended installation")
-
-        dummy_medium_re = r'\bKVM_TEST_MEDIUM\b'
-        if self.medium == "cdrom":
-            content = "cdrom"
-        elif self.medium == "url":
-            content = "url --url %s" % self.url
-        elif self.medium == "nfs":
-            content = "nfs --server=%s --dir=%s" % (self.nfs_server,
-                                                    self.nfs_dir)
-        else:
-            raise SetupError("Unexpected installation medium %s" % self.url)
-
-        unattended_contents = re.sub(dummy_medium_re, content,
-                                     unattended_contents)
-
-        def replace_virtio_key(contents, dummy_re, env):
-            """
-            Replace a virtio dummy string with contents.
-
-            If install_virtio is not set, replace it with a dummy string.
-
-            @param contents: Contents of the unattended file
-            @param dummy_re: Regular expression used to search on the.
-                    unattended file contents.
-            @param env: Name of the environment variable.
-            """
-            dummy_path = "C:"
-            driver = os.environ.get(env, '')
-
-            if re.search(dummy_re, contents):
-                if self.install_virtio == "yes":
-                    if driver.endswith("msi"):
-                        driver = 'msiexec /passive /package ' + driver
-                    else:
-                        try:
-                            # Let's escape windows style paths properly
-                            drive, path = driver.split(":")
-                            driver = drive + ":" + re.escape(path)
-                        except:
-                            pass
-                    contents = re.sub(dummy_re, driver, contents)
-                else:
-                    contents = re.sub(dummy_re, dummy_path, contents)
-            return contents
-
-        vdict = {r'\bKVM_TEST_STORAGE_DRIVER_PATH\b':
-                 'KVM_TEST_virtio_storage_path',
-                 r'\bKVM_TEST_NETWORK_DRIVER_PATH\b':
-                 'KVM_TEST_virtio_network_path',
-                 r'\bKVM_TEST_VIRTIO_NETWORK_INSTALLER\b':
-                 'KVM_TEST_virtio_network_installer_path'}
-
-        for vkey in vdict:
-            unattended_contents = replace_virtio_key(unattended_contents,
-                                                     vkey, vdict[vkey])
-
-        print "Unattended install contents:"
-        print unattended_contents
-        return unattended_contents
-
-
-    def setup_boot_disk(self):
-        answer_contents = self.render_answer_file()
-
-        if self.unattended_file.endswith('.sif'):
-            dest_fname = 'winnt.sif'
-            setup_file = 'winnt.bat'
-            boot_disk = FloppyDisk(self.floppy)
-            boot_disk.setup_answer_file(dest_fname, answer_contents)
-            setup_file_path = os.path.join(self.unattended_dir, setup_file)
-            boot_disk.copy_to(setup_file_path)
-            if self.install_virtio == "yes":
-                boot_disk.setup_virtio_win2003(self.virtio_floppy,
-                                               self.virtio_oemsetup_id)
-            boot_disk.copy_to(self.finish_program)
-
-        elif self.unattended_file.endswith('.ks'):
-            # Red Hat kickstart install
-            dest_fname = 'ks.cfg'
-            if self.cdrom_unattended:
-                boot_disk = CdromDisk(self.cdrom_unattended)
-            elif self.floppy:
-                boot_disk = FloppyDisk(self.floppy)
-            else:
-                raise SetupError("Neither cdrom_unattended nor floppy set "
-                                 "on the config file, please verify")
-            boot_disk.setup_answer_file(dest_fname, answer_contents)
-
-        elif self.unattended_file.endswith('.xml'):
-            if "autoyast" in self.extra_params:
-                # SUSE autoyast install
-                dest_fname = "autoinst.xml"
-                if self.cdrom_unattended:
-                    boot_disk = CdromDisk(self.cdrom_unattended)
-                elif self.floppy:
-                    boot_disk = FloppyDisk(self.floppy)
-                else:
-                    raise SetupError("Neither cdrom_unattended nor floppy set "
-                                     "on the config file, please verify")
-                boot_disk.setup_answer_file(dest_fname, answer_contents)
-
-            else:
-                # Windows unattended install
-                dest_fname = "autounattend.xml"
-                boot_disk = FloppyDisk(self.floppy)
-                boot_disk.setup_answer_file(dest_fname, answer_contents)
-                if self.install_virtio == "yes":
-                    boot_disk.setup_virtio_win2008(self.virtio_floppy)
-                boot_disk.copy_to(self.finish_program)
-
-        else:
-            raise SetupError('Unknown answer file %s' %
-                             self.unattended_file)
-
-        boot_disk.close()
-
-
-    def setup_cdrom(self):
-        """
-        Mount cdrom and copy vmlinuz and initrd.img.
-        """
-        print "Copying vmlinuz and initrd.img from cdrom"
-        m_cmd = ('mount -t iso9660 -v -o loop,ro %s %s' %
-                 (self.cdrom_cd1, self.cdrom_cd1_mount))
-        run(m_cmd, info='Could not mount CD image %s.' % self.cdrom_cd1)
-
-        try:
-            img_path_cmd = ("mkdir -p %s" % self.image_path)
-            run(img_path_cmd, info=("Could not create image path dir %s" %
-                                    self.image_path))
-            kernel_fetch_cmd = ("cp %s/%s/%s %s" %
-                                (self.cdrom_cd1_mount, self.boot_path,
-                                 os.path.basename(self.kernel), self.kernel))
-            run(kernel_fetch_cmd, info=("Could not copy the vmlinuz from %s" %
-                                        self.cdrom_cd1_mount))
-            initrd_fetch_cmd = ("cp %s/%s/%s %s" %
-                                (self.cdrom_cd1_mount, self.boot_path,
-                                 os.path.basename(self.initrd), self.initrd))
-            run(initrd_fetch_cmd, info=("Could not copy the initrd.img from "
-                                        "%s" % self.cdrom_cd1_mount))
-        finally:
-            cleanup(self.cdrom_cd1_mount)
-
-
-    def setup_url(self):
-        """
-        Download the vmlinuz and initrd.img from URL.
-        """
-        print "Downloading vmlinuz and initrd.img from URL"
-        os.chdir(self.image_path)
-
-        kernel_fetch_cmd = "wget -q %s/%s/%s" % (self.url, self.boot_path,
-                                                 os.path.basename(self.kernel))
-        initrd_fetch_cmd = "wget -q %s/%s/%s" % (self.url, self.boot_path,
-                                                 os.path.basename(self.initrd))
-
-        if os.path.exists(self.kernel):
-            os.unlink(self.kernel)
-        if os.path.exists(self.initrd):
-            os.unlink(self.initrd)
-
-        run(kernel_fetch_cmd, info="Could not fetch vmlinuz from %s" % self.url)
-        run(initrd_fetch_cmd, info=("Could not fetch initrd.img from %s" %
-                                    self.url))
-
-
-    def setup_nfs(self):
-        """
-        Copy the vmlinuz and initrd.img from nfs.
-        """
-        print "Copying the vmlinuz and initrd.img from nfs"
-
-        m_cmd = ("mount %s:%s %s -o ro" %
-                 (self.nfs_server, self.nfs_dir, self.nfs_mount))
-        run(m_cmd, info='Could not mount nfs server')
-
-        try:
-            kernel_fetch_cmd = ("cp %s/%s/%s %s" %
-                                (self.nfs_mount, self.boot_path,
-                                os.path.basename(self.kernel), self.image_path))
-            run(kernel_fetch_cmd, info=("Could not copy the vmlinuz from %s" %
-                                        self.nfs_mount))
-            initrd_fetch_cmd = ("cp %s/%s/%s %s" %
-                                (self.nfs_mount, self.boot_path,
-                                os.path.basename(self.initrd), self.image_path))
-            run(initrd_fetch_cmd, info=("Could not copy the initrd.img from "
-                                        "%s" % self.nfs_mount))
-        finally:
-            cleanup(self.nfs_mount)
-
-
-    def setup(self):
-        """
-        Configure the environment for unattended install.
-
-        Uses an appropriate strategy according to each install model.
-        """
-        print "Starting unattended install setup"
-        print
-
-        print "Variables set:"
-        for member in inspect.getmembers(self):
-            name, value = member
-            attribute = getattr(self, name)
-            if not (name.startswith("__") or callable(attribute) or not value):
-                print "    %s: %s" % (name, value)
-        print
-
-        if self.unattended_file and (self.floppy or self.cdrom_unattended):
-            self.setup_boot_disk()
-        if self.medium == "cdrom":
-            if self.kernel and self.initrd:
-                self.setup_cdrom()
-        elif self.medium == "url":
-            self.setup_url()
-        elif self.medium == "nfs":
-            self.setup_nfs()
-        else:
-            raise SetupError("Unexpected installation method %s" %
-                             self.medium)
-        print "Unattended install setup finished successfuly"
-
-
-if __name__ == "__main__":
-    os_install = UnattendedInstall()
-    os_install.setup()
diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
index c727c32..f90d551 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -1199,8 +1199,6 @@ variants:
                             md5sum_cd1 = f2ebf941dc45f99ee3e8a457c9544552
                             md5sum_1m_cd1 = df029f9cffbc3517937a91124a1e0c3a
 
-
-
             - DSL-4.2.5:
                 no setup dbench bonnie linux_s3
                 image_name = dsl-4.2.5
-- 
1.7.3.4

[PATCH 4/6] KVM config: Move HugePageConfig() to test_setup

[Lucas Meneghel Rodrigues]

So we concentrate the setup classes together.

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/kvm_preprocessing.py |    2 +-
 client/tests/kvm/kvm_utils.py         |  101 ---------------------------------
 client/tests/kvm/test_setup.py        |  101 +++++++++++++++++++++++++++++++++
 3 files changed, 102 insertions(+), 102 deletions(-)

diff --git a/client/tests/kvm/kvm_preprocessing.py b/client/tests/kvm/kvm_preprocessing.py
index 12adb6a..081a13f 100644
--- a/client/tests/kvm/kvm_preprocessing.py
+++ b/client/tests/kvm/kvm_preprocessing.py
@@ -255,7 +255,7 @@ def preprocess(test, params, env):
     test.write_test_keyval({"kvm_userspace_version": kvm_userspace_version})
 
     if params.get("setup_hugepages") == "yes":
-        h = kvm_utils.HugePageConfig(params)
+        h = test_setup.HugePageConfig(params)
         h.setup()
 
     if params.get("type") == "unattended_install":
diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py
index 632badb..78c9f25 100644
--- a/client/tests/kvm/kvm_utils.py
+++ b/client/tests/kvm/kvm_utils.py
@@ -1265,107 +1265,6 @@ class KvmLoggingConfig(logging_config.LoggingConfig):
                                                         verbose=verbose)
 
 
-class HugePageConfig:
-    def __init__(self, params):
-        """
-        Gets environment variable values and calculates the target number
-        of huge memory pages.
-
-        @param params: Dict like object containing parameters for the test.
-        """
-        self.vms = len(params.objects("vms"))
-        self.mem = int(params.get("mem"))
-        self.max_vms = int(params.get("max_vms", 0))
-        self.hugepage_path = '/mnt/kvm_hugepage'
-        self.hugepage_size = self.get_hugepage_size()
-        self.target_hugepages = self.get_target_hugepages()
-        self.kernel_hp_file = '/proc/sys/vm/nr_hugepages'
-
-
-    def get_hugepage_size(self):
-        """
-        Get the current system setting for huge memory page size.
-        """
-        meminfo = open('/proc/meminfo', 'r').readlines()
-        huge_line_list = [h for h in meminfo if h.startswith("Hugepagesize")]
-        try:
-            return int(huge_line_list[0].split()[1])
-        except ValueError, e:
-            raise ValueError("Could not get huge page size setting from "
-                             "/proc/meminfo: %s" % e)
-
-
-    def get_target_hugepages(self):
-        """
-        Calculate the target number of hugepages for testing purposes.
-        """
-        if self.vms < self.max_vms:
-            self.vms = self.max_vms
-        # memory of all VMs plus qemu overhead of 64MB per guest
-        vmsm = (self.vms * self.mem) + (self.vms * 64)
-        return int(vmsm * 1024 / self.hugepage_size)
-
-
-    @error.context_aware
-    def set_hugepages(self):
-        """
-        Sets the hugepage limit to the target hugepage value calculated.
-        """
-        error.context("setting hugepages limit to %s" % self.target_hugepages)
-        hugepage_cfg = open(self.kernel_hp_file, "r+")
-        hp = hugepage_cfg.readline()
-        while int(hp) < self.target_hugepages:
-            loop_hp = hp
-            hugepage_cfg.write(str(self.target_hugepages))
-            hugepage_cfg.flush()
-            hugepage_cfg.seek(0)
-            hp = int(hugepage_cfg.readline())
-            if loop_hp == hp:
-                raise ValueError("Cannot set the kernel hugepage setting "
-                                 "to the target value of %d hugepages." %
-                                 self.target_hugepages)
-        hugepage_cfg.close()
-        logging.debug("Successfuly set %s large memory pages on host ",
-                      self.target_hugepages)
-
-
-    @error.context_aware
-    def mount_hugepage_fs(self):
-        """
-        Verify if there's a hugetlbfs mount set. If there's none, will set up
-        a hugetlbfs mount using the class attribute that defines the mount
-        point.
-        """
-        error.context("mounting hugepages path")
-        if not os.path.ismount(self.hugepage_path):
-            if not os.path.isdir(self.hugepage_path):
-                os.makedirs(self.hugepage_path)
-            cmd = "mount -t hugetlbfs none %s" % self.hugepage_path
-            utils.system(cmd)
-
-
-    def setup(self):
-        logging.debug("Number of VMs this test will use: %d", self.vms)
-        logging.debug("Amount of memory used by each vm: %s", self.mem)
-        logging.debug("System setting for large memory page size: %s",
-                      self.hugepage_size)
-        logging.debug("Number of large memory pages needed for this test: %s",
-                      self.target_hugepages)
-        self.set_hugepages()
-        self.mount_hugepage_fs()
-
-
-    @error.context_aware
-    def cleanup(self):
-        error.context("trying to dealocate hugepage memory")
-        try:
-            utils.system("umount %s" % self.hugepage_path)
-        except error.CmdError:
-            return
-        utils.system("echo 0 > %s" % self.kernel_hp_file)
-        logging.debug("Hugepage memory successfuly dealocated")
-
-
 class PciAssignable(object):
     """
     Request PCI assignable devices on host. It will check whether to request
diff --git a/client/tests/kvm/test_setup.py b/client/tests/kvm/test_setup.py
index 7b7ef14..b17c473 100644
--- a/client/tests/kvm/test_setup.py
+++ b/client/tests/kvm/test_setup.py
@@ -492,3 +492,104 @@ class UnattendedInstallConfig(object):
         else:
             raise ValueError("Unexpected installation method %s" %
                              self.medium)
+
+
+class HugePageConfig(object):
+    def __init__(self, params):
+        """
+        Gets environment variable values and calculates the target number
+        of huge memory pages.
+
+        @param params: Dict like object containing parameters for the test.
+        """
+        self.vms = len(params.objects("vms"))
+        self.mem = int(params.get("mem"))
+        self.max_vms = int(params.get("max_vms", 0))
+        self.hugepage_path = '/mnt/kvm_hugepage'
+        self.hugepage_size = self.get_hugepage_size()
+        self.target_hugepages = self.get_target_hugepages()
+        self.kernel_hp_file = '/proc/sys/vm/nr_hugepages'
+
+
+    def get_hugepage_size(self):
+        """
+        Get the current system setting for huge memory page size.
+        """
+        meminfo = open('/proc/meminfo', 'r').readlines()
+        huge_line_list = [h for h in meminfo if h.startswith("Hugepagesize")]
+        try:
+            return int(huge_line_list[0].split()[1])
+        except ValueError, e:
+            raise ValueError("Could not get huge page size setting from "
+                             "/proc/meminfo: %s" % e)
+
+
+    def get_target_hugepages(self):
+        """
+        Calculate the target number of hugepages for testing purposes.
+        """
+        if self.vms < self.max_vms:
+            self.vms = self.max_vms
+        # memory of all VMs plus qemu overhead of 64MB per guest
+        vmsm = (self.vms * self.mem) + (self.vms * 64)
+        return int(vmsm * 1024 / self.hugepage_size)
+
+
+    @error.context_aware
+    def set_hugepages(self):
+        """
+        Sets the hugepage limit to the target hugepage value calculated.
+        """
+        error.context("setting hugepages limit to %s" % self.target_hugepages)
+        hugepage_cfg = open(self.kernel_hp_file, "r+")
+        hp = hugepage_cfg.readline()
+        while int(hp) < self.target_hugepages:
+            loop_hp = hp
+            hugepage_cfg.write(str(self.target_hugepages))
+            hugepage_cfg.flush()
+            hugepage_cfg.seek(0)
+            hp = int(hugepage_cfg.readline())
+            if loop_hp == hp:
+                raise ValueError("Cannot set the kernel hugepage setting "
+                                 "to the target value of %d hugepages." %
+                                 self.target_hugepages)
+        hugepage_cfg.close()
+        logging.debug("Successfuly set %s large memory pages on host ",
+                      self.target_hugepages)
+
+
+    @error.context_aware
+    def mount_hugepage_fs(self):
+        """
+        Verify if there's a hugetlbfs mount set. If there's none, will set up
+        a hugetlbfs mount using the class attribute that defines the mount
+        point.
+        """
+        error.context("mounting hugepages path")
+        if not os.path.ismount(self.hugepage_path):
+            if not os.path.isdir(self.hugepage_path):
+                os.makedirs(self.hugepage_path)
+            cmd = "mount -t hugetlbfs none %s" % self.hugepage_path
+            utils.system(cmd)
+
+
+    def setup(self):
+        logging.debug("Number of VMs this test will use: %d", self.vms)
+        logging.debug("Amount of memory used by each vm: %s", self.mem)
+        logging.debug("System setting for large memory page size: %s",
+                      self.hugepage_size)
+        logging.debug("Number of large memory pages needed for this test: %s",
+                      self.target_hugepages)
+        self.set_hugepages()
+        self.mount_hugepage_fs()
+
+
+    @error.context_aware
+    def cleanup(self):
+        error.context("trying to dealocate hugepage memory")
+        try:
+            utils.system("umount %s" % self.hugepage_path)
+        except error.CmdError:
+            return
+        utils.system("echo 0 > %s" % self.kernel_hp_file)
+        logging.debug("Hugepage memory successfuly dealocated")
-- 
1.7.3.4

[PATCH 5/6] KVM test: Turn enospc test pre/post actions into infrastructure

[Lucas Meneghel Rodrigues]

So we can get rid of the pre/post scripts. Wit the rearrangement
we were able to achieve several advantages:

- More rigorous and paranoid cleanup phase
- Better identification of the lvm devices, less likely to
originate conflicts with devices in the host
- Use the shared autotest API avoiding code duplication

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/kvm_preprocessing.py  |    8 ++
 client/tests/kvm/test_setup.py         |  116 +++++++++++++++++++++++++++++--
 client/tests/kvm/tests/enospc.py       |    6 ++-
 client/tests/kvm/tests_base.cfg.sample |    5 +-
 4 files changed, 124 insertions(+), 11 deletions(-)

diff --git a/client/tests/kvm/kvm_preprocessing.py b/client/tests/kvm/kvm_preprocessing.py
index 081a13f..2713805 100644
--- a/client/tests/kvm/kvm_preprocessing.py
+++ b/client/tests/kvm/kvm_preprocessing.py
@@ -262,6 +262,10 @@ def preprocess(test, params, env):
         u = test_setup.UnattendedInstallConfig(test, params)
         u.setup()
 
+    if params.get("type") == "enospc":
+        e = test_setup.EnospcConfig(test, params)
+        e.setup()
+
     # Execute any pre_commands
     if params.get("pre_command"):
         process_command(test, params, env, params.get("pre_command"),
@@ -362,6 +366,10 @@ def postprocess(test, params, env):
         h = kvm_utils.HugePageConfig(params)
         h.cleanup()
 
+    if params.get("type") == "enospc":
+        e = test_setup.EnospcConfig(test, params)
+        e.cleanup()
+
     # Execute any post_commands
     if params.get("post_command"):
         process_command(test, params, env, params.get("post_command"),
diff --git a/client/tests/kvm/test_setup.py b/client/tests/kvm/test_setup.py
index b17c473..e906e18 100644
--- a/client/tests/kvm/test_setup.py
+++ b/client/tests/kvm/test_setup.py
@@ -2,7 +2,7 @@
 Library to perform pre/post test setup for KVM autotest.
 """
 import os, sys, shutil, tempfile, re, ConfigParser, glob, inspect, commands
-import logging
+import logging, time
 from autotest_lib.client.common_lib import error
 from autotest_lib.client.bin import utils
 
@@ -42,6 +42,19 @@ def clean_old_image(image):
         os.remove(image)
 
 
+def display_attributes(instance):
+    """
+    Inspects a given class instance attributes and displays them, convenient
+    for debugging.
+    """
+    logging.debug("Attributes set:")
+    for member in inspect.getmembers(instance):
+        name, value = member
+        attribute = getattr(instance, name)
+        if not (name.startswith("__") or callable(attribute) or not value):
+            logging.debug("    %s: %s", name, value)
+
+
 class Disk(object):
     """
     Abstract class for Disk objects, with the common methods implemented.
@@ -472,13 +485,7 @@ class UnattendedInstallConfig(object):
         Uses an appropriate strategy according to each install model.
         """
         logging.info("Starting unattended install setup")
-
-        logging.debug("Variables set:")
-        for member in inspect.getmembers(self):
-            name, value = member
-            attribute = getattr(self, name)
-            if not (name.startswith("__") or callable(attribute) or not value):
-                logging.debug("    %s: %s", name, value)
+        display_attributes(self)
 
         if self.unattended_file and (self.floppy or self.cdrom_unattended):
             self.setup_boot_disk()
@@ -593,3 +600,96 @@ class HugePageConfig(object):
             return
         utils.system("echo 0 > %s" % self.kernel_hp_file)
         logging.debug("Hugepage memory successfuly dealocated")
+
+
+class EnospcConfig(object):
+    """
+    Performs setup for the test enospc. This is a borg class, similar to a
+    singleton. The idea is to keep state in memory for when we call cleanup()
+    on postprocessing.
+    """
+    __shared_state = {}
+    def __init__(self, test, params):
+        self.__dict__ = self.__shared_state
+        root_dir = test.bindir
+        self.tmpdir = test.tmpdir
+        self.qemu_img_binary = params.get('qemu_img_binary')
+        if not os.path.isfile(self.qemu_img_binary):
+            self.qemu_img_binary = os.path.join(root_dir,
+                                                self.qemu_img_binary)
+        self.raw_file_path = os.path.join(self.tmpdir, 'enospc.raw')
+        # Here we're trying to choose fairly explanatory names so it's less
+        # likely that we run in conflict with other devices in the system
+        self.vgtest_name = params.get("vgtest_name")
+        self.lvtest_name = params.get("lvtest_name")
+        self.lvtest_device = "/dev/%s/%s" % (self.vgtest_name, self.lvtest_name)
+        image_dir = os.path.dirname(params.get("image_name"))
+        self.qcow_file_path = os.path.join(image_dir, 'enospc.qcow2')
+        try:
+            getattr(self, 'loopback')
+        except AttributeError:
+            self.loopback = ''
+
+
+    @error.context_aware
+    def setup(self):
+        logging.debug("Starting enospc setup")
+        error.context("performing enospc setup")
+        display_attributes(self)
+        # Double check if there aren't any leftovers
+        self.cleanup()
+        try:
+            utils.run("%s create -f raw %s 10G" %
+                      (self.qemu_img_binary, self.raw_file_path))
+            # Associate a loopback device with the raw file.
+            # Subject to race conditions, that's why try here to associate
+            # it with the raw file as quickly as possible
+            l_result = utils.run("losetup -f")
+            utils.run("losetup -f %s" % self.raw_file_path)
+            self.loopback = l_result.stdout.strip()
+            # Add the loopback device configured to the list of pvs
+            # recognized by LVM
+            utils.run("pvcreate %s" % self.loopback)
+            utils.run("vgcreate %s %s" % (self.vgtest_name, self.loopback))
+            # Create an lv inside the vg with starting size of 200M
+            utils.run("lvcreate -L 200M -n %s %s" %
+                      (self.lvtest_name, self.vgtest_name))
+            # Create a 10GB qcow2 image in the logical volume
+            utils.run("%s create -f qcow2 %s 10G" %
+                      (self.qemu_img_binary, self.lvtest_device))
+            # Let's symlink the logical volume with the image name that autotest
+            # expects this device to have
+            os.symlink(self.lvtest_device, self.qcow_file_path)
+        except Exception, e:
+            self.cleanup()
+            raise
+
+    @error.context_aware
+    def cleanup(self):
+        error.context("performing enospc cleanup")
+        if os.path.isfile(self.lvtest_device):
+            utils.run("fuser -k %s" % self.lvtest_device)
+            time.sleep(2)
+        l_result = utils.run("lvdisplay")
+        # Let's remove all volumes inside the volume group created
+        if self.lvtest_name in l_result.stdout:
+            utils.run("lvremove -f %s" % self.lvtest_device)
+        # Now, removing the volume group itself
+        v_result = utils.run("vgdisplay")
+        if self.vgtest_name in v_result.stdout:
+            utils.run("vgremove -f %s" % self.vgtest_name)
+        # Now, if we can, let's remove the physical volume from lvm list
+        if self.loopback:
+            p_result = utils.run("pvdisplay")
+            if self.loopback in p_result.stdout:
+                utils.run("pvremove -f %s" % self.loopback)
+        l_result = utils.run('losetup -a')
+        if self.loopback and (self.loopback in l_result.stdout):
+            try:
+                utils.run("losetup -d %s" % self.loopback)
+            except error.CmdError:
+                logging.error("Failed to liberate loopback %s", self.loopback)
+        if os.path.islink(self.qcow_file_path):
+            os.remove(self.qcow_file_path)
+        if os.path.isfile(self.raw_file_path):
+            os.remove(self.raw_file_path)
diff --git a/client/tests/kvm/tests/enospc.py b/client/tests/kvm/tests/enospc.py
index 6a149f9..3868cc4 100644
--- a/client/tests/kvm/tests/enospc.py
+++ b/client/tests/kvm/tests/enospc.py
@@ -24,6 +24,10 @@ def run_enospc(test, params, env):
     login_timeout = int(params.get("login_timeout", 360))
     session_serial = vm.wait_for_serial_login(timeout=login_timeout)
 
+    vgtest_name = params.get("vgtest_name")
+    lvtest_name = params.get("lvtest_name")
+    logical_volume = "/dev/%s/%s" % (vgtest_name, lvtest_name)
+
     drive_format = params.get("drive_format")
     if drive_format == "virtio":
         devname = "/dev/vdb"
@@ -54,7 +58,7 @@ def run_enospc(test, params, env):
                     logging.error(e)
             logging.info("Guest paused, extending Logical Volume size")
             try:
-                cmd_result = utils.run("lvextend -L +200M /dev/vgtest/lvtest")
+                cmd_result = utils.run("lvextend -L +200M %s" % logical_volume)
             except error.CmdError, e:
                 logging.debug(e.result_obj.stdout)
             vm.monitor.cmd("cont")
diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
index f90d551..bd2f720 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -601,9 +601,10 @@ variants:
         image_format_stg = qcow2
         image_boot_stg = no
         image_snapshot_stg = no
+        check_image_stg = no
+        vgtest_name = vg_kvm_test_enospc
+        lvtest_name = lv_kvm_test_enospc
         background_cmd = "nohup dd if=/dev/zero of=%s bs=1024 &"
-        pre_command += " scripts/enospc-pre.py;"
-        post_command += " scripts/enospc-post.py;"
         kill_vm = yes
 
     - qmp_basic: install setup unattended_install.cdrom
-- 
1.7.3.4

[PATCH 6/6] KVM test: Removing enospc pre and post scripts

[Lucas Meneghel Rodrigues]

As their functionality has been reimplemented as
framework functionality

Signed-off-by: Lucas Meneghel Rodrigues <lmr@redhat.com>
---
 client/tests/kvm/scripts/enospc-post.py |   77 -------------------------------
 client/tests/kvm/scripts/enospc-pre.py  |   73 -----------------------------
 2 files changed, 0 insertions(+), 150 deletions(-)
 delete mode 100755 client/tests/kvm/scripts/enospc-post.py
 delete mode 100755 client/tests/kvm/scripts/enospc-pre.py

diff --git a/client/tests/kvm/scripts/enospc-post.py b/client/tests/kvm/scripts/enospc-post.py
deleted file mode 100755
index c6714f2..0000000
--- a/client/tests/kvm/scripts/enospc-post.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/python
-"""
-Simple script to setup enospc test environment
-"""
-import os, commands, sys
-
-SCRIPT_DIR = os.path.dirname(sys.modules[__name__].__file__)
-KVM_TEST_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
-
-class SetupError(Exception):
-    """
-    Simple wrapper for the builtin Exception class.
-    """
-    pass
-
-
-def find_command(cmd):
-    """
-    Searches for a command on common paths, error if it can't find it.
-
-    @param cmd: Command to be found.
-    """
-    if os.path.exists(cmd):
-        return cmd
-    for dir in ["/usr/local/sbin", "/usr/local/bin",
-                "/usr/sbin", "/usr/bin", "/sbin", "/bin"]:
-        file = os.path.join(dir, cmd)
-        if os.path.exists(file):
-            return file
-    raise ValueError('Missing command: %s' % cmd)
-
-
-def run(cmd, info=None):
-    """
-    Run a command and throw an exception if it fails.
-    Optionally, you can provide additional contextual info.
-
-    @param cmd: Command string.
-    @param reason: Optional string that explains the context of the failure.
-
-    @raise: SetupError if command fails.
-    """
-    print "Running '%s'" % cmd
-    cmd_name = cmd.split(' ')[0]
-    find_command(cmd_name)
-    status, output = commands.getstatusoutput(cmd)
-    if status:
-        e_msg = ('Command %s failed.\nStatus:%s\nOutput:%s' %
-                 (cmd, status, output))
-        if info is not None:
-            e_msg += '\nAdditional Info:%s' % info
-        raise SetupError(e_msg)
-
-    return (status, output)
-
-
-if __name__ == "__main__":
-    qemu_img_binary = os.environ['KVM_TEST_qemu_img_binary']
-    if not os.path.isabs(qemu_img_binary):
-        qemu_img_binary = os.path.join(KVM_TEST_DIR, qemu_img_binary)
-    if not os.path.exists(qemu_img_binary):
-        raise SetupError('The qemu-img binary that is supposed to be used '
-                         '(%s) does not exist. Please verify your '
-                         'configuration' % qemu_img_binary)
-
-    run("lvremove -f vgtest")
-    status, output = run("losetup -a")
-    loopback_device = None
-    if output:
-        for line in output.splitlines():
-            device = line.split(":")[0]
-            if "/tmp/enospc.raw" in line:
-                loopback_device = device
-                break
-    if loopback_device is not None:
-        run("losetup -d %s" % loopback_device)
-    run("rm -rf /tmp/enospc.raw /tmp/kvm_autotest_root/images/enospc.qcow2")
diff --git a/client/tests/kvm/scripts/enospc-pre.py b/client/tests/kvm/scripts/enospc-pre.py
deleted file mode 100755
index 1313de3..0000000
--- a/client/tests/kvm/scripts/enospc-pre.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/python
-"""
-Simple script to setup enospc test environment
-"""
-import os, commands, sys
-
-SCRIPT_DIR = os.path.dirname(sys.modules[__name__].__file__)
-KVM_TEST_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, ".."))
-
-class SetupError(Exception):
-    """
-    Simple wrapper for the builtin Exception class.
-    """
-    pass
-
-
-def find_command(cmd):
-    """
-    Searches for a command on common paths, error if it can't find it.
-
-    @param cmd: Command to be found.
-    """
-    if os.path.exists(cmd):
-        return cmd
-    for dir in ["/usr/local/sbin", "/usr/local/bin",
-                "/usr/sbin", "/usr/bin", "/sbin", "/bin"]:
-        file = os.path.join(dir, cmd)
-        if os.path.exists(file):
-            return file
-    raise ValueError('Missing command: %s' % cmd)
-
-
-def run(cmd, info=None):
-    """
-    Run a command and throw an exception if it fails.
-    Optionally, you can provide additional contextual info.
-
-    @param cmd: Command string.
-    @param reason: Optional string that explains the context of the failure.
-
-    @raise: SetupError if command fails.
-    """
-    print "Running '%s'" % cmd
-    cmd_name = cmd.split(' ')[0]
-    find_command(cmd_name)
-    status, output = commands.getstatusoutput(cmd)
-    if status:
-        e_msg = ('Command %s failed.\nStatus:%s\nOutput:%s' %
-                 (cmd, status, output))
-        if info is not None:
-            e_msg += '\nAdditional Info:%s' % info
-        raise SetupError(e_msg)
-
-    return (status, output.strip())
-
-
-if __name__ == "__main__":
-    qemu_img_binary = os.environ['KVM_TEST_qemu_img_binary']
-    if not os.path.isabs(qemu_img_binary):
-        qemu_img_binary = os.path.join(KVM_TEST_DIR, qemu_img_binary)
-    if not os.path.exists(qemu_img_binary):
-        raise SetupError('The qemu-img binary that is supposed to be used '
-                         '(%s) does not exist. Please verify your '
-                         'configuration' % qemu_img_binary)
-
-    run("%s create -f raw /tmp/enospc.raw 10G" % qemu_img_binary)
-    status, loopback_device = run("losetup -f")
-    run("losetup -f /tmp/enospc.raw")
-    run("pvcreate %s" % loopback_device)
-    run("vgcreate vgtest %s" % loopback_device)
-    run("lvcreate -L200M -n lvtest vgtest")
-    run("%s create -f qcow2 /dev/vgtest/lvtest 10G" % qemu_img_binary)
-    run("ln -s /dev/vgtest/lvtest /tmp/kvm_autotest_root/images/enospc.qcow2")
-- 
1.7.3.4