* [Qemu-devel] [RFC 1/3] image-fuzzer: Added execution of multiple tests to the test runner
2014-06-18 16:14 [Qemu-devel] [RFC 0/3] image-fuzzer: Initial image generator and extended runner Maria Kustova
@ 2014-06-18 16:14 ` Maria Kustova
2014-06-20 1:49 ` Fam Zheng
2014-06-18 16:14 ` [Qemu-devel] [RFC 2/3] image-fuzzer: Initial generator of qcow2 fuzzed images Maria Kustova
` (2 subsequent siblings)
3 siblings, 1 reply; 7+ messages in thread
From: Maria Kustova @ 2014-06-18 16:14 UTC (permalink / raw)
To: qemu-devel; +Cc: kwolf, famz, Maria Kustova, stefanha
Apart from fixes this patch allows to run multiple tests in a row. If 'seed'
argument is not specified the runner generates and executes new tests one by
one till keyboard interruption. Specified seed forces the runner to execute
only one test with current seed and exit.
Signed-off-by: Maria Kustova <maria.k@catit.be>
---
tests/image-fuzzer/runner/runner.py | 260 ++++++++++++++++++++++++++++++++++++
1 file changed, 260 insertions(+)
create mode 100644 tests/image-fuzzer/runner/runner.py
diff --git a/tests/image-fuzzer/runner/runner.py b/tests/image-fuzzer/runner/runner.py
new file mode 100644
index 0000000..5d09b2e
--- /dev/null
+++ b/tests/image-fuzzer/runner/runner.py
@@ -0,0 +1,260 @@
+# Tool for running fuzz tests
+#
+# Copyright (C) 2014 Maria Kustova <maria.k@catit.be>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys, os, signal
+from time import time
+import subprocess
+import random
+from itertools import count
+from shutil import rmtree
+import getopt
+import resource
+resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
+
+
+def multilog(msg, *output):
+ """ Write an object to all of specified file descriptors
+ """
+
+ for fd in output:
+ fd.write(msg)
+ fd.flush()
+
+
+def str_signal(sig):
+ """ Convert a numeric value of a system signal to the string one
+ defined by the current operational system
+ """
+
+ for k, v in signal.__dict__.items():
+ if v == sig:
+ return k
+
+
+class TestException(Exception):
+ """Exception for errors risen by TestEnv objects"""
+ pass
+
+
+class TestEnv(object):
+ """ Trivial test object
+
+ The class sets up test environment, generates a test image and executes
+ application under tests with specified arguments and a test image provided.
+ All logs are collected.
+ Summary log will contain short descriptions and statuses of tests in
+ a run.
+ Test log will include application (e.g. 'qemu-img') logs besides info sent
+ to the summary log.
+ """
+
+ def __init__(self, test_id, seed, work_dir, run_log, exec_bin=None,
+ cleanup=True, log_all=False):
+ """Set test environment in a specified work directory.
+
+ Path to qemu_img will be retrieved from 'QEMU_IMG' environment
+ variable, if a test binary is not specified.
+ """
+
+ if seed is not None:
+ self.seed = seed
+ else:
+ self.seed = hash(time())
+
+ self.init_path = os.getcwd()
+ self.work_dir = work_dir
+ self.current_dir = os.path.join(work_dir, 'test-' + test_id)
+ if exec_bin is not None:
+ self.exec_bin = exec_bin.strip().split(' ')
+ else:
+ self.exec_bin = \
+ os.environ.get('QEMU_IMG', 'qemu-img').strip().split(' ')
+
+ try:
+ os.makedirs(self.current_dir)
+ except OSError:
+ e = sys.exc_info()[1]
+ print >>sys.stderr, \
+ "Error: The working directory '%s' cannot be used. Reason: %s"\
+ % (self.work_dir, e[1])
+ raise TestException
+ self.log = open(os.path.join(self.current_dir, "test.log"), "w")
+ self.parent_log = open(run_log, "a")
+ self.result = False
+ self.cleanup = cleanup
+ self.log_all = log_all
+
+ def _test_app(self, q_args):
+ """ Start application under test with specified arguments and return
+ an exit code or a kill signal depending on result of an execution.
+ """
+ devnull = open('/dev/null', 'r+')
+ return subprocess.call(self.exec_bin + q_args + ['test_image'],
+ stdin=devnull, stdout=self.log, stderr=self.log)
+
+ def execute(self, q_args):
+ """ Execute a test.
+
+ The method creates a test image, runs test app and analyzes its exit
+ status. If the application was killed by a signal, the test is marked
+ as failed.
+ """
+ os.chdir(self.current_dir)
+ # Seed initialization should be as close to image generation call
+ # as posssible to avoid a corruption of random sequence
+ random.seed(self.seed)
+ image_generator.create_image('test_image')
+ test_summary = "Seed: %s\nCommand: %s\nTest directory: %s\n" \
+ % (self.seed, " ".join(q_args), self.current_dir)
+ try:
+ retcode = self._test_app(q_args)
+ except OSError:
+ e = sys.exc_info()[1]
+ multilog(test_summary + "Error: Start of '%s' failed. " \
+ "Reason: %s\n\n" % (os.path.basename(self.exec_bin[0]),
+ e[1]),
+ sys.stderr, self.log, self.parent_log)
+ raise TestException
+
+ if retcode < 0:
+ multilog(test_summary + "FAIL: Test terminated by signal %s\n\n"
+ % str_signal(-retcode), sys.stderr, self.log,
+ self.parent_log)
+ elif self.log_all:
+ multilog(test_summary + "PASS: Application exited with the code" +
+ " '%d'\n\n" % retcode, sys.stdout, self.log,
+ self.parent_log)
+ self.result = True
+ else:
+ self.result = True
+
+ def finish(self):
+ """ Restore environment after a test execution. Remove folders of
+ passed tests
+ """
+ self.log.close()
+ self.parent_log.close()
+ os.chdir(self.init_path)
+ if self.result and self.cleanup:
+ rmtree(self.current_dir)
+
+if __name__ == '__main__':
+
+ def usage():
+ print """
+ Usage: runner.py [OPTION...] DIRECTORY PATH
+
+ Set up test environment in DIRECTORY and run a test in it. Test image
+ generator should be specified via PATH to it.
+
+ Optional arguments:
+ -h, --help display this help and exit
+ -b, --binary=PATH path to the application under test,
+ by default "qemu-img" in PATH or
+ QEMU_IMG environment variables
+ -c, --command=STRING execute the tested application
+ with arguments specified,
+ by default STRING="check"
+ -s, --seed=STRING seed for a test image generation,
+ by default will be generated randomly
+ -k, --keep_passed don't remove folders of passed tests
+ -v, --verbose log information about passed tests
+ """
+
+ def run_test(test_id, seed, work_dir, run_log, test_bin, cleanup, log_all,
+ command):
+ """Setup environment for one test and execute this test"""
+ try:
+ test = TestEnv(test_id, seed, work_dir, run_log, test_bin, cleanup,
+ log_all)
+ except TestException:
+ sys.exit(1)
+
+ # Python 2.4 doesn't support 'finally' and 'except' in the same 'try'
+ # block
+ try:
+ try:
+ test.execute(command)
+ # Silent exit on user break
+ except TestException:
+ sys.exit(1)
+ finally:
+ test.finish()
+
+ try:
+ opts, args = getopt.gnu_getopt(sys.argv[1:], 'c:hb:s:kv',
+ ['command=', 'help', 'binary=', 'seed=',
+ 'keep_passed', 'verbose'])
+ except getopt.error:
+ e = sys.exc_info()[1]
+ print "Error: %s\n\nTry 'runner.py --help' for more information" % e
+ sys.exit(1)
+
+ command = ['check']
+ cleanup = True
+ log_all = False
+ test_bin = None
+ seed = None
+ for opt, arg in opts:
+ if opt in ('-h', '--help'):
+ usage()
+ sys.exit()
+ elif opt in ('-c', '--command'):
+ command = arg.split(" ")
+ elif opt in ('-k', '--keep_passed'):
+ cleanup = False
+ elif opt in ('-v', '--verbose'):
+ log_all = True
+ elif opt in ('-b', '--binary'):
+ test_bin = os.path.realpath(arg)
+ elif opt in ('-s', '--seed'):
+ seed = arg
+
+ if not len(args) == 2:
+ print "Missed parameter\nTry 'runner.py --help' " \
+ "for more information"
+ sys.exit(1)
+
+ work_dir = os.path.realpath(args[0])
+ # run_log is created in 'main', because multiple tests are expected to \
+ # log in it
+ run_log = os.path.join(work_dir, 'run.log')
+
+ # Add the module path to sys.path
+ sys.path.append(os.path.dirname(os.path.realpath(args[1])))
+ # Remove a script extension if any
+ generator_name = os.path.splitext(os.path.basename(args[1]))[0]
+ try:
+ image_generator = __import__(generator_name)
+ except ImportError:
+ e = sys.exc_info()[1]
+ print "Error: The image generator '%s' cannot be imported.\n" \
+ "Reason: %s" % (generator_name, e)
+ sys.exit(1)
+
+ # If a seed is specified, only one test will be executed.
+ # Otherwise runner will terminate after a keyboard interruption
+ for test_id in count(1):
+ try:
+ run_test(str(test_id), seed, work_dir, run_log, test_bin, cleanup,
+ log_all, command)
+ except (KeyboardInterrupt, SystemExit):
+ sys.exit(1)
+
+ if seed is not None:
+ break
--
1.9.3
^ permalink raw reply related [flat|nested] 7+ messages in thread* [Qemu-devel] [RFC 2/3] image-fuzzer: Initial generator of qcow2 fuzzed images
2014-06-18 16:14 [Qemu-devel] [RFC 0/3] image-fuzzer: Initial image generator and extended runner Maria Kustova
2014-06-18 16:14 ` [Qemu-devel] [RFC 1/3] image-fuzzer: Added execution of multiple tests to the test runner Maria Kustova
@ 2014-06-18 16:14 ` Maria Kustova
2014-06-18 16:14 ` [Qemu-devel] [RFC 3/3] image-fuzzer: Added description for the qcow2 image generator Maria Kustova
2014-06-20 6:09 ` [Qemu-devel] [RFC 0/3] image-fuzzer: Initial image generator and extended runner Fam Zheng
3 siblings, 0 replies; 7+ messages in thread
From: Maria Kustova @ 2014-06-18 16:14 UTC (permalink / raw)
To: qemu-devel; +Cc: kwolf, famz, Maria Kustova, stefanha
Qcow2 image generator is a python package providing create_image(img_path)
method required by the test runner.
It generates files containing fuzzed qcow2 image headers. Files are randomly
variable not only in fuzzed fields but in valid structure elements like image
and cluster size.
Signed-off-by: Maria Kustova <maria.k@catit.be>
---
tests/image-fuzzer/qcow2/__init__.py | 1 +
tests/image-fuzzer/qcow2/fuzz.py | 271 +++++++++++++++++++++++++++++++++++
tests/image-fuzzer/qcow2/layout.py | 125 ++++++++++++++++
3 files changed, 397 insertions(+)
create mode 100644 tests/image-fuzzer/qcow2/__init__.py
create mode 100644 tests/image-fuzzer/qcow2/fuzz.py
create mode 100644 tests/image-fuzzer/qcow2/layout.py
diff --git a/tests/image-fuzzer/qcow2/__init__.py b/tests/image-fuzzer/qcow2/__init__.py
new file mode 100644
index 0000000..e2ebe19
--- /dev/null
+++ b/tests/image-fuzzer/qcow2/__init__.py
@@ -0,0 +1 @@
+from layout import create_image
diff --git a/tests/image-fuzzer/qcow2/fuzz.py b/tests/image-fuzzer/qcow2/fuzz.py
new file mode 100644
index 0000000..6edf84b
--- /dev/null
+++ b/tests/image-fuzzer/qcow2/fuzz.py
@@ -0,0 +1,271 @@
+# Fuzzing functions for qcow2 fields
+#
+# Copyright (C) 2014 Maria Kustova <maria.k@catit.be>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import random
+
+
+UINT32 = 2**32 - 1
+UINT64 = 2**64 - 1
+# Most significant bit orders
+UINT32_M = 31
+UINT64_M = 63
+
+
+def random_from_intervals(intervals):
+ """Select a random integer number from the list of specified intervals
+
+ Each interval is a tuple of lower and upper limits of the interval. The
+ limits are included. Intervals in a list should not overlap.
+ """
+ total = reduce(lambda x, y: x + y[1] - y[0] + 1, intervals, 0)
+ r = random.randint(0, total-1) + intervals[0][0]
+ temp = zip(intervals, intervals[1:])
+ for x in temp:
+ r = r + (r > x[0][1])*(x[1][0] - x[0][1] - 1)
+ return r
+
+
+def random_bits(bit_ranges):
+ """Generate random binary mask with ones in the specified bit ranges
+
+ Each bit_ranges is a list of tuples of lower and upper limits of bit
+ positions will be fuzzed. The limits are included. Random amount of bits
+ in range limits will be set to ones. The mask is returned in decimal
+ integer format.
+ """
+ bit_numbers = []
+ # Select random amount of random positions in bit_ranges
+ for rng in bit_ranges:
+ bit_numbers += random.sample(range(rng[0], rng[1] + 1),
+ random.randint(0, rng[1] - rng[0] + 1))
+ val = 0
+ # Set bits on selected possitions to ones
+ for bit in bit_numbers:
+ val |= 1 << bit
+ return val
+
+
+def validator(current, intervals):
+ """Return a random value from intervals not equal to the current.
+
+ This function is useful for selection from valid values except current one.
+ """
+ val = random_from_intervals(intervals)
+ if val == current:
+ return validator(current, intervals)
+ else:
+ return val
+
+
+def bit_validator(current, bit_ranges):
+ """Return a random bit mask not equal to the current.
+
+ This function is useful for selection from valid values except current one.
+ """
+
+ val = random_bits(bit_ranges)
+ if val == current:
+ return bit_validator(current, bit_ranges)
+ else:
+ return val
+
+
+def selector(current, constraints, is_bitmask=None):
+ """Select one value from all defined by constraints
+
+ Each constraint produces one random value satisfying to it. The function
+ randomly selects one value satisfying at least one constraint (depending on
+ constraints overlaps).
+ """
+ if is_bitmask is None:
+ validate = validator
+ else:
+ validate = bit_validator
+
+ def iter_validate(c):
+ """Apply validate() only to constraints represented as lists
+
+ This auxiliary function replaces short circuit conditions not supported
+ in Python 2.4
+ """
+ if type(c) == list:
+ return validate(current, c)
+ else:
+ return c
+ fuzz_values = [iter_validate(c) for c in constraints]
+ # Remove current for cases it's implicitly specified in constraints
+ # Duplicate validator functionality to prevent decreasing of probability
+ # to get one of allowable values
+ # TODO: remove validators after implementation of intelligent selection
+ # of fields will be fuzzed
+ try:
+ fuzz_values.remove(current)
+ except ValueError:
+ pass
+ return random.choice(fuzz_values)
+
+
+def magic(current):
+ """Fuzz magic header field
+
+ The function just returns the current magic value and provides uniformity
+ of calls for all fuzzing functions
+ """
+ return current
+
+
+def version(current):
+ """Fuzz version header field"""
+ constraints = [
+ [(2, 3)], # correct values
+ [(0, 1), (4, UINT32)]
+ ]
+ return selector(current, constraints)
+
+
+def backing_file_offset(current):
+ """Fuzz backing file offset header field"""
+ constraints = [
+ [(0, UINT64)]
+ ]
+ return selector(current, constraints)
+
+
+def backing_file_size(current):
+ """Fuzz backing file size header field"""
+ constraints = [
+ [(0, UINT32)]
+ ]
+ return selector(current, constraints)
+
+
+def cluster_bits(current):
+ """Fuzz cluster bits header field"""
+ constraints = [
+ [(9, 20)], # correct values
+ [(0, 9), (20, UINT32)]
+ ]
+ return selector(current, constraints)
+
+
+def size(current):
+ """Fuzz image size header field"""
+ constraints = [
+ [(0, UINT64)]
+ ]
+ return selector(current, constraints)
+
+
+def crypt_method(current):
+ """Fuzz crypt method header field"""
+ constraints = [
+ [(0, 1)],
+ [(2, UINT32)]
+ ]
+ return selector(current, constraints)
+
+
+def l1_size(current):
+ """Fuzz L1 table size header field"""
+ constraints = [
+ [(0, UINT32)]
+ ]
+ return selector(current, constraints)
+
+
+def l1_table_offset(current):
+ """Fuzz L1 table offset header field"""
+ constraints = [
+ [(0, UINT64)]
+ ]
+ return selector(current, constraints)
+
+
+def refcount_table_offset(current):
+ """Fuzz refcount table offset header field"""
+ constraints = [
+ [(0, UINT64)]
+ ]
+ return selector(current, constraints)
+
+
+def refcount_table_clusters(current):
+ """Fuzz refcount table clusters header field"""
+ constraints = [
+ [(0, UINT32)]
+ ]
+ return selector(current, constraints)
+
+
+def nb_snapshots(current):
+ """Fuzz number of snapshots header field"""
+ constraints = [
+ [(0, UINT32)]
+ ]
+ return selector(current, constraints)
+
+
+def snapshots_offset(current):
+ """Fuzz snapshots offset header field"""
+ constraints = [
+ [(0, UINT64)]
+ ]
+ return selector(current, constraints)
+
+
+def incompatible_features(current):
+ """Fuzz incompatible features header field"""
+ constraints = [
+ [(0, 1)], # allowable values
+ [(0, UINT64_M)]
+ ]
+ return selector(current, constraints, 1)
+
+
+def compatible_features(current):
+ """Fuzz compatible features header field"""
+ constraints = [
+ [(0, UINT64_M)]
+ ]
+ return selector(current, constraints, 1)
+
+
+def autoclear_features(current):
+ """Fuzz autoclear features header field"""
+ constraints = [
+ [(0, UINT64_M)]
+ ]
+ return selector(current, constraints, 1)
+
+
+def refcount_order(current):
+ """Fuzz number of refcount order header field"""
+ constraints = [
+ [(0, UINT32)]
+ ]
+ return selector(current, constraints)
+
+
+def header_length(current):
+ """Fuzz number of refcount order header field"""
+ constraints = [
+ 72,
+ 104,
+ [(0, UINT32)]
+ ]
+ return selector(current, constraints)
diff --git a/tests/image-fuzzer/qcow2/layout.py b/tests/image-fuzzer/qcow2/layout.py
new file mode 100644
index 0000000..9eb11e9
--- /dev/null
+++ b/tests/image-fuzzer/qcow2/layout.py
@@ -0,0 +1,125 @@
+# Generator of fuzzed qcow2 images
+#
+# Copyright (C) 2014 Maria Kustova <maria.k@catit.be>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import random
+import struct
+import fuzz
+
+
+MAX_IMAGE_SIZE = 10*2**20
+
+
+def fuzz_struct(structure):
+ """Select part of fields in the specified structure and assign them invalid
+ values
+
+ From 20% to 50% of all fields will be randomly selected and fuzzed
+ """
+ extract = random.sample(structure,
+ random.randint(len(structure)/5, len(structure)/2))
+
+ def iter_fuzz(field):
+ """Fuzz field value if it's selected
+
+ This auxiliary function replaces short circuit conditions not supported
+ in Python 2.4
+ """
+ if field in extract:
+ return field[0:2] + [getattr(fuzz, field[3])(field[2])] \
+ + field[-1:]
+ else:
+ return field
+
+ return [iter_fuzz(field) for field in structure]
+
+
+def image_size():
+ """Generate a random file size aligned to a random correct cluster size"""
+ cluster_bits = random.randrange(9, 21)
+ cluster_size = 1 << cluster_bits
+ file_size = random.randrange(5*cluster_size,
+ MAX_IMAGE_SIZE + 1,
+ cluster_size)
+ return [cluster_bits, file_size]
+
+
+def header(cluster_bits, img_size):
+ """Generate a random valid header"""
+ magic = "QFI\xfb"
+ version = random.randint(2, 3)
+ # Next two set to zero while qcow emulation is not supported
+ backing_file_offset = 0
+ backing_file_size = 0
+ crypt_method = random.randint(0, 1)
+ # All below are zeroes while a corresponding feature is not supported
+ l1_size = 0
+ l1_table_offset = 0
+ refcount_table_offset = 0
+ refcount_table_clusters = 0
+ nb_snapshots = 0
+ snapshots_offset = 0
+ # From the e-mail thread for [PATCH] docs: Define refcount_bits value:
+ # Only refcount_order = 4 is supported by QEMU at the moment
+ refcount_order = 4
+ autoclear_features = 0 # doesn't depend on version
+ if version == 2:
+ incompatible_features = 0
+ compatible_features = 0
+ header_length = 72
+ else:
+ incompatible_features = random.getrandbits(2)
+ compatible_features = random.getrandbits(1)
+ header_length = 104
+
+ return [
+ [0, '>4s', magic, 'magic'],
+ [4, '>I', version, 'version'],
+ [8, '>Q', backing_file_offset, 'backing_file_offset'],
+ [16, '>I', backing_file_size, 'backing_file_size'],
+ [20, '>I', cluster_bits, 'cluster_bits'],
+ [24, '>Q', img_size, 'size'],
+ [32, '>I', crypt_method, 'crypt_method'],
+ [36, '>I', l1_size, 'l1_size'],
+ [40, '>Q', l1_table_offset, 'l1_table_offset'],
+ [48, '>Q', refcount_table_offset, 'refcount_table_offset'],
+ [56, '>I', refcount_table_clusters, 'refcount_table_clusters'],
+ [60, '>I', nb_snapshots, 'nb_snapshots'],
+ [64, '>Q', snapshots_offset, 'snapshots_offset'],
+ [72, '>Q', incompatible_features, 'incompatible_features'],
+ [80, '>Q', compatible_features, 'compatible_features'],
+ [88, '>Q', autoclear_features, 'autoclear_features'],
+ [96, '>I', refcount_order, 'refcount_order'],
+ [100, '>I', header_length, 'header_length']
+ ]
+
+
+def create_image(test_img_path):
+ """Write a fuzzed image to the specified file"""
+ image_file = open(test_img_path, 'w')
+ cluster_bits, v_image_size = image_size()
+ # Create an empty image
+ # (sparse if FS supports it or preallocated otherwise)
+ image_file.seek(v_image_size - 1)
+ image_file.write("\0")
+ v_header = header(cluster_bits, v_image_size) # create a valid header
+ v_header = fuzz_struct(v_header) # fuzz the header
+
+ for field in v_header:
+ image_file.seek(field[0])
+ image_file.write(struct.pack(field[1], field[2]))
+ image_file.close()
--
1.9.3
^ permalink raw reply related [flat|nested] 7+ messages in thread* [Qemu-devel] [RFC 3/3] image-fuzzer: Added description for the qcow2 image generator
2014-06-18 16:14 [Qemu-devel] [RFC 0/3] image-fuzzer: Initial image generator and extended runner Maria Kustova
2014-06-18 16:14 ` [Qemu-devel] [RFC 1/3] image-fuzzer: Added execution of multiple tests to the test runner Maria Kustova
2014-06-18 16:14 ` [Qemu-devel] [RFC 2/3] image-fuzzer: Initial generator of qcow2 fuzzed images Maria Kustova
@ 2014-06-18 16:14 ` Maria Kustova
2014-06-20 6:09 ` [Qemu-devel] [RFC 0/3] image-fuzzer: Initial image generator and extended runner Fam Zheng
3 siblings, 0 replies; 7+ messages in thread
From: Maria Kustova @ 2014-06-18 16:14 UTC (permalink / raw)
To: qemu-devel; +Cc: kwolf, famz, Maria Kustova, stefanha
Apart from fixes the description for image generator was added.
Signed-off-by: Maria Kustova <maria.k@catit.be>
---
tests/image-fuzzer/docs/image-fuzzer.txt | 178 +++++++++++++++++++++++++++++++
1 file changed, 178 insertions(+)
create mode 100644 tests/image-fuzzer/docs/image-fuzzer.txt
diff --git a/tests/image-fuzzer/docs/image-fuzzer.txt b/tests/image-fuzzer/docs/image-fuzzer.txt
new file mode 100644
index 0000000..917107d
--- /dev/null
+++ b/tests/image-fuzzer/docs/image-fuzzer.txt
@@ -0,0 +1,178 @@
+Image fuzzer
+============
+
+Description
+-----------
+
+The goal of the image fuzzer is to catch crashes of qemu-io/qemu-img providing
+to them randomly corrupted images.
+Test images are generated from scratch and have valid inner structure with some
+elements, e.g. L1/L2 tables, having random invalid values.
+
+
+Test runner
+-----------
+
+The test runner generates test images, executes tests utilizing generated
+images, indicates their results and collect all test related artifacts (logs,
+core dumps, test image).
+The test means one start of a system under test (SUT), e.g. qemu-io, with
+specified arguments and one test image.
+By default, the test runner generates new tests and executes them until
+keyboard interruption. But if a test seed is specified via '-s' runner
+parameter, then only one test with this seed will be executed, after its finish
+the runner will exit.
+
+The runner uses an external image fuzzer to generate test images. An image
+generator should be specified as a mandatory parameter of the test runner.
+Details about interactions between the runner and fuzzers see "Module
+interfaces".
+
+The runner activates generation of core dumps during test executions, but it
+assumes that core dumps will be generated in the current working directory.
+For comprehensive test results, please, set up your test environment
+properly.
+
+Path to a binary under test can be specified via environment variables (for now
+only qemu-img) or the runner parameter. For details about environment variables
+see qemu-iotests/check.
+
+
+Qcow2 image generator
+---------------------
+
+The 'qcow2' generator is a Python package providing
+'create_image(test_img_path)' method as a single public API. Other methods can
+be accessed by specifying a submodule prefix, e.g.
+
+ import qcow2
+
+ qcow2.create_image('test_image')
+ qcow2.fuzz.version()'
+
+
+Qcow2 contains two submodules: fuzz.py and layout.py.
+
+'fuzz.py' contains all fuzzing functions one per image field. It's assumed that
+after code analysis every field will have own constraints for its value.
+
+'layout.py' creates a random valid image, fuzzes a random subset of the image
+fields based on their already generated valid values and writes the fuzzed
+image to the file specified.
+
+For now only header fields are generated, the remaining file is filled with
+zeros.
+
+
+Module interfaces
+-----------------
+
+* Test runner/image fuzzer
+
+The runner calls an image generator specifying path to a test image file.
+An image generator is expected to provide 'create_image(test_img_path)' method
+that creates a test image and writes it to the specified file. The file should
+be created if it doesn't exist or overwritten otherwise.
+Random seed is set by the runner at every test execution for the regression
+purpose, so an image generator is not recommended to modify it internally.
+
+* Test runner/SUT
+
+For now a full test command is composed from the SUT, SUT arguments specified
+via '-c' runner parameter and the name of generated image, e.g. for qemu-img
+as a SUT and 'check' command the result call will be
+
+ qemu-img check test_image
+
+where 'test_image' is predefined name for any generated image.
+
+
+Overall fuzzer requirements
+===========================
+
+Input data:
+----------
+
+ - image structure template
+ - test run duration (optional)
+ - action vector (optional)
+ - seed (optional)
+ - fuzzing type (optional)
+ - qemu-img arguments (optional)
+
+
+Fuzzer requirements:
+-------------------
+
+1. Should be able to inject random data
+2. Should be able to permutate part of specified data
+3. Should be able to select a random value from the manually pregenerated
+ vector (boundary values, e.g. max/min cluster size)
+4. Image template should describe a general structure invariant for all
+ test images (image format description)
+5. Image template should be autonomous and other fuzzer parts should not
+ relate on it
+6. Image template should contain reference rules (not only block+size
+ description)
+7. Should generate the test image with the correct structure based on an image
+ template
+8. Should accept a seed as an argument (for regression purpose)
+9. Should generate a seed if it is not specified as an input parameter.
+10. The same seed should generate the same image, if no or the same action
+ vector and fuzzing type are specified
+11. Should accept a vector of actions as an argument (for test reproducing and
+ for test case specification, e.g. group of tests for header structure,
+ group of test for snapshots, etc)
+12. Action vector should be randomly generated from the pool of available
+ actions, if it is not specified as an input parameter
+13. Pool of actions should be defined automatically based on an image template
+14. Should accept a fuzzing type: random values or pregenerated ones
+ (if possible).
+15. Should accept qemu-img call parameters as an argument and select them
+ randomly otherwise. As far as it's expected to be rarely changed, the list
+ of possible values will be available in the test runner internally.
+16. Should accept a test run duration as an argument. Tests should be executed
+ during a minimum period from a test run duration and time while fuzzer can
+ generate different test images
+17. Should support an external cancellation of a test run
+18. Seed, action vector and fuzzing type should be logged (for regression
+ purpose)
+19. All files related to test result should be collected: a test image,
+ qemu logs, fuzzer logs and crash dumps
+20. Should be compatible with python version 2.4-2.7
+21. Usage of external libraries should be limited as much as possible.
+
+
+Image formats:
+-------------
+
+Main target image format is qcow2, but support of image templates should
+provide an ability to add any other image format.
+
+
+Effectiveness:
+-------------
+
+Fuzzer can be controlled via template, seed and action vector;
+this make the fuzzer itself invariant to an image format and test logic.
+It should be able to perform rather complex and precise tests, that can be
+specified via action vector. Otherwise, knowledge about an image structure
+allows the fuzzer to generate the pool of all available areas can be fuzzed
+and randomly select some of them and so compose its own action vector.
+Also complexity of a template defines complexity of the fuzzer, so its
+functionality can be varied from simple model-independent fuzzing to smart
+model-based one.
+
+
+Glossary:
+--------
+
+Action vector is a sequence of structure elements retrieved from an image
+format, each of which will be fuzzed for the test image. It's a subset of
+elements of the action pool. Example: header, refcount block, etc.
+Action pool is all available elements of an image structure that generated
+automatically from an image template.
+Image template is a formal description of an image structure and relations
+between image blocks
+Test image is an output image of fuzzer defined by the current seed, fuzz type
+and action vector.
--
1.9.3
^ permalink raw reply related [flat|nested] 7+ messages in thread