nvdimm.lists.linux.dev archive mirror
 help / color / mirror / Atom feed
From: Brendan Higgins <brendanhiggins-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>
To: gregkh-hQyY1W1yCW8ekmWlsbkhG0B+6BGkLq7r@public.gmane.org,
	keescook-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org,
	mcgrof-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org,
	shuah-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org
Cc: brakmo-b10kYP2dOMg@public.gmane.org,
	jdike-OPE4K8JWMJJBDgjK7y7TUQ@public.gmane.org,
	dri-devel-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org,
	linux-kselftest-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
	robh-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org,
	linux-nvdimm-hn68Rpc1hR1g9hUCZPvPmw@public.gmane.org,
	richard-/L3Ra7n9ekc@public.gmane.org,
	kieran.bingham-ryLnwIuWjnjg/C1BVhZhaw@public.gmane.org,
	Felix Guo
	<felixguoxiuping-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>,
	joel-U3u1mxZcP9KHXe+LvDLADg@public.gmane.org,
	khilman-rdvid1DuHRBWk0Htik3J/w@public.gmane.org,
	Brendan Higgins
	<brendanhiggins-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>,
	Tim.Bird-7U/KSKJipcs@public.gmane.org,
	linux-um-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r@public.gmane.org,
	rostedt-nx8X9YLhiw1AfugRpC6u6w@public.gmane.org,
	julia.lawall-L2FTfq7BK8M@public.gmane.org,
	kunit-dev-/JYPxA39Uh5TLH3MbocFFw@public.gmane.org,
	linux-kernel-u79uwXL29TY76Z2rM5mHXA@public.gmane.org,
	daniel-/w4YWyX8dFk@public.gmane.org,
	mpe-Gsx/Oe8HsFggBc27wqDAHg@public.gmane.org,
	joe-6d6DIl74uiNBDgjK7y7TUQ@public.gmane.org
Subject: [RFC v2 10/14] kunit: add Python libraries for handing KUnit config and kernel
Date: Tue, 23 Oct 2018 16:57:46 -0700	[thread overview]
Message-ID: <20181023235750.103146-11-brendanhiggins@google.com> (raw)
In-Reply-To: <20181023235750.103146-1-brendanhiggins-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>

The ultimate goal is to create minimal isolated test binaries; in the
meantime we are using UML to provide the infrastructure to run tests, so
define an abstract way to configure and run tests that allow us to
change the context in which tests are built without affecting the user.
This also makes pretty and dynamic error reporting, and a lot of other
nice features easier.

kunit_config.py:
  - parse .config and Kconfig files.

kunit_kernel.py: provides helper functions to:
  - configure the kernel using kunitconfig.
  - build the kernel with the appropriate configuration.
  - provide function to invoke the kernel and stream the output back.

Signed-off-by: Felix Guo <felixguoxiuping-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>
Signed-off-by: Brendan Higgins <brendanhiggins-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>
---
 tools/testing/kunit/.gitignore      |   3 +
 tools/testing/kunit/kunit_config.py |  60 ++++++++++++++
 tools/testing/kunit/kunit_kernel.py | 123 ++++++++++++++++++++++++++++
 3 files changed, 186 insertions(+)
 create mode 100644 tools/testing/kunit/.gitignore
 create mode 100644 tools/testing/kunit/kunit_config.py
 create mode 100644 tools/testing/kunit/kunit_kernel.py

diff --git a/tools/testing/kunit/.gitignore b/tools/testing/kunit/.gitignore
new file mode 100644
index 0000000000000..c791ff59a37a9
--- /dev/null
+++ b/tools/testing/kunit/.gitignore
@@ -0,0 +1,3 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
\ No newline at end of file
diff --git a/tools/testing/kunit/kunit_config.py b/tools/testing/kunit/kunit_config.py
new file mode 100644
index 0000000000000..183bd5e758762
--- /dev/null
+++ b/tools/testing/kunit/kunit_config.py
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: GPL-2.0
+
+import collections
+import re
+
+CONFIG_IS_NOT_SET_PATTERN = r'^# CONFIG_\w+ is not set$'
+CONFIG_PATTERN = r'^CONFIG_\w+=\S+$'
+
+KconfigEntryBase = collections.namedtuple('KconfigEntry', ['raw_entry'])
+
+
+class KconfigEntry(KconfigEntryBase):
+
+	def __str__(self) -> str:
+		return self.raw_entry
+
+
+class KconfigParseError(Exception):
+	"""Error parsing Kconfig defconfig or .config."""
+
+
+class Kconfig(object):
+	"""Represents defconfig or .config specified using the Kconfig language."""
+
+	def __init__(self):
+		self._entries = []
+
+	def entries(self):
+		return set(self._entries)
+
+	def add_entry(self, entry: KconfigEntry) -> None:
+		self._entries.append(entry)
+
+	def is_subset_of(self, other: "Kconfig") -> bool:
+		return self.entries().issubset(other.entries())
+
+	def write_to_file(self, path: str) -> None:
+		with open(path, 'w') as f:
+			for entry in self.entries():
+				f.write(str(entry) + '\n')
+
+	def parse_from_string(self, blob: str) -> None:
+		"""Parses a string containing KconfigEntrys and populates this Kconfig."""
+		self._entries = []
+		is_not_set_matcher = re.compile(CONFIG_IS_NOT_SET_PATTERN)
+		config_matcher = re.compile(CONFIG_PATTERN)
+		for line in blob.split('\n'):
+			line = line.strip()
+			if not line:
+				continue
+			elif config_matcher.match(line) or is_not_set_matcher.match(line):
+				self._entries.append(KconfigEntry(line))
+			elif line[0] == '#':
+				continue
+			else:
+				raise KconfigParseError('Failed to parse: ' + line)
+
+	def read_from_file(self, path: str) -> None:
+		with open(path, 'r') as f:
+			self.parse_from_string(f.read())
diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
new file mode 100644
index 0000000000000..87abaede50513
--- /dev/null
+++ b/tools/testing/kunit/kunit_kernel.py
@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: GPL-2.0
+
+import logging
+import subprocess
+import os
+
+import kunit_config
+
+KCONFIG_PATH = '.config'
+
+class ConfigError(Exception):
+	"""Represents an error trying to configure the Linux kernel."""
+
+
+class BuildError(Exception):
+	"""Represents an error trying to build the Linux kernel."""
+
+
+class LinuxSourceTreeOperations(object):
+	"""An abstraction over command line operations performed on a source tree."""
+
+	def make_mrproper(self):
+		try:
+			subprocess.check_output(['make', 'mrproper'])
+		except OSError as e:
+			raise ConfigError('Could not call make command: ' + e)
+		except subprocess.CalledProcessError as e:
+			raise ConfigError(e.output)
+
+	def make_olddefconfig(self):
+		try:
+			subprocess.check_output(['make', 'ARCH=um', 'olddefconfig'])
+		except OSError as e:
+			raise ConfigError('Could not call make command: ' + e)
+		except subprocess.CalledProcessError as e:
+			raise ConfigError(e.output)
+
+	def make(self):
+		try:
+			subprocess.check_output(['make', 'ARCH=um'])
+		except OSError as e:
+			raise BuildError('Could not call execute make: ' + e)
+		except subprocess.CalledProcessError as e:
+			raise BuildError(e.output)
+
+	def linux_bin(self, params, timeout):
+		"""Runs the Linux UML binary. Must be named 'linux'."""
+		process = subprocess.Popen(
+			['./linux'] + params,
+			stdin=subprocess.PIPE,
+			stdout=subprocess.PIPE,
+			stderr=subprocess.PIPE)
+		process.wait(timeout=timeout)
+		return process
+
+
+class LinuxSourceTree(object):
+	"""Represents a Linux kernel source tree with KUnit tests."""
+
+	def __init__(self):
+		self._kconfig = kunit_config.Kconfig()
+		self._kconfig.read_from_file('kunitconfig')
+		self._ops = LinuxSourceTreeOperations()
+
+	def clean(self):
+		try:
+			self._ops.make_mrproper()
+		except ConfigError as e:
+			logging.error(e)
+			return False
+		return True
+
+	def build_config(self):
+		self._kconfig.write_to_file(KCONFIG_PATH)
+		try:
+			self._ops.make_olddefconfig()
+		except ConfigError as e:
+			logging.error(e)
+			return False
+		validated_kconfig = kunit_config.Kconfig()
+		validated_kconfig.read_from_file(KCONFIG_PATH)
+		if not self._kconfig.is_subset_of(validated_kconfig):
+			logging.error('Provided Kconfig is not contained in validated .config!')
+			return False
+		return True
+
+	def build_reconfig(self):
+		"""Creates a new .config if it is not a subset of the kunitconfig."""
+		if os.path.exists(KCONFIG_PATH):
+			existing_kconfig = kunit_config.Kconfig()
+			existing_kconfig.read_from_file(KCONFIG_PATH)
+			if not self._kconfig.is_subset_of(existing_kconfig):
+				print('Regenerating .config ...')
+				os.remove(KCONFIG_PATH)
+				return self.build_config()
+			else:
+				return True
+		else:
+			print('Generating .config ...')
+			return self.build_config()
+
+	def build_um_kernel(self):
+		try:
+			self._ops.make_olddefconfig()
+			self._ops.make()
+		except (ConfigError, BuildError) as e:
+			logging.error(e)
+			return False
+		used_kconfig = kunit_config.Kconfig()
+		used_kconfig.read_from_file(KCONFIG_PATH)
+		if not self._kconfig.is_subset_of(used_kconfig):
+			logging.error('Provided Kconfig is not contained in final config!')
+			return False
+		return True
+
+	def run_kernel(self, args=[]):
+		timeout = None
+		args.extend(['mem=256M'])
+		process = self._ops.linux_bin(args, timeout)
+		with open('test.log', 'w') as f:
+			for line in process.stdout:
+				f.write(line.rstrip().decode('ascii') + '\n')
+				yield line.rstrip().decode('ascii')
-- 
2.19.1.568.g152ad8e336-goog

  parent reply	other threads:[~2018-10-23 23:57 UTC|newest]

Thread overview: 30+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2018-10-23 23:57 [RFC v2 00/14] kunit: introduce KUnit, the Linux kernel unit testing framework Brendan Higgins
     [not found] ` <20181023235750.103146-1-brendanhiggins-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org>
2018-10-23 23:57   ` [RFC v2 01/14] kunit: test: add KUnit test runner core Brendan Higgins
2018-11-02 18:44     ` Shuah Khan
     [not found]       ` <017b111f-d960-c1ef-46ae-eb0eb639fe5b-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
2018-11-07  1:28         ` Brendan Higgins
2018-11-07 20:02           ` Shuah Khan
2018-10-23 23:57   ` [RFC v2 02/14] kunit: test: add test resource management API Brendan Higgins
2018-10-23 23:57   ` [RFC v2 03/14] kunit: test: add string_stream a std::stream like string builder Brendan Higgins
2018-10-23 23:57   ` [RFC v2 04/14] kunit: test: add test_stream a std::stream like logger Brendan Higgins
2018-10-23 23:57   ` [RFC v2 05/14] kunit: test: add the concept of expectations Brendan Higgins
2018-10-23 23:57   ` [RFC v2 06/14] arch: um: enable running kunit from User Mode Linux Brendan Higgins
2018-10-23 23:57   ` [RFC v2 07/14] kunit: test: add initial tests Brendan Higgins
2018-10-23 23:57   ` [RFC v2 08/14] arch: um: add shim to trap to allow installing a fault catcher for tests Brendan Higgins
2018-10-23 23:57   ` [RFC v2 09/14] kunit: test: add the concept of assertions Brendan Higgins
2018-10-23 23:57   ` Brendan Higgins [this message]
2018-10-23 23:57   ` [RFC v2 11/14] kunit: add KUnit wrapper script and simple output parser Brendan Higgins
2018-10-23 23:57   ` [RFC v2 12/14] kunit.py: improve output from python wrapper Brendan Higgins
2018-10-23 23:57   ` [RFC v2 13/14] Documentation: kunit: add documentation for KUnit Brendan Higgins
2018-10-23 23:57   ` [RFC v2 14/14] MAINTAINERS: add entry for KUnit the unit testing framework Brendan Higgins
2018-10-24  9:14 ` [RFC v2 00/14] kunit: introduce KUnit, the Linux kernel " Daniel Vetter
2018-10-25 21:25   ` Brendan Higgins
2018-10-25 17:40 ` Shuah Khan
2018-11-02 18:23 ` Shuah Khan
2018-11-07  1:17   ` Brendan Higgins
2018-11-07 17:46     ` Frank Rowand
     [not found]       ` <04f677b1-bc44-f004-cf2a-51b47baf0965-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>
2018-11-13 10:10         ` Brendan Higgins
2018-11-24  5:15 ` Knut Omang
     [not found]   ` <1543036529.4680.655.camel-QHcLZuEGTsvQT0dZR+AlfA@public.gmane.org>
2018-11-27  1:41     ` Brendan Higgins
2018-11-28 19:54       ` Knut Omang
2018-11-28 20:50         ` shuah
2018-11-30  0:59           ` Luis Chamberlain

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20181023235750.103146-11-brendanhiggins@google.com \
    --to=brendanhiggins-hpiqsd4aklfqt0dzr+alfa@public.gmane.org \
    --cc=Tim.Bird-7U/KSKJipcs@public.gmane.org \
    --cc=brakmo-b10kYP2dOMg@public.gmane.org \
    --cc=daniel-/w4YWyX8dFk@public.gmane.org \
    --cc=dri-devel-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW@public.gmane.org \
    --cc=felixguoxiuping-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org \
    --cc=gregkh-hQyY1W1yCW8ekmWlsbkhG0B+6BGkLq7r@public.gmane.org \
    --cc=jdike-OPE4K8JWMJJBDgjK7y7TUQ@public.gmane.org \
    --cc=joe-6d6DIl74uiNBDgjK7y7TUQ@public.gmane.org \
    --cc=joel-U3u1mxZcP9KHXe+LvDLADg@public.gmane.org \
    --cc=julia.lawall-L2FTfq7BK8M@public.gmane.org \
    --cc=keescook-hpIqsD4AKlfQT0dZR+AlfA@public.gmane.org \
    --cc=khilman-rdvid1DuHRBWk0Htik3J/w@public.gmane.org \
    --cc=kieran.bingham-ryLnwIuWjnjg/C1BVhZhaw@public.gmane.org \
    --cc=kunit-dev-/JYPxA39Uh5TLH3MbocFFw@public.gmane.org \
    --cc=linux-kernel-u79uwXL29TY76Z2rM5mHXA@public.gmane.org \
    --cc=linux-kselftest-u79uwXL29TY76Z2rM5mHXA@public.gmane.org \
    --cc=linux-nvdimm-hn68Rpc1hR1g9hUCZPvPmw@public.gmane.org \
    --cc=linux-um-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r@public.gmane.org \
    --cc=mcgrof-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org \
    --cc=mpe-Gsx/Oe8HsFggBc27wqDAHg@public.gmane.org \
    --cc=richard-/L3Ra7n9ekc@public.gmane.org \
    --cc=robh-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org \
    --cc=rostedt-nx8X9YLhiw1AfugRpC6u6w@public.gmane.org \
    --cc=shuah-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org \
    --subject='Re: [RFC v2 10/14] kunit: add Python libraries for handing KUnit config and kernel' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).