[PATCH bpf-next] bpf: Helper script for running BPF presubmit tests

[PATCH bpf-next] bpf: Helper script for running BPF presubmit tests

[KP Singh]

The script runs the BPF selftests locally on the same kernel image
as they would run post submit in the BPF continuous integration
framework. The goal of the script is to run selftests locally in the
same environment to check if their changes would end up breaking the
BPF CI and reduce the back-and-forth between the maintainers and the
developers.

Signed-off-by: KP Singh <kpsingh@kernel.org>
---
 tools/testing/selftests/bpf/run_in_vm.sh | 346 +++++++++++++++++++++++
 1 file changed, 346 insertions(+)
 create mode 100755 tools/testing/selftests/bpf/run_in_vm.sh

diff --git a/tools/testing/selftests/bpf/run_in_vm.sh b/tools/testing/selftests/bpf/run_in_vm.sh
new file mode 100755
index 000000000000..a4f28f5cdd52
--- /dev/null
+++ b/tools/testing/selftests/bpf/run_in_vm.sh
@@ -0,0 +1,346 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+set -u
+set -e
+
+QEMU_BINARY="${QEMU_BINARY:="qemu-system-x86_64"}"
+X86_BZIMAGE="arch/x86/boot/bzImage"
+DEFAULT_COMMAND="./test_progs"
+MOUNT_DIR="mnt"
+ROOTFS_IMAGE="root.img"
+OUTPUT_DIR="$HOME/.bpf_selftests"
+KCONFIG_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/latest.config"
+INDEX_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/INDEX"
+
+usage()
+{
+	cat <<EOF
+Usage: $0 [-k] [-i] [-d <output_dir>] -- [<command>]
+
+<command> is the command you would normally run when you are in
+tools/testing/selftests/bpf. e.g:
+
+	$0 -- ./test_progs -t test_lsm
+
+If no command is specified, "${DEFAULT_COMMAND}" will be run by
+default.
+
+If you build your kernel using KBUILD_OUTPUT= or O= options, these
+can be passed as environment variables to the script:
+
+  O=<path_relative_to_cwd> $0 -- ./test_progs -t test_lsm
+
+or
+
+  KBUILD_OUTPUT=<path_relative_to_cwd> $0 -- ./test_progs -t test_lsm
+
+Options:
+
+	-k)		Recompile the kernel with the config for selftests.
+	-i)		Update the rootfs image with a newer version.
+	-d)		Update the output directory (default: ${OUTPUT_DIR})
+EOF
+}
+
+unset URLS
+populate_url_map()
+{
+	if ! declare -p URLS &> /dev/null; then
+		# URLS contain the mapping from file names to URLs where
+		# those files can be downloaded from.
+		declare -gA URLS
+		while IFS=$'\t' read -r name url; do
+			URLS["$name"]="$url"
+		done < <(curl -Lsf ${INDEX_URL})
+	fi
+	echo "${URLS[*]}"
+}
+
+download()
+{
+	local file="$1"
+
+	if [[ ! -v URLS[$file] ]]; then
+		echo "$file not found" >&2
+		return 1
+	fi
+
+	echo "Downloading $file..." >&2
+	curl -Lf "${URLS[$file]}" "${@:2}"
+}
+
+newest_rootfs_version()
+{
+	{
+	for file in "${!URLS[@]}"; do
+		if [[ $file =~ ^libbpf-vmtest-rootfs-(.*)\.tar\.zst$ ]]; then
+			echo "${BASH_REMATCH[1]}"
+		fi
+	done
+	} | sort -rV | head -1
+}
+
+download_rootfs()
+{
+	local rootfsversion="$1"
+	local dir="$2"
+
+	if ! which zstd &> /dev/null; then
+		echo 'Could not find "zstd" on the system, please install zstd'
+		exit 1
+	fi
+
+	download "libbpf-vmtest-rootfs-$rootfsversion.tar.zst" |
+		zstd -d | sudo tar -C "$dir" -x
+}
+
+recompile_kernel()
+{
+	local kernel_checkout="$1"
+	local make_command="$2"
+
+	cd "${kernel_checkout}"
+
+	${make_command} olddefconfig
+	${make_command}
+}
+
+mount_image()
+{
+	local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
+	local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
+
+	sudo mount -o loop "${rootfs_img}" "${mount_dir}"
+}
+
+unmount_image()
+{
+	local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
+
+	sudo umount "${mount_dir}" &> /dev/null
+}
+
+update_selftests()
+{
+	local kernel_checkout="$1"
+	local selftests_dir="${kernel_checkout}/tools/testing/selftests/bpf"
+
+	cd "${selftests_dir}"
+	${make_command}
+
+	# Mount the image and copy the selftests to the image.
+	mount_image
+	sudo rm -rf "${mount_dir}/root/bpf"
+	sudo cp -r "${selftests_dir}" "${mount_dir}/root"
+	unmount_image
+}
+
+update_init_script()
+{
+	local init_script_dir="${OUTPUT_DIR}/${MOUNT_DIR}/etc/rcS.d"
+	local init_script="${init_script_dir}/S50-startup"
+	local command="$1"
+	local log_file="$2"
+
+	mount_image
+
+	if [[ ! -d "${init_script_dir}" ]]; then
+		cat <<EOF
+Could not find ${init_script_dir} in the mounted image.
+This likely indicates a bad rootfs image, Please download
+a new image by passing "-i" to the script
+EOF
+		exit 1
+
+	fi
+
+	cat <<EOF | sudo tee "${init_script}"
+#!/bin/sh
+
+exec 2>&1
+
+cd /root/bpf
+echo ${command} | tee /root/${log_file}
+${command} | tee -a /root/${log_file}
+poweroff -f
+EOF
+
+	sudo chmod a+x "${init_script}"
+	unmount_image
+}
+
+create_vm_image()
+{
+	local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
+	local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
+
+	rm -rf "${rootfs_img}"
+	touch "${rootfs_img}"
+	chattr +C "${rootfs_img}" >/dev/null 2>&1 || true
+
+	truncate -s 2G "${rootfs_img}"
+	mkfs.ext4 -q "${rootfs_img}"
+
+	mount_image
+	download_rootfs "$(newest_rootfs_version)" "${mount_dir}"
+	unmount_image
+}
+
+run_vm()
+{
+	local kernel_bzimage="$1"
+	local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
+
+	if ! which "${QEMU_BINARY}" &> /dev/null; then
+		cat <<EOF
+Could not find ${QEMU_BINARY}
+Please install qemu or set the QEMU_BINARY environment variable.
+EOF
+		exit 1
+	fi
+
+	${QEMU_BINARY} \
+		-nodefaults \
+		-display none \
+		-serial mon:stdio \
+		-cpu kvm64 \
+		-enable-kvm \
+		-smp 4 \
+		-m 2G \
+		-drive file="${rootfs_img}",format=raw,index=1,media=disk,if=virtio,cache=none \
+		-kernel "${kernel_bzimage}" \
+		-append "root=/dev/vda rw console=ttyS0,115200"
+}
+
+copy_logs()
+{
+	local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
+	local log_file="${mount_dir}/root/$1"
+
+	mount_image
+	sudo cp ${log_file} "${OUTPUT_DIR}"
+	sudo rm -f ${log_file}
+	unmount_image
+}
+
+is_rel_path()
+{
+	local path="$1"
+
+	[[ ${path:0:1} != "/" ]]
+}
+
+main()
+{
+	local script_dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+	local kernel_checkout=$(realpath "${script_dir}"/../../../../)
+	local log_file="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S.log")"
+	# By default the script searches for the kernel in the checkout directory but
+	# it also obeys environment variables O= and KBUILD_OUTPUT=
+	local kernel_bzimage="${kernel_checkout}/${X86_BZIMAGE}"
+	local command="${DEFAULT_COMMAND}"
+	local kernel_recompile="no"
+	local update_image="no"
+
+	while getopts 'hkid:' opt; do
+		case ${opt} in
+		k)
+			kernel_recompile="yes"
+			;;
+		i)
+			update_image="yes"
+			;;
+		d)
+			OUTPUT_DIR="$OPTARG"
+			;;
+		h)
+			usage
+			exit 0
+			;;
+		\? )
+			echo "Invalid Option: -$OPTARG"
+			usage
+			exit 1
+			;;
+      		: )
+        		echo "Invalid Option: -$OPTARG requires an argument"
+			usage
+			exit 1
+			;;
+		esac
+	done
+	shift $((OPTIND -1))
+
+	if [[ $# -eq 0 ]]; then
+		echo "No command specified, will run ${DEFAULT_COMMAND} in the vm"
+	else
+		command="$@"
+	fi
+
+	local kconfig_file="${OUTPUT_DIR}/latest.config"
+	local make_command="make -j KCONFIG_CONFIG=${kconfig_file}"
+
+	# Figure out where the kernel is being built.
+	# O takes precedence over KBUILD_OUTPUT.
+	if [[ "${O:=""}" != "" ]]; then
+		if is_rel_path "${O}"; then
+			O="$(realpath "${PWD}/${O}")"
+		fi
+		kernel_bzimage="${O}/${X86_BZIMAGE}"
+		make_command="${make_command} O=${O}"
+	elif [[ "${KBUILD_OUTPUT:=""}" != "" ]]; then
+		if is_rel_path "${KBUILD_OUTPUT}"; then
+			KBUILD_OUTPUT="$(realpath "${PWD}/${KBUILD_OUTPUT}")"
+		fi
+		kernel_bzimage="${KBUILD_OUTPUT}/${X86_BZIMAGE}"
+		make_command="${make_command} KBUILD_OUTPUT=${KBUILD_OUTPUT}"
+	fi
+
+	populate_url_map
+
+	local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}"
+	local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}"
+
+	echo "Output directory: ${OUTPUT_DIR}"
+
+	mkdir -p "${OUTPUT_DIR}"
+	mkdir -p "${mount_dir}"
+	curl -Lf "${KCONFIG_URL}" -o "${kconfig_file}"
+
+	if [[ "${kernel_recompile}" == "no" && ! -f "${kernel_bzimage}" ]]; then
+		echo "Kernel image not found in ${kernel_bzimage}, kernel will be recompiled"
+		kernel_recompile="yes"
+	fi
+
+	if [[ "${kernel_recompile}" == "yes" ]]; then
+		recompile_kernel "${kernel_checkout}" "${make_command}"
+	fi
+
+	if [[ "${update_image}" == "no" && ! -f "${rootfs_img}" ]]; then
+		echo "rootfs image not found in ${rootfs_img}"
+		update_image="yes"
+	fi
+
+	if [[ "${update_image}" == "yes" ]]; then
+		create_vm_image
+	fi
+
+	update_selftests "${kernel_checkout}" "${make_command}"
+	update_init_script "${command}" "${log_file}"
+	run_vm "${kernel_bzimage}"
+	copy_logs "${log_file}"
+	echo "Logs saved in ${OUTPUT_DIR}/${log_file}"
+}
+
+catch()
+{
+	local exit_code=$1
+
+	unmount_image
+	exit ${exit_code}
+}
+
+trap 'catch "$?"' EXIT
+
+main "$@"
-- 
2.30.0.284.gd98b1dd5eaa7-goog

Re: [PATCH bpf-next] bpf: Helper script for running BPF presubmit tests

[Daniel Borkmann]

On 1/18/21 3:19 PM, KP Singh wrote:
> The script runs the BPF selftests locally on the same kernel image
> as they would run post submit in the BPF continuous integration
> framework. The goal of the script is to run selftests locally in the
> same environment to check if their changes would end up breaking the
> BPF CI and reduce the back-and-forth between the maintainers and the
> developers.
> 
> Signed-off-by: KP Singh <kpsingh@kernel.org>

Looks really nice! I gave it a run, some minor feedback:

> ---
>   tools/testing/selftests/bpf/run_in_vm.sh | 346 +++++++++++++++++++++++
>   1 file changed, 346 insertions(+)
>   create mode 100755 tools/testing/selftests/bpf/run_in_vm.sh
> 
> diff --git a/tools/testing/selftests/bpf/run_in_vm.sh b/tools/testing/selftests/bpf/run_in_vm.sh
> new file mode 100755
> index 000000000000..a4f28f5cdd52
> --- /dev/null
> +++ b/tools/testing/selftests/bpf/run_in_vm.sh
> @@ -0,0 +1,346 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +
> +set -u
> +set -e
> +
> +QEMU_BINARY="${QEMU_BINARY:="qemu-system-x86_64"}"
> +X86_BZIMAGE="arch/x86/boot/bzImage"
> +DEFAULT_COMMAND="./test_progs"
> +MOUNT_DIR="mnt"
> +ROOTFS_IMAGE="root.img"
> +OUTPUT_DIR="$HOME/.bpf_selftests"
> +KCONFIG_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/latest.config"
> +INDEX_URL="https://raw.githubusercontent.com/libbpf/libbpf/master/travis-ci/vmtest/configs/INDEX"
> +
> +usage()
> +{
> +	cat <<EOF
> +Usage: $0 [-k] [-i] [-d <output_dir>] -- [<command>]
> +
> +<command> is the command you would normally run when you are in
> +tools/testing/selftests/bpf. e.g:
> +
> +	$0 -- ./test_progs -t test_lsm
> +
> +If no command is specified, "${DEFAULT_COMMAND}" will be run by
> +default.
> +
> +If you build your kernel using KBUILD_OUTPUT= or O= options, these
> +can be passed as environment variables to the script:
> +
> +  O=<path_relative_to_cwd> $0 -- ./test_progs -t test_lsm
> +
> +or
> +
> +  KBUILD_OUTPUT=<path_relative_to_cwd> $0 -- ./test_progs -t test_lsm
> +
> +Options:
> +
> +	-k)		Recompile the kernel with the config for selftests.
> +	-i)		Update the rootfs image with a newer version.
> +	-d)		Update the output directory (default: ${OUTPUT_DIR})

Probably best to have a small howto in tools/testing/selftests/bpf/README.rst for
people to have a 'getting started' point. Initially I forgot the -k, so VM paniced
on boot, but after that it was working great modulo a small change below.

[...]
> +main()
> +{
> +	local script_dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
> +	local kernel_checkout=$(realpath "${script_dir}"/../../../../)
> +	local log_file="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S.log")"
> +	# By default the script searches for the kernel in the checkout directory but
> +	# it also obeys environment variables O= and KBUILD_OUTPUT=
> +	local kernel_bzimage="${kernel_checkout}/${X86_BZIMAGE}"
> +	local command="${DEFAULT_COMMAND}"
> +	local kernel_recompile="no"
> +	local update_image="no"
> +
> +	while getopts 'hkid:' opt; do
> +		case ${opt} in
> +		k)
> +			kernel_recompile="yes"
> +			;;
> +		i)
> +			update_image="yes"
> +			;;
> +		d)
> +			OUTPUT_DIR="$OPTARG"
> +			;;
> +		h)
> +			usage
> +			exit 0
> +			;;
> +		\? )
> +			echo "Invalid Option: -$OPTARG"
> +			usage
> +			exit 1
> +			;;
> +      		: )
> +        		echo "Invalid Option: -$OPTARG requires an argument"
> +			usage
> +			exit 1
> +			;;
> +		esac
> +	done
> +	shift $((OPTIND -1))
> +
> +	if [[ $# -eq 0 ]]; then
> +		echo "No command specified, will run ${DEFAULT_COMMAND} in the vm"
> +	else
> +		command="$@"
> +	fi
> +
> +	local kconfig_file="${OUTPUT_DIR}/latest.config"
> +	local make_command="make -j KCONFIG_CONFIG=${kconfig_file}"

I had to fix/limit this locally to -j <num-cpus> as otherwise this was OOM killing
my box. make man page says 'if the -j option is given without an argument, make will
not limit the number of jobs that can run simultaneously.'

> +
> +	# Figure out where the kernel is being built.
> +	# O takes precedence over KBUILD_OUTPUT.
> +	if [[ "${O:=""}" != "" ]]; then
> +		if is_rel_path "${O}"; then
> +			O="$(realpath "${PWD}/${O}")"
> +		fi

For future follow-up, would be amazing to auto-grab nightly build of llvm + pahole
as well. And even further out maybe also to allow cross-compilation + testing on
other archs. :)

Thanks,
Daniel

Re: [PATCH bpf-next] bpf: Helper script for running BPF presubmit tests

[KP Singh]

On Wed, Jan 20, 2021 at 3:01 PM Daniel Borkmann <daniel@iogearbox.net> wrote:
>
> On 1/18/21 3:19 PM, KP Singh wrote:
> > The script runs the BPF selftests locally on the same kernel image
> > as they would run post submit in the BPF continuous integration
> > framework. The goal of the script is to run selftests locally in the
> > same environment to check if their changes would end up breaking the
> > BPF CI and reduce the back-and-forth between the maintainers and the
> > developers.
> >
> > Signed-off-by: KP Singh <kpsingh@kernel.org>
>
> Looks really nice! I gave it a run, some minor feedback:

Thanks! :)

>
> > ---
> >   tools/testing/selftests/bpf/run_in_vm.sh | 346 +++++++++++++++++++++++
> >   1 file changed, 346 insertions(+)
> >   create mode 100755 tools/testing/selftests/bpf/run_in_vm.sh
> >
> > diff --git a/tools/testing/selftests/bpf/run_in_vm.sh b/tools/testing/selftests/bpf/run_in_vm.sh
> > new file mode 100755
> > index 000000000000..a4f28f5cdd52
> > --- /dev/null
> > +++ b/tools/testing/selftests/bpf/run_in_vm.sh
> > @@ -0,0 +1,346 @@
> > +#!/bin/bash
> > +# SPDX-License-Identifier: GPL-2.0
> > +
> > +set -u
> > +set -e

[...]

> > +
> > +  KBUILD_OUTPUT=<path_relative_to_cwd> $0 -- ./test_progs -t test_lsm
> > +
> > +Options:
> > +
> > +     -k)             Recompile the kernel with the config for selftests.
> > +     -i)             Update the rootfs image with a newer version.
> > +     -d)             Update the output directory (default: ${OUTPUT_DIR})
>
> Probably best to have a small howto in tools/testing/selftests/bpf/README.rst for
> people to have a 'getting started' point. Initially I forgot the -k, so VM paniced
> on boot, but after that it was working great modulo a small change below.

Makes sense, I totally forgot about the case when one already has a
precompiled kernel
and it does not work well with the image.

Will update the docs.

>
> [...]
> > +main()
> > +{
> > +     local script_dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
> > +     local kernel_checkout=$(realpath "${script_dir}"/../../../../)
> > +     local log_file="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S.log")"

[...]

> > +             esac
> > +     done
> > +     shift $((OPTIND -1))
> > +
> > +     if [[ $# -eq 0 ]]; then
> > +             echo "No command specified, will run ${DEFAULT_COMMAND} in the vm"
> > +     else
> > +             command="$@"
> > +     fi
> > +
> > +     local kconfig_file="${OUTPUT_DIR}/latest.config"
> > +     local make_command="make -j KCONFIG_CONFIG=${kconfig_file}"
>
> I had to fix/limit this locally to -j <num-cpus> as otherwise this was OOM killing
> my box. make man page says 'if the -j option is given without an argument, make will
> not limit the number of jobs that can run simultaneously.'

I thought that -j without an option did something smart. I will update
it to be -j `<num_cpus>`
Thanks!

>
> > +
> > +     # Figure out where the kernel is being built.
> > +     # O takes precedence over KBUILD_OUTPUT.
> > +     if [[ "${O:=""}" != "" ]]; then
> > +             if is_rel_path "${O}"; then
> > +                     O="$(realpath "${PWD}/${O}")"
> > +             fi
>
> For future follow-up, would be amazing to auto-grab nightly build of llvm + pahole
> as well. And even further out maybe also to allow cross-compilation + testing on
> other archs. :)

Indeed, these are definitely on the radar :)

- KP

>
> Thanks,
> Daniel