[PATCH 05/10] guestfs: add initial debian trixie support with custom URLs

[Luis Chamberlain <mcgrof@kernel.org>]

debian does not yet provide an index file for virt-builder, but we
have URLS with qcow2 and raw files. Using them in guestfs is actually
not quite trivial. So we do the handy work to enable others to also
use custom URLs and build a virt-builder local source and index file
for you. All we need really, is to check the checksums.

Sadly this does not yet work, as it seems we get stuck on the grub
prompt for some reason. This needs some more investigation.

Signed-off-by: Luis Chamberlain <mcgrof@kernel.org>
---
 kconfigs/Kconfig.defaults    |   4 ++
 kconfigs/Kconfig.distro      |  12 ++++
 kconfigs/Kconfig.guestfs     |  94 ++++++++++++++++++++++++-
 scripts/bringup_guestfs.sh   | 132 +++++++++++++++++++++++++++++++++++
 scripts/os-debian-version.sh |  19 +++++
 5 files changed, 259 insertions(+), 2 deletions(-)
 create mode 100755 scripts/os-debian-version.sh

diff --git a/kconfigs/Kconfig.defaults b/kconfigs/Kconfig.defaults
index 854a92e40ed2..41529dc86ce1 100644
--- a/kconfigs/Kconfig.defaults
+++ b/kconfigs/Kconfig.defaults
@@ -4,6 +4,10 @@ config DEFAULT_BLKTRACE_URL
 	string
 	default "https://git.kernel.dk/blktrace"
 
+config DEFAULT_DATE_ISO8601
+	string
+	default $(shell, date -I)
+
 config DEFAULT_DBENCH_URL
 	string
 	default "https://github.com/linux-kdevops/dbench.git"
diff --git a/kconfigs/Kconfig.distro b/kconfigs/Kconfig.distro
index b064e31a7c29..18e582eda40b 100644
--- a/kconfigs/Kconfig.distro
+++ b/kconfigs/Kconfig.distro
@@ -2,6 +2,18 @@ config DISTRO_DEBIAN
 	bool
 	default $(shell, scripts/os-release-check.sh debian)
 
+if DISTRO_DEBIAN
+
+config DISTRO_DEBIAN_BUSTER
+	bool
+	default $(shell, scripts/os-debian-version.sh buster)
+
+config DISTRO_DEBIAN_TRIXIE
+	bool
+	default $(shell, scripts/os-debian-version.sh trixie)
+
+endif
+
 config DISTRO_FEDORA
 	bool
 	default $(shell, scripts/os-release-check.sh fedora)
diff --git a/kconfigs/Kconfig.guestfs b/kconfigs/Kconfig.guestfs
index 5838522908e8..5839fbedfd08 100644
--- a/kconfigs/Kconfig.guestfs
+++ b/kconfigs/Kconfig.guestfs
@@ -1,5 +1,30 @@
 if GUESTFS
 
+config GUESTFS_HAS_CUSTOM_RAW_IMAGE
+	bool
+
+config GUESTFS_HAS_CUSTOM_RAW_IMAGE_URL
+	bool
+
+config GUESTFS_CUSTOM_RAW_IMAGE_URL
+	depends on GUESTFS_HAS_CUSTOM_RAW_IMAGE
+	depends on GUESTFS_HAS_CUSTOM_RAW_IMAGE_URL
+	string
+	default "https://cloud.debian.org/images/cloud/trixie/daily/latest/debian-13-generic-amd64-daily" if GUESTFS_DEBIAN_TRIXIE_GENERIC_AMD64
+	default "https://cloud.debian.org/images/cloud/trixie/daily/latest/debian-13-genericcloud-amd64-daily" if GUESTFS_DEBIAN_TRIXIE_GENERIC_CLOUD_AMD64
+	default "https://cloud.debian.org/images/cloud/trixie/daily/latest/debian-13-nocloud-amd64-daily" if GUESTFS_DEBIAN_TRIXIE_NOCLOUD_AMD64
+
+config GUESTFS_HAS_CUSTOM_RAW_IMAGE_SHA512SUMS
+	bool
+
+config GUESTFS_CUSTOM_RAW_IMAGE_SHA512SUMS_URL
+	string
+	depends on GUESTFS_HAS_CUSTOM_RAW_IMAGE_SHA512SUMS
+	default "https://cloud.debian.org/images/cloud/trixie/daily/latest/SHA512SUMS" if GUESTFS_DEBIAN_TRIXIE
+
+config GUESTFS_HAS_CUSTOM_RAW_IMAGE_ROLLING
+	bool
+
 choice
 	prompt "Guestfs Linux distribution to use"
 	default GUESTFS_FEDORA if DISTRO_FEDORA || DISTRO_REDHAT
@@ -33,16 +58,81 @@ endchoice
 
 if GUESTFS_DEBIAN
 
+choice
+	prompt "Debian version"
+	default GUESTFS_DEBIAN_BUSTER
+
+config GUESTFS_DEBIAN_TRIXIE
+	bool "Debian 13 - Trixie"
+	select GUESTFS_HAS_CUSTOM_RAW_IMAGE
+	select GUESTFS_HAS_CUSTOM_RAW_IMAGE_URL
+	select GUESTFS_HAS_CUSTOM_RAW_IMAGE_SHA512SUMS
+	select GUESTFS_HAS_CUSTOM_RAW_IMAGE_ROLLING
+	help
+	  Select this for debian trixie, debian 13. Since debian does not yet
+	  provie a virt-builder repo, we must download mock up our own URL
+	  for trixie release, as the default guestfs repo only has debian buster.
+	  We download the image for you once, and once its there we don't refresh
+	  it. We can later add support to force a refresh using the latest URL,
+	  as trixie is debian-testing and debian testing always has an new
+	  up to date image release, that is, its a rolling distribution release.
+
+	  This doesn't work yet.. it gets stuck on the grub prompt.
+
+config GUESTFS_DEBIAN_BUSTER
+	bool "Debian 12 - Buster"
+	help
+	  Select this for debian buster, debian 12.
+
+endchoice
+
+if GUESTFS_DEBIAN_TRIXIE
+
+choice
+	prompt "Debian trixie flavor"
+	default GUESTFS_DEBIAN_TRIXIE_GENERIC_CLOUD_AMD64
+
+config GUESTFS_DEBIAN_TRIXIE_GENERIC_AMD64
+	bool "debian-13-generic-amd64-daily"
+	help
+	  Should run in any environment using cloud-init, for e.g. OpenStack,
+	  DigitalOcean and also on bare metal.
+
+config GUESTFS_DEBIAN_TRIXIE_GENERIC_CLOUD_AMD64
+	bool "debian-13-genericcloud-amd64-daily"
+	help
+	  Similar to generic. Should run in any virtualised environment. Is
+	  smaller than generic by excluding drivers for physical hardware.
+
+config GUESTFS_DEBIAN_TRIXIE_NOCLOUD_AMD64
+	bool "debian-13-nocloud-amd64-daily"
+	help
+	  Mostly useful for testing the build process itself. Doesn't have
+	  cloud-init installed, but instead allows root login without a
+	  password.
+
+endchoice
+
+endif # GUESTFS_DEBIAN_TRIXIE
+
+config GUESTFS_DEBIAN_IMAGE_NAME
+	string
+	default "debian-13-generic-amd64-daily" if GUESTFS_DEBIAN_TRIXIE_GENERIC_AMD64
+	default "debian-13-genericcloud-amd64-daily" if GUESTFS_DEBIAN_TRIXIE_GENERIC_CLOUD_AMD64
+	default "debian-13-nocloud-amd64-daily" if GUESTFS_DEBIAN_TRIXIE_NOCLOUD_AMD64
+	default "debian-12" if GUESTFS_DEBIAN_BUSTER
+
 config GUESTFS_DEBIAN_BOX_SHORT
         string
-        default "debian12" if GUESTFS_DEBIAN
+        default "debian13" if GUESTFS_DEBIAN_TRIXIE
+        default "debian12" if GUESTFS_DEBIAN_BUSTER
 
 endif # GUESTFS_DEBIAN
 
 config VIRT_BUILDER_OS_VERSION
        string "virt-builder os-version"
        default "fedora-39" if GUESTFS_FEDORA
-       default "debian-12" if GUESTFS_DEBIAN
+       default GUESTFS_DEBIAN_IMAGE_NAME if GUESTFS_DEBIAN
        help
          Have virt-builder use this os-version string to
          build a root image for the guest. Run "virt-builder -l"
diff --git a/scripts/bringup_guestfs.sh b/scripts/bringup_guestfs.sh
index f90ed499051b..514a26a60436 100755
--- a/scripts/bringup_guestfs.sh
+++ b/scripts/bringup_guestfs.sh
@@ -30,8 +30,135 @@ GUESTFSDIR="${TOPDIR}/guestfs"
 OS_VERSION=${CONFIG_VIRT_BUILDER_OS_VERSION}
 BASE_IMAGE_DIR="${STORAGEDIR}/base_images"
 BASE_IMAGE="${BASE_IMAGE_DIR}/${OS_VERSION}.raw"
+
+build_custom_source()
+{
+	SOURCE_TMP=$(mktemp)
+	cat <<_EOT >$SOURCE_TMP
+[local]
+uri=file:///${CUSTOM_INDEX}
+proxy=off
+_EOT
+	sudo mv $SOURCE_TMP $CUSTOM_SOURCE
+}
+
+build_custom_index()
+{
+	cat <<_EOT >$CUSTOM_INDEX
+[$OS_VERSION]
+file=${OS_VERSION}.raw
+_EOT
+}
+
+fetch_custom_image()
+{
+	wget --directory-prefix=$CUSTOM_IMAGE_DIR $CONFIG_GUESTFS_CUSTOM_RAW_IMAGE_URL
+	if [[ $? -ne 0 ]]; then
+		echo -e "Could not download:\n$CONFIG_GUESTFS_CUSTOM_RAW_IMAGE_URL"
+		exit 1
+	fi
+}
+
+check_custom_image()
+{
+	SHA512SUMS_FILE="$(basename $CONFIG_GUESTFS_CUSTOM_RAW_IMAGE_SHA512SUMS_URL)"
+	CUSTOM_IMAGE_SHA512SUM="$CUSTOM_IMAGE_DIR/$SHA512SUMS_FILE"
+	if [[ ! -f $CUSTOM_IMAGE_SHA512SUM ]]; then
+		wget --directory-prefix=$CUSTOM_IMAGE_DIR $CONFIG_GUESTFS_CUSTOM_RAW_IMAGE_SHA512SUMS_URL
+		if [[ $? -ne 0 ]]; then
+			echo "Could not get sha512sum file: $CONFIG_GUESTFS_CUSTOM_RAW_IMAGE_SHA512SUMS_URL"
+			exit 1
+		fi
+	fi
+	echo "Checking $CUSTOM_IMAGE_DIR/$SHA512SUMS_FILE"
+
+	# This subshell let's us keep below in the current directory.
+	# sha512sum files are relative to the local directory
+	(cd $CUSTOM_IMAGE_DIR && sha512sum --ignore-missing -c $SHA512SUMS_FILE)
+	if [[ $? -ne 0 ]]; then
+		echo "Invalid SHA512SUM checksum for $CUSTOM_IMAGE as per $SHA512SUMS_FILE"
+		exit 1
+	fi
+	touch $CUSTOM_IMAGE_OK
+}
+
+# Ensure folks are not surprised. If you're using rolling distros you know what
+# you are doing. This gives us the right later to change this at will.
+#
+# In the future we can make this smoother, as we used to have it with vagrant
+# update, but for now downloading *once* for a rolling distro seems ok to start.
+# We give enough information so you can update.
+build_warn_rolling_distro()
+{
+	echo "------------------------------------------------------------------"
+	echo "This is a rolling distribution release! To upgrade just do:"
+	echo
+	echo "rm -rf ${CUSTOM_IMAGE}/*"
+	echo "rm -f  ${CUSTOM_SOURCE}"
+	echo "rm -f  ${CUSTOM_INDEX}"
+	echo
+	echo "Running guests always use their own copy. To rebuild your custom"
+	echo "base image from the custom image, also remov the base image:"
+	echo
+	echo "rm -f  ${BASE_IMAGE}"
+	echo
+	echo "This can always be done safely without affecting running guests."
+	echo "------------------------------------------------------------------"
+}
+
+build_custom_image()
+{
+	CUSTOM_IMAGE_DIR="${STORAGEDIR}/custom_images/${OS_VERSION}"
+	CUSTOM_IMAGE="${CUSTOM_IMAGE_DIR}/${OS_VERSION}.raw"
+	CUSTOM_IMAGE_OK="${CUSTOM_IMAGE_DIR}.ok"
+	CUSTOM_SOURCE="/etc/virt-builder/repos.d/kdevops-custom-images-${OS_VERSION}.conf"
+	CUSTOM_INDEX="${CUSTOM_IMAGE_DIR}/index"
+
+	mkdir -p ${CUSTOM_IMAGE_DIR}
+
+	if [[ ! -f $CUSTOM_IMAGE && "$CONFIG_GUESTFS_HAS_CUSTOM_RAW_IMAGE_URL" == "y" ]]; then
+		fetch_custom_image
+	fi
+
+	if [[ ! -f $CUSTOM_IMAGE_OK && "$CONFIG_GUESTFS_HAS_CUSTOM_RAW_IMAGE_SHA512SUMS" == "y" ]]; then
+		check_custom_image
+	fi
+
+	if [[ ! -f $CUSTOM_IMAGE ]]; then
+		echo "Custom image on path $CUSTOM_IMAGE not found"
+		exit 1
+	fi
+
+	if [[ ! -f $CUSTOM_SOURCE ]]; then
+		build_custom_source
+	fi
+
+	if [[ ! -f $CUSTOM_INDEX ]]; then
+		build_custom_index
+	fi
+
+	echo "Custom virt-builder source: $CUSTOM_SOURCE"
+	echo "Custom virt-builder index:  $CUSTOM_INDEX"
+	echo "Custom image source:        $CUSTOM_IMAGE"
+
+	if [[ "$CONFIG_GUESTFS_HAS_CUSTOM_RAW_IMAGE_ROLLING" == "y" ]]; then
+		build_warn_rolling_distro
+	fi
+
+	echo "Going to build index for $OS_VERSION ..."
+	virt-builder-repository --no-compression $CUSTOM_IMAGE_DIR
+	if [[ $? -ne 0 ]]; then
+		echo "Failed to build repository ..."
+		exit 1
+	fi
+
+	# Note, we don't build $BASE_IMAGE, virt-builder does that later. We
+	# just build $virt-builder, which is the pristine upstream image.
+}
+
 mkdir -p $STORAGEDIR
 mkdir -p $BASE_IMAGE_DIR
+
 if [[ "$CONFIG_LIBVIRT_URI_SYSTEM" == "y" ]]; then
 	sudo chgrp -R $QEMU_GROUP $STORAGEDIR
 	sudo chmod -R g+rw $STORAGEDIR
@@ -41,6 +168,10 @@ fi
 cmdfile=$(mktemp)
 
 if [ ! -f $BASE_IMAGE ]; then
+	if [[ "$CONFIG_GUESTFS_HAS_CUSTOM_RAW_IMAGE" == "y" ]]; then
+		build_custom_image
+	fi
+
 	DO_UNREG=0
 	if echo $OS_VERSION | grep -q '^rhel'; then
 		if [ -n "$CONFIG_RHEL_ORG_ID" -a -n "$CONFIG_RHEL_ACTIVATION_KEY" ]; then
@@ -77,6 +208,7 @@ _EOT
 # Hope we get that interface name right!
 	if echo $OS_VERSION | grep -q '^debian'; then
 		cat <<_EOT >>$cmdfile
+mkdir /etc/network/interfaces.d/
 append-line /etc/network/interfaces.d/enp1s0:auto enp1s0
 append-line /etc/network/interfaces.d/enp1s0:allow-hotplug enp1s0
 append-line /etc/network/interfaces.d/enp1s0:iface enp1s0 inet dhcp
diff --git a/scripts/os-debian-version.sh b/scripts/os-debian-version.sh
new file mode 100755
index 000000000000..f6b01d689b87
--- /dev/null
+++ b/scripts/os-debian-version.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+DEBIAN_VERSION_FILE="/etc/debian_version"
+
+if [[ ! -f $DEBIAN_VERSION_FILE ]]; then
+	echo n
+fi
+
+check_debian_version()
+{
+	grep -qi $1 $DEBIAN_VERSION_FILE
+	if [[ $? -eq 0 ]]; then
+		echo y
+		exit
+	fi
+	echo n
+	exit
+}
+
+check_debian_version $1
-- 
2.43.0