kvm.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: Tony Krowiak <akrowiak@linux.ibm.com>
To: linux-s390@vger.kernel.org, linux-kernel@vger.kernel.org,
	kvm@vger.kernel.org
Cc: jjherne@linux.ibm.com, freude@linux.ibm.com,
	borntraeger@de.ibm.com, cohuck@redhat.com,
	mjrosato@linux.ibm.com, pasic@linux.ibm.com,
	alex.williamson@redhat.com, kwankhede@nvidia.com,
	fiuczy@linux.ibm.com, Tony Krowiak <akrowiak@linux.ibm.com>
Subject: [PATCH v16 09/14] s390/vfio-ap: reset queues after adapter/domain unassignment
Date: Mon, 10 May 2021 12:44:18 -0400	[thread overview]
Message-ID: <20210510164423.346858-10-akrowiak@linux.ibm.com> (raw)
In-Reply-To: <20210510164423.346858-1-akrowiak@linux.ibm.com>

When an adapter or domain is unassigned from an mdev providing the AP
configuration to a KVM guest, one or more of the guest's queues may get
dynamically removed. Since the removed queues could get re-assigned to
another mdev, they need to be reset. So, when an adapter or domain is
unassigned from the mdev, the queues that are removed from the guest's
AP configuration will be reset.

Signed-off-by: Tony Krowiak <akrowiak@linux.ibm.com>
---
 drivers/s390/crypto/vfio_ap_ops.c     | 155 +++++++++++++++++++++++---
 drivers/s390/crypto/vfio_ap_private.h |   1 +
 2 files changed, 138 insertions(+), 18 deletions(-)

diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index e388eaf4f601..48e3db2f1c28 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -25,6 +25,8 @@
 #define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
 
 static int vfio_ap_mdev_reset_queues(struct ap_matrix *matrix);
+static int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q,
+				    unsigned int retry);
 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);
 
 static struct vfio_ap_queue *
@@ -488,12 +490,6 @@ static void vfio_ap_mdev_unlink_fr_queue(struct vfio_ap_queue *q)
 	q->matrix_mdev = NULL;
 }
 
-static void vfio_ap_mdev_unlink_queue(struct vfio_ap_queue *q)
-{
-	vfio_ap_mdev_unlink_queue_fr_mdev(q);
-	vfio_ap_mdev_unlink_fr_queue(q);
-}
-
 static void vfio_ap_mdev_unlink_fr_queues(struct ap_matrix_mdev *matrix_mdev)
 {
 	struct vfio_ap_queue *q;
@@ -740,8 +736,20 @@ static ssize_t assign_adapter_store(struct device *dev,
 }
 static DEVICE_ATTR_WO(assign_adapter);
 
-static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev,
-					unsigned long apid)
+/*
+ * vfio_ap_unlink_adapter_fr_mdev
+ *
+ * @matrix_mdev: a pointer to the mdev currently in use by the KVM guest
+ * @apqi: the APID of the adapter that was unassigned from the mdev
+ * @qlist: a pointer to a list to which the queues unlinked from the mdev
+ *         will be stored.
+ *
+ * Unlinks the queues associated with the adapter from the mdev. Each queue that
+ * is unlinked will be added to @qlist.
+ */
+static void vfio_ap_unlink_adapter_fr_mdev(struct ap_matrix_mdev *matrix_mdev,
+					   unsigned long apid,
+					   struct list_head *qlist)
 {
 	unsigned long apqi;
 	struct vfio_ap_queue *q;
@@ -749,11 +757,79 @@ static void vfio_ap_mdev_unlink_adapter(struct ap_matrix_mdev *matrix_mdev,
 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {
 		q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));
 
-		if (q)
-			vfio_ap_mdev_unlink_queue(q);
+		if (q) {
+			vfio_ap_mdev_unlink_queue_fr_mdev(q);
+			list_add(&q->qlist_node, qlist);
+		}
 	}
 }
 
+/*
+ * vfio_ap_reset_queues_removed
+ *
+ * @guest_apcb: a pointer to a matrix object containing the values from the
+ *		  guest's APCB prior to refreshing the guest's AP configuration
+ *		  as a result of unassigning an adapter or domain from an mdev.
+ * @qlist: the list of queues unlinked from the mdev as a result of unassigning
+ *	   an adapter or domain from an mdev.
+ *
+ * Resets the queues that were removed from the guest's AP configuration.
+ */
+static void vfio_ap_reset_queues_removed(struct ap_matrix *guest_apcb,
+					 struct list_head *qlist)
+{
+	struct vfio_ap_queue *q;
+	unsigned long apid, apqi;
+
+	list_for_each_entry(q, qlist, qlist_node) {
+		apid = AP_QID_CARD(q->apqn);
+		apqi = AP_QID_QUEUE(q->apqn);
+
+		if (test_bit_inv(apid, guest_apcb->apm) &&
+		    test_bit_inv(apqi, guest_apcb->aqm))
+			vfio_ap_mdev_reset_queue(q, 1);
+	}
+}
+
+/*
+ * vfio_ap_unlink_mdev_fr_queues
+ *
+ * @qlist: the list of queues unlinked from the mdev to which they were
+ *	   previously assigned.
+ *
+ * Unlink the mdev from each queue.
+ */
+static void vfio_ap_unlink_mdev_fr_queues(struct list_head *qlist)
+{
+	struct vfio_ap_queue *q;
+
+	list_for_each_entry(q, qlist, qlist_node)
+		vfio_ap_mdev_unlink_fr_queue(q);
+}
+
+/*
+ * vfio_ap_mdev_rem_adapter_refresh
+ *
+ * @matrix_mdev: the mdev currently in use by the KVM guest
+ * @apqi: the APID of the adapter unassigned from the mdev
+ *
+ * Refreshes the KVM guest's APCB and resets any queues that may have been
+ * dynamically removed from the guest's AP configuration.
+ */
+static void vfio_ap_mdev_rem_adapter_refresh(struct ap_matrix_mdev *matrix_mdev,
+					     unsigned long apid)
+{
+	LIST_HEAD(qlist);
+	struct ap_matrix shadow_matrix;
+
+	memcpy(&shadow_matrix, &matrix_mdev->shadow_apcb,
+	       sizeof(shadow_matrix));
+	vfio_ap_unlink_adapter_fr_mdev(matrix_mdev, apid, &qlist);
+	vfio_ap_mdev_refresh_apcb(matrix_mdev);
+	vfio_ap_reset_queues_removed(&shadow_matrix, &qlist);
+	vfio_ap_unlink_mdev_fr_queues(&qlist);
+}
+
 /**
  * unassign_adapter_store
  *
@@ -801,8 +877,7 @@ static ssize_t unassign_adapter_store(struct device *dev,
 	}
 
 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
-	vfio_ap_mdev_unlink_adapter(matrix_mdev, apid);
-	vfio_ap_mdev_refresh_apcb(matrix_mdev);
+	vfio_ap_mdev_rem_adapter_refresh(matrix_mdev, apid);
 	ret = count;
 done:
 	mutex_unlock(&matrix_dev->lock);
@@ -898,8 +973,20 @@ static ssize_t assign_domain_store(struct device *dev,
 }
 static DEVICE_ATTR_WO(assign_domain);
 
-static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev,
-				       unsigned long apqi)
+/*
+ * vfio_ap_unlink_domain_fr_mdev
+ *
+ * @matrix_mdev: a pointer to the mdev currently in use by the KVM guest
+ * @apqi: the APQI of the domain that was unassigned from the mdev
+ * @qlist: a pointer to a list to which the queues unlinked from the mdev
+ *         will be stored.
+ *
+ * Unlinks the queues associated with the domain from the mdev. Each queue that
+ * is unlinked will be added to @qlist.
+ */
+static void vfio_ap_unlink_domain_fr_mdev(struct ap_matrix_mdev *matrix_mdev,
+					  unsigned long apqi,
+					  struct list_head *qlist)
 {
 	unsigned long apid;
 	struct vfio_ap_queue *q;
@@ -907,11 +994,44 @@ static void vfio_ap_mdev_unlink_domain(struct ap_matrix_mdev *matrix_mdev,
 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {
 		q = vfio_ap_mdev_get_queue(matrix_mdev, AP_MKQID(apid, apqi));
 
-		if (q)
-			vfio_ap_mdev_unlink_queue(q);
+		if (q) {
+			vfio_ap_mdev_unlink_queue_fr_mdev(q);
+			list_add(&q->qlist_node, qlist);
+		}
 	}
 }
 
+/*
+ * vfio_ap_mdev_rem_domain_refresh
+ *
+ * @matrix_mdev: the mdev currently in use by the KVM guest
+ * @apqi: the APQI of the domain unassigned from the mdev
+ *
+ * Refreshes the KVM guest's APCB and resets any queues that may have been
+ * dynamically removed from the guest's AP configuration.
+ */
+static void vfio_ap_mdev_rem_domain_refresh(struct ap_matrix_mdev *matrix_mdev,
+					    unsigned long apqi)
+{
+	LIST_HEAD(qlist);
+	struct ap_matrix shadow_matrix;
+
+	memcpy(&shadow_matrix, &matrix_mdev->shadow_apcb,
+	       sizeof(shadow_matrix));
+	/*
+	 * Unlink the queues associated with the domain from the mdev
+	 * so the refresh can determine what adapter to unplug from the guest.
+	 */
+	vfio_ap_unlink_domain_fr_mdev(matrix_mdev, apqi, &qlist);
+	vfio_ap_mdev_refresh_apcb(matrix_mdev);
+	vfio_ap_reset_queues_removed(&shadow_matrix, &qlist);
+	/*
+	 * Complete the unlinking of the queues associated with the domain
+	 * by unlinking the mdev from each queue.
+	 */
+	vfio_ap_unlink_mdev_fr_queues(&qlist);
+}
+
 /**
  * unassign_domain_store
  *
@@ -959,8 +1079,7 @@ static ssize_t unassign_domain_store(struct device *dev,
 	}
 
 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);
-	vfio_ap_mdev_unlink_domain(matrix_mdev, apqi);
-	vfio_ap_mdev_refresh_apcb(matrix_mdev);
+	vfio_ap_mdev_rem_domain_refresh(matrix_mdev, apqi);
 	ret = count;
 
 done:
diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
index 6f4f1f5bd611..1b95486fccf0 100644
--- a/drivers/s390/crypto/vfio_ap_private.h
+++ b/drivers/s390/crypto/vfio_ap_private.h
@@ -101,6 +101,7 @@ struct vfio_ap_queue {
 #define VFIO_AP_ISC_INVALID 0xff
 	unsigned char saved_isc;
 	struct hlist_node mdev_qnode;
+	struct list_head qlist_node;
 };
 
 int vfio_ap_mdev_register(void);
-- 
2.30.2


  parent reply	other threads:[~2021-05-10 16:45 UTC|newest]

Thread overview: 27+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-05-10 16:44 [PATCH v16 00/14] s390/vfio-ap: dynamic configuration support Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 01/14] s390/vfio-ap: fix memory leak in mdev remove callback Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 02/14] s390/vfio-ap: use new AP bus interface to search for queue devices Tony Krowiak
2021-06-09 13:52   ` Jason J. Herne
2021-06-09 18:23     ` Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 03/14] s390/vfio-ap: move probe and remove callbacks to vfio_ap_ops.c Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 04/14] s390/vfio-ap: manage link between queue struct and matrix mdev Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 05/14] s390/vfio-ap: introduce shadow APCB Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 06/14] s390/vfio-ap: refresh guest's APCB by filtering APQNs assigned to mdev Tony Krowiak
2021-05-24 16:15   ` Halil Pasic
2021-06-01 11:25     ` Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 07/14] s390/vfio-ap: allow assignment of unavailable AP queues to mdev device Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 08/14] s390/vfio-ap: allow hot plug/unplug of AP resources using " Tony Krowiak
2021-05-10 16:44 ` Tony Krowiak [this message]
2021-05-10 16:44 ` [PATCH v16 10/14] s390/zcrypt: driver callback to indicate resource in use Tony Krowiak
2021-05-19 11:58   ` Julian Wiedmann
2021-05-20 11:58     ` Harald Freudenberger
2021-05-20 15:28     ` Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 11/14] s390/vfio-ap: implement in-use callback for vfio_ap driver Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 12/14] s390/vfio-ap: sysfs attribute to display the guest's matrix Tony Krowiak
2021-05-10 16:44 ` [PATCH v16 13/14] s390/zcrypt: notify drivers on config changed and scan complete callbacks Tony Krowiak
2021-05-10 20:02   ` kernel test robot
2021-05-10 16:44 ` [PATCH v16 14/14] s390/vfio-ap: update docs to include dynamic config support Tony Krowiak
2021-05-18 13:26 ` [PATCH v16 00/14] s390/vfio-ap: dynamic configuration support Tony Krowiak
2021-05-18 16:54   ` Halil Pasic
2021-05-19 11:52     ` Tony Krowiak
2021-06-09 11:36 ` Tony Krowiak

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20210510164423.346858-10-akrowiak@linux.ibm.com \
    --to=akrowiak@linux.ibm.com \
    --cc=alex.williamson@redhat.com \
    --cc=borntraeger@de.ibm.com \
    --cc=cohuck@redhat.com \
    --cc=fiuczy@linux.ibm.com \
    --cc=freude@linux.ibm.com \
    --cc=jjherne@linux.ibm.com \
    --cc=kvm@vger.kernel.org \
    --cc=kwankhede@nvidia.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-s390@vger.kernel.org \
    --cc=mjrosato@linux.ibm.com \
    --cc=pasic@linux.ibm.com \
    --subject='Re: [PATCH v16 09/14] s390/vfio-ap: reset queues after adapter/domain unassignment' \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).