Re: [PATCH v2] PCI/MSI: Re-add checks for skip masking MSI-X on Xen PV

[Josef Johansson <josef@oderland.se>]

On 10/25/21 03:25, Jason Andryuk wrote:
> On Sun, Oct 24, 2021 at 2:55 PM Josef Johansson <josef@oderland.se> wrote:
>
>> I ended up with this patch, I also masked pci_set_mask and
>> pci_set_unmask, even though patching __pci_restore_msi_state and
>> __pci_restore_msi_state solved this problem, I found that it did not
>> properly make the system be able to survive flip_done timeout related
>> problems during suspend/resume. Would this be something you had in mind
>> Marc? I will make one more try with just patching
>> __pci_restore_msi_state and __pci_restore_msix_state just to make sure.
>> diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index
>> 4b4792940e86..0b2225066778 100644 --- a/drivers/pci/msi.c +++
>> b/drivers/pci/msi.c @@ -420,7 +420,8 @@ static void
>> __pci_restore_msi_state(struct pci_dev *dev) arch_restore_msi_irqs(dev);
>> pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); -
>> pci_msi_update_mask(entry, 0, 0); + if (!(pci_msi_ignore_mask ||
>> entry->msi_attrib.is_virtual)) + pci_msi_update_mask(entry, 0, 0);
>> control &= ~PCI_MSI_FLAGS_QSIZE; control |= (entry->msi_attrib.multiple
> This patch was mangled.
Thunderbird dislikes me plenty. Let's hope this turns out better.

I ended up with this patch, I also masked pci_set_mask and
pci_set_unmask, even though patching __pci_restore_msi_state and
__pci_restore_msi_state solved this problem, I found that it did not
properly make the system be able to survive flip_done timeout related
problems during suspend/resume. Would this be something you had in mind
Marc? I will make one more try with just patching
__pci_restore_msi_state and __pci_restore_msix_state just to make sure.


diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 4b4792940e86..0b2225066778 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -420,7 +420,8 @@ static void __pci_restore_msi_state(struct pci_dev *dev)
 	arch_restore_msi_irqs(dev);
 
 	pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
-	pci_msi_update_mask(entry, 0, 0);
+	if (!(pci_msi_ignore_mask || entry->msi_attrib.is_virtual))
+		pci_msi_update_mask(entry, 0, 0);
 	control &= ~PCI_MSI_FLAGS_QSIZE;
 	control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE;
 	pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
@@ -450,8 +451,9 @@ static void __pci_restore_msix_state(struct pci_dev *dev)
 				PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL);
 
 	arch_restore_msi_irqs(dev);
-	for_each_pci_msi_entry(entry, dev)
-		pci_msix_write_vector_ctrl(entry, entry->msix_ctrl);
+	if (!(pci_msi_ignore_mask || entry->msi_attrib.is_virtual))
+		for_each_pci_msi_entry(entry, dev)
+			pci_msix_write_vector_ctrl(entry, entry->msix_ctrl);
 
 	pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
 }
@@ -546,7 +548,8 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
 		return -ENOMEM;
 
 	/* All MSIs are unmasked by default; mask them all */
-	pci_msi_mask(entry, msi_multi_mask(entry));
+	if (!pci_msi_ignore_mask)
+		pci_msi_mask(entry, msi_multi_mask(entry));
 
 	list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
 
@@ -577,7 +580,8 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
 	return 0;
 
 err:
-	pci_msi_unmask(entry, msi_multi_mask(entry));
+	if (!pci_msi_ignore_mask)
+		pci_msi_unmask(entry, msi_multi_mask(entry));
 	free_msi_irqs(dev);
 	return ret;
 }
@@ -865,7 +868,8 @@ static void pci_msi_shutdown(struct pci_dev *dev)
 	dev->msi_enabled = 0;
 
 	/* Return the device with MSI unmasked as initial states */
-	pci_msi_unmask(desc, msi_multi_mask(desc));
+	if (!pci_msi_ignore_mask)
+		pci_msi_unmask(desc, msi_multi_mask(desc));
 
 	/* Restore dev->irq to its default pin-assertion IRQ */
 	dev->irq = desc->msi_attrib.default_irq;
@@ -950,8 +954,9 @@ static void pci_msix_shutdown(struct pci_dev *dev)
 	}
 
 	/* Return the device with MSI-X masked as initial states */
-	for_each_pci_msi_entry(entry, dev)
-		pci_msix_mask(entry);
+	if (!pci_msi_ignore_mask)
+		for_each_pci_msi_entry(entry, dev)
+			pci_msix_mask(entry);
 
 	pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
 	pci_intx_for_msi(dev, 1);



>> This makes sense the patch would be like so, I'm testing this out now
>> hoping it will
>>
>> perform as good. Now the check is performed in four places
> Close.  I'll reply with my compiled, but untested patch of what I was thinking.
>> That leaves me with a though, will this set masked, and should be checked as well?
>>
>> void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
>> {
>>         struct pci_dev *dev = msi_desc_to_pci_dev(entry);
>>
>>         if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) {
>>                 /* Don't touch the hardware now */
>>         } else if (entry->msi_attrib.is_msix) {
>>                 void __iomem *base = pci_msix_desc_addr(entry);
>>                 u32 ctrl = entry->msix_ctrl;
>>                 bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT);
>>
>>                 if (entry->msi_attrib.is_virtual)
>>                         goto skip;
>>
>>                 /*
>>                  * The specification mandates that the entry is masked
>>                  * when the message is modified:
>>                  *
>>                  * "If software changes the Address or Data value of an
>>                  * entry while the entry is unmasked, the result is
>>                  * undefined."
>>                  */
>>                 if (unmasked)
>>>>>                     pci_msix_write_vector_ctrl(entry, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT);
> My patch adds a check in pci_msix_write_vector_ctrl(), but the comment
> above means PV Xen's behavior may be incorrect if Linux is calling
> this function and modifying the message.
>
> Regards,
> Jason
Turns out it seems to mess things up. I'm compiling this patch right now
with config flags below ( for anyone trying the same ). It should
perform ok I hope.

CONFIG_AMD_PMC=y
#CONFIG_HSA_AMD is not set
#CONFIG_DRM_AMD_SECURE_DISPLAY is not set
#CONFIG_CRYPTO_DEV_CCP is not set

Moving checks pci_msix_mask/pci_msix_unmask to ensure that init/shutdown
gets the checks as well. Avoiding
pci_msix_write_vector_ctrl/__pci_write_msi_msg
since it seems to have odd effects, like the comment in __pci_write_msi_msg
tells us. Just applying checks in __pci_restore_msi_state and
__pci_restore_msix_state
did not do the trick.

diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 4b4792940e86..acf14a4708e6 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -186,6 +186,9 @@ static void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl)
 
 static inline void pci_msix_mask(struct msi_desc *desc)
 {
+	if (pci_msi_ignore_mask)
+		return;
+
 	desc->msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
 	pci_msix_write_vector_ctrl(desc, desc->msix_ctrl);
 	/* Flush write to device */
@@ -194,13 +197,16 @@ static inline void pci_msix_mask(struct msi_desc *desc)
 
 static inline void pci_msix_unmask(struct msi_desc *desc)
 {
+	if (pci_msi_ignore_mask)
+		return;
+
 	desc->msix_ctrl &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
 	pci_msix_write_vector_ctrl(desc, desc->msix_ctrl);
 }
 
 static void __pci_msi_mask_desc(struct msi_desc *desc, u32 mask)
 {
-	if (pci_msi_ignore_mask || desc->msi_attrib.is_virtual)
+	if (desc->msi_attrib.is_virtual)
 		return;
 
 	if (desc->msi_attrib.is_msix)
@@ -211,7 +217,7 @@ static void __pci_msi_mask_desc(struct msi_desc *desc, u32 mask)
 
 static void __pci_msi_unmask_desc(struct msi_desc *desc, u32 mask)
 {
-	if (pci_msi_ignore_mask || desc->msi_attrib.is_virtual)
+	if (desc->msi_attrib.is_virtual)
 		return;
 
 	if (desc->msi_attrib.is_msix)
@@ -420,7 +426,8 @@ static void __pci_restore_msi_state(struct pci_dev *dev)
 	arch_restore_msi_irqs(dev);
 
 	pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
-	pci_msi_update_mask(entry, 0, 0);
+	if (!(pci_msi_ignore_mask || entry->msi_attrib.is_virtual))
+		pci_msi_update_mask(entry, 0, 0);
 	control &= ~PCI_MSI_FLAGS_QSIZE;
 	control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE;
 	pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
@@ -450,8 +457,9 @@ static void __pci_restore_msix_state(struct pci_dev *dev)
 				PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL);
 
 	arch_restore_msi_irqs(dev);
-	for_each_pci_msi_entry(entry, dev)
-		pci_msix_write_vector_ctrl(entry, entry->msix_ctrl);
+	if (!(pci_msi_ignore_mask || entry->msi_attrib.is_virtual))
+		for_each_pci_msi_entry(entry, dev)
+			pci_msix_write_vector_ctrl(entry, entry->msix_ctrl);
 
 	pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
 }

Please let me know if I should submit any of the two, or make changes to them.

Regards
- Josef