* [PATCH] PCI: Add pci reset quirk for Cavium VNIC
@ 2017-03-07 15:04 ` Radoslaw Biernacki
0 siblings, 0 replies; 11+ messages in thread
From: Radoslaw Biernacki @ 2017-03-07 15:04 UTC (permalink / raw)
To: Bjorn Helgaas, Sunil.Goutham, Robert Richter, David Daney,
Robert Richter, David Miller
Cc: linux-pci, linux-kernel, netdev, linux-arm-kernel,
Radoslaw Biernacki, Tomasz Nowicki, Marcin Wojtas, Maciej Czekaj,
Linu.Cherian, Radoslaw Biernacki
From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
PCI reset quirk is needed for Cavium Function NIC since it does not
handle a function level reset.
This cause problems when VNIC is used from userspace by vfio.
If application (or VM) does not stop the VNIC queues, HW may cause
overwrite of memory locations when next application that use it will
establish new SMMU mappings. More likely HW it will cause SMMU exception,
when network packet will came before new SMMU mappings will be made.
Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
---
drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 90 insertions(+)
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index f754453..f7cdbe5 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
return 0;
}
+#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
+#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
+#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
+#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
+#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
+#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
+#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
+
+#define CAVIUM_VNIC_Q_SHIFT (18)
+#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
+#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
+#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
+#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
+
+/* Poll a register for a specific value */
+static int cavium_vnic_poll(struct pci_dev *pdev,
+ void __iomem *addr, size_t bit_pos,
+ size_t bits, u64 val)
+{
+ u64 bit_mask;
+ u64 reg_val;
+ size_t timeout = 10;
+
+ bit_mask = (1ULL << bits) - 1;
+ bit_mask = (bit_mask << bit_pos);
+
+ while (timeout) {
+ reg_val = readq(addr);
+ if (((reg_val & bit_mask) >> bit_pos) == val)
+ return 0;
+ usleep_range(1000, 2000);
+ timeout--;
+ }
+ dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
+ return -1;
+}
+
+static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
+{
+ size_t qidx;
+ void __iomem *bar_base;
+ void __iomem *qset_base;
+
+ bar_base = pci_iomap(pdev, 0, 0);
+ if (!bar_base)
+ return -ENOMEM;
+
+ /* For each of 8 RQ/CQ/SQ (queues) in VF */
+ for (qidx = 0; qidx < 8; qidx++) {
+ /* Disable receive queue */
+ qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
+
+ /* Disable timer threshold (doesn't get reset upon CQ reset */
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
+ /* Disable completion queue */
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
+ /* Reset completion queue */
+ writeq(CAVIUM_VNIC_CQ_RESET,
+ qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
+
+ /* Disable send queue */
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
+ /* Reset send queue */
+ writeq(CAVIUM_VNIC_SQ_RESET,
+ qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
+ }
+
+ /* Reset and disable both RBDR's */
+ for (qidx = 0; qidx < 2; qidx++) {
+ qset_base = bar_base +
+ (qidx << CAVIUM_VNIC_Q_SHIFT);
+ writeq(CAVIUM_VNIC_RBDR_RESET,
+ qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
+ if (cavium_vnic_poll(pdev, qset_base +
+ CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
+ CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
+ 2, 0x00))
+ dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
+ }
+
+ pci_iounmap(pdev, bar_base);
+ return 0;
+}
+
#define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
#define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
#define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
+#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
+
static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
reset_intel_82599_sfp_virtfn },
@@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
reset_ivb_igd },
{ PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
reset_chelsio_generic_dev },
+ { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
+ cavium_vnic_reset },
{ 0 }
};
--
1.9.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [PATCH] PCI: Add pci reset quirk for Cavium VNIC
@ 2017-03-07 15:04 ` Radoslaw Biernacki
0 siblings, 0 replies; 11+ messages in thread
From: Radoslaw Biernacki @ 2017-03-07 15:04 UTC (permalink / raw)
To: Bjorn Helgaas, Sunil.Goutham, Robert Richter, David Daney,
Robert Richter, David Miller
Cc: Linu.Cherian, Radoslaw Biernacki, Maciej Czekaj, Tomasz Nowicki,
Radoslaw Biernacki, linux-kernel, netdev, linux-pci,
Marcin Wojtas, linux-arm-kernel
From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
PCI reset quirk is needed for Cavium Function NIC since it does not
handle a function level reset.
This cause problems when VNIC is used from userspace by vfio.
If application (or VM) does not stop the VNIC queues, HW may cause
overwrite of memory locations when next application that use it will
establish new SMMU mappings. More likely HW it will cause SMMU exception,
when network packet will came before new SMMU mappings will be made.
Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
---
drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 90 insertions(+)
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index f754453..f7cdbe5 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
return 0;
}
+#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
+#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
+#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
+#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
+#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
+#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
+#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
+
+#define CAVIUM_VNIC_Q_SHIFT (18)
+#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
+#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
+#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
+#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
+
+/* Poll a register for a specific value */
+static int cavium_vnic_poll(struct pci_dev *pdev,
+ void __iomem *addr, size_t bit_pos,
+ size_t bits, u64 val)
+{
+ u64 bit_mask;
+ u64 reg_val;
+ size_t timeout = 10;
+
+ bit_mask = (1ULL << bits) - 1;
+ bit_mask = (bit_mask << bit_pos);
+
+ while (timeout) {
+ reg_val = readq(addr);
+ if (((reg_val & bit_mask) >> bit_pos) == val)
+ return 0;
+ usleep_range(1000, 2000);
+ timeout--;
+ }
+ dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
+ return -1;
+}
+
+static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
+{
+ size_t qidx;
+ void __iomem *bar_base;
+ void __iomem *qset_base;
+
+ bar_base = pci_iomap(pdev, 0, 0);
+ if (!bar_base)
+ return -ENOMEM;
+
+ /* For each of 8 RQ/CQ/SQ (queues) in VF */
+ for (qidx = 0; qidx < 8; qidx++) {
+ /* Disable receive queue */
+ qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
+
+ /* Disable timer threshold (doesn't get reset upon CQ reset */
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
+ /* Disable completion queue */
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
+ /* Reset completion queue */
+ writeq(CAVIUM_VNIC_CQ_RESET,
+ qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
+
+ /* Disable send queue */
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
+ /* Reset send queue */
+ writeq(CAVIUM_VNIC_SQ_RESET,
+ qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
+ }
+
+ /* Reset and disable both RBDR's */
+ for (qidx = 0; qidx < 2; qidx++) {
+ qset_base = bar_base +
+ (qidx << CAVIUM_VNIC_Q_SHIFT);
+ writeq(CAVIUM_VNIC_RBDR_RESET,
+ qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
+ if (cavium_vnic_poll(pdev, qset_base +
+ CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
+ CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
+ 2, 0x00))
+ dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
+ }
+
+ pci_iounmap(pdev, bar_base);
+ return 0;
+}
+
#define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
#define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
#define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
+#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
+
static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
reset_intel_82599_sfp_virtfn },
@@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
reset_ivb_igd },
{ PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
reset_chelsio_generic_dev },
+ { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
+ cavium_vnic_reset },
{ 0 }
};
--
1.9.1
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [PATCH] PCI: Add pci reset quirk for Cavium VNIC
@ 2017-03-07 15:04 ` Radoslaw Biernacki
0 siblings, 0 replies; 11+ messages in thread
From: Radoslaw Biernacki @ 2017-03-07 15:04 UTC (permalink / raw)
To: linux-arm-kernel
From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
PCI reset quirk is needed for Cavium Function NIC since it does not
handle a function level reset.
This cause problems when VNIC is used from userspace by vfio.
If application (or VM) does not stop the VNIC queues, HW may cause
overwrite of memory locations when next application that use it will
establish new SMMU mappings. More likely HW it will cause SMMU exception,
when network packet will came before new SMMU mappings will be made.
Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
---
drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 90 insertions(+)
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index f754453..f7cdbe5 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
return 0;
}
+#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
+#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
+#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
+#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
+#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
+#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
+#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
+
+#define CAVIUM_VNIC_Q_SHIFT (18)
+#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
+#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
+#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
+#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
+
+/* Poll a register for a specific value */
+static int cavium_vnic_poll(struct pci_dev *pdev,
+ void __iomem *addr, size_t bit_pos,
+ size_t bits, u64 val)
+{
+ u64 bit_mask;
+ u64 reg_val;
+ size_t timeout = 10;
+
+ bit_mask = (1ULL << bits) - 1;
+ bit_mask = (bit_mask << bit_pos);
+
+ while (timeout) {
+ reg_val = readq(addr);
+ if (((reg_val & bit_mask) >> bit_pos) == val)
+ return 0;
+ usleep_range(1000, 2000);
+ timeout--;
+ }
+ dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
+ return -1;
+}
+
+static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
+{
+ size_t qidx;
+ void __iomem *bar_base;
+ void __iomem *qset_base;
+
+ bar_base = pci_iomap(pdev, 0, 0);
+ if (!bar_base)
+ return -ENOMEM;
+
+ /* For each of 8 RQ/CQ/SQ (queues) in VF */
+ for (qidx = 0; qidx < 8; qidx++) {
+ /* Disable receive queue */
+ qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
+
+ /* Disable timer threshold (doesn't get reset upon CQ reset */
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
+ /* Disable completion queue */
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
+ /* Reset completion queue */
+ writeq(CAVIUM_VNIC_CQ_RESET,
+ qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
+
+ /* Disable send queue */
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
+ /* Reset send queue */
+ writeq(CAVIUM_VNIC_SQ_RESET,
+ qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
+ }
+
+ /* Reset and disable both RBDR's */
+ for (qidx = 0; qidx < 2; qidx++) {
+ qset_base = bar_base +
+ (qidx << CAVIUM_VNIC_Q_SHIFT);
+ writeq(CAVIUM_VNIC_RBDR_RESET,
+ qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
+ writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
+ if (cavium_vnic_poll(pdev, qset_base +
+ CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
+ CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
+ 2, 0x00))
+ dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
+ }
+
+ pci_iounmap(pdev, bar_base);
+ return 0;
+}
+
#define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
#define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
#define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
+#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
+
static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
{ PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
reset_intel_82599_sfp_virtfn },
@@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
reset_ivb_igd },
{ PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
reset_chelsio_generic_dev },
+ { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
+ cavium_vnic_reset },
{ 0 }
};
--
1.9.1
^ permalink raw reply related [flat|nested] 11+ messages in thread
* Re: [PATCH] PCI: Add pci reset quirk for Cavium VNIC
2017-03-07 15:04 ` Radoslaw Biernacki
(?)
(?)
@ 2017-03-07 16:16 ` Robin Murphy
-1 siblings, 0 replies; 11+ messages in thread
From: Robin Murphy @ 2017-03-07 16:16 UTC (permalink / raw)
To: Radoslaw Biernacki, Bjorn Helgaas, Sunil.Goutham, Robert Richter,
David Daney, Robert Richter, David Miller
Cc: Linu.Cherian, Radoslaw Biernacki, Maciej Czekaj, Tomasz Nowicki,
linux-kernel, netdev, linux-pci, Marcin Wojtas, linux-arm-kernel
On 07/03/17 15:04, Radoslaw Biernacki wrote:
> From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>
> PCI reset quirk is needed for Cavium Function NIC since it does not
> handle a function level reset.
> This cause problems when VNIC is used from userspace by vfio.
> If application (or VM) does not stop the VNIC queues, HW may cause
> overwrite of memory locations when next application that use it will
> establish new SMMU mappings. More likely HW it will cause SMMU exception,
> when network packet will came before new SMMU mappings will be made.
>
> Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
> Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
> ---
> drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 90 insertions(+)
>
> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
> index f754453..f7cdbe5 100644
> --- a/drivers/pci/quirks.c
> +++ b/drivers/pci/quirks.c
> @@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
> return 0;
> }
>
> +#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
> +#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
> +#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
> +#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
> +#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
> +
> +#define CAVIUM_VNIC_Q_SHIFT (18)
> +#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
> +#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
> +#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
> +#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
> +
> +/* Poll a register for a specific value */
> +static int cavium_vnic_poll(struct pci_dev *pdev,
> + void __iomem *addr, size_t bit_pos,
> + size_t bits, u64 val)
> +{
> + u64 bit_mask;
> + u64 reg_val;
> + size_t timeout = 10;
> +
> + bit_mask = (1ULL << bits) - 1;
> + bit_mask = (bit_mask << bit_pos);
GENMASK_ULL(bit_pos, bit_pos + bits - 1)? Admittedly it's not an awful
lot tidier :/
> +
> + while (timeout) {
> + reg_val = readq(addr);
> + if (((reg_val & bit_mask) >> bit_pos) == val)
> + return 0;
> + usleep_range(1000, 2000);
> + timeout--;
> + }
This is essentially an open-coded version of readq_poll_timeout() from
<linux/iopoll.h>, which could be used instead. Doing "val <<= bitpos"
first might help simplify the condition, although since it's only called
with val == 0 it seems that "(reg_val & bit_mask)" would suffice.
Robin.
> + dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
> + return -1;
> +}
> +
> +static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
> +{
> + size_t qidx;
> + void __iomem *bar_base;
> + void __iomem *qset_base;
> +
> + bar_base = pci_iomap(pdev, 0, 0);
> + if (!bar_base)
> + return -ENOMEM;
> +
> + /* For each of 8 RQ/CQ/SQ (queues) in VF */
> + for (qidx = 0; qidx < 8; qidx++) {
> + /* Disable receive queue */
> + qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
> +
> + /* Disable timer threshold (doesn't get reset upon CQ reset */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
> + /* Disable completion queue */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
> + /* Reset completion queue */
> + writeq(CAVIUM_VNIC_CQ_RESET,
> + qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
> +
> + /* Disable send queue */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
> + /* Reset send queue */
> + writeq(CAVIUM_VNIC_SQ_RESET,
> + qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
> + }
> +
> + /* Reset and disable both RBDR's */
> + for (qidx = 0; qidx < 2; qidx++) {
> + qset_base = bar_base +
> + (qidx << CAVIUM_VNIC_Q_SHIFT);
> + writeq(CAVIUM_VNIC_RBDR_RESET,
> + qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
> + if (cavium_vnic_poll(pdev, qset_base +
> + CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
> + CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
> + 2, 0x00))
> + dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
> + }
> +
> + pci_iounmap(pdev, bar_base);
> + return 0;
> +}
> +
> #define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
> #define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
> #define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
>
> +#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
> +
> static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
> { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
> reset_intel_82599_sfp_virtfn },
> @@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
> reset_ivb_igd },
> { PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
> reset_chelsio_generic_dev },
> + { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
> + cavium_vnic_reset },
> { 0 }
> };
>
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH] PCI: Add pci reset quirk for Cavium VNIC
@ 2017-03-07 16:16 ` Robin Murphy
0 siblings, 0 replies; 11+ messages in thread
From: Robin Murphy @ 2017-03-07 16:16 UTC (permalink / raw)
To: Radoslaw Biernacki, Bjorn Helgaas, Sunil.Goutham, Robert Richter,
David Daney, Robert Richter, David Miller
Cc: Radoslaw Biernacki, Maciej Czekaj, Tomasz Nowicki, Linu.Cherian,
linux-pci, linux-kernel, netdev, Marcin Wojtas, linux-arm-kernel
On 07/03/17 15:04, Radoslaw Biernacki wrote:
> From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>
> PCI reset quirk is needed for Cavium Function NIC since it does not
> handle a function level reset.
> This cause problems when VNIC is used from userspace by vfio.
> If application (or VM) does not stop the VNIC queues, HW may cause
> overwrite of memory locations when next application that use it will
> establish new SMMU mappings. More likely HW it will cause SMMU exception,
> when network packet will came before new SMMU mappings will be made.
>
> Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
> Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
> ---
> drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 90 insertions(+)
>
> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
> index f754453..f7cdbe5 100644
> --- a/drivers/pci/quirks.c
> +++ b/drivers/pci/quirks.c
> @@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
> return 0;
> }
>
> +#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
> +#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
> +#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
> +#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
> +#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
> +
> +#define CAVIUM_VNIC_Q_SHIFT (18)
> +#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
> +#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
> +#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
> +#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
> +
> +/* Poll a register for a specific value */
> +static int cavium_vnic_poll(struct pci_dev *pdev,
> + void __iomem *addr, size_t bit_pos,
> + size_t bits, u64 val)
> +{
> + u64 bit_mask;
> + u64 reg_val;
> + size_t timeout = 10;
> +
> + bit_mask = (1ULL << bits) - 1;
> + bit_mask = (bit_mask << bit_pos);
GENMASK_ULL(bit_pos, bit_pos + bits - 1)? Admittedly it's not an awful
lot tidier :/
> +
> + while (timeout) {
> + reg_val = readq(addr);
> + if (((reg_val & bit_mask) >> bit_pos) == val)
> + return 0;
> + usleep_range(1000, 2000);
> + timeout--;
> + }
This is essentially an open-coded version of readq_poll_timeout() from
<linux/iopoll.h>, which could be used instead. Doing "val <<= bitpos"
first might help simplify the condition, although since it's only called
with val == 0 it seems that "(reg_val & bit_mask)" would suffice.
Robin.
> + dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
> + return -1;
> +}
> +
> +static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
> +{
> + size_t qidx;
> + void __iomem *bar_base;
> + void __iomem *qset_base;
> +
> + bar_base = pci_iomap(pdev, 0, 0);
> + if (!bar_base)
> + return -ENOMEM;
> +
> + /* For each of 8 RQ/CQ/SQ (queues) in VF */
> + for (qidx = 0; qidx < 8; qidx++) {
> + /* Disable receive queue */
> + qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
> +
> + /* Disable timer threshold (doesn't get reset upon CQ reset */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
> + /* Disable completion queue */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
> + /* Reset completion queue */
> + writeq(CAVIUM_VNIC_CQ_RESET,
> + qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
> +
> + /* Disable send queue */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
> + /* Reset send queue */
> + writeq(CAVIUM_VNIC_SQ_RESET,
> + qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
> + }
> +
> + /* Reset and disable both RBDR's */
> + for (qidx = 0; qidx < 2; qidx++) {
> + qset_base = bar_base +
> + (qidx << CAVIUM_VNIC_Q_SHIFT);
> + writeq(CAVIUM_VNIC_RBDR_RESET,
> + qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
> + if (cavium_vnic_poll(pdev, qset_base +
> + CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
> + CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
> + 2, 0x00))
> + dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
> + }
> +
> + pci_iounmap(pdev, bar_base);
> + return 0;
> +}
> +
> #define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
> #define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
> #define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
>
> +#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
> +
> static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
> { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
> reset_intel_82599_sfp_virtfn },
> @@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
> reset_ivb_igd },
> { PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
> reset_chelsio_generic_dev },
> + { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
> + cavium_vnic_reset },
> { 0 }
> };
>
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH] PCI: Add pci reset quirk for Cavium VNIC
@ 2017-03-07 16:16 ` Robin Murphy
0 siblings, 0 replies; 11+ messages in thread
From: Robin Murphy @ 2017-03-07 16:16 UTC (permalink / raw)
To: Radoslaw Biernacki, Bjorn Helgaas, Sunil.Goutham, Robert Richter,
David Daney, Robert Richter, David Miller
Cc: Radoslaw Biernacki, Maciej Czekaj, Tomasz Nowicki, Linu.Cherian,
linux-pci, linux-kernel, netdev, Marcin Wojtas, linux-arm-kernel
On 07/03/17 15:04, Radoslaw Biernacki wrote:
> From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>
> PCI reset quirk is needed for Cavium Function NIC since it does not
> handle a function level reset.
> This cause problems when VNIC is used from userspace by vfio.
> If application (or VM) does not stop the VNIC queues, HW may cause
> overwrite of memory locations when next application that use it will
> establish new SMMU mappings. More likely HW it will cause SMMU exception,
> when network packet will came before new SMMU mappings will be made.
>
> Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
> Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
> ---
> drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 90 insertions(+)
>
> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
> index f754453..f7cdbe5 100644
> --- a/drivers/pci/quirks.c
> +++ b/drivers/pci/quirks.c
> @@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
> return 0;
> }
>
> +#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
> +#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
> +#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
> +#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
> +#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
> +
> +#define CAVIUM_VNIC_Q_SHIFT (18)
> +#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
> +#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
> +#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
> +#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
> +
> +/* Poll a register for a specific value */
> +static int cavium_vnic_poll(struct pci_dev *pdev,
> + void __iomem *addr, size_t bit_pos,
> + size_t bits, u64 val)
> +{
> + u64 bit_mask;
> + u64 reg_val;
> + size_t timeout = 10;
> +
> + bit_mask = (1ULL << bits) - 1;
> + bit_mask = (bit_mask << bit_pos);
GENMASK_ULL(bit_pos, bit_pos + bits - 1)? Admittedly it's not an awful
lot tidier :/
> +
> + while (timeout) {
> + reg_val = readq(addr);
> + if (((reg_val & bit_mask) >> bit_pos) == val)
> + return 0;
> + usleep_range(1000, 2000);
> + timeout--;
> + }
This is essentially an open-coded version of readq_poll_timeout() from
<linux/iopoll.h>, which could be used instead. Doing "val <<= bitpos"
first might help simplify the condition, although since it's only called
with val == 0 it seems that "(reg_val & bit_mask)" would suffice.
Robin.
> + dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
> + return -1;
> +}
> +
> +static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
> +{
> + size_t qidx;
> + void __iomem *bar_base;
> + void __iomem *qset_base;
> +
> + bar_base = pci_iomap(pdev, 0, 0);
> + if (!bar_base)
> + return -ENOMEM;
> +
> + /* For each of 8 RQ/CQ/SQ (queues) in VF */
> + for (qidx = 0; qidx < 8; qidx++) {
> + /* Disable receive queue */
> + qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
> +
> + /* Disable timer threshold (doesn't get reset upon CQ reset */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
> + /* Disable completion queue */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
> + /* Reset completion queue */
> + writeq(CAVIUM_VNIC_CQ_RESET,
> + qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
> +
> + /* Disable send queue */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
> + /* Reset send queue */
> + writeq(CAVIUM_VNIC_SQ_RESET,
> + qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
> + }
> +
> + /* Reset and disable both RBDR's */
> + for (qidx = 0; qidx < 2; qidx++) {
> + qset_base = bar_base +
> + (qidx << CAVIUM_VNIC_Q_SHIFT);
> + writeq(CAVIUM_VNIC_RBDR_RESET,
> + qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
> + if (cavium_vnic_poll(pdev, qset_base +
> + CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
> + CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
> + 2, 0x00))
> + dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
> + }
> +
> + pci_iounmap(pdev, bar_base);
> + return 0;
> +}
> +
> #define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
> #define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
> #define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
>
> +#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
> +
> static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
> { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
> reset_intel_82599_sfp_virtfn },
> @@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
> reset_ivb_igd },
> { PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
> reset_chelsio_generic_dev },
> + { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
> + cavium_vnic_reset },
> { 0 }
> };
>
>
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH] PCI: Add pci reset quirk for Cavium VNIC
@ 2017-03-07 16:16 ` Robin Murphy
0 siblings, 0 replies; 11+ messages in thread
From: Robin Murphy @ 2017-03-07 16:16 UTC (permalink / raw)
To: linux-arm-kernel
On 07/03/17 15:04, Radoslaw Biernacki wrote:
> From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>
> PCI reset quirk is needed for Cavium Function NIC since it does not
> handle a function level reset.
> This cause problems when VNIC is used from userspace by vfio.
> If application (or VM) does not stop the VNIC queues, HW may cause
> overwrite of memory locations when next application that use it will
> establish new SMMU mappings. More likely HW it will cause SMMU exception,
> when network packet will came before new SMMU mappings will be made.
>
> Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
> Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
> ---
> drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 90 insertions(+)
>
> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
> index f754453..f7cdbe5 100644
> --- a/drivers/pci/quirks.c
> +++ b/drivers/pci/quirks.c
> @@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
> return 0;
> }
>
> +#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
> +#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
> +#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
> +#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
> +#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
> +
> +#define CAVIUM_VNIC_Q_SHIFT (18)
> +#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
> +#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
> +#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
> +#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
> +
> +/* Poll a register for a specific value */
> +static int cavium_vnic_poll(struct pci_dev *pdev,
> + void __iomem *addr, size_t bit_pos,
> + size_t bits, u64 val)
> +{
> + u64 bit_mask;
> + u64 reg_val;
> + size_t timeout = 10;
> +
> + bit_mask = (1ULL << bits) - 1;
> + bit_mask = (bit_mask << bit_pos);
GENMASK_ULL(bit_pos, bit_pos + bits - 1)? Admittedly it's not an awful
lot tidier :/
> +
> + while (timeout) {
> + reg_val = readq(addr);
> + if (((reg_val & bit_mask) >> bit_pos) == val)
> + return 0;
> + usleep_range(1000, 2000);
> + timeout--;
> + }
This is essentially an open-coded version of readq_poll_timeout() from
<linux/iopoll.h>, which could be used instead. Doing "val <<= bitpos"
first might help simplify the condition, although since it's only called
with val == 0 it seems that "(reg_val & bit_mask)" would suffice.
Robin.
> + dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
> + return -1;
> +}
> +
> +static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
> +{
> + size_t qidx;
> + void __iomem *bar_base;
> + void __iomem *qset_base;
> +
> + bar_base = pci_iomap(pdev, 0, 0);
> + if (!bar_base)
> + return -ENOMEM;
> +
> + /* For each of 8 RQ/CQ/SQ (queues) in VF */
> + for (qidx = 0; qidx < 8; qidx++) {
> + /* Disable receive queue */
> + qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
> +
> + /* Disable timer threshold (doesn't get reset upon CQ reset */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
> + /* Disable completion queue */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
> + /* Reset completion queue */
> + writeq(CAVIUM_VNIC_CQ_RESET,
> + qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
> +
> + /* Disable send queue */
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
> + /* Reset send queue */
> + writeq(CAVIUM_VNIC_SQ_RESET,
> + qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
> + }
> +
> + /* Reset and disable both RBDR's */
> + for (qidx = 0; qidx < 2; qidx++) {
> + qset_base = bar_base +
> + (qidx << CAVIUM_VNIC_Q_SHIFT);
> + writeq(CAVIUM_VNIC_RBDR_RESET,
> + qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
> + if (cavium_vnic_poll(pdev, qset_base +
> + CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
> + CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
> + 2, 0x00))
> + dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
> + }
> +
> + pci_iounmap(pdev, bar_base);
> + return 0;
> +}
> +
> #define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
> #define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
> #define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
>
> +#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
> +
> static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
> { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
> reset_intel_82599_sfp_virtfn },
> @@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
> reset_ivb_igd },
> { PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
> reset_chelsio_generic_dev },
> + { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
> + cavium_vnic_reset },
> { 0 }
> };
>
>
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH] PCI: Add pci reset quirk for Cavium VNIC
2017-03-07 16:16 ` Robin Murphy
(?)
(?)
@ 2017-03-08 14:21 ` rad
-1 siblings, 0 replies; 11+ messages in thread
From: rad @ 2017-03-08 14:21 UTC (permalink / raw)
To: Robin Murphy, Bjorn Helgaas, Sunil.Goutham, Robert Richter,
David Daney, Robert Richter, David Miller
Cc: Linu.Cherian, Radoslaw Biernacki, Maciej Czekaj, Tomasz Nowicki,
linux-kernel, netdev, linux-pci, Marcin Wojtas, linux-arm-kernel
On 07.03.2017 17:16, Robin Murphy wrote:
> On 07/03/17 15:04, Radoslaw Biernacki wrote:
>> From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>>
>> PCI reset quirk is needed for Cavium Function NIC since it does not
>> handle a function level reset.
>> This cause problems when VNIC is used from userspace by vfio.
>> If application (or VM) does not stop the VNIC queues, HW may cause
>> overwrite of memory locations when next application that use it will
>> establish new SMMU mappings. More likely HW it will cause SMMU exception,
>> when network packet will came before new SMMU mappings will be made.
>>
>> Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>> Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
>> ---
>> drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>> 1 file changed, 90 insertions(+)
>>
>> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
>> index f754453..f7cdbe5 100644
>> --- a/drivers/pci/quirks.c
>> +++ b/drivers/pci/quirks.c
>> @@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
>> return 0;
>> }
>>
>> +#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
>> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
>> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
>> +#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
>> +#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
>> +#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
>> +#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
>> +
>> +#define CAVIUM_VNIC_Q_SHIFT (18)
>> +#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
>> +#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
>> +#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
>> +#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
>> +
>> +/* Poll a register for a specific value */
>> +static int cavium_vnic_poll(struct pci_dev *pdev,
>> + void __iomem *addr, size_t bit_pos,
>> + size_t bits, u64 val)
>> +{
>> + u64 bit_mask;
>> + u64 reg_val;
>> + size_t timeout = 10;
>> +
>> + bit_mask = (1ULL << bits) - 1;
>> + bit_mask = (bit_mask << bit_pos);
> GENMASK_ULL(bit_pos, bit_pos + bits - 1)? Admittedly it's not an awful
> lot tidier :/
>
>> +
>> + while (timeout) {
>> + reg_val = readq(addr);
>> + if (((reg_val & bit_mask) >> bit_pos) == val)
>> + return 0;
>> + usleep_range(1000, 2000);
>> + timeout--;
>> + }
> This is essentially an open-coded version of readq_poll_timeout() from
> <linux/iopoll.h>, which could be used instead. Doing "val <<= bitpos"
> first might help simplify the condition, although since it's only called
> with val == 0 it seems that "(reg_val & bit_mask)" would suffice.
>
> Robin.
Thanks for pointing this out.
I will try to simplify the condition and use readq_poll_timeout().
>
>> + dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
>> + return -1;
>> +}
>> +
>> +static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
>> +{
>> + size_t qidx;
>> + void __iomem *bar_base;
>> + void __iomem *qset_base;
>> +
>> + bar_base = pci_iomap(pdev, 0, 0);
>> + if (!bar_base)
>> + return -ENOMEM;
>> +
>> + /* For each of 8 RQ/CQ/SQ (queues) in VF */
>> + for (qidx = 0; qidx < 8; qidx++) {
>> + /* Disable receive queue */
>> + qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
>> +
>> + /* Disable timer threshold (doesn't get reset upon CQ reset */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
>> + /* Disable completion queue */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
>> + /* Reset completion queue */
>> + writeq(CAVIUM_VNIC_CQ_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
>> +
>> + /* Disable send queue */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
>> + /* Reset send queue */
>> + writeq(CAVIUM_VNIC_SQ_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
>> + }
>> +
>> + /* Reset and disable both RBDR's */
>> + for (qidx = 0; qidx < 2; qidx++) {
>> + qset_base = bar_base +
>> + (qidx << CAVIUM_VNIC_Q_SHIFT);
>> + writeq(CAVIUM_VNIC_RBDR_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
>> + if (cavium_vnic_poll(pdev, qset_base +
>> + CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
>> + CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
>> + 2, 0x00))
>> + dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
>> + }
>> +
>> + pci_iounmap(pdev, bar_base);
>> + return 0;
>> +}
>> +
>> #define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
>> #define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
>> #define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
>>
>> +#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
>> +
>> static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
>> { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
>> reset_intel_82599_sfp_virtfn },
>> @@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
>> reset_ivb_igd },
>> { PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
>> reset_chelsio_generic_dev },
>> + { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
>> + cavium_vnic_reset },
>> { 0 }
>> };
>>
>>
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH] PCI: Add pci reset quirk for Cavium VNIC
@ 2017-03-08 14:21 ` rad
0 siblings, 0 replies; 11+ messages in thread
From: rad @ 2017-03-08 14:21 UTC (permalink / raw)
To: Robin Murphy, Bjorn Helgaas, Sunil.Goutham, Robert Richter,
David Daney, Robert Richter, David Miller
Cc: Radoslaw Biernacki, Maciej Czekaj, Tomasz Nowicki, Linu.Cherian,
linux-pci, linux-kernel, netdev, Marcin Wojtas, linux-arm-kernel
On 07.03.2017 17:16, Robin Murphy wrote:
> On 07/03/17 15:04, Radoslaw Biernacki wrote:
>> From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>>
>> PCI reset quirk is needed for Cavium Function NIC since it does not
>> handle a function level reset.
>> This cause problems when VNIC is used from userspace by vfio.
>> If application (or VM) does not stop the VNIC queues, HW may cause
>> overwrite of memory locations when next application that use it will
>> establish new SMMU mappings. More likely HW it will cause SMMU exception,
>> when network packet will came before new SMMU mappings will be made.
>>
>> Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>> Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
>> ---
>> drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>> 1 file changed, 90 insertions(+)
>>
>> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
>> index f754453..f7cdbe5 100644
>> --- a/drivers/pci/quirks.c
>> +++ b/drivers/pci/quirks.c
>> @@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
>> return 0;
>> }
>>
>> +#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
>> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
>> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
>> +#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
>> +#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
>> +#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
>> +#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
>> +
>> +#define CAVIUM_VNIC_Q_SHIFT (18)
>> +#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
>> +#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
>> +#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
>> +#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
>> +
>> +/* Poll a register for a specific value */
>> +static int cavium_vnic_poll(struct pci_dev *pdev,
>> + void __iomem *addr, size_t bit_pos,
>> + size_t bits, u64 val)
>> +{
>> + u64 bit_mask;
>> + u64 reg_val;
>> + size_t timeout = 10;
>> +
>> + bit_mask = (1ULL << bits) - 1;
>> + bit_mask = (bit_mask << bit_pos);
> GENMASK_ULL(bit_pos, bit_pos + bits - 1)? Admittedly it's not an awful
> lot tidier :/
>
>> +
>> + while (timeout) {
>> + reg_val = readq(addr);
>> + if (((reg_val & bit_mask) >> bit_pos) == val)
>> + return 0;
>> + usleep_range(1000, 2000);
>> + timeout--;
>> + }
> This is essentially an open-coded version of readq_poll_timeout() from
> <linux/iopoll.h>, which could be used instead. Doing "val <<= bitpos"
> first might help simplify the condition, although since it's only called
> with val == 0 it seems that "(reg_val & bit_mask)" would suffice.
>
> Robin.
Thanks for pointing this out.
I will try to simplify the condition and use readq_poll_timeout().
>
>> + dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
>> + return -1;
>> +}
>> +
>> +static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
>> +{
>> + size_t qidx;
>> + void __iomem *bar_base;
>> + void __iomem *qset_base;
>> +
>> + bar_base = pci_iomap(pdev, 0, 0);
>> + if (!bar_base)
>> + return -ENOMEM;
>> +
>> + /* For each of 8 RQ/CQ/SQ (queues) in VF */
>> + for (qidx = 0; qidx < 8; qidx++) {
>> + /* Disable receive queue */
>> + qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
>> +
>> + /* Disable timer threshold (doesn't get reset upon CQ reset */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
>> + /* Disable completion queue */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
>> + /* Reset completion queue */
>> + writeq(CAVIUM_VNIC_CQ_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
>> +
>> + /* Disable send queue */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
>> + /* Reset send queue */
>> + writeq(CAVIUM_VNIC_SQ_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
>> + }
>> +
>> + /* Reset and disable both RBDR's */
>> + for (qidx = 0; qidx < 2; qidx++) {
>> + qset_base = bar_base +
>> + (qidx << CAVIUM_VNIC_Q_SHIFT);
>> + writeq(CAVIUM_VNIC_RBDR_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
>> + if (cavium_vnic_poll(pdev, qset_base +
>> + CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
>> + CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
>> + 2, 0x00))
>> + dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
>> + }
>> +
>> + pci_iounmap(pdev, bar_base);
>> + return 0;
>> +}
>> +
>> #define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
>> #define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
>> #define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
>>
>> +#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
>> +
>> static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
>> { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
>> reset_intel_82599_sfp_virtfn },
>> @@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
>> reset_ivb_igd },
>> { PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
>> reset_chelsio_generic_dev },
>> + { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
>> + cavium_vnic_reset },
>> { 0 }
>> };
>>
>>
^ permalink raw reply [flat|nested] 11+ messages in thread
* Re: [PATCH] PCI: Add pci reset quirk for Cavium VNIC
@ 2017-03-08 14:21 ` rad
0 siblings, 0 replies; 11+ messages in thread
From: rad @ 2017-03-08 14:21 UTC (permalink / raw)
To: Robin Murphy, Bjorn Helgaas, Sunil.Goutham, Robert Richter,
David Daney, Robert Richter, David Miller
Cc: Radoslaw Biernacki, Maciej Czekaj, Tomasz Nowicki, Linu.Cherian,
linux-pci, linux-kernel, netdev, Marcin Wojtas, linux-arm-kernel
On 07.03.2017 17:16, Robin Murphy wrote:
> On 07/03/17 15:04, Radoslaw Biernacki wrote:
>> From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>>
>> PCI reset quirk is needed for Cavium Function NIC since it does not
>> handle a function level reset.
>> This cause problems when VNIC is used from userspace by vfio.
>> If application (or VM) does not stop the VNIC queues, HW may cause
>> overwrite of memory locations when next application that use it will
>> establish new SMMU mappings. More likely HW it will cause SMMU exception,
>> when network packet will came before new SMMU mappings will be made.
>>
>> Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>> Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
>> ---
>> drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>> 1 file changed, 90 insertions(+)
>>
>> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
>> index f754453..f7cdbe5 100644
>> --- a/drivers/pci/quirks.c
>> +++ b/drivers/pci/quirks.c
>> @@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
>> return 0;
>> }
>>
>> +#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
>> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
>> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
>> +#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
>> +#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
>> +#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
>> +#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
>> +
>> +#define CAVIUM_VNIC_Q_SHIFT (18)
>> +#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
>> +#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
>> +#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
>> +#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
>> +
>> +/* Poll a register for a specific value */
>> +static int cavium_vnic_poll(struct pci_dev *pdev,
>> + void __iomem *addr, size_t bit_pos,
>> + size_t bits, u64 val)
>> +{
>> + u64 bit_mask;
>> + u64 reg_val;
>> + size_t timeout = 10;
>> +
>> + bit_mask = (1ULL << bits) - 1;
>> + bit_mask = (bit_mask << bit_pos);
> GENMASK_ULL(bit_pos, bit_pos + bits - 1)? Admittedly it's not an awful
> lot tidier :/
>
>> +
>> + while (timeout) {
>> + reg_val = readq(addr);
>> + if (((reg_val & bit_mask) >> bit_pos) == val)
>> + return 0;
>> + usleep_range(1000, 2000);
>> + timeout--;
>> + }
> This is essentially an open-coded version of readq_poll_timeout() from
> <linux/iopoll.h>, which could be used instead. Doing "val <<= bitpos"
> first might help simplify the condition, although since it's only called
> with val == 0 it seems that "(reg_val & bit_mask)" would suffice.
>
> Robin.
Thanks for pointing this out.
I will try to simplify the condition and use readq_poll_timeout().
>
>> + dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
>> + return -1;
>> +}
>> +
>> +static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
>> +{
>> + size_t qidx;
>> + void __iomem *bar_base;
>> + void __iomem *qset_base;
>> +
>> + bar_base = pci_iomap(pdev, 0, 0);
>> + if (!bar_base)
>> + return -ENOMEM;
>> +
>> + /* For each of 8 RQ/CQ/SQ (queues) in VF */
>> + for (qidx = 0; qidx < 8; qidx++) {
>> + /* Disable receive queue */
>> + qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
>> +
>> + /* Disable timer threshold (doesn't get reset upon CQ reset */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
>> + /* Disable completion queue */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
>> + /* Reset completion queue */
>> + writeq(CAVIUM_VNIC_CQ_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
>> +
>> + /* Disable send queue */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
>> + /* Reset send queue */
>> + writeq(CAVIUM_VNIC_SQ_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
>> + }
>> +
>> + /* Reset and disable both RBDR's */
>> + for (qidx = 0; qidx < 2; qidx++) {
>> + qset_base = bar_base +
>> + (qidx << CAVIUM_VNIC_Q_SHIFT);
>> + writeq(CAVIUM_VNIC_RBDR_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
>> + if (cavium_vnic_poll(pdev, qset_base +
>> + CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
>> + CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
>> + 2, 0x00))
>> + dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
>> + }
>> +
>> + pci_iounmap(pdev, bar_base);
>> + return 0;
>> +}
>> +
>> #define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
>> #define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
>> #define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
>>
>> +#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
>> +
>> static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
>> { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
>> reset_intel_82599_sfp_virtfn },
>> @@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
>> reset_ivb_igd },
>> { PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
>> reset_chelsio_generic_dev },
>> + { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
>> + cavium_vnic_reset },
>> { 0 }
>> };
>>
>>
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply [flat|nested] 11+ messages in thread
* [PATCH] PCI: Add pci reset quirk for Cavium VNIC
@ 2017-03-08 14:21 ` rad
0 siblings, 0 replies; 11+ messages in thread
From: rad @ 2017-03-08 14:21 UTC (permalink / raw)
To: linux-arm-kernel
On 07.03.2017 17:16, Robin Murphy wrote:
> On 07/03/17 15:04, Radoslaw Biernacki wrote:
>> From: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>>
>> PCI reset quirk is needed for Cavium Function NIC since it does not
>> handle a function level reset.
>> This cause problems when VNIC is used from userspace by vfio.
>> If application (or VM) does not stop the VNIC queues, HW may cause
>> overwrite of memory locations when next application that use it will
>> establish new SMMU mappings. More likely HW it will cause SMMU exception,
>> when network packet will came before new SMMU mappings will be made.
>>
>> Signed-off-by: Radoslaw Biernacki <rbiernacki@caviumnetworks.com>
>> Reviewed-by: Sunil Goutham <Sunil.Goutham@cavium.com>
>> ---
>> drivers/pci/quirks.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>> 1 file changed, 90 insertions(+)
>>
>> diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
>> index f754453..f7cdbe5 100644
>> --- a/drivers/pci/quirks.c
>> +++ b/drivers/pci/quirks.c
>> @@ -3783,10 +3783,98 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
>> return 0;
>> }
>>
>> +#define CAVIUM_VNIC_QSET_RQ_0_7_CFG (0x010600)
>> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG (0x010400)
>> +#define CAVIUM_VNIC_QSET_CQ_0_7_CFG2 (0x010408)
>> +#define CAVIUM_VNIC_QSET_SQ_0_7_CFG (0x010800)
>> +#define CAVIUM_VNIC_QSET_SQ_0_7_STATUS (0x010840)
>> +#define CAVIUM_VNIC_QSET_RBDR_0_1_CFG (0x010C00)
>> +#define CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0 (0x010C40)
>> +
>> +#define CAVIUM_VNIC_Q_SHIFT (18)
>> +#define CAVIUM_VNIC_CQ_RESET (1ULL << 41)
>> +#define CAVIUM_VNIC_SQ_RESET (1ULL << 17)
>> +#define CAVIUM_VNIC_RBDR_RESET (1ULL << 43)
>> +#define CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT (62)
>> +
>> +/* Poll a register for a specific value */
>> +static int cavium_vnic_poll(struct pci_dev *pdev,
>> + void __iomem *addr, size_t bit_pos,
>> + size_t bits, u64 val)
>> +{
>> + u64 bit_mask;
>> + u64 reg_val;
>> + size_t timeout = 10;
>> +
>> + bit_mask = (1ULL << bits) - 1;
>> + bit_mask = (bit_mask << bit_pos);
> GENMASK_ULL(bit_pos, bit_pos + bits - 1)? Admittedly it's not an awful
> lot tidier :/
>
>> +
>> + while (timeout) {
>> + reg_val = readq(addr);
>> + if (((reg_val & bit_mask) >> bit_pos) == val)
>> + return 0;
>> + usleep_range(1000, 2000);
>> + timeout--;
>> + }
> This is essentially an open-coded version of readq_poll_timeout() from
> <linux/iopoll.h>, which could be used instead. Doing "val <<= bitpos"
> first might help simplify the condition, although since it's only called
> with val == 0 it seems that "(reg_val & bit_mask)" would suffice.
>
> Robin.
Thanks for pointing this out.
I will try to simplify the condition and use readq_poll_timeout().
>
>> + dev_err(&pdev->dev, "Poll on addr %p failed\n", addr);
>> + return -1;
>> +}
>> +
>> +static int cavium_vnic_reset(struct pci_dev *pdev, int probe)
>> +{
>> + size_t qidx;
>> + void __iomem *bar_base;
>> + void __iomem *qset_base;
>> +
>> + bar_base = pci_iomap(pdev, 0, 0);
>> + if (!bar_base)
>> + return -ENOMEM;
>> +
>> + /* For each of 8 RQ/CQ/SQ (queues) in VF */
>> + for (qidx = 0; qidx < 8; qidx++) {
>> + /* Disable receive queue */
>> + qset_base = bar_base + (qidx << CAVIUM_VNIC_Q_SHIFT);
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RQ_0_7_CFG);
>> +
>> + /* Disable timer threshold (doesn't get reset upon CQ reset */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG2);
>> + /* Disable completion queue */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
>> + /* Reset completion queue */
>> + writeq(CAVIUM_VNIC_CQ_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_CQ_0_7_CFG);
>> +
>> + /* Disable send queue */
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
>> + /* Reset send queue */
>> + writeq(CAVIUM_VNIC_SQ_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_SQ_0_7_CFG);
>> + }
>> +
>> + /* Reset and disable both RBDR's */
>> + for (qidx = 0; qidx < 2; qidx++) {
>> + qset_base = bar_base +
>> + (qidx << CAVIUM_VNIC_Q_SHIFT);
>> + writeq(CAVIUM_VNIC_RBDR_RESET,
>> + qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
>> + writeq(0, qset_base + CAVIUM_VNIC_QSET_RBDR_0_1_CFG);
>> + if (cavium_vnic_poll(pdev, qset_base +
>> + CAVIUM_VNIC_QSET_RBDR_0_1_STATUS0,
>> + CAVIUM_VNIC_RBDR_FIFO_STATE_SHIFT,
>> + 2, 0x00))
>> + dev_err(&pdev->dev, "Timeout on RBDR reset sequence");
>> + }
>> +
>> + pci_iounmap(pdev, bar_base);
>> + return 0;
>> +}
>> +
>> #define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
>> #define PCI_DEVICE_ID_INTEL_IVB_M_VGA 0x0156
>> #define PCI_DEVICE_ID_INTEL_IVB_M2_VGA 0x0166
>>
>> +#define PCI_DEVICE_ID_CAVIUM_NIC_VF 0xA034
>> +
>> static const struct pci_dev_reset_methods pci_dev_reset_methods[] = {
>> { PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82599_SFP_VF,
>> reset_intel_82599_sfp_virtfn },
>> @@ -3796,6 +3884,8 @@ static int reset_chelsio_generic_dev(struct pci_dev *dev, int probe)
>> reset_ivb_igd },
>> { PCI_VENDOR_ID_CHELSIO, PCI_ANY_ID,
>> reset_chelsio_generic_dev },
>> + { PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_CAVIUM_NIC_VF,
>> + cavium_vnic_reset },
>> { 0 }
>> };
>>
>>
^ permalink raw reply [flat|nested] 11+ messages in thread
end of thread, other threads:[~2017-03-08 14:22 UTC | newest]
Thread overview: 11+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2017-03-07 15:04 [PATCH] PCI: Add pci reset quirk for Cavium VNIC Radoslaw Biernacki
2017-03-07 15:04 ` Radoslaw Biernacki
2017-03-07 15:04 ` Radoslaw Biernacki
2017-03-07 16:16 ` Robin Murphy
2017-03-07 16:16 ` Robin Murphy
2017-03-07 16:16 ` Robin Murphy
2017-03-07 16:16 ` Robin Murphy
2017-03-08 14:21 ` rad
2017-03-08 14:21 ` rad
2017-03-08 14:21 ` rad
2017-03-08 14:21 ` rad
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.