[PATCH for-next 00/10] Fixes for 5.13

[PATCH for-next 00/10] Fixes for 5.13

[dennis.dalessandro]

From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>

Here are some fixes, clean-up and debugging type patches for-next. There are
some changs to the way AIP/VNIC netdevs are used and a timeout handler is added
for rdma_netdev. 

The MTU patch from Kaike will to add a physical MTU to query_port is also of
particualr note.

Kaike Wan (2):
  rdma: Set physical MTU for query_port function
  IB/hfi1: Remove unused function

Mike Marciniszyn (8):
  IB/hfi1: Add AIP tx traces
  IB/{ipoib,hfi1}: Add a timeout handler for rdma_netdev
  IB/hfi1: Correct oversized ring allocation
  IB/hfi1: Use napi_schedule_irqoff() for tx napi
  IB/hfi1: Remove indirect call to hfi1_ipoib_send_dma()
  IB/hfi1: Add additional usdma traces
  IB/hfi1: Use kzalloc() for mmu_rb_handler allocation
  IB/hfi1: Rework AIP and VNIC dummy netdev usage

 drivers/infiniband/hw/bnxt_re/ib_verbs.c        |   1 +
 drivers/infiniband/hw/cxgb4/provider.c          |   1 +
 drivers/infiniband/hw/efa/efa_verbs.c           |   1 +
 drivers/infiniband/hw/hfi1/chip.c               |   6 +-
 drivers/infiniband/hw/hfi1/hfi.h                |   4 +-
 drivers/infiniband/hw/hfi1/init.c               |   2 +-
 drivers/infiniband/hw/hfi1/ipoib.h              |  13 +-
 drivers/infiniband/hw/hfi1/ipoib_main.c         |   9 +-
 drivers/infiniband/hw/hfi1/ipoib_tx.c           |  71 ++++++++--
 drivers/infiniband/hw/hfi1/mmu_rb.c             |   2 +-
 drivers/infiniband/hw/hfi1/netdev.h             |  39 ++----
 drivers/infiniband/hw/hfi1/netdev_rx.c          | 177 +++++++++++------------
 drivers/infiniband/hw/hfi1/sdma.h               |  18 ---
 drivers/infiniband/hw/hfi1/trace_tx.h           | 179 ++++++++++++++++++++++++
 drivers/infiniband/hw/hfi1/user_sdma.c          |  12 +-
 drivers/infiniband/hw/hfi1/user_sdma.h          |   1 +
 drivers/infiniband/hw/hns/hns_roce_main.c       |   1 +
 drivers/infiniband/hw/i40iw/i40iw_verbs.c       |   1 +
 drivers/infiniband/hw/mlx4/main.c               |   1 +
 drivers/infiniband/hw/mlx5/mad.c                |   1 +
 drivers/infiniband/hw/mlx5/main.c               |   2 +
 drivers/infiniband/hw/mthca/mthca_provider.c    |   1 +
 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |   1 +
 drivers/infiniband/hw/qib/qib_verbs.c           |   1 +
 drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |   1 +
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |   1 +
 drivers/infiniband/sw/siw/siw_verbs.c           |   1 +
 drivers/infiniband/ulp/ipoib/ipoib_main.c       |   7 +-
 include/rdma/ib_verbs.h                         |  19 +--
 29 files changed, 386 insertions(+), 188 deletions(-)

-- 
1.8.3.1

[PATCH for-next 01/10] IB/hfi1: Add AIP tx traces

[dennis.dalessandro]

From: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>

Add traces to allow for debugging issues with AIP
tx.

Reviewed-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
---
 drivers/infiniband/hw/hfi1/ipoib_tx.c |  18 +++++-
 drivers/infiniband/hw/hfi1/trace_tx.h | 104 ++++++++++++++++++++++++++++++++++
 2 files changed, 119 insertions(+), 3 deletions(-)

diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c
index edd4eea..5129dc9 100644
--- a/drivers/infiniband/hw/hfi1/ipoib_tx.c
+++ b/drivers/infiniband/hw/hfi1/ipoib_tx.c
@@ -15,6 +15,7 @@
 #include "verbs.h"
 #include "trace_ibhdrs.h"
 #include "ipoib.h"
+#include "trace_tx.h"
 
 /* Add a convenience helper */
 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1))
@@ -63,12 +64,14 @@ static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq)
 
 static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq)
 {
+	trace_hfi1_txq_stop(txq);
 	if (atomic_inc_return(&txq->stops) == 1)
 		netif_stop_subqueue(txq->priv->netdev, txq->q_idx);
 }
 
 static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq)
 {
+	trace_hfi1_txq_wake(txq);
 	if (atomic_dec_and_test(&txq->stops))
 		netif_wake_subqueue(txq->priv->netdev, txq->q_idx);
 }
@@ -89,8 +92,10 @@ static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq)
 {
 	++txq->sent_txreqs;
 	if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) &&
-	    !atomic_xchg(&txq->ring_full, 1))
+	    !atomic_xchg(&txq->ring_full, 1)) {
+		trace_hfi1_txq_full(txq);
 		hfi1_ipoib_stop_txq(txq);
+	}
 }
 
 static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq)
@@ -112,8 +117,10 @@ static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq)
 	 * to protect against ring overflow.
 	 */
 	if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) &&
-	    atomic_xchg(&txq->ring_full, 0))
+	    atomic_xchg(&txq->ring_full, 0)) {
+		trace_hfi1_txq_xmit_unstopped(txq);
 		hfi1_ipoib_wake_txq(txq);
+	}
 }
 
 static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget)
@@ -405,6 +412,7 @@ static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev,
 				sdma_select_engine_sc(priv->dd,
 						      txp->flow.tx_queue,
 						      txp->flow.sc5);
+			trace_hfi1_flow_switch(txp->txq);
 		}
 
 		return tx;
@@ -525,6 +533,7 @@ static int hfi1_ipoib_send_dma_list(struct net_device *dev,
 	if (txq->flow.as_int != txp->flow.as_int) {
 		int ret;
 
+		trace_hfi1_flow_flush(txq);
 		ret = hfi1_ipoib_flush_tx_list(dev, txq);
 		if (unlikely(ret)) {
 			if (ret == -EBUSY)
@@ -635,8 +644,10 @@ static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde,
 			/* came from non-list submit */
 			list_add_tail(&txreq->list, &txq->tx_list);
 		if (list_empty(&txq->wait.list)) {
-			if (!atomic_xchg(&txq->no_desc, 1))
+			if (!atomic_xchg(&txq->no_desc, 1)) {
+				trace_hfi1_txq_queued(txq);
 				hfi1_ipoib_stop_txq(txq);
+			}
 			iowait_queue(pkts_sent, wait->iow, &sde->dmawait);
 		}
 
@@ -659,6 +670,7 @@ static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason)
 	struct hfi1_ipoib_txq *txq =
 		container_of(wait, struct hfi1_ipoib_txq, wait);
 
+	trace_hfi1_txq_wakeup(txq);
 	if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED))
 		iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND);
 }
diff --git a/drivers/infiniband/hw/hfi1/trace_tx.h b/drivers/infiniband/hw/hfi1/trace_tx.h
index 769e5e4..8476541 100644
--- a/drivers/infiniband/hw/hfi1/trace_tx.h
+++ b/drivers/infiniband/hw/hfi1/trace_tx.h
@@ -53,6 +53,7 @@
 #include "hfi.h"
 #include "mad.h"
 #include "sdma.h"
+#include "ipoib.h"
 
 const char *parse_sdma_flags(struct trace_seq *p, u64 desc0, u64 desc1);
 
@@ -858,6 +859,109 @@
 	TP_ARGS(qp, flag)
 );
 
+DECLARE_EVENT_CLASS(/* AIP  */
+	hfi1_ipoib_txq_template,
+	TP_PROTO(struct hfi1_ipoib_txq *txq),
+	TP_ARGS(txq),
+	TP_STRUCT__entry(/* entry */
+		DD_DEV_ENTRY(txq->priv->dd)
+		__field(struct hfi1_ipoib_txq *, txq)
+		__field(struct sdma_engine *, sde)
+		__field(ulong, head)
+		__field(ulong, tail)
+		__field(uint, used)
+		__field(uint, flow)
+		__field(int, stops)
+		__field(int, no_desc)
+		__field(u8, idx)
+		__field(u8, stopped)
+	),
+	TP_fast_assign(/* assign */
+		DD_DEV_ASSIGN(txq->priv->dd)
+		__entry->txq = txq;
+		__entry->sde = txq->sde;
+		__entry->head = txq->tx_ring.head;
+		__entry->tail = txq->tx_ring.tail;
+		__entry->idx = txq->q_idx;
+		__entry->used =
+			txq->sent_txreqs -
+			atomic64_read(&txq->complete_txreqs);
+		__entry->flow = txq->flow.as_int;
+		__entry->stops = atomic_read(&txq->stops);
+		__entry->no_desc = atomic_read(&txq->no_desc);
+		__entry->stopped =
+		 __netif_subqueue_stopped(txq->priv->netdev, txq->q_idx);
+	),
+	TP_printk(/* print  */
+		"[%s] txq %llx idx %u sde %llx head %lx tail %lx flow %x used %u stops %d no_desc %d stopped %u",
+		__get_str(dev),
+		(unsigned long long)__entry->txq,
+		__entry->idx,
+		(unsigned long long)__entry->sde,
+		__entry->head,
+		__entry->tail,
+		__entry->flow,
+		__entry->used,
+		__entry->stops,
+		__entry->no_desc,
+		__entry->stopped
+	)
+);
+
+DEFINE_EVENT(/* queue stop */
+	hfi1_ipoib_txq_template, hfi1_txq_stop,
+	TP_PROTO(struct hfi1_ipoib_txq *txq),
+	TP_ARGS(txq)
+);
+
+DEFINE_EVENT(/* queue wake */
+	hfi1_ipoib_txq_template, hfi1_txq_wake,
+	TP_PROTO(struct hfi1_ipoib_txq *txq),
+	TP_ARGS(txq)
+);
+
+DEFINE_EVENT(/* flow flush */
+	hfi1_ipoib_txq_template, hfi1_flow_flush,
+	TP_PROTO(struct hfi1_ipoib_txq *txq),
+	TP_ARGS(txq)
+);
+
+DEFINE_EVENT(/* flow switch */
+	hfi1_ipoib_txq_template, hfi1_flow_switch,
+	TP_PROTO(struct hfi1_ipoib_txq *txq),
+	TP_ARGS(txq)
+);
+
+DEFINE_EVENT(/* wakeup */
+	hfi1_ipoib_txq_template, hfi1_txq_wakeup,
+	TP_PROTO(struct hfi1_ipoib_txq *txq),
+	TP_ARGS(txq)
+);
+
+DEFINE_EVENT(/* full */
+	hfi1_ipoib_txq_template, hfi1_txq_full,
+	TP_PROTO(struct hfi1_ipoib_txq *txq),
+	TP_ARGS(txq)
+);
+
+DEFINE_EVENT(/* queued */
+	hfi1_ipoib_txq_template, hfi1_txq_queued,
+	TP_PROTO(struct hfi1_ipoib_txq *txq),
+	TP_ARGS(txq)
+);
+
+DEFINE_EVENT(/* xmit_stopped */
+	hfi1_ipoib_txq_template, hfi1_txq_xmit_stopped,
+	TP_PROTO(struct hfi1_ipoib_txq *txq),
+	TP_ARGS(txq)
+);
+
+DEFINE_EVENT(/* xmit_unstopped */
+	hfi1_ipoib_txq_template, hfi1_txq_xmit_unstopped,
+	TP_PROTO(struct hfi1_ipoib_txq *txq),
+	TP_ARGS(txq)
+);
+
 #endif /* __HFI1_TRACE_TX_H */
 
 #undef TRACE_INCLUDE_PATH
-- 
1.8.3.1

[PATCH for-next 02/10] IB/{ipoib,hfi1}: Add a timeout handler for rdma_netdev

[dennis.dalessandro]

From: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>

The current rdma_netdev handling in ipoib hooks the
tx_timeout handler, but prints out a totally useless
message that prevents effective debugging especially when
multiple transmit queues are being used.

Add a tx_timeout rdma_netdev hook and implement the
callback in the hfi1 to print additional information.

The existing non-helpful message is avoided when the driver
has presented a callback.

Reviewed-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
---
 drivers/infiniband/hw/hfi1/ipoib.h        |  2 ++
 drivers/infiniband/hw/hfi1/ipoib_main.c   |  1 +
 drivers/infiniband/hw/hfi1/ipoib_tx.c     | 29 +++++++++++++++++++++++++++++
 drivers/infiniband/ulp/ipoib/ipoib_main.c |  5 +++++
 include/rdma/ib_verbs.h                   |  2 ++
 5 files changed, 39 insertions(+)

diff --git a/drivers/infiniband/hw/hfi1/ipoib.h b/drivers/infiniband/hw/hfi1/ipoib.h
index f650cac..1659beb 100644
--- a/drivers/infiniband/hw/hfi1/ipoib.h
+++ b/drivers/infiniband/hw/hfi1/ipoib.h
@@ -147,4 +147,6 @@ int hfi1_ipoib_rn_get_params(struct ib_device *device,
 			     enum rdma_netdev_t type,
 			     struct rdma_netdev_alloc_params *params);
 
+void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q);
+
 #endif /* _IPOIB_H */
diff --git a/drivers/infiniband/hw/hfi1/ipoib_main.c b/drivers/infiniband/hw/hfi1/ipoib_main.c
index 3242290..b8838fa 100644
--- a/drivers/infiniband/hw/hfi1/ipoib_main.c
+++ b/drivers/infiniband/hw/hfi1/ipoib_main.c
@@ -204,6 +204,7 @@ static int hfi1_ipoib_setup_rn(struct ib_device *device,
 	int rc;
 
 	rn->send = hfi1_ipoib_send;
+	rn->tx_timeout = hfi1_ipoib_tx_timeout;
 	rn->attach_mcast = hfi1_ipoib_mcast_attach;
 	rn->detach_mcast = hfi1_ipoib_mcast_detach;
 	rn->set_id = hfi1_ipoib_set_id;
diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c
index 5129dc9..e8dece5 100644
--- a/drivers/infiniband/hw/hfi1/ipoib_tx.c
+++ b/drivers/infiniband/hw/hfi1/ipoib_tx.c
@@ -866,3 +866,32 @@ void hfi1_ipoib_napi_tx_disable(struct net_device *dev)
 		(void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
 	}
 }
+
+void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q)
+{
+	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
+	struct hfi1_ipoib_txq *txq = &priv->txqs[q];
+	u64 completed = atomic64_read(&txq->complete_txreqs);
+
+	dd_dev_info(priv->dd, "timeout txq %llx q %u stopped %u stops %d no_desc %d ring_full %d\n",
+		    (unsigned long long)txq, q,
+		    __netif_subqueue_stopped(dev, txq->q_idx),
+		    atomic_read(&txq->stops),
+		    atomic_read(&txq->no_desc),
+		    atomic_read(&txq->ring_full));
+	dd_dev_info(priv->dd, "sde %llx engine %u\n",
+		    (unsigned long long)txq->sde,
+		    txq->sde ? txq->sde->this_idx : 0);
+	dd_dev_info(priv->dd, "flow %x\n", txq->flow.as_int);
+	dd_dev_info(priv->dd, "sent %llu completed %llu used %llu\n",
+		    txq->sent_txreqs, completed, hfi1_ipoib_used(txq));
+	dd_dev_info(priv->dd, "tx_queue_len %u max_items %lu\n",
+		    dev->tx_queue_len, txq->tx_ring.max_items);
+	dd_dev_info(priv->dd, "head %lu tail %lu\n",
+		    txq->tx_ring.head, txq->tx_ring.tail);
+	dd_dev_info(priv->dd, "wait queued %u\n",
+		    !list_empty(&txq->wait.list));
+	dd_dev_info(priv->dd, "tx_list empty %u\n",
+		    list_empty(&txq->tx_list));
+}
+
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index e16b40c..2fb2fa16 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1181,7 +1181,12 @@ static netdev_tx_t ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
 static void ipoib_timeout(struct net_device *dev, unsigned int txqueue)
 {
 	struct ipoib_dev_priv *priv = ipoib_priv(dev);
+	struct rdma_netdev *rn = netdev_priv(dev);
 
+	if (rn->tx_timeout) {
+		rn->tx_timeout(dev, txqueue);
+		return;
+	}
 	ipoib_warn(priv, "transmit timeout: latency %d msecs\n",
 		   jiffies_to_msecs(jiffies - dev_trans_start(dev)));
 	ipoib_warn(priv,
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 21c19b1..84f7084 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -2215,6 +2215,8 @@ struct rdma_netdev {
 			    int set_qkey, u32 qkey);
 	int (*detach_mcast)(struct net_device *dev, struct ib_device *hca,
 			    union ib_gid *gid, u16 mlid);
+	/* timeout */
+	void (*tx_timeout)(struct net_device *dev, unsigned int txqueue);
 };
 
 struct rdma_netdev_alloc_params {
-- 
1.8.3.1

[PATCH for-next 03/10] IB/hfi1: Correct oversized ring allocation

[dennis.dalessandro]

From: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>

The completion ring for tx is using the wrong size to size the
ring, oversizing the ring by two orders of magniture.

Correct the allocation size and use kcalloc_node() to allocate
the ring.  Fix mistaken GFP defines in similar allocations.

Reviewed-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
---
 drivers/infiniband/hw/hfi1/ipoib.h    |  3 ++-
 drivers/infiniband/hw/hfi1/ipoib_tx.c | 14 +++++++-------
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/drivers/infiniband/hw/hfi1/ipoib.h b/drivers/infiniband/hw/hfi1/ipoib.h
index 1659beb..39c8909 100644
--- a/drivers/infiniband/hw/hfi1/ipoib.h
+++ b/drivers/infiniband/hw/hfi1/ipoib.h
@@ -52,8 +52,9 @@
  * @producer_lock: producer sync lock
  * @consumer_lock: consumer sync lock
  */
+struct ipoib_txreq;
 struct hfi1_ipoib_circ_buf {
-	void **items;
+	struct ipoib_txreq **items;
 	unsigned long head;
 	unsigned long tail;
 	unsigned long max_items;
diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c
index e8dece5..1c38c38 100644
--- a/drivers/infiniband/hw/hfi1/ipoib_tx.c
+++ b/drivers/infiniband/hw/hfi1/ipoib_tx.c
@@ -714,14 +714,14 @@ int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
 
 	priv->tx_napis = kcalloc_node(dev->num_tx_queues,
 				      sizeof(struct napi_struct),
-				      GFP_ATOMIC,
+				      GFP_KERNEL,
 				      priv->dd->node);
 	if (!priv->tx_napis)
 		goto free_txreq_cache;
 
 	priv->txqs = kcalloc_node(dev->num_tx_queues,
 				  sizeof(struct hfi1_ipoib_txq),
-				  GFP_ATOMIC,
+				  GFP_KERNEL,
 				  priv->dd->node);
 	if (!priv->txqs)
 		goto free_tx_napis;
@@ -753,9 +753,9 @@ int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
 					     priv->dd->node);
 
 		txq->tx_ring.items =
-			vzalloc_node(array_size(tx_ring_size,
-						sizeof(struct ipoib_txreq)),
-				     priv->dd->node);
+			kcalloc_node(tx_ring_size,
+				     sizeof(struct ipoib_txreq *),
+				     GFP_KERNEL, priv->dd->node);
 		if (!txq->tx_ring.items)
 			goto free_txqs;
 
@@ -776,7 +776,7 @@ int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
 		struct hfi1_ipoib_txq *txq = &priv->txqs[i];
 
 		netif_napi_del(txq->napi);
-		vfree(txq->tx_ring.items);
+		kfree(txq->tx_ring.items);
 	}
 
 	kfree(priv->txqs);
@@ -829,7 +829,7 @@ void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv)
 		hfi1_ipoib_drain_tx_list(txq);
 		netif_napi_del(txq->napi);
 		(void)hfi1_ipoib_drain_tx_ring(txq, txq->tx_ring.max_items);
-		vfree(txq->tx_ring.items);
+		kfree(txq->tx_ring.items);
 	}
 
 	kfree(priv->txqs);
-- 
1.8.3.1

[PATCH for-next 04/10] IB/hfi1: Use napi_schedule_irqoff() for tx napi

[dennis.dalessandro]

From: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>

The call is from an ISR context and napi_schedule_irqoff() can
be used.

Change the call to the more efficient type.

Reviewed-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
---
 drivers/infiniband/hw/hfi1/ipoib_tx.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c
index 1c38c38..8ebb653 100644
--- a/drivers/infiniband/hw/hfi1/ipoib_tx.c
+++ b/drivers/infiniband/hw/hfi1/ipoib_tx.c
@@ -209,7 +209,7 @@ static void hfi1_ipoib_add_tx(struct ipoib_txreq *tx)
 
 		/* Finish storing txreq before incrementing head. */
 		smp_store_release(&tx_ring->head, CIRC_ADD(head, 1, max_tx));
-		napi_schedule(tx->txq->napi);
+		napi_schedule_irqoff(tx->txq->napi);
 	} else {
 		struct hfi1_ipoib_txq *txq = tx->txq;
 		struct hfi1_ipoib_dev_priv *priv = tx->priv;
-- 
1.8.3.1

[PATCH for-next 05/10] IB/hfi1: Remove indirect call to hfi1_ipoib_send_dma()

[dennis.dalessandro]

From: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>

hfi1_ipoib_send() directly calls hfi1_ipoib_send_dma() with
no value add.

Fix by renaming hfi1_ipoib_send_dma() to hfi1_ipoib_send().

Reviewed-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
---
 drivers/infiniband/hw/hfi1/ipoib.h      | 8 ++++----
 drivers/infiniband/hw/hfi1/ipoib_main.c | 8 --------
 drivers/infiniband/hw/hfi1/ipoib_tx.c   | 8 ++++----
 3 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/drivers/infiniband/hw/hfi1/ipoib.h b/drivers/infiniband/hw/hfi1/ipoib.h
index 39c8909..bd0c9af 100644
--- a/drivers/infiniband/hw/hfi1/ipoib.h
+++ b/drivers/infiniband/hw/hfi1/ipoib.h
@@ -126,10 +126,10 @@ struct hfi1_ipoib_rdma_netdev {
 	return &((struct hfi1_ipoib_rdma_netdev *)netdev_priv(dev))->dev_priv;
 }
 
-int hfi1_ipoib_send_dma(struct net_device *dev,
-			struct sk_buff *skb,
-			struct ib_ah *address,
-			u32 dqpn);
+int hfi1_ipoib_send(struct net_device *dev,
+		    struct sk_buff *skb,
+		    struct ib_ah *address,
+		    u32 dqpn);
 
 int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv);
 void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv);
diff --git a/drivers/infiniband/hw/hfi1/ipoib_main.c b/drivers/infiniband/hw/hfi1/ipoib_main.c
index b8838fa..b737354 100644
--- a/drivers/infiniband/hw/hfi1/ipoib_main.c
+++ b/drivers/infiniband/hw/hfi1/ipoib_main.c
@@ -101,14 +101,6 @@ static int hfi1_ipoib_dev_stop(struct net_device *dev)
 	.ndo_get_stats64  = dev_get_tstats64,
 };
 
-static int hfi1_ipoib_send(struct net_device *dev,
-			   struct sk_buff *skb,
-			   struct ib_ah *address,
-			   u32 dqpn)
-{
-	return hfi1_ipoib_send_dma(dev, skb, address, dqpn);
-}
-
 static int hfi1_ipoib_mcast_attach(struct net_device *dev,
 				   struct ib_device *device,
 				   union ib_gid *mgid,
diff --git a/drivers/infiniband/hw/hfi1/ipoib_tx.c b/drivers/infiniband/hw/hfi1/ipoib_tx.c
index 8ebb653..993f983 100644
--- a/drivers/infiniband/hw/hfi1/ipoib_tx.c
+++ b/drivers/infiniband/hw/hfi1/ipoib_tx.c
@@ -581,10 +581,10 @@ static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb)
 	return (u8)skb_get_queue_mapping(skb);
 }
 
-int hfi1_ipoib_send_dma(struct net_device *dev,
-			struct sk_buff *skb,
-			struct ib_ah *address,
-			u32 dqpn)
+int hfi1_ipoib_send(struct net_device *dev,
+		    struct sk_buff *skb,
+		    struct ib_ah *address,
+		    u32 dqpn)
 {
 	struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
 	struct ipoib_txparms txp;
-- 
1.8.3.1

[PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[dennis.dalessandro]

From: Kaike Wan <kaike.wan@intel.com>

This is a follow on patch to add a phys_mtu field to the
ib_port_attr structure to indicate the maximum physical MTU
the underlying device supports.

Extends the following:
commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's upper limit")

Reviewed-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Signed-off-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
---
 drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
 drivers/infiniband/hw/cxgb4/provider.c          |  1 +
 drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
 drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
 drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
 drivers/infiniband/hw/mlx4/main.c               |  1 +
 drivers/infiniband/hw/mlx5/mad.c                |  1 +
 drivers/infiniband/hw/mlx5/main.c               |  2 ++
 drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
 drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
 drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
 drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
 drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
 drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
 include/rdma/ib_verbs.h                         | 17 -----------------
 16 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index ba515ef..17d4871 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -205,6 +205,7 @@ int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
 		port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 	}
 	port_attr->max_mtu = IB_MTU_4096;
+	port_attr->phys_mtu = ib_mtu_enum_to_int(port_attr->max_mtu);
 	port_attr->active_mtu = iboe_get_mtu(rdev->netdev->mtu);
 	port_attr->gid_tbl_len = dev_attr->max_sgid;
 	port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 1f1f856..09a995b 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -311,6 +311,7 @@ static int c4iw_query_port(struct ib_device *ibdev, u8 port,
 	    IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
 	props->gid_tbl_len = 1;
 	props->max_msg_sz = -1;
+	props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu);
 
 	return ret;
 }
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 479b604..1a426c0 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -261,6 +261,7 @@ int efa_query_port(struct ib_device *ibdev, u8 port,
 	props->active_speed = IB_SPEED_EDR;
 	props->active_width = IB_WIDTH_4X;
 	props->max_mtu = ib_mtu_int_to_enum(dev->dev_attr.mtu);
+	props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu);
 	props->active_mtu = ib_mtu_int_to_enum(dev->dev_attr.mtu);
 	props->max_msg_sz = dev->dev_attr.mtu;
 	props->max_vl_num = 1;
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 1a747f7..0fb46eb 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -228,6 +228,7 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num,
 	/* props being zeroed by the caller, avoid zeroing it here */
 
 	props->max_mtu = hr_dev->caps.max_mtu;
+	props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu);
 	props->gid_tbl_len = hr_dev->caps.gid_table_len[port];
 	props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
 				IB_PORT_VENDOR_CLASS_SUP |
diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
index 375b774..0e76968 100644
--- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c
+++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c
@@ -104,6 +104,7 @@ static int i40iw_query_port(struct ib_device *ibdev,
 	props->active_width = IB_WIDTH_4X;
 	props->active_speed = 1;
 	props->max_msg_sz = I40IW_MAX_OUTBOUND_MESSAGE_SIZE;
+	props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu);
 	return 0;
 }
 
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index f26a0d9..a33a7b7 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -702,6 +702,7 @@ static int ib_link_query_port(struct ib_device *ibdev, u8 port,
 	props->active_width	= out_mad->data[31] & 0xf;
 	props->active_speed	= out_mad->data[35] >> 4;
 	props->max_mtu		= out_mad->data[41] & 0xf;
+	props->phys_mtu		= ib_mtu_enum_to_int(props->max_mtu);
 	props->active_mtu	= out_mad->data[36] >> 4;
 	props->subnet_timeout	= out_mad->data[51] & 0x1f;
 	props->max_vl_num	= out_mad->data[37] >> 4;
diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c
index 652c6cc..9a1806d 100644
--- a/drivers/infiniband/hw/mlx5/mad.c
+++ b/drivers/infiniband/hw/mlx5/mad.c
@@ -555,6 +555,7 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
 	props->active_width	= out_mad->data[31] & 0xf;
 	props->active_speed	= out_mad->data[35] >> 4;
 	props->max_mtu		= out_mad->data[41] & 0xf;
+	props->phys_mtu		= ib_mtu_enum_to_int(props->max_mtu);
 	props->active_mtu	= out_mad->data[36] >> 4;
 	props->subnet_timeout	= out_mad->data[51] & 0x1f;
 	props->max_vl_num	= out_mad->data[37] >> 4;
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 9ff28f7..732e5ce 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -511,6 +511,7 @@ static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
 		props->qkey_viol_cntr = qkey_viol_cntr;
 	}
 	props->max_mtu          = IB_MTU_4096;
+	props->phys_mtu		= ib_mtu_enum_to_int(props->max_mtu);
 	props->max_msg_sz       = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
 	props->pkey_tbl_len     = 1;
 	props->state            = IB_PORT_DOWN;
@@ -1321,6 +1322,7 @@ static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,
 	mlx5_query_port_max_mtu(mdev, &max_mtu, port);
 
 	props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
+	props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu);
 
 	mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
 
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 1a3dd07..87f7b11 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -164,6 +164,7 @@ static int mthca_query_port(struct ib_device *ibdev,
 	props->active_width      = out_mad->data[31] & 0xf;
 	props->active_speed      = out_mad->data[35] >> 4;
 	props->max_mtu           = out_mad->data[41] & 0xf;
+	props->phys_mtu		 = ib_mtu_enum_to_int(props->max_mtu);
 	props->active_mtu        = out_mad->data[36] >> 4;
 	props->subnet_timeout    = out_mad->data[51] & 0x1f;
 	props->max_vl_num        = out_mad->data[37] >> 4;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index 3acb5c1..d0496c1 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -167,6 +167,7 @@ int ocrdma_query_port(struct ib_device *ibdev,
 		props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 	}
 	props->max_mtu = IB_MTU_4096;
+	props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu);
 	props->active_mtu = iboe_get_mtu(netdev->mtu);
 	props->lid = 0;
 	props->lmc = 0;
diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c
index 8e0de26..0534a12 100644
--- a/drivers/infiniband/hw/qib/qib_verbs.c
+++ b/drivers/infiniband/hw/qib/qib_verbs.c
@@ -1209,6 +1209,7 @@ static int qib_query_port(struct rvt_dev_info *rdi, u8 port_num,
 	props->max_vl_num = qib_num_vls(ppd->vls_supported);
 
 	props->max_mtu = qib_ibmtu ? qib_ibmtu : IB_MTU_4096;
+	props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu);
 	switch (ppd->ibmtu) {
 	case 4096:
 		mtu = IB_MTU_4096;
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index 3705c6b..a83eaf0 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -373,6 +373,7 @@ int usnic_ib_query_port(struct ib_device *ibdev, u8 port,
 	props->bad_pkey_cntr = 0;
 	props->qkey_viol_cntr = 0;
 	props->max_mtu = IB_MTU_4096;
+	props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu);
 	props->active_mtu = iboe_get_mtu(us_ibdev->ufdev->mtu);
 	/* Userspace will adjust for hdrs */
 	props->max_msg_sz = us_ibdev->ufdev->mtu;
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
index fc412cb..a40083ae 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
@@ -150,6 +150,7 @@ int pvrdma_query_port(struct ib_device *ibdev, u8 port,
 
 	props->state = pvrdma_port_state_to_ib(resp->attrs.state);
 	props->max_mtu = pvrdma_mtu_to_ib(resp->attrs.max_mtu);
+	props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu);
 	props->active_mtu = pvrdma_mtu_to_ib(resp->attrs.active_mtu);
 	props->gid_tbl_len = resp->attrs.gid_tbl_len;
 	props->port_cap_flags =
diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c
index e389d44..8a5df31 100644
--- a/drivers/infiniband/sw/siw/siw_verbs.c
+++ b/drivers/infiniband/sw/siw/siw_verbs.c
@@ -173,6 +173,7 @@ int siw_query_port(struct ib_device *base_dev, u8 port,
 	attr->gid_tbl_len = 1;
 	attr->max_msg_sz = -1;
 	attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
+	attr->phys_mtu = ib_mtu_enum_to_int(attr->max_mtu);
 	attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 	attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
 		IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 2fb2fa16..4ca6150 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1865,7 +1865,7 @@ static int ipoib_parent_init(struct net_device *ndev)
 			priv->port);
 		return result;
 	}
-	priv->max_ib_mtu = rdma_mtu_from_attr(priv->ca, priv->port, &attr);
+	priv->max_ib_mtu = attr.phys_mtu;
 
 	result = ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey);
 	if (result) {
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 84f7084..221c5b2 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -3338,23 +3338,6 @@ static inline int rdma_mtu_enum_to_int(struct ib_device *device, u8 port,
 		return ib_mtu_enum_to_int((enum ib_mtu)mtu);
 }
 
-/**
- * rdma_mtu_from_attr - Return the mtu of the port from the port attribute.
- * @device: Device
- * @port_num: Port number
- * @attr: port attribute
- *
- * Return the MTU size supported by the port as an integer value.
- */
-static inline int rdma_mtu_from_attr(struct ib_device *device, u8 port,
-				     struct ib_port_attr *attr)
-{
-	if (rdma_core_cap_opa_port(device, port))
-		return attr->phys_mtu;
-	else
-		return ib_mtu_enum_to_int(attr->max_mtu);
-}
-
 int ib_set_vf_link_state(struct ib_device *device, int vf, u8 port,
 			 int state);
 int ib_get_vf_config(struct ib_device *device, int vf, u8 port,
-- 
1.8.3.1

[PATCH for-next 07/10] IB/hfi1: Add additional usdma traces

[dennis.dalessandro]

From: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>

Add traces that were vital in isolating an issue with pq waitlist.  See
URL below.

Reviewed-by: Kaike Wan <kaike.wan@intel.com>
URL: https://lore.kernel.org/linux-rdma/20200504130917.175613.43231.stgit@awfm-01.aw.intel.com/
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
---
 drivers/infiniband/hw/hfi1/trace_tx.h  | 75 ++++++++++++++++++++++++++++++++++
 drivers/infiniband/hw/hfi1/user_sdma.c | 12 ++++--
 drivers/infiniband/hw/hfi1/user_sdma.h |  1 +
 3 files changed, 85 insertions(+), 3 deletions(-)

diff --git a/drivers/infiniband/hw/hfi1/trace_tx.h b/drivers/infiniband/hw/hfi1/trace_tx.h
index 8476541..d44fc54 100644
--- a/drivers/infiniband/hw/hfi1/trace_tx.h
+++ b/drivers/infiniband/hw/hfi1/trace_tx.h
@@ -54,6 +54,7 @@
 #include "mad.h"
 #include "sdma.h"
 #include "ipoib.h"
+#include "user_sdma.h"
 
 const char *parse_sdma_flags(struct trace_seq *p, u64 desc0, u64 desc1);
 
@@ -654,6 +655,80 @@
 		      __entry->code)
 );
 
+TRACE_EVENT(hfi1_usdma_defer,
+	    TP_PROTO(struct hfi1_user_sdma_pkt_q *pq,
+		     struct sdma_engine *sde,
+		     struct iowait *wait),
+	    TP_ARGS(pq, sde, wait),
+	    TP_STRUCT__entry(DD_DEV_ENTRY(pq->dd)
+			     __field(struct hfi1_user_sdma_pkt_q *, pq)
+			     __field(struct sdma_engine *, sde)
+			     __field(struct iowait *, wait)
+			     __field(int, engine)
+			     __field(int, empty)
+			     ),
+	     TP_fast_assign(DD_DEV_ASSIGN(pq->dd);
+			    __entry->pq = pq;
+			    __entry->sde = sde;
+			    __entry->wait = wait;
+			    __entry->engine = sde->this_idx;
+			    __entry->empty = list_empty(&__entry->wait->list);
+			    ),
+	     TP_printk("[%s] pq %llx sde %llx wait %llx engine %d empty %d",
+		       __get_str(dev),
+		       (unsigned long long)__entry->pq,
+		       (unsigned long long)__entry->sde,
+		       (unsigned long long)__entry->wait,
+		       __entry->engine,
+		       __entry->empty
+		)
+);
+
+TRACE_EVENT(hfi1_usdma_activate,
+	    TP_PROTO(struct hfi1_user_sdma_pkt_q *pq,
+		     struct iowait *wait,
+		     int reason),
+	    TP_ARGS(pq, wait, reason),
+	    TP_STRUCT__entry(DD_DEV_ENTRY(pq->dd)
+			     __field(struct hfi1_user_sdma_pkt_q *, pq)
+			     __field(struct iowait *, wait)
+			     __field(int, reason)
+			     ),
+	     TP_fast_assign(DD_DEV_ASSIGN(pq->dd);
+			    __entry->pq = pq;
+			    __entry->wait = wait;
+			    __entry->reason = reason;
+			    ),
+	     TP_printk("[%s] pq %llx wait %llx reason %d",
+		       __get_str(dev),
+		       (unsigned long long)__entry->pq,
+		       (unsigned long long)__entry->wait,
+		       __entry->reason
+		)
+);
+
+TRACE_EVENT(hfi1_usdma_we,
+	    TP_PROTO(struct hfi1_user_sdma_pkt_q *pq,
+		     int we_ret),
+	    TP_ARGS(pq, we_ret),
+	    TP_STRUCT__entry(DD_DEV_ENTRY(pq->dd)
+			     __field(struct hfi1_user_sdma_pkt_q *, pq)
+			     __field(int, state)
+			     __field(int, we_ret)
+			     ),
+	     TP_fast_assign(DD_DEV_ASSIGN(pq->dd);
+			    __entry->pq = pq;
+			    __entry->state = pq->state;
+			    __entry->we_ret = we_ret;
+			    ),
+	     TP_printk("[%s] pq %llx state %d we_ret %d",
+		       __get_str(dev),
+		       (unsigned long long)__entry->pq,
+		       __entry->state,
+		       __entry->we_ret
+		)
+);
+
 const char *print_u32_array(struct trace_seq *, u32 *, int);
 #define __print_u32_hex(arr, len) print_u32_array(p, arr, len)
 
diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c
index 4a4956f9..da5b2e3 100644
--- a/drivers/infiniband/hw/hfi1/user_sdma.c
+++ b/drivers/infiniband/hw/hfi1/user_sdma.c
@@ -133,6 +133,7 @@ static int defer_packet_queue(
 		container_of(wait->iow, struct hfi1_user_sdma_pkt_q, busy);
 
 	write_seqlock(&sde->waitlock);
+	trace_hfi1_usdma_defer(pq, sde, &pq->busy);
 	if (sdma_progress(sde, seq, txreq))
 		goto eagain;
 	/*
@@ -157,7 +158,8 @@ static void activate_packet_queue(struct iowait *wait, int reason)
 {
 	struct hfi1_user_sdma_pkt_q *pq =
 		container_of(wait, struct hfi1_user_sdma_pkt_q, busy);
-	pq->busy.lock = NULL;
+
+	trace_hfi1_usdma_activate(pq, wait, reason);
 	xchg(&pq->state, SDMA_PKT_Q_ACTIVE);
 	wake_up(&wait->wait_dma);
 };
@@ -599,13 +601,17 @@ int hfi1_user_sdma_process_request(struct hfi1_filedata *fd,
 	while (req->seqsubmitted != req->info.npkts) {
 		ret = user_sdma_send_pkts(req, pcount);
 		if (ret < 0) {
+			int we_ret;
+
 			if (ret != -EBUSY)
 				goto free_req;
-			if (wait_event_interruptible_timeout(
+			we_ret = wait_event_interruptible_timeout(
 				pq->busy.wait_dma,
 				pq->state == SDMA_PKT_Q_ACTIVE,
 				msecs_to_jiffies(
-					SDMA_IOWAIT_TIMEOUT)) <= 0)
+					SDMA_IOWAIT_TIMEOUT));
+			trace_hfi1_usdma_we(pq, we_ret);
+			if (we_ret <= 0)
 				flush_pq_iowait(pq);
 		}
 	}
diff --git a/drivers/infiniband/hw/hfi1/user_sdma.h b/drivers/infiniband/hw/hfi1/user_sdma.h
index 1e8c02f..fabe581 100644
--- a/drivers/infiniband/hw/hfi1/user_sdma.h
+++ b/drivers/infiniband/hw/hfi1/user_sdma.h
@@ -53,6 +53,7 @@
 #include "common.h"
 #include "iowait.h"
 #include "user_exp_rcv.h"
+#include "mmu_rb.h"
 
 /* The maximum number of Data io vectors per message/request */
 #define MAX_VECTORS_PER_REQ 8
-- 
1.8.3.1

[PATCH for-next 08/10] IB/hfi1: Use kzalloc() for mmu_rb_handler allocation

[dennis.dalessandro]

From: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>

The code currently assumes that the mmu_notifier struct
embedded in mmu_rb_handler only contains two fields.

There are now extra fields:

struct mmu_notifier {
        struct hlist_node hlist;
        const struct mmu_notifier_ops *ops;
        struct mm_struct *mm;
        struct rcu_head rcu;
        unsigned int users;
};

Given that there in no init for the mmu_notifier, a kzalloc() should
be used to insure that any newly added fields are given a predictable
initial value of zero.

Fixes: 06e0ffa69312 ("IB/hfi1: Re-factor MMU notification code")
Reviewed-by: Adam Goldman <adam.goldman@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
---
 drivers/infiniband/hw/hfi1/mmu_rb.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/infiniband/hw/hfi1/mmu_rb.c b/drivers/infiniband/hw/hfi1/mmu_rb.c
index 375a881..dccb4c1 100644
--- a/drivers/infiniband/hw/hfi1/mmu_rb.c
+++ b/drivers/infiniband/hw/hfi1/mmu_rb.c
@@ -89,7 +89,7 @@ int hfi1_mmu_rb_register(void *ops_arg,
 	struct mmu_rb_handler *h;
 	int ret;
 
-	h = kmalloc(sizeof(*h), GFP_KERNEL);
+	h = kzalloc(sizeof(*h), GFP_KERNEL);
 	if (!h)
 		return -ENOMEM;
 
-- 
1.8.3.1

[PATCH for-next 09/10] IB/hfi1: Remove unused function

[dennis.dalessandro]

From: Kaike Wan <kaike.wan@intel.com>

Remove the unused function sdma_iowait_schedule().

Fixes: 7724105686e7 ("IB/hfi1: add driver files")
Reviewed-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Signed-off-by: Kaike Wan <kaike.wan@cornelisnetworks.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
---
 drivers/infiniband/hw/hfi1/sdma.h | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h
index 7a85119..f57d552 100644
--- a/drivers/infiniband/hw/hfi1/sdma.h
+++ b/drivers/infiniband/hw/hfi1/sdma.h
@@ -907,24 +907,6 @@ static inline unsigned sdma_progress(struct sdma_engine *sde, unsigned seq,
 	return 0;
 }
 
-/**
- * sdma_iowait_schedule() - initialize wait structure
- * @sde: sdma_engine to schedule
- * @wait: wait struct to schedule
- *
- * This function initializes the iowait
- * structure embedded in the QP or PQ.
- *
- */
-static inline void sdma_iowait_schedule(
-	struct sdma_engine *sde,
-	struct iowait *wait)
-{
-	struct hfi1_pportdata *ppd = sde->dd->pport;
-
-	iowait_schedule(wait, ppd->hfi1_wq, sde->cpu);
-}
-
 /* for use by interrupt handling */
 void sdma_engine_error(struct sdma_engine *sde, u64 status);
 void sdma_engine_interrupt(struct sdma_engine *sde, u64 status);
-- 
1.8.3.1

[PATCH for-next 10/10] IB/hfi1: Rework AIP and VNIC dummy netdev usage

[dennis.dalessandro]

From: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>

All other users of the dummy netdevice embed the netdev in other
structures:

init_dummy_netdev(&mal->dummy_dev);
init_dummy_netdev(&eth->dummy_dev);
init_dummy_netdev(&ar->napi_dev);
init_dummy_netdev(&irq_grp->napi_ndev);
init_dummy_netdev(&wil->napi_ndev);
init_dummy_netdev(&trans_pcie->napi_dev);
init_dummy_netdev(&dev->napi_dev);
init_dummy_netdev(&bus->mux_dev);

The AIP and VNIC implementation turns that model inside out and used a
kfree() to free what appears to be a netdev struct when in reality, it is
a struct that enbodies the rx state as well as the dummy netdev used to
support napi_poll across disparate receive contexts.  The relationship is
infered by the odd allocation:

	const int netdev_size = sizeof(*dd->dummy_netdev) +
		sizeof(struct hfi1_netdev_priv);
	<snip>
	dd->dummy_netdev = kcalloc_node(1, netdev_size, GFP_KERNEL, dd->node);


Correct the issue by:
- Correctly naming the alloc and free functions
- Renaming hfi1_netdev_priv to hfi1_netdev_rx
- Replacing dd dummy_netdev with a netdev_rx pointer
- Embedding the net_device in hfi1_netdev_rx
- Moving the init_dummy_netdev to the alloc routine
- Adjusting wrappers to fit the new model

Fixes: 6991abcb993c ("IB/hfi1: Add functions to receive accelerated ipoib packets")
Reviewed-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
---
 drivers/infiniband/hw/hfi1/chip.c      |   6 +-
 drivers/infiniband/hw/hfi1/hfi.h       |   4 +-
 drivers/infiniband/hw/hfi1/init.c      |   2 +-
 drivers/infiniband/hw/hfi1/netdev.h    |  39 +++-----
 drivers/infiniband/hw/hfi1/netdev_rx.c | 177 +++++++++++++++++----------------
 5 files changed, 108 insertions(+), 120 deletions(-)

diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c
index a156522..5eeae8d 100644
--- a/drivers/infiniband/hw/hfi1/chip.c
+++ b/drivers/infiniband/hw/hfi1/chip.c
@@ -15243,8 +15243,8 @@ int hfi1_init_dd(struct hfi1_devdata *dd)
 		 (dd->revision >> CCE_REVISION_SW_SHIFT)
 		    & CCE_REVISION_SW_MASK);
 
-	/* alloc netdev data */
-	ret = hfi1_netdev_alloc(dd);
+	/* alloc VNIC/AIP rx data */
+	ret = hfi1_alloc_rx(dd);
 	if (ret)
 		goto bail_cleanup;
 
@@ -15348,7 +15348,7 @@ int hfi1_init_dd(struct hfi1_devdata *dd)
 	hfi1_comp_vectors_clean_up(dd);
 	msix_clean_up_interrupts(dd);
 bail_cleanup:
-	hfi1_netdev_free(dd);
+	hfi1_free_rx(dd);
 	hfi1_pcie_ddcleanup(dd);
 bail_free:
 	hfi1_free_devdata(dd);
diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h
index d341b8a..8afb757 100644
--- a/drivers/infiniband/hw/hfi1/hfi.h
+++ b/drivers/infiniband/hw/hfi1/hfi.h
@@ -69,7 +69,6 @@
 #include <rdma/ib_hdrs.h>
 #include <rdma/opa_addr.h>
 #include <linux/rhashtable.h>
-#include <linux/netdevice.h>
 #include <rdma/rdma_vt.h>
 
 #include "chip_registers.h"
@@ -1060,6 +1059,7 @@ struct hfi1_vnic_data {
 #define SERIAL_MAX 16 /* length of the serial number */
 
 typedef int (*send_routine)(struct rvt_qp *, struct hfi1_pkt_state *, u64);
+struct hfi1_netdev_rx;
 struct hfi1_devdata {
 	struct hfi1_ibdev verbs_dev;     /* must be first */
 	/* pointers to related structs for this device */
@@ -1402,7 +1402,7 @@ struct hfi1_devdata {
 	/* Lock to protect IRQ SRC register access */
 	spinlock_t irq_src_lock;
 	int vnic_num_vports;
-	struct net_device *dummy_netdev;
+	struct hfi1_netdev_rx *netdev_rx;
 	struct hfi1_affinity_node *affinity_entry;
 
 	/* Keeps track of IPoIB RSM rule users */
diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c
index 6d03aa0..447b5dab 100644
--- a/drivers/infiniband/hw/hfi1/init.c
+++ b/drivers/infiniband/hw/hfi1/init.c
@@ -1775,7 +1775,7 @@ static void remove_one(struct pci_dev *pdev)
 	hfi1_unregister_ib_device(dd);
 
 	/* free netdev data */
-	hfi1_netdev_free(dd);
+	hfi1_free_rx(dd);
 
 	/*
 	 * Disable the IB link, disable interrupts on the device,
diff --git a/drivers/infiniband/hw/hfi1/netdev.h b/drivers/infiniband/hw/hfi1/netdev.h
index 947543a..8aa0746 100644
--- a/drivers/infiniband/hw/hfi1/netdev.h
+++ b/drivers/infiniband/hw/hfi1/netdev.h
@@ -14,15 +14,14 @@
 
 /**
  * struct hfi1_netdev_rxq - Receive Queue for HFI
- * dummy netdev. Both IPoIB and VNIC netdevices will be working on
- * top of this device.
+ * Both IPoIB and VNIC netdevices will be working on the rx abstraction.
  * @napi: napi object
- * @priv: ptr to netdev_priv
+ * @rx: ptr to netdev_rx
  * @rcd:  ptr to receive context data
  */
 struct hfi1_netdev_rxq {
 	struct napi_struct napi;
-	struct hfi1_netdev_priv *priv;
+	struct hfi1_netdev_rx *rx;
 	struct hfi1_ctxtdata *rcd;
 };
 
@@ -36,7 +35,8 @@ struct hfi1_netdev_rxq {
 #define NUM_NETDEV_MAP_ENTRIES HFI1_MAX_NETDEV_CTXTS
 
 /**
- * struct hfi1_netdev_priv: data required to setup and run HFI netdev.
+ * struct hfi1_netdev_rx: data required to setup and run HFI netdev.
+ * @rx_napi:	the dummy netdevice to support "polling" the receive contexts
  * @dd:		hfi1_devdata
  * @rxq:	pointer to dummy netdev receive queues.
  * @num_rx_q:	number of receive queues
@@ -48,7 +48,8 @@ struct hfi1_netdev_rxq {
  * @netdevs:	atomic counter of netdevs using dummy netdev.
  *		When 0 receive queues will be freed.
  */
-struct hfi1_netdev_priv {
+struct hfi1_netdev_rx {
+	struct net_device rx_napi;
 	struct hfi1_devdata *dd;
 	struct hfi1_netdev_rxq *rxq;
 	int num_rx_q;
@@ -61,41 +62,27 @@ struct hfi1_netdev_priv {
 };
 
 static inline
-struct hfi1_netdev_priv *hfi1_netdev_priv(struct net_device *dev)
-{
-	return (struct hfi1_netdev_priv *)&dev[1];
-}
-
-static inline
 int hfi1_netdev_ctxt_count(struct hfi1_devdata *dd)
 {
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
-
-	return priv->num_rx_q;
+	return dd->netdev_rx->num_rx_q;
 }
 
 static inline
 struct hfi1_ctxtdata *hfi1_netdev_get_ctxt(struct hfi1_devdata *dd, int ctxt)
 {
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
-
-	return priv->rxq[ctxt].rcd;
+	return dd->netdev_rx->rxq[ctxt].rcd;
 }
 
 static inline
 int hfi1_netdev_get_free_rmt_idx(struct hfi1_devdata *dd)
 {
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
-
-	return priv->rmt_start;
+	return dd->netdev_rx->rmt_start;
 }
 
 static inline
 void hfi1_netdev_set_free_rmt_idx(struct hfi1_devdata *dd, int rmt_idx)
 {
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
-
-	priv->rmt_start = rmt_idx;
+	dd->netdev_rx->rmt_start = rmt_idx;
 }
 
 u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,
@@ -105,8 +92,8 @@ u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,
 void hfi1_netdev_disable_queues(struct hfi1_devdata *dd);
 int hfi1_netdev_rx_init(struct hfi1_devdata *dd);
 int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd);
-int hfi1_netdev_alloc(struct hfi1_devdata *dd);
-void hfi1_netdev_free(struct hfi1_devdata *dd);
+int hfi1_alloc_rx(struct hfi1_devdata *dd);
+void hfi1_free_rx(struct hfi1_devdata *dd);
 int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data);
 void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id);
 void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id);
diff --git a/drivers/infiniband/hw/hfi1/netdev_rx.c b/drivers/infiniband/hw/hfi1/netdev_rx.c
index c1fa53d..f8995f3 100644
--- a/drivers/infiniband/hw/hfi1/netdev_rx.c
+++ b/drivers/infiniband/hw/hfi1/netdev_rx.c
@@ -17,11 +17,11 @@
 #include <linux/etherdevice.h>
 #include <rdma/ib_verbs.h>
 
-static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_priv *priv,
+static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_rx *rx,
 				  struct hfi1_ctxtdata *uctxt)
 {
 	unsigned int rcvctrl_ops;
-	struct hfi1_devdata *dd = priv->dd;
+	struct hfi1_devdata *dd = rx->dd;
 	int ret;
 
 	uctxt->rhf_rcv_function_map = netdev_rhf_rcv_functions;
@@ -118,11 +118,11 @@ static void hfi1_netdev_deallocate_ctxt(struct hfi1_devdata *dd,
 	hfi1_free_ctxt(uctxt);
 }
 
-static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_priv *priv,
+static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_rx *rx,
 				  struct hfi1_ctxtdata **ctxt)
 {
 	int rc;
-	struct hfi1_devdata *dd = priv->dd;
+	struct hfi1_devdata *dd = rx->dd;
 
 	rc = hfi1_netdev_allocate_ctxt(dd, ctxt);
 	if (rc) {
@@ -130,7 +130,7 @@ static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_priv *priv,
 		return rc;
 	}
 
-	rc = hfi1_netdev_setup_ctxt(priv, *ctxt);
+	rc = hfi1_netdev_setup_ctxt(rx, *ctxt);
 	if (rc) {
 		dd_dev_err(dd, "netdev ctxt setup failed %d\n", rc);
 		hfi1_netdev_deallocate_ctxt(dd, *ctxt);
@@ -183,31 +183,31 @@ u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,
 		    (u32)HFI1_MAX_NETDEV_CTXTS);
 }
 
-static int hfi1_netdev_rxq_init(struct net_device *dev)
+static int hfi1_netdev_rxq_init(struct hfi1_netdev_rx *rx)
 {
 	int i;
 	int rc;
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev);
-	struct hfi1_devdata *dd = priv->dd;
+	struct hfi1_devdata *dd = rx->dd;
+	struct net_device *dev = &rx->rx_napi;
 
-	priv->num_rx_q = dd->num_netdev_contexts;
-	priv->rxq = kcalloc_node(priv->num_rx_q, sizeof(struct hfi1_netdev_rxq),
-				 GFP_KERNEL, dd->node);
+	rx->num_rx_q = dd->num_netdev_contexts;
+	rx->rxq = kcalloc_node(rx->num_rx_q, sizeof(*rx->rxq),
+			       GFP_KERNEL, dd->node);
 
-	if (!priv->rxq) {
+	if (!rx->rxq) {
 		dd_dev_err(dd, "Unable to allocate netdev queue data\n");
 		return (-ENOMEM);
 	}
 
-	for (i = 0; i < priv->num_rx_q; i++) {
-		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
+	for (i = 0; i < rx->num_rx_q; i++) {
+		struct hfi1_netdev_rxq *rxq = &rx->rxq[i];
 
-		rc = hfi1_netdev_allot_ctxt(priv, &rxq->rcd);
+		rc = hfi1_netdev_allot_ctxt(rx, &rxq->rcd);
 		if (rc)
 			goto bail_context_irq_failure;
 
 		hfi1_rcd_get(rxq->rcd);
-		rxq->priv = priv;
+		rxq->rx = rx;
 		rxq->rcd->napi = &rxq->napi;
 		dd_dev_info(dd, "Setting rcv queue %d napi to context %d\n",
 			    i, rxq->rcd->ctxt);
@@ -227,7 +227,7 @@ static int hfi1_netdev_rxq_init(struct net_device *dev)
 bail_context_irq_failure:
 	dd_dev_err(dd, "Unable to allot receive context\n");
 	for (; i >= 0; i--) {
-		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
+		struct hfi1_netdev_rxq *rxq = &rx->rxq[i];
 
 		if (rxq->rcd) {
 			hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);
@@ -235,20 +235,19 @@ static int hfi1_netdev_rxq_init(struct net_device *dev)
 			rxq->rcd = NULL;
 		}
 	}
-	kfree(priv->rxq);
-	priv->rxq = NULL;
+	kfree(rx->rxq);
+	rx->rxq = NULL;
 
 	return rc;
 }
 
-static void hfi1_netdev_rxq_deinit(struct net_device *dev)
+static void hfi1_netdev_rxq_deinit(struct hfi1_netdev_rx *rx)
 {
 	int i;
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev);
-	struct hfi1_devdata *dd = priv->dd;
+	struct hfi1_devdata *dd = rx->dd;
 
-	for (i = 0; i < priv->num_rx_q; i++) {
-		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
+	for (i = 0; i < rx->num_rx_q; i++) {
+		struct hfi1_netdev_rxq *rxq = &rx->rxq[i];
 
 		netif_napi_del(&rxq->napi);
 		hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);
@@ -256,41 +255,41 @@ static void hfi1_netdev_rxq_deinit(struct net_device *dev)
 		rxq->rcd = NULL;
 	}
 
-	kfree(priv->rxq);
-	priv->rxq = NULL;
-	priv->num_rx_q = 0;
+	kfree(rx->rxq);
+	rx->rxq = NULL;
+	rx->num_rx_q = 0;
 }
 
-static void enable_queues(struct hfi1_netdev_priv *priv)
+static void enable_queues(struct hfi1_netdev_rx *rx)
 {
 	int i;
 
-	for (i = 0; i < priv->num_rx_q; i++) {
-		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
+	for (i = 0; i < rx->num_rx_q; i++) {
+		struct hfi1_netdev_rxq *rxq = &rx->rxq[i];
 
-		dd_dev_info(priv->dd, "enabling queue %d on context %d\n", i,
+		dd_dev_info(rx->dd, "enabling queue %d on context %d\n", i,
 			    rxq->rcd->ctxt);
 		napi_enable(&rxq->napi);
-		hfi1_rcvctrl(priv->dd,
+		hfi1_rcvctrl(rx->dd,
 			     HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB,
 			     rxq->rcd);
 	}
 }
 
-static void disable_queues(struct hfi1_netdev_priv *priv)
+static void disable_queues(struct hfi1_netdev_rx *rx)
 {
 	int i;
 
-	msix_netdev_synchronize_irq(priv->dd);
+	msix_netdev_synchronize_irq(rx->dd);
 
-	for (i = 0; i < priv->num_rx_q; i++) {
-		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
+	for (i = 0; i < rx->num_rx_q; i++) {
+		struct hfi1_netdev_rxq *rxq = &rx->rxq[i];
 
-		dd_dev_info(priv->dd, "disabling queue %d on context %d\n", i,
+		dd_dev_info(rx->dd, "disabling queue %d on context %d\n", i,
 			    rxq->rcd->ctxt);
 
 		/* wait for napi if it was scheduled */
-		hfi1_rcvctrl(priv->dd,
+		hfi1_rcvctrl(rx->dd,
 			     HFI1_RCVCTRL_CTXT_DIS | HFI1_RCVCTRL_INTRAVAIL_DIS,
 			     rxq->rcd);
 		napi_synchronize(&rxq->napi);
@@ -307,15 +306,14 @@ static void disable_queues(struct hfi1_netdev_priv *priv)
  */
 int hfi1_netdev_rx_init(struct hfi1_devdata *dd)
 {
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+	struct hfi1_netdev_rx *rx = dd->netdev_rx;
 	int res;
 
-	if (atomic_fetch_inc(&priv->netdevs))
+	if (atomic_fetch_inc(&rx->netdevs))
 		return 0;
 
 	mutex_lock(&hfi1_mutex);
-	init_dummy_netdev(dd->dummy_netdev);
-	res = hfi1_netdev_rxq_init(dd->dummy_netdev);
+	res = hfi1_netdev_rxq_init(rx);
 	mutex_unlock(&hfi1_mutex);
 	return res;
 }
@@ -328,12 +326,12 @@ int hfi1_netdev_rx_init(struct hfi1_devdata *dd)
  */
 int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd)
 {
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+	struct hfi1_netdev_rx *rx = dd->netdev_rx;
 
 	/* destroy the RX queues only if it is the last netdev going away */
-	if (atomic_fetch_add_unless(&priv->netdevs, -1, 0) == 1) {
+	if (atomic_fetch_add_unless(&rx->netdevs, -1, 0) == 1) {
 		mutex_lock(&hfi1_mutex);
-		hfi1_netdev_rxq_deinit(dd->dummy_netdev);
+		hfi1_netdev_rxq_deinit(rx);
 		mutex_unlock(&hfi1_mutex);
 	}
 
@@ -341,43 +339,46 @@ int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd)
 }
 
 /**
- * hfi1_netdev_alloc - Allocates netdev and private data. It is required
- * because RMT index and MSI-X interrupt can be set only
- * during driver initialization.
- *
+ * hfi1_alloc_rx - Allocates the rx support structure
  * @dd: hfi1 dev data
+ *
+ * Allocate the rx structure to support gathering the receive
+ * resources and the dummy netdev.
+ *
+ * Updates dd struct pointer upon success.
+ *
+ * Return: 0 (success) -error on failure
+ *
  */
-int hfi1_netdev_alloc(struct hfi1_devdata *dd)
+int hfi1_alloc_rx(struct hfi1_devdata *dd)
 {
-	struct hfi1_netdev_priv *priv;
-	const int netdev_size = sizeof(*dd->dummy_netdev) +
-		sizeof(struct hfi1_netdev_priv);
+	struct hfi1_netdev_rx *rx;
 
-	dd_dev_info(dd, "allocating netdev size %d\n", netdev_size);
-	dd->dummy_netdev = kcalloc_node(1, netdev_size, GFP_KERNEL, dd->node);
+	dd_dev_info(dd, "allocating rx size %ld\n", sizeof(*rx));
+	rx = kzalloc_node(sizeof(*rx), GFP_KERNEL, dd->node);
 
-	if (!dd->dummy_netdev)
+	if (!rx)
 		return -ENOMEM;
+	rx->dd = dd;
+	init_dummy_netdev(&rx->rx_napi);
 
-	priv = hfi1_netdev_priv(dd->dummy_netdev);
-	priv->dd = dd;
-	xa_init(&priv->dev_tbl);
-	atomic_set(&priv->enabled, 0);
-	atomic_set(&priv->netdevs, 0);
+	xa_init(&rx->dev_tbl);
+	atomic_set(&rx->enabled, 0);
+	atomic_set(&rx->netdevs, 0);
+	dd->netdev_rx = rx;
 
 	return 0;
 }
 
-void hfi1_netdev_free(struct hfi1_devdata *dd)
+void hfi1_free_rx(struct hfi1_devdata *dd)
 {
-	if (dd->dummy_netdev) {
-		struct hfi1_netdev_priv *priv =
-			hfi1_netdev_priv(dd->dummy_netdev);
-
-		dd_dev_info(dd, "hfi1 netdev freed\n");
-		xa_destroy(&priv->dev_tbl);
-		kfree(dd->dummy_netdev);
-		dd->dummy_netdev = NULL;
+	if (dd->netdev_rx) {
+		struct hfi1_netdev_rx *rx = dd->netdev_rx;
+
+		dd_dev_info(dd, "hfi1 rx freed\n");
+		xa_destroy(&rx->dev_tbl);
+		kfree(dd->netdev_rx);
+		dd->netdev_rx = NULL;
 	}
 }
 
@@ -392,33 +393,33 @@ void hfi1_netdev_free(struct hfi1_devdata *dd)
  */
 void hfi1_netdev_enable_queues(struct hfi1_devdata *dd)
 {
-	struct hfi1_netdev_priv *priv;
+	struct hfi1_netdev_rx *rx;
 
-	if (!dd->dummy_netdev)
+	if (!dd->netdev_rx)
 		return;
 
-	priv = hfi1_netdev_priv(dd->dummy_netdev);
-	if (atomic_fetch_inc(&priv->enabled))
+	rx = dd->netdev_rx;
+	if (atomic_fetch_inc(&rx->enabled))
 		return;
 
 	mutex_lock(&hfi1_mutex);
-	enable_queues(priv);
+	enable_queues(rx);
 	mutex_unlock(&hfi1_mutex);
 }
 
 void hfi1_netdev_disable_queues(struct hfi1_devdata *dd)
 {
-	struct hfi1_netdev_priv *priv;
+	struct hfi1_netdev_rx *rx;
 
-	if (!dd->dummy_netdev)
+	if (!dd->netdev_rx)
 		return;
 
-	priv = hfi1_netdev_priv(dd->dummy_netdev);
-	if (atomic_dec_if_positive(&priv->enabled))
+	rx = dd->netdev_rx;
+	if (atomic_dec_if_positive(&rx->enabled))
 		return;
 
 	mutex_lock(&hfi1_mutex);
-	disable_queues(priv);
+	disable_queues(rx);
 	mutex_unlock(&hfi1_mutex);
 }
 
@@ -434,9 +435,9 @@ void hfi1_netdev_disable_queues(struct hfi1_devdata *dd)
  */
 int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data)
 {
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+	struct hfi1_netdev_rx *rx = dd->netdev_rx;
 
-	return xa_insert(&priv->dev_tbl, id, data, GFP_NOWAIT);
+	return xa_insert(&rx->dev_tbl, id, data, GFP_NOWAIT);
 }
 
 /**
@@ -448,9 +449,9 @@ int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data)
  */
 void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id)
 {
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+	struct hfi1_netdev_rx *rx = dd->netdev_rx;
 
-	return xa_erase(&priv->dev_tbl, id);
+	return xa_erase(&rx->dev_tbl, id);
 }
 
 /**
@@ -461,9 +462,9 @@ void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id)
  */
 void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id)
 {
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+	struct hfi1_netdev_rx *rx = dd->netdev_rx;
 
-	return xa_load(&priv->dev_tbl, id);
+	return xa_load(&rx->dev_tbl, id);
 }
 
 /**
@@ -474,11 +475,11 @@ void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id)
  */
 void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id)
 {
-	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
+	struct hfi1_netdev_rx *rx = dd->netdev_rx;
 	unsigned long index = *start_id;
 	void *ret;
 
-	ret = xa_find(&priv->dev_tbl, &index, UINT_MAX, XA_PRESENT);
+	ret = xa_find(&rx->dev_tbl, &index, UINT_MAX, XA_PRESENT);
 	*start_id = (int)index;
 	return ret;
 }
-- 
1.8.3.1

[PATCH for-next v2 09/10] IB/hfi1: Remove unused function

[dennis.dalessandro]

From: Kaike Wan <kaike.wan@intel.com>

Remove the unused function sdma_iowait_schedule().

Fixes: 7724105686e7 ("IB/hfi1: add driver files")
Reviewed-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
Signed-off-by: Kaike Wan <kaike.wan@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>

---
Changes from v0: Fix Kaike's email address
---
 drivers/infiniband/hw/hfi1/sdma.h | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/drivers/infiniband/hw/hfi1/sdma.h b/drivers/infiniband/hw/hfi1/sdma.h
index 7a85119..f57d552 100644
--- a/drivers/infiniband/hw/hfi1/sdma.h
+++ b/drivers/infiniband/hw/hfi1/sdma.h
@@ -907,24 +907,6 @@ static inline unsigned sdma_progress(struct sdma_engine *sde, unsigned seq,
 	return 0;
 }
 
-/**
- * sdma_iowait_schedule() - initialize wait structure
- * @sde: sdma_engine to schedule
- * @wait: wait struct to schedule
- *
- * This function initializes the iowait
- * structure embedded in the QP or PQ.
- *
- */
-static inline void sdma_iowait_schedule(
-	struct sdma_engine *sde,
-	struct iowait *wait)
-{
-	struct hfi1_pportdata *ppd = sde->dd->pport;
-
-	iowait_schedule(wait, ppd->hfi1_wq, sde->cpu);
-}
-
 /* for use by interrupt handling */
 void sdma_engine_error(struct sdma_engine *sde, u64 status);
 void sdma_engine_interrupt(struct sdma_engine *sde, u64 status);
-- 
1.8.3.1

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Leon Romanovsky]

On Mon, Mar 29, 2021 at 09:54:12AM -0400, dennis.dalessandro@cornelisnetworks.com wrote:
> From: Kaike Wan <kaike.wan@intel.com>
> 
> This is a follow on patch to add a phys_mtu field to the
> ib_port_attr structure to indicate the maximum physical MTU
> the underlying device supports.
> 
> Extends the following:
> commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's upper limit")
> 
> Reviewed-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
> Signed-off-by: Kaike Wan <kaike.wan@intel.com>
> Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> ---
>  drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
>  drivers/infiniband/hw/cxgb4/provider.c          |  1 +
>  drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
>  drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
>  drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
>  drivers/infiniband/hw/mlx4/main.c               |  1 +
>  drivers/infiniband/hw/mlx5/mad.c                |  1 +
>  drivers/infiniband/hw/mlx5/main.c               |  2 ++
>  drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
>  drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
>  drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
>  drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
>  drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
>  drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
>  include/rdma/ib_verbs.h                         | 17 -----------------
>  16 files changed, 16 insertions(+), 18 deletions(-)

But why? What will it give us that almost all drivers have same 
props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?

Thanks

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Jason Gunthorpe]

On Thu, Apr 01, 2021 at 11:42:41AM +0300, Leon Romanovsky wrote:
> On Mon, Mar 29, 2021 at 09:54:12AM -0400, dennis.dalessandro@cornelisnetworks.com wrote:
> > From: Kaike Wan <kaike.wan@intel.com>
> > 
> > This is a follow on patch to add a phys_mtu field to the
> > ib_port_attr structure to indicate the maximum physical MTU
> > the underlying device supports.
> > 
> > Extends the following:
> > commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's upper limit")
> > 
> > Reviewed-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
> > Signed-off-by: Kaike Wan <kaike.wan@intel.com>
> > Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> >  drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
> >  drivers/infiniband/hw/cxgb4/provider.c          |  1 +
> >  drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
> >  drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
> >  drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
> >  drivers/infiniband/hw/mlx4/main.c               |  1 +
> >  drivers/infiniband/hw/mlx5/mad.c                |  1 +
> >  drivers/infiniband/hw/mlx5/main.c               |  2 ++
> >  drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
> >  drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
> >  drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
> >  drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
> >  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
> >  drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
> >  drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
> >  include/rdma/ib_verbs.h                         | 17 -----------------
> >  16 files changed, 16 insertions(+), 18 deletions(-)
> 
> But why? What will it give us that almost all drivers have same 
> props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?

Dennis?

Jason

Re: [PATCH for-next 00/10] Fixes for 5.13

[Jason Gunthorpe]

On Mon, Mar 29, 2021 at 09:54:06AM -0400, dennis.dalessandro@cornelisnetworks.com wrote:
> From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> 
> Here are some fixes, clean-up and debugging type patches for-next. There are
> some changs to the way AIP/VNIC netdevs are used and a timeout handler is added
> for rdma_netdev. 
> 
> The MTU patch from Kaike will to add a physical MTU to query_port is also of
> particualr note.
> 
> Kaike Wan (2):
>   rdma: Set physical MTU for query_port function

I left this one for now

>   IB/hfi1: Rework AIP and VNIC dummy netdev usage

And this one didn't apply, it is on top of the -rc xa_destroy patches
that I'm still waiting to know if they are real bugs or not

>   IB/hfi1: Remove unused function
> 
> Mike Marciniszyn (8):
>   IB/hfi1: Add AIP tx traces
>   IB/{ipoib,hfi1}: Add a timeout handler for rdma_netdev
>   IB/hfi1: Correct oversized ring allocation
>   IB/hfi1: Use napi_schedule_irqoff() for tx napi
>   IB/hfi1: Remove indirect call to hfi1_ipoib_send_dma()
>   IB/hfi1: Add additional usdma traces
>   IB/hfi1: Use kzalloc() for mmu_rb_handler allocation

Applied these to for-rc, thanks

Jason

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Dennis Dalessandro]

On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
> On Mon, Mar 29, 2021 at 09:54:12AM -0400, dennis.dalessandro@cornelisnetworks.com wrote:
>> From: Kaike Wan <kaike.wan@intel.com>
>>
>> This is a follow on patch to add a phys_mtu field to the
>> ib_port_attr structure to indicate the maximum physical MTU
>> the underlying device supports.
>>
>> Extends the following:
>> commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's upper limit")
>>
>> Reviewed-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
>> Signed-off-by: Kaike Wan <kaike.wan@intel.com>
>> Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
>> ---
>>   drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
>>   drivers/infiniband/hw/cxgb4/provider.c          |  1 +
>>   drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
>>   drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
>>   drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
>>   drivers/infiniband/hw/mlx4/main.c               |  1 +
>>   drivers/infiniband/hw/mlx5/mad.c                |  1 +
>>   drivers/infiniband/hw/mlx5/main.c               |  2 ++
>>   drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
>>   drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
>>   drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
>>   drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
>>   drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
>>   drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
>>   drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
>>   include/rdma/ib_verbs.h                         | 17 -----------------
>>   16 files changed, 16 insertions(+), 18 deletions(-)
> 
> But why? What will it give us that almost all drivers have same
> props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
> 

Almost is not all. Alternative idea to convey this? Seemed like a 
sensible thing to at least have support for but open to other approaches.

-Denny

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Leon Romanovsky]

On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
> On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
> > On Mon, Mar 29, 2021 at 09:54:12AM -0400, dennis.dalessandro@cornelisnetworks.com wrote:
> > > From: Kaike Wan <kaike.wan@intel.com>
> > > 
> > > This is a follow on patch to add a phys_mtu field to the
> > > ib_port_attr structure to indicate the maximum physical MTU
> > > the underlying device supports.
> > > 
> > > Extends the following:
> > > commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's upper limit")
> > > 
> > > Reviewed-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
> > > Signed-off-by: Kaike Wan <kaike.wan@intel.com>
> > > Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> > > ---
> > >   drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
> > >   drivers/infiniband/hw/cxgb4/provider.c          |  1 +
> > >   drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
> > >   drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
> > >   drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
> > >   drivers/infiniband/hw/mlx4/main.c               |  1 +
> > >   drivers/infiniband/hw/mlx5/mad.c                |  1 +
> > >   drivers/infiniband/hw/mlx5/main.c               |  2 ++
> > >   drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
> > >   drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
> > >   drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
> > >   drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
> > >   drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
> > >   drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
> > >   drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
> > >   include/rdma/ib_verbs.h                         | 17 -----------------
> > >   16 files changed, 16 insertions(+), 18 deletions(-)
> > 
> > But why? What will it give us that almost all drivers have same
> > props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
> > 
> 
> Almost is not all. Alternative idea to convey this? Seemed like a sensible
> thing to at least have support for but open to other approaches.

What about leave it as is? 

I'm struggling to get the rationale behind this patch., the code already works
and set the phys_mtu correctly, isn't it?

Thanks

> 
> -Denny

Re: [PATCH for-next 00/10] Fixes for 5.13

[Dennis Dalessandro]

On 4/7/2021 7:21 PM, Jason Gunthorpe wrote:
> On Mon, Mar 29, 2021 at 09:54:06AM -0400, dennis.dalessandro@cornelisnetworks.com wrote:
>> From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
>>
>> Here are some fixes, clean-up and debugging type patches for-next. There are
>> some changs to the way AIP/VNIC netdevs are used and a timeout handler is added
>> for rdma_netdev.
>>
>> The MTU patch from Kaike will to add a physical MTU to query_port is also of
>> particualr note.
>>
>> Kaike Wan (2):
>>    rdma: Set physical MTU for query_port function
> 
> I left this one for now
> 
>>    IB/hfi1: Rework AIP and VNIC dummy netdev usage
> 
> And this one didn't apply, it is on top of the -rc xa_destroy patches
> that I'm still waiting to know if they are real bugs or not

Fair enough. We will certainly rework if we drop those other two.

>>    IB/hfi1: Remove unused function
>>
>> Mike Marciniszyn (8):
>>    IB/hfi1: Add AIP tx traces
>>    IB/{ipoib,hfi1}: Add a timeout handler for rdma_netdev
>>    IB/hfi1: Correct oversized ring allocation
>>    IB/hfi1: Use napi_schedule_irqoff() for tx napi
>>    IB/hfi1: Remove indirect call to hfi1_ipoib_send_dma()
>>    IB/hfi1: Add additional usdma traces
>>    IB/hfi1: Use kzalloc() for mmu_rb_handler allocation
> 
> Applied these to for-rc, thanks

for-next?

-Denny

Re: [PATCH for-next 00/10] Fixes for 5.13

[Jason Gunthorpe]

On Thu, Apr 08, 2021 at 08:19:22AM -0400, Dennis Dalessandro wrote:

> > > Mike Marciniszyn (8):
> > >    IB/hfi1: Add AIP tx traces
> > >    IB/{ipoib,hfi1}: Add a timeout handler for rdma_netdev
> > >    IB/hfi1: Correct oversized ring allocation
> > >    IB/hfi1: Use napi_schedule_irqoff() for tx napi
> > >    IB/hfi1: Remove indirect call to hfi1_ipoib_send_dma()
> > >    IB/hfi1: Add additional usdma traces
> > >    IB/hfi1: Use kzalloc() for mmu_rb_handler allocation
> > 
> > Applied these to for-rc, thanks
> 
> for-next?
 

Yes, typo here

Jason

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Dennis Dalessandro]

On 4/8/2021 8:14 AM, Leon Romanovsky wrote:
> On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
>> On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
>>> On Mon, Mar 29, 2021 at 09:54:12AM -0400, dennis.dalessandro@cornelisnetworks.com wrote:
>>>> From: Kaike Wan <kaike.wan@intel.com>
>>>>
>>>> This is a follow on patch to add a phys_mtu field to the
>>>> ib_port_attr structure to indicate the maximum physical MTU
>>>> the underlying device supports.
>>>>
>>>> Extends the following:
>>>> commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's upper limit")
>>>>
>>>> Reviewed-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
>>>> Signed-off-by: Kaike Wan <kaike.wan@intel.com>
>>>> Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
>>>> ---
>>>>    drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
>>>>    drivers/infiniband/hw/cxgb4/provider.c          |  1 +
>>>>    drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
>>>>    drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
>>>>    drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
>>>>    drivers/infiniband/hw/mlx4/main.c               |  1 +
>>>>    drivers/infiniband/hw/mlx5/mad.c                |  1 +
>>>>    drivers/infiniband/hw/mlx5/main.c               |  2 ++
>>>>    drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
>>>>    drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
>>>>    drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
>>>>    drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
>>>>    drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
>>>>    drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
>>>>    drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
>>>>    include/rdma/ib_verbs.h                         | 17 -----------------
>>>>    16 files changed, 16 insertions(+), 18 deletions(-)
>>>
>>> But why? What will it give us that almost all drivers have same
>>> props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
>>>
>>
>> Almost is not all. Alternative idea to convey this? Seemed like a sensible
>> thing to at least have support for but open to other approaches.
> 
> What about leave it as is?
> 
> I'm struggling to get the rationale behind this patch., the code already works
> and set the phys_mtu correctly, isn't it?

I see what you are saying now. Kaike, correct me if I'm wrong, but the 
intent of this patch is just to make everything behave the same in the 
sense that a device could have a different physical MTU. The field got 
added to the ib_port_attr previously so this is giving it an initial 
value vs leaving it unset.

-Denny

Re: [PATCH for-next 10/10] IB/hfi1: Rework AIP and VNIC dummy netdev usage

[Jason Gunthorpe]

On Mon, Mar 29, 2021 at 09:54:16AM -0400, dennis.dalessandro@cornelisnetworks.com wrote:
> From: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
> 
> All other users of the dummy netdevice embed the netdev in other
> structures:
> 
> init_dummy_netdev(&mal->dummy_dev);
> init_dummy_netdev(&eth->dummy_dev);
> init_dummy_netdev(&ar->napi_dev);
> init_dummy_netdev(&irq_grp->napi_ndev);
> init_dummy_netdev(&wil->napi_ndev);
> init_dummy_netdev(&trans_pcie->napi_dev);
> init_dummy_netdev(&dev->napi_dev);
> init_dummy_netdev(&bus->mux_dev);
> 
> The AIP and VNIC implementation turns that model inside out and used a
> kfree() to free what appears to be a netdev struct when in reality, it is
> a struct that enbodies the rx state as well as the dummy netdev used to
> support napi_poll across disparate receive contexts.  The relationship is
> infered by the odd allocation:
> 
> 	const int netdev_size = sizeof(*dd->dummy_netdev) +
> 		sizeof(struct hfi1_netdev_priv);
> 	<snip>
> 	dd->dummy_netdev = kcalloc_node(1, netdev_size, GFP_KERNEL, dd->node);
> 
> 
> Correct the issue by:
> - Correctly naming the alloc and free functions
> - Renaming hfi1_netdev_priv to hfi1_netdev_rx
> - Replacing dd dummy_netdev with a netdev_rx pointer
> - Embedding the net_device in hfi1_netdev_rx
> - Moving the init_dummy_netdev to the alloc routine
> - Adjusting wrappers to fit the new model
> 
> Fixes: 6991abcb993c ("IB/hfi1: Add functions to receive accelerated ipoib packets")
> Reviewed-by: Kaike Wan <kaike.wan@intel.com>
> Signed-off-by: Mike Marciniszyn <mike.marciniszyn@cornelisnetworks.com>
> Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> ---
>  drivers/infiniband/hw/hfi1/chip.c      |   6 +-
>  drivers/infiniband/hw/hfi1/hfi.h       |   4 +-
>  drivers/infiniband/hw/hfi1/init.c      |   2 +-
>  drivers/infiniband/hw/hfi1/netdev.h    |  39 +++-----
>  drivers/infiniband/hw/hfi1/netdev_rx.c | 177 +++++++++++++++++----------------
>  5 files changed, 108 insertions(+), 120 deletions(-)

Applied to for-next

Thanks,
Jason

RE: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Wan, Kaike]

> -----Original Message-----
> From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> Sent: Thursday, April 08, 2021 8:31 AM
> To: Leon Romanovsky <leon@kernel.org>
> Cc: dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org; Wan,
> Kaike <kaike.wan@intel.com>
> Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port
> function
> 
> On 4/8/2021 8:14 AM, Leon Romanovsky wrote:
> > On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
> >> On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
> >>> On Mon, Mar 29, 2021 at 09:54:12AM -0400,
> dennis.dalessandro@cornelisnetworks.com wrote:
> >>>> From: Kaike Wan <kaike.wan@intel.com>
> >>>>
> >>>> This is a follow on patch to add a phys_mtu field to the
> >>>> ib_port_attr structure to indicate the maximum physical MTU the
> >>>> underlying device supports.
> >>>>
> >>>> Extends the following:
> >>>> commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's
> >>>> upper limit")
> >>>>
> >>>> Reviewed-by: Mike Marciniszyn
> >>>> <mike.marciniszyn@cornelisnetworks.com>
> >>>> Signed-off-by: Kaike Wan <kaike.wan@intel.com>
> >>>> Signed-off-by: Dennis Dalessandro
> >>>> <dennis.dalessandro@cornelisnetworks.com>
> >>>> ---
> >>>>    drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
> >>>>    drivers/infiniband/hw/cxgb4/provider.c          |  1 +
> >>>>    drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
> >>>>    drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
> >>>>    drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
> >>>>    drivers/infiniband/hw/mlx4/main.c               |  1 +
> >>>>    drivers/infiniband/hw/mlx5/mad.c                |  1 +
> >>>>    drivers/infiniband/hw/mlx5/main.c               |  2 ++
> >>>>    drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
> >>>>    drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
> >>>>    drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
> >>>>    drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
> >>>>    drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
> >>>>    drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
> >>>>    drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
> >>>>    include/rdma/ib_verbs.h                         | 17 -----------------
> >>>>    16 files changed, 16 insertions(+), 18 deletions(-)
> >>>
> >>> But why? What will it give us that almost all drivers have same
> >>> props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
> >>>
> >>
> >> Almost is not all. Alternative idea to convey this? Seemed like a
> >> sensible thing to at least have support for but open to other approaches.
> >
> > What about leave it as is?
> >
> > I'm struggling to get the rationale behind this patch., the code
> > already works and set the phys_mtu correctly, isn't it?
> 
> I see what you are saying now. Kaike, correct me if I'm wrong, but the intent
> of this patch is just to make everything behave the same in the sense that a
> device could have a different physical MTU. The field got added to the
> ib_port_attr previously so this is giving it an initial value vs leaving it unset.
[Wan, Kaike]  Correct.
  
> 
> -Denny

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Leon Romanovsky]

On Mon, Apr 19, 2021 at 12:20:33PM +0000, Wan, Kaike wrote:
> 
> 
> > -----Original Message-----
> > From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> > Sent: Thursday, April 08, 2021 8:31 AM
> > To: Leon Romanovsky <leon@kernel.org>
> > Cc: dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org; Wan,
> > Kaike <kaike.wan@intel.com>
> > Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port
> > function
> > 
> > On 4/8/2021 8:14 AM, Leon Romanovsky wrote:
> > > On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
> > >> On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
> > >>> On Mon, Mar 29, 2021 at 09:54:12AM -0400,
> > dennis.dalessandro@cornelisnetworks.com wrote:
> > >>>> From: Kaike Wan <kaike.wan@intel.com>
> > >>>>
> > >>>> This is a follow on patch to add a phys_mtu field to the
> > >>>> ib_port_attr structure to indicate the maximum physical MTU the
> > >>>> underlying device supports.
> > >>>>
> > >>>> Extends the following:
> > >>>> commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's
> > >>>> upper limit")
> > >>>>
> > >>>> Reviewed-by: Mike Marciniszyn
> > >>>> <mike.marciniszyn@cornelisnetworks.com>
> > >>>> Signed-off-by: Kaike Wan <kaike.wan@intel.com>
> > >>>> Signed-off-by: Dennis Dalessandro
> > >>>> <dennis.dalessandro@cornelisnetworks.com>
> > >>>> ---
> > >>>>    drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
> > >>>>    drivers/infiniband/hw/cxgb4/provider.c          |  1 +
> > >>>>    drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
> > >>>>    drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
> > >>>>    drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
> > >>>>    drivers/infiniband/hw/mlx4/main.c               |  1 +
> > >>>>    drivers/infiniband/hw/mlx5/mad.c                |  1 +
> > >>>>    drivers/infiniband/hw/mlx5/main.c               |  2 ++
> > >>>>    drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
> > >>>>    drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
> > >>>>    drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
> > >>>>    drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
> > >>>>    drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
> > >>>>    drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
> > >>>>    drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
> > >>>>    include/rdma/ib_verbs.h                         | 17 -----------------
> > >>>>    16 files changed, 16 insertions(+), 18 deletions(-)
> > >>>
> > >>> But why? What will it give us that almost all drivers have same
> > >>> props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
> > >>>
> > >>
> > >> Almost is not all. Alternative idea to convey this? Seemed like a
> > >> sensible thing to at least have support for but open to other approaches.
> > >
> > > What about leave it as is?
> > >
> > > I'm struggling to get the rationale behind this patch., the code
> > > already works and set the phys_mtu correctly, isn't it?
> > 
> > I see what you are saying now. Kaike, correct me if I'm wrong, but the intent
> > of this patch is just to make everything behave the same in the sense that a
> > device could have a different physical MTU. The field got added to the
> > ib_port_attr previously so this is giving it an initial value vs leaving it unset.
> [Wan, Kaike]  Correct.

No one is using this "phys_mtu" field, except one place in ipoib.

Thanks

>   
> > 
> > -Denny
> 

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Dennis Dalessandro]

On 4/19/2021 8:29 AM, Leon Romanovsky wrote:
> On Mon, Apr 19, 2021 at 12:20:33PM +0000, Wan, Kaike wrote:
>>
>>
>>> -----Original Message-----
>>> From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
>>> Sent: Thursday, April 08, 2021 8:31 AM
>>> To: Leon Romanovsky <leon@kernel.org>
>>> Cc: dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org; Wan,
>>> Kaike <kaike.wan@intel.com>
>>> Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port
>>> function
>>>
>>> On 4/8/2021 8:14 AM, Leon Romanovsky wrote:
>>>> On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
>>>>> On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
>>>>>> On Mon, Mar 29, 2021 at 09:54:12AM -0400,
>>> dennis.dalessandro@cornelisnetworks.com wrote:
>>>>>>> From: Kaike Wan <kaike.wan@intel.com>
>>>>>>>
>>>>>>> This is a follow on patch to add a phys_mtu field to the
>>>>>>> ib_port_attr structure to indicate the maximum physical MTU the
>>>>>>> underlying device supports.
>>>>>>>
>>>>>>> Extends the following:
>>>>>>> commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's
>>>>>>> upper limit")
>>>>>>>
>>>>>>> Reviewed-by: Mike Marciniszyn
>>>>>>> <mike.marciniszyn@cornelisnetworks.com>
>>>>>>> Signed-off-by: Kaike Wan <kaike.wan@intel.com>
>>>>>>> Signed-off-by: Dennis Dalessandro
>>>>>>> <dennis.dalessandro@cornelisnetworks.com>
>>>>>>> ---
>>>>>>>     drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
>>>>>>>     drivers/infiniband/hw/cxgb4/provider.c          |  1 +
>>>>>>>     drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
>>>>>>>     drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
>>>>>>>     drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
>>>>>>>     drivers/infiniband/hw/mlx4/main.c               |  1 +
>>>>>>>     drivers/infiniband/hw/mlx5/mad.c                |  1 +
>>>>>>>     drivers/infiniband/hw/mlx5/main.c               |  2 ++
>>>>>>>     drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
>>>>>>>     drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
>>>>>>>     drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
>>>>>>>     drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
>>>>>>>     drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
>>>>>>>     drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
>>>>>>>     drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
>>>>>>>     include/rdma/ib_verbs.h                         | 17 -----------------
>>>>>>>     16 files changed, 16 insertions(+), 18 deletions(-)
>>>>>>
>>>>>> But why? What will it give us that almost all drivers have same
>>>>>> props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
>>>>>>
>>>>>
>>>>> Almost is not all. Alternative idea to convey this? Seemed like a
>>>>> sensible thing to at least have support for but open to other approaches.
>>>>
>>>> What about leave it as is?
>>>>
>>>> I'm struggling to get the rationale behind this patch., the code
>>>> already works and set the phys_mtu correctly, isn't it?
>>>
>>> I see what you are saying now. Kaike, correct me if I'm wrong, but the intent
>>> of this patch is just to make everything behave the same in the sense that a
>>> device could have a different physical MTU. The field got added to the
>>> ib_port_attr previously so this is giving it an initial value vs leaving it unset.
>> [Wan, Kaike]  Correct.
> 
> No one is using this "phys_mtu" field, except one place in ipoib.

Today. I think it would be better to formalize the idea though and have 
a cleaner interface. Does this cause some problem?

-Denny

RE: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Wan, Kaike]

> -----Original Message-----
> From: Leon Romanovsky <leon@kernel.org>
> Sent: Monday, April 19, 2021 8:29 AM
> To: Wan, Kaike <kaike.wan@intel.com>
> Cc: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>;
> dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org
> Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port
> function
> 
> On Mon, Apr 19, 2021 at 12:20:33PM +0000, Wan, Kaike wrote:
> >
> >
> > > -----Original Message-----
> > > From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> > > Sent: Thursday, April 08, 2021 8:31 AM
> > > To: Leon Romanovsky <leon@kernel.org>
> > > Cc: dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org;
> > > Wan, Kaike <kaike.wan@intel.com>
> > > Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for
> > > query_port function
> > >
> > > On 4/8/2021 8:14 AM, Leon Romanovsky wrote:
> > > > On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
> > > >> On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
> > > >>> On Mon, Mar 29, 2021 at 09:54:12AM -0400,
> > > dennis.dalessandro@cornelisnetworks.com wrote:
> > > >>>> From: Kaike Wan <kaike.wan@intel.com>
> > > >>>>
> > > >>>> This is a follow on patch to add a phys_mtu field to the
> > > >>>> ib_port_attr structure to indicate the maximum physical MTU the
> > > >>>> underlying device supports.
> > > >>>>
> > > >>>> Extends the following:
> > > >>>> commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode
> > > >>>> MTU's upper limit")
> > > >>>>
> > > >>>> Reviewed-by: Mike Marciniszyn
> > > >>>> <mike.marciniszyn@cornelisnetworks.com>
> > > >>>> Signed-off-by: Kaike Wan <kaike.wan@intel.com>
> > > >>>> Signed-off-by: Dennis Dalessandro
> > > >>>> <dennis.dalessandro@cornelisnetworks.com>
> > > >>>> ---
> > > >>>>    drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
> > > >>>>    drivers/infiniband/hw/cxgb4/provider.c          |  1 +
> > > >>>>    drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
> > > >>>>    drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
> > > >>>>    drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
> > > >>>>    drivers/infiniband/hw/mlx4/main.c               |  1 +
> > > >>>>    drivers/infiniband/hw/mlx5/mad.c                |  1 +
> > > >>>>    drivers/infiniband/hw/mlx5/main.c               |  2 ++
> > > >>>>    drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
> > > >>>>    drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
> > > >>>>    drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
> > > >>>>    drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
> > > >>>>    drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
> > > >>>>    drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
> > > >>>>    drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
> > > >>>>    include/rdma/ib_verbs.h                         | 17 -----------------
> > > >>>>    16 files changed, 16 insertions(+), 18 deletions(-)
> > > >>>
> > > >>> But why? What will it give us that almost all drivers have same
> > > >>> props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
> > > >>>
> > > >>
> > > >> Almost is not all. Alternative idea to convey this? Seemed like a
> > > >> sensible thing to at least have support for but open to other
> approaches.
> > > >
> > > > What about leave it as is?
> > > >
> > > > I'm struggling to get the rationale behind this patch., the code
> > > > already works and set the phys_mtu correctly, isn't it?
> > >
> > > I see what you are saying now. Kaike, correct me if I'm wrong, but
> > > the intent of this patch is just to make everything behave the same
> > > in the sense that a device could have a different physical MTU. The
> > > field got added to the ib_port_attr previously so this is giving it an initial
> value vs leaving it unset.
> > [Wan, Kaike]  Correct.
> 
> No one is using this "phys_mtu" field, except one place in ipoib.
[Wan, Kaike]  Since phys_mtu was introduced into ib_port_attr and every query_port call will  return it in ib_port_attr. Rather than leaving it set to 0, setting it correctly in each hw driver seem more reasonable to me. 

> 
> Thanks
> 
> >
> > >
> > > -Denny
> >

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Leon Romanovsky]

On Mon, Apr 19, 2021 at 09:08:55AM -0400, Dennis Dalessandro wrote:
> On 4/19/2021 8:29 AM, Leon Romanovsky wrote:
> > On Mon, Apr 19, 2021 at 12:20:33PM +0000, Wan, Kaike wrote:
> > > 
> > > 
> > > > -----Original Message-----
> > > > From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> > > > Sent: Thursday, April 08, 2021 8:31 AM
> > > > To: Leon Romanovsky <leon@kernel.org>
> > > > Cc: dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org; Wan,
> > > > Kaike <kaike.wan@intel.com>
> > > > Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port
> > > > function
> > > > 
> > > > On 4/8/2021 8:14 AM, Leon Romanovsky wrote:
> > > > > On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
> > > > > > On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
> > > > > > > On Mon, Mar 29, 2021 at 09:54:12AM -0400,
> > > > dennis.dalessandro@cornelisnetworks.com wrote:
> > > > > > > > From: Kaike Wan <kaike.wan@intel.com>
> > > > > > > > 
> > > > > > > > This is a follow on patch to add a phys_mtu field to the
> > > > > > > > ib_port_attr structure to indicate the maximum physical MTU the
> > > > > > > > underlying device supports.
> > > > > > > > 
> > > > > > > > Extends the following:
> > > > > > > > commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's
> > > > > > > > upper limit")
> > > > > > > > 
> > > > > > > > Reviewed-by: Mike Marciniszyn
> > > > > > > > <mike.marciniszyn@cornelisnetworks.com>
> > > > > > > > Signed-off-by: Kaike Wan <kaike.wan@intel.com>
> > > > > > > > Signed-off-by: Dennis Dalessandro
> > > > > > > > <dennis.dalessandro@cornelisnetworks.com>
> > > > > > > > ---
> > > > > > > >     drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
> > > > > > > >     drivers/infiniband/hw/cxgb4/provider.c          |  1 +
> > > > > > > >     drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
> > > > > > > >     drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
> > > > > > > >     drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
> > > > > > > >     drivers/infiniband/hw/mlx4/main.c               |  1 +
> > > > > > > >     drivers/infiniband/hw/mlx5/mad.c                |  1 +
> > > > > > > >     drivers/infiniband/hw/mlx5/main.c               |  2 ++
> > > > > > > >     drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
> > > > > > > >     drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
> > > > > > > >     drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
> > > > > > > >     drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
> > > > > > > >     drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
> > > > > > > >     drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
> > > > > > > >     drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
> > > > > > > >     include/rdma/ib_verbs.h                         | 17 -----------------
> > > > > > > >     16 files changed, 16 insertions(+), 18 deletions(-)
> > > > > > > 
> > > > > > > But why? What will it give us that almost all drivers have same
> > > > > > > props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
> > > > > > > 
> > > > > > 
> > > > > > Almost is not all. Alternative idea to convey this? Seemed like a
> > > > > > sensible thing to at least have support for but open to other approaches.
> > > > > 
> > > > > What about leave it as is?
> > > > > 
> > > > > I'm struggling to get the rationale behind this patch., the code
> > > > > already works and set the phys_mtu correctly, isn't it?
> > > > 
> > > > I see what you are saying now. Kaike, correct me if I'm wrong, but the intent
> > > > of this patch is just to make everything behave the same in the sense that a
> > > > device could have a different physical MTU. The field got added to the
> > > > ib_port_attr previously so this is giving it an initial value vs leaving it unset.
> > > [Wan, Kaike]  Correct.
> > 
> > No one is using this "phys_mtu" field, except one place in ipoib.
> 
> Today. I think it would be better to formalize the idea though and have a
> cleaner interface. Does this cause some problem?

Not directly, but yes.

Before your change, drivers don't need to care about this field because
it is not in use at all, after your change all drivers need to carry same
line. This is prone to errors.

Thanks

> 
> -Denny

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Leon Romanovsky]

On Mon, Apr 19, 2021 at 01:09:13PM +0000, Wan, Kaike wrote:
> 
> 
> > -----Original Message-----
> > From: Leon Romanovsky <leon@kernel.org>
> > Sent: Monday, April 19, 2021 8:29 AM
> > To: Wan, Kaike <kaike.wan@intel.com>
> > Cc: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>;
> > dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org
> > Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port
> > function
> > 
> > On Mon, Apr 19, 2021 at 12:20:33PM +0000, Wan, Kaike wrote:
> > >
> > >
> > > > -----Original Message-----
> > > > From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> > > > Sent: Thursday, April 08, 2021 8:31 AM
> > > > To: Leon Romanovsky <leon@kernel.org>
> > > > Cc: dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org;
> > > > Wan, Kaike <kaike.wan@intel.com>
> > > > Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for
> > > > query_port function
> > > >
> > > > On 4/8/2021 8:14 AM, Leon Romanovsky wrote:
> > > > > On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
> > > > >> On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
> > > > >>> On Mon, Mar 29, 2021 at 09:54:12AM -0400,
> > > > dennis.dalessandro@cornelisnetworks.com wrote:
> > > > >>>> From: Kaike Wan <kaike.wan@intel.com>
> > > > >>>>
> > > > >>>> This is a follow on patch to add a phys_mtu field to the
> > > > >>>> ib_port_attr structure to indicate the maximum physical MTU the
> > > > >>>> underlying device supports.
> > > > >>>>
> > > > >>>> Extends the following:
> > > > >>>> commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode
> > > > >>>> MTU's upper limit")
> > > > >>>>
> > > > >>>> Reviewed-by: Mike Marciniszyn
> > > > >>>> <mike.marciniszyn@cornelisnetworks.com>
> > > > >>>> Signed-off-by: Kaike Wan <kaike.wan@intel.com>
> > > > >>>> Signed-off-by: Dennis Dalessandro
> > > > >>>> <dennis.dalessandro@cornelisnetworks.com>
> > > > >>>> ---
> > > > >>>>    drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
> > > > >>>>    drivers/infiniband/hw/cxgb4/provider.c          |  1 +
> > > > >>>>    drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
> > > > >>>>    drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
> > > > >>>>    drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
> > > > >>>>    drivers/infiniband/hw/mlx4/main.c               |  1 +
> > > > >>>>    drivers/infiniband/hw/mlx5/mad.c                |  1 +
> > > > >>>>    drivers/infiniband/hw/mlx5/main.c               |  2 ++
> > > > >>>>    drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
> > > > >>>>    drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
> > > > >>>>    drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
> > > > >>>>    drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
> > > > >>>>    drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
> > > > >>>>    drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
> > > > >>>>    drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
> > > > >>>>    include/rdma/ib_verbs.h                         | 17 -----------------
> > > > >>>>    16 files changed, 16 insertions(+), 18 deletions(-)
> > > > >>>
> > > > >>> But why? What will it give us that almost all drivers have same
> > > > >>> props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
> > > > >>>
> > > > >>
> > > > >> Almost is not all. Alternative idea to convey this? Seemed like a
> > > > >> sensible thing to at least have support for but open to other
> > approaches.
> > > > >
> > > > > What about leave it as is?
> > > > >
> > > > > I'm struggling to get the rationale behind this patch., the code
> > > > > already works and set the phys_mtu correctly, isn't it?
> > > >
> > > > I see what you are saying now. Kaike, correct me if I'm wrong, but
> > > > the intent of this patch is just to make everything behave the same
> > > > in the sense that a device could have a different physical MTU. The
> > > > field got added to the ib_port_attr previously so this is giving it an initial
> > value vs leaving it unset.
> > > [Wan, Kaike]  Correct.
> > 
> > No one is using this "phys_mtu" field, except one place in ipoib.
> [Wan, Kaike]  Since phys_mtu was introduced into ib_port_attr and every query_port call will  return it in ib_port_attr. Rather than leaving it set to 0, setting it correctly in each hw driver seem more reasonable to me. 

Yes, if the drivers set this field differently, it is not the case here.
All of them (except one) set same value.

If you really want to have phys_mtu be correct all the time, change
query_port to set it.

Something like that:
query_port():
  if (!phys_mtu)
     phys_mtu = ib_mtu_enum_to_int(..)

Thanks

> 
> > 
> > Thanks
> > 
> > >
> > > >
> > > > -Denny
> > >

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Dennis Dalessandro]

On 4/20/2021 1:11 AM, Leon Romanovsky wrote:
> On Mon, Apr 19, 2021 at 09:08:55AM -0400, Dennis Dalessandro wrote:
>> On 4/19/2021 8:29 AM, Leon Romanovsky wrote:
>>> On Mon, Apr 19, 2021 at 12:20:33PM +0000, Wan, Kaike wrote:
>>>>
>>>>
>>>>> -----Original Message-----
>>>>> From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
>>>>> Sent: Thursday, April 08, 2021 8:31 AM
>>>>> To: Leon Romanovsky <leon@kernel.org>
>>>>> Cc: dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org; Wan,
>>>>> Kaike <kaike.wan@intel.com>
>>>>> Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port
>>>>> function
>>>>>
>>>>> On 4/8/2021 8:14 AM, Leon Romanovsky wrote:
>>>>>> On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
>>>>>>> On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
>>>>>>>> On Mon, Mar 29, 2021 at 09:54:12AM -0400,
>>>>> dennis.dalessandro@cornelisnetworks.com wrote:
>>>>>>>>> From: Kaike Wan <kaike.wan@intel.com>
>>>>>>>>>
>>>>>>>>> This is a follow on patch to add a phys_mtu field to the
>>>>>>>>> ib_port_attr structure to indicate the maximum physical MTU the
>>>>>>>>> underlying device supports.
>>>>>>>>>
>>>>>>>>> Extends the following:
>>>>>>>>> commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's
>>>>>>>>> upper limit")
>>>>>>>>>
>>>>>>>>> Reviewed-by: Mike Marciniszyn
>>>>>>>>> <mike.marciniszyn@cornelisnetworks.com>
>>>>>>>>> Signed-off-by: Kaike Wan <kaike.wan@intel.com>
>>>>>>>>> Signed-off-by: Dennis Dalessandro
>>>>>>>>> <dennis.dalessandro@cornelisnetworks.com>
>>>>>>>>> ---
>>>>>>>>>      drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
>>>>>>>>>      drivers/infiniband/hw/cxgb4/provider.c          |  1 +
>>>>>>>>>      drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
>>>>>>>>>      drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
>>>>>>>>>      drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
>>>>>>>>>      drivers/infiniband/hw/mlx4/main.c               |  1 +
>>>>>>>>>      drivers/infiniband/hw/mlx5/mad.c                |  1 +
>>>>>>>>>      drivers/infiniband/hw/mlx5/main.c               |  2 ++
>>>>>>>>>      drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
>>>>>>>>>      drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
>>>>>>>>>      drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
>>>>>>>>>      drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
>>>>>>>>>      drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
>>>>>>>>>      drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
>>>>>>>>>      drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
>>>>>>>>>      include/rdma/ib_verbs.h                         | 17 -----------------
>>>>>>>>>      16 files changed, 16 insertions(+), 18 deletions(-)
>>>>>>>>
>>>>>>>> But why? What will it give us that almost all drivers have same
>>>>>>>> props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
>>>>>>>>
>>>>>>>
>>>>>>> Almost is not all. Alternative idea to convey this? Seemed like a
>>>>>>> sensible thing to at least have support for but open to other approaches.
>>>>>>
>>>>>> What about leave it as is?
>>>>>>
>>>>>> I'm struggling to get the rationale behind this patch., the code
>>>>>> already works and set the phys_mtu correctly, isn't it?
>>>>>
>>>>> I see what you are saying now. Kaike, correct me if I'm wrong, but the intent
>>>>> of this patch is just to make everything behave the same in the sense that a
>>>>> device could have a different physical MTU. The field got added to the
>>>>> ib_port_attr previously so this is giving it an initial value vs leaving it unset.
>>>> [Wan, Kaike]  Correct.
>>>
>>> No one is using this "phys_mtu" field, except one place in ipoib.
>>
>> Today. I think it would be better to formalize the idea though and have a
>> cleaner interface. Does this cause some problem?
> 
> Not directly, but yes.
> 
> Before your change, drivers don't need to care about this field because
> it is not in use at all, after your change all drivers need to carry same
> line. This is prone to errors.

Perhaps a more common place to set this in the core is appropriate.

-Denny

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Leon Romanovsky]

On Tue, Apr 20, 2021 at 08:17:00AM -0400, Dennis Dalessandro wrote:
> On 4/20/2021 1:11 AM, Leon Romanovsky wrote:
> > On Mon, Apr 19, 2021 at 09:08:55AM -0400, Dennis Dalessandro wrote:
> > > On 4/19/2021 8:29 AM, Leon Romanovsky wrote:
> > > > On Mon, Apr 19, 2021 at 12:20:33PM +0000, Wan, Kaike wrote:
> > > > > 
> > > > > 
> > > > > > -----Original Message-----
> > > > > > From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
> > > > > > Sent: Thursday, April 08, 2021 8:31 AM
> > > > > > To: Leon Romanovsky <leon@kernel.org>
> > > > > > Cc: dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org; Wan,
> > > > > > Kaike <kaike.wan@intel.com>
> > > > > > Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port
> > > > > > function
> > > > > > 
> > > > > > On 4/8/2021 8:14 AM, Leon Romanovsky wrote:
> > > > > > > On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
> > > > > > > > On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
> > > > > > > > > On Mon, Mar 29, 2021 at 09:54:12AM -0400,
> > > > > > dennis.dalessandro@cornelisnetworks.com wrote:
> > > > > > > > > > From: Kaike Wan <kaike.wan@intel.com>
> > > > > > > > > > 
> > > > > > > > > > This is a follow on patch to add a phys_mtu field to the
> > > > > > > > > > ib_port_attr structure to indicate the maximum physical MTU the
> > > > > > > > > > underlying device supports.
> > > > > > > > > > 
> > > > > > > > > > Extends the following:
> > > > > > > > > > commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's
> > > > > > > > > > upper limit")
> > > > > > > > > > 
> > > > > > > > > > Reviewed-by: Mike Marciniszyn
> > > > > > > > > > <mike.marciniszyn@cornelisnetworks.com>
> > > > > > > > > > Signed-off-by: Kaike Wan <kaike.wan@intel.com>
> > > > > > > > > > Signed-off-by: Dennis Dalessandro
> > > > > > > > > > <dennis.dalessandro@cornelisnetworks.com>
> > > > > > > > > > ---
> > > > > > > > > >      drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
> > > > > > > > > >      drivers/infiniband/hw/cxgb4/provider.c          |  1 +
> > > > > > > > > >      drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
> > > > > > > > > >      drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
> > > > > > > > > >      drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
> > > > > > > > > >      drivers/infiniband/hw/mlx4/main.c               |  1 +
> > > > > > > > > >      drivers/infiniband/hw/mlx5/mad.c                |  1 +
> > > > > > > > > >      drivers/infiniband/hw/mlx5/main.c               |  2 ++
> > > > > > > > > >      drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
> > > > > > > > > >      drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
> > > > > > > > > >      drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
> > > > > > > > > >      drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
> > > > > > > > > >      drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
> > > > > > > > > >      drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
> > > > > > > > > >      drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
> > > > > > > > > >      include/rdma/ib_verbs.h                         | 17 -----------------
> > > > > > > > > >      16 files changed, 16 insertions(+), 18 deletions(-)
> > > > > > > > > 
> > > > > > > > > But why? What will it give us that almost all drivers have same
> > > > > > > > > props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
> > > > > > > > > 
> > > > > > > > 
> > > > > > > > Almost is not all. Alternative idea to convey this? Seemed like a
> > > > > > > > sensible thing to at least have support for but open to other approaches.
> > > > > > > 
> > > > > > > What about leave it as is?
> > > > > > > 
> > > > > > > I'm struggling to get the rationale behind this patch., the code
> > > > > > > already works and set the phys_mtu correctly, isn't it?
> > > > > > 
> > > > > > I see what you are saying now. Kaike, correct me if I'm wrong, but the intent
> > > > > > of this patch is just to make everything behave the same in the sense that a
> > > > > > device could have a different physical MTU. The field got added to the
> > > > > > ib_port_attr previously so this is giving it an initial value vs leaving it unset.
> > > > > [Wan, Kaike]  Correct.
> > > > 
> > > > No one is using this "phys_mtu" field, except one place in ipoib.
> > > 
> > > Today. I think it would be better to formalize the idea though and have a
> > > cleaner interface. Does this cause some problem?
> > 
> > Not directly, but yes.
> > 
> > Before your change, drivers don't need to care about this field because
> > it is not in use at all, after your change all drivers need to carry same
> > line. This is prone to errors.
> 
> Perhaps a more common place to set this in the core is appropriate.

This is basically what I suggested to Kaike.

Thanks

> 
> -Denny
> 
> 

Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port function

[Dennis Dalessandro]

On 4/20/2021 8:31 AM, Leon Romanovsky wrote:
> On Tue, Apr 20, 2021 at 08:17:00AM -0400, Dennis Dalessandro wrote:
>> On 4/20/2021 1:11 AM, Leon Romanovsky wrote:
>>> On Mon, Apr 19, 2021 at 09:08:55AM -0400, Dennis Dalessandro wrote:
>>>> On 4/19/2021 8:29 AM, Leon Romanovsky wrote:
>>>>> On Mon, Apr 19, 2021 at 12:20:33PM +0000, Wan, Kaike wrote:
>>>>>>
>>>>>>
>>>>>>> -----Original Message-----
>>>>>>> From: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
>>>>>>> Sent: Thursday, April 08, 2021 8:31 AM
>>>>>>> To: Leon Romanovsky <leon@kernel.org>
>>>>>>> Cc: dledford@redhat.com; jgg@ziepe.ca; linux-rdma@vger.kernel.org; Wan,
>>>>>>> Kaike <kaike.wan@intel.com>
>>>>>>> Subject: Re: [PATCH for-next 06/10] rdma: Set physical MTU for query_port
>>>>>>> function
>>>>>>>
>>>>>>> On 4/8/2021 8:14 AM, Leon Romanovsky wrote:
>>>>>>>> On Thu, Apr 08, 2021 at 08:06:46AM -0400, Dennis Dalessandro wrote:
>>>>>>>>> On 4/1/2021 4:42 AM, Leon Romanovsky wrote:
>>>>>>>>>> On Mon, Mar 29, 2021 at 09:54:12AM -0400,
>>>>>>> dennis.dalessandro@cornelisnetworks.com wrote:
>>>>>>>>>>> From: Kaike Wan <kaike.wan@intel.com>
>>>>>>>>>>>
>>>>>>>>>>> This is a follow on patch to add a phys_mtu field to the
>>>>>>>>>>> ib_port_attr structure to indicate the maximum physical MTU the
>>>>>>>>>>> underlying device supports.
>>>>>>>>>>>
>>>>>>>>>>> Extends the following:
>>>>>>>>>>> commit 6d72344cf6c4 ("IB/ipoib: Increase ipoib Datagram mode MTU's
>>>>>>>>>>> upper limit")
>>>>>>>>>>>
>>>>>>>>>>> Reviewed-by: Mike Marciniszyn
>>>>>>>>>>> <mike.marciniszyn@cornelisnetworks.com>
>>>>>>>>>>> Signed-off-by: Kaike Wan <kaike.wan@intel.com>
>>>>>>>>>>> Signed-off-by: Dennis Dalessandro
>>>>>>>>>>> <dennis.dalessandro@cornelisnetworks.com>
>>>>>>>>>>> ---
>>>>>>>>>>>       drivers/infiniband/hw/bnxt_re/ib_verbs.c        |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/cxgb4/provider.c          |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/efa/efa_verbs.c           |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/hns/hns_roce_main.c       |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/i40iw/i40iw_verbs.c       |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/mlx4/main.c               |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/mlx5/mad.c                |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/mlx5/main.c               |  2 ++
>>>>>>>>>>>       drivers/infiniband/hw/mthca/mthca_provider.c    |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/ocrdma/ocrdma_verbs.c     |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/qib/qib_verbs.c           |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/usnic/usnic_ib_verbs.c    |  1 +
>>>>>>>>>>>       drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c |  1 +
>>>>>>>>>>>       drivers/infiniband/sw/siw/siw_verbs.c           |  1 +
>>>>>>>>>>>       drivers/infiniband/ulp/ipoib/ipoib_main.c       |  2 +-
>>>>>>>>>>>       include/rdma/ib_verbs.h                         | 17 -----------------
>>>>>>>>>>>       16 files changed, 16 insertions(+), 18 deletions(-)
>>>>>>>>>>
>>>>>>>>>> But why? What will it give us that almost all drivers have same
>>>>>>>>>> props->phys_mtu = ib_mtu_enum_to_int(props->max_mtu); line?
>>>>>>>>>>
>>>>>>>>>
>>>>>>>>> Almost is not all. Alternative idea to convey this? Seemed like a
>>>>>>>>> sensible thing to at least have support for but open to other approaches.
>>>>>>>>
>>>>>>>> What about leave it as is?
>>>>>>>>
>>>>>>>> I'm struggling to get the rationale behind this patch., the code
>>>>>>>> already works and set the phys_mtu correctly, isn't it?
>>>>>>>
>>>>>>> I see what you are saying now. Kaike, correct me if I'm wrong, but the intent
>>>>>>> of this patch is just to make everything behave the same in the sense that a
>>>>>>> device could have a different physical MTU. The field got added to the
>>>>>>> ib_port_attr previously so this is giving it an initial value vs leaving it unset.
>>>>>> [Wan, Kaike]  Correct.
>>>>>
>>>>> No one is using this "phys_mtu" field, except one place in ipoib.
>>>>
>>>> Today. I think it would be better to formalize the idea though and have a
>>>> cleaner interface. Does this cause some problem?
>>>
>>> Not directly, but yes.
>>>
>>> Before your change, drivers don't need to care about this field because
>>> it is not in use at all, after your change all drivers need to carry same
>>> line. This is prone to errors.
>>
>> Perhaps a more common place to set this in the core is appropriate.
> 
> This is basically what I suggested to Kaike.

Yep, just seen that after I hit send.