[PATCH for-next 0/2] RDMA/hns: Add support of net device event reporting to ULP

[PATCH for-next 0/2] RDMA/hns: Add support of net device event reporting to ULP

[Weihang Li]

When net link up/down, driver of hip08 will notify ULP.

Lang Cheng (2):
  {topost} RDMA/hns: Deliver net device event to ofed
  {topost} RDMA/hns: Add support for sending port down event quickly

 drivers/infiniband/hw/hns/hns_roce_device.h | 12 +++++++++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 34 ++++++++++++++++++++++++
 drivers/infiniband/hw/hns/hns_roce_main.c   | 40 +++++++++++++++++++++--------
 3 files changed, 76 insertions(+), 10 deletions(-)

-- 
2.8.1

[PATCH for-next 1/2] {topost} RDMA/hns: Deliver net device event to ofed

[Weihang Li]

From: Lang Cheng <chenglang@huawei.com>

Driver can notify ULP with IB event when net link down/up.

Signed-off-by: Lang Cheng <chenglang@huawei.com>
Signed-off-by: Weihang Li <liweihang@hisilicon.com>
---
 drivers/infiniband/hw/hns/hns_roce_device.h | 12 +++++++++
 drivers/infiniband/hw/hns/hns_roce_main.c   | 40 +++++++++++++++++++++--------
 2 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 05d375a..47c209f 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -711,6 +711,7 @@ struct hns_roce_ib_iboe {
 	struct net_device      *netdevs[HNS_ROCE_MAX_PORTS];
 	struct notifier_block	nb;
 	u8			phy_port[HNS_ROCE_MAX_PORTS];
+	enum ib_port_state	port_state[HNS_ROCE_MAX_PORTS];
 };
 
 enum {
@@ -1135,6 +1136,17 @@ static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset)
 		       (offset & (page_size - 1));
 }
 
+static inline u8 to_rdma_port_num(u8 phy_port_num)
+{
+	return phy_port_num + 1;
+}
+
+static inline enum ib_port_state get_port_state(struct net_device *net_dev)
+{
+	return (netif_running(net_dev) && netif_carrier_ok(net_dev)) ?
+		IB_PORT_ACTIVE : IB_PORT_DOWN;
+}
+
 int hns_roce_init_uar_table(struct hns_roce_dev *dev);
 int hns_roce_uar_alloc(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
 void hns_roce_uar_free(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 665ce24..742a36e 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -103,10 +103,13 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context)
 }
 
 static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
-			   unsigned long event)
+			   unsigned long dev_event)
 {
 	struct device *dev = hr_dev->dev;
+	enum ib_port_state port_state;
 	struct net_device *netdev;
+	struct ib_event event;
+	unsigned long flags;
 	int ret = 0;
 
 	netdev = hr_dev->iboe.netdevs[port];
@@ -115,20 +118,38 @@ static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
 		return -ENODEV;
 	}
 
-	switch (event) {
-	case NETDEV_UP:
-	case NETDEV_CHANGE:
+	switch (dev_event) {
 	case NETDEV_REGISTER:
 	case NETDEV_CHANGEADDR:
 		ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr);
 		break;
+	case NETDEV_UP:
+	case NETDEV_CHANGE:
+		ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr);
+		if (ret)
+			return ret;
+		/* port up/down events need send ib events */
 	case NETDEV_DOWN:
-		/*
-		 * In v1 engine, only support all ports closed together.
-		 */
+		port_state = get_port_state(netdev);
+
+		spin_lock_irqsave(&hr_dev->iboe.lock, flags);
+		if (hr_dev->iboe.port_state[port] == port_state) {
+			spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+			return NOTIFY_DONE;
+		}
+		hr_dev->iboe.port_state[port] = port_state;
+		spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+
+		event.device = &hr_dev->ib_dev;
+		event.event = (port_state == IB_PORT_ACTIVE) ?
+			      IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
+		event.element.port_num = to_rdma_port_num(port);
+		ib_dispatch_event(&event);
+		break;
+	case NETDEV_UNREGISTER:
 		break;
 	default:
-		dev_dbg(dev, "NETDEV event = 0x%x!\n", (u32)(event));
+		dev_dbg(dev, "NETDEV event = 0x%x!\n", (u32)(dev_event));
 		break;
 	}
 
@@ -259,8 +280,7 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num,
 
 	mtu = iboe_get_mtu(net_dev->mtu);
 	props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256;
-	props->state = (netif_running(net_dev) && netif_carrier_ok(net_dev)) ?
-			IB_PORT_ACTIVE : IB_PORT_DOWN;
+	props->state = get_port_state(net_dev);
 	props->phys_state = (props->state == IB_PORT_ACTIVE) ?
 			     IB_PORT_PHYS_STATE_LINK_UP :
 			     IB_PORT_PHYS_STATE_DISABLED;
-- 
2.8.1

[PATCH for-next 2/2] {topost} RDMA/hns: Add support for sending port down event quickly

[Weihang Li]

From: Lang Cheng <chenglang@huawei.com>

When the netdev port status changes, the roce driver sends a port down
event by parsing the netdev event dispatched by IB_CORE, which takes about
a few hundred milliseconds. It is not fast enough for ULP sometimes.

The HNS NIC driver can directly notify the ROCE driver to send port event
via callback function, which will only take a few milliseconds.
This patch implements above callback function.

Signed-off-by: Lang Cheng <chenglang@huawei.com>
Signed-off-by: Weihang Li <liweihang@hisilicon.com>
---
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 34 ++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 20918f8..3196a11 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -6727,9 +6727,43 @@ static int hns_roce_hw_v2_reset_notify(struct hnae3_handle *handle,
 	return ret;
 }
 
+static void hns_roce_hw_v2_link_status_change(struct hnae3_handle *handle,
+					      bool linkup)
+{
+	struct hns_roce_dev *hr_dev = (struct hns_roce_dev *)handle->priv;
+	struct net_device *netdev = handle->rinfo.netdev;
+	struct ib_event event;
+	unsigned long flags;
+	u8 phy_port;
+
+	if (linkup || !hr_dev)
+		return;
+
+	for (phy_port = 0; phy_port < hr_dev->caps.num_ports; phy_port++)
+		if (netdev == hr_dev->iboe.netdevs[phy_port])
+			break;
+
+	if (phy_port == hr_dev->caps.num_ports)
+		return;
+
+	spin_lock_irqsave(&hr_dev->iboe.lock, flags);
+	if (hr_dev->iboe.port_state[phy_port] == IB_PORT_DOWN) {
+		spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+		return;
+	}
+	hr_dev->iboe.port_state[phy_port] = IB_PORT_DOWN;
+	spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
+
+	event.device = &hr_dev->ib_dev;
+	event.element.port_num = to_rdma_port_num(phy_port);
+	event.event = IB_EVENT_PORT_ERR;
+	ib_dispatch_event(&event);
+}
+
 static const struct hnae3_client_ops hns_roce_hw_v2_ops = {
 	.init_instance = hns_roce_hw_v2_init_instance,
 	.uninit_instance = hns_roce_hw_v2_uninit_instance,
+	.link_status_change = hns_roce_hw_v2_link_status_change,
 	.reset_notify = hns_roce_hw_v2_reset_notify,
 };
 
-- 
2.8.1

Re: [PATCH for-next 1/2] {topost} RDMA/hns: Deliver net device event to ofed

[Jason Gunthorpe]

On Fri, Oct 04, 2019 at 06:29:13PM +0800, Weihang Li wrote:
> From: Lang Cheng <chenglang@huawei.com>
> 
> Driver can notify ULP with IB event when net link down/up.
> 
> Signed-off-by: Lang Cheng <chenglang@huawei.com>
> Signed-off-by: Weihang Li <liweihang@hisilicon.com>
>  drivers/infiniband/hw/hns/hns_roce_device.h | 12 +++++++++
>  drivers/infiniband/hw/hns/hns_roce_main.c   | 40 +++++++++++++++++++++--------
>  2 files changed, 42 insertions(+), 10 deletions(-)
> 
> diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
> index 05d375a..47c209f 100644
> +++ b/drivers/infiniband/hw/hns/hns_roce_device.h
> @@ -711,6 +711,7 @@ struct hns_roce_ib_iboe {
>  	struct net_device      *netdevs[HNS_ROCE_MAX_PORTS];
>  	struct notifier_block	nb;
>  	u8			phy_port[HNS_ROCE_MAX_PORTS];
> +	enum ib_port_state	port_state[HNS_ROCE_MAX_PORTS];
>  };
>  
>  enum {
> @@ -1135,6 +1136,17 @@ static inline void *hns_roce_buf_offset(struct hns_roce_buf *buf, int offset)
>  		       (offset & (page_size - 1));
>  }
>  
> +static inline u8 to_rdma_port_num(u8 phy_port_num)
> +{
> +	return phy_port_num + 1;
> +}
> +
> +static inline enum ib_port_state get_port_state(struct net_device *net_dev)
> +{
> +	return (netif_running(net_dev) && netif_carrier_ok(net_dev)) ?
> +		IB_PORT_ACTIVE : IB_PORT_DOWN;
> +}
> +
>  int hns_roce_init_uar_table(struct hns_roce_dev *dev);
>  int hns_roce_uar_alloc(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
>  void hns_roce_uar_free(struct hns_roce_dev *dev, struct hns_roce_uar *uar);
> diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
> index 665ce24..742a36e 100644
> +++ b/drivers/infiniband/hw/hns/hns_roce_main.c
> @@ -103,10 +103,13 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context)
>  }
>  
>  static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
> -			   unsigned long event)
> +			   unsigned long dev_event)
>  {
>  	struct device *dev = hr_dev->dev;
> +	enum ib_port_state port_state;
>  	struct net_device *netdev;
> +	struct ib_event event;
> +	unsigned long flags;
>  	int ret = 0;
>  
>  	netdev = hr_dev->iboe.netdevs[port];
> @@ -115,20 +118,38 @@ static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
>  		return -ENODEV;
>  	}
>  
> -	switch (event) {
> -	case NETDEV_UP:
> -	case NETDEV_CHANGE:
> +	switch (dev_event) {
>  	case NETDEV_REGISTER:
>  	case NETDEV_CHANGEADDR:
>  		ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr);
>  		break;
> +	case NETDEV_UP:
> +	case NETDEV_CHANGE:
> +		ret = hns_roce_set_mac(hr_dev, port, netdev->dev_addr);
> +		if (ret)
> +			return ret;
> +		/* port up/down events need send ib events */
>  	case NETDEV_DOWN:
> -		/*
> -		 * In v1 engine, only support all ports closed together.
> -		 */
> +		port_state = get_port_state(netdev);
> +
> +		spin_lock_irqsave(&hr_dev->iboe.lock, flags);
> +		if (hr_dev->iboe.port_state[port] == port_state) {
> +			spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
> +			return NOTIFY_DONE;
> +		}
> +		hr_dev->iboe.port_state[port] = port_state;
> +		spin_unlock_irqrestore(&hr_dev->iboe.lock, flags);
> +
> +		event.device = &hr_dev->ib_dev;
> +		event.event = (port_state == IB_PORT_ACTIVE) ?
> +			      IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
> +		event.element.port_num = to_rdma_port_num(port);
> +		ib_dispatch_event(&event);
> +		break;

Every roce driver shouldn't have this kind of stuff, please try to
make a version to add this to the core code. Now that we have the
ib_device_set_netdev() it should be possible

Also, this driver should be using ib_device_set_netdev and the sketchy
code in hns_roce_netdev_event() removed

Jason