Netdev Archive on lore.kernel.org
 help / color / Atom feed
From: Saeed Mahameed <saeedm@mellanox.com>
To: Jason Gunthorpe <jgg@mellanox.com>,
	"leon@kernel.org" <leon@kernel.org>,
	"dledford@redhat.com" <dledford@redhat.com>
Cc: Yishai Hadas <yishaih@mellanox.com>,
	"netdev@vger.kernel.org" <netdev@vger.kernel.org>,
	Leon Romanovsky <leonro@mellanox.com>,
	"linux-rdma@vger.kernel.org" <linux-rdma@vger.kernel.org>
Subject: Re: [PATCH mlx5-next v1 02/12] net/mlx5: Use event mask based on device capabilities
Date: Thu, 27 Jun 2019 00:23:55 +0000
Message-ID: <4ac3d24606b712514c0265d0073850f3f06ec9c3.camel@mellanox.com> (raw)
In-Reply-To: <20190618171540.11729-3-leon@kernel.org>

On Tue, 2019-06-18 at 20:15 +0300, Leon Romanovsky wrote:
> From: Yishai Hadas <yishaih@mellanox.com>
> 
> Use the reported device capabilities for the supported user events
> (i.e.
> affiliated and un-affiliated) to set the EQ mask.
> 
> As the event mask can be up to 256 defined by 4 entries of u64 change
> the applicable code to work accordingly.
> 
> Signed-off-by: Yishai Hadas <yishaih@mellanox.com>
> Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
> ---
>  drivers/infiniband/hw/mlx5/odp.c             |  3 +-
>  drivers/net/ethernet/mellanox/mlx5/core/eq.c | 45 ++++++++++++++++
> ----
>  drivers/net/ethernet/mellanox/mlx5/core/fw.c |  6 +++
>  include/linux/mlx5/device.h                  |  6 ++-
>  include/linux/mlx5/eq.h                      |  4 +-
>  include/linux/mlx5/mlx5_ifc.h                | 13 ++++--
>  6 files changed, 63 insertions(+), 14 deletions(-)
> 
> diff --git a/drivers/infiniband/hw/mlx5/odp.c
> b/drivers/infiniband/hw/mlx5/odp.c
> index 600fe23e2eae..a6740ec308ed 100644
> --- a/drivers/infiniband/hw/mlx5/odp.c
> +++ b/drivers/infiniband/hw/mlx5/odp.c
> @@ -1559,10 +1559,11 @@ mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev,
> struct mlx5_ib_pf_eq *eq)
>  	eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int;
>  	param = (struct mlx5_eq_param) {
>  		.irq_index = 0,
> -		.mask = 1 << MLX5_EVENT_TYPE_PAGE_FAULT,
>  		.nent = MLX5_IB_NUM_PF_EQE,
>  	};
>  	eq->core = mlx5_eq_create_generic(dev->mdev, &param);
> +
> +	param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT;

As Yishai already pointer out, there is a regression here,
the line above was merged in the wrong order, mask should be setup
before calling mlx5_eq_create_generic.

I will expect V3.

>  	if (IS_ERR(eq->core)) {
>  		err = PTR_ERR(eq->core);
>  		goto err_wq;
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
> b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
> index 8000d2a4a7e2..9d07add38940 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
> @@ -256,6 +256,7 @@ create_map_eq(struct mlx5_core_dev *dev, struct
> mlx5_eq *eq,
>  	int inlen;
>  	u32 *in;
>  	int err;
> +	int i;
>  
>  	/* Init CQ table */
>  	memset(cq_table, 0, sizeof(*cq_table));
> @@ -283,10 +284,12 @@ create_map_eq(struct mlx5_core_dev *dev, struct
> mlx5_eq *eq,
>  	mlx5_fill_page_array(&eq->buf, pas);
>  
>  	MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
> -	if (!param->mask && MLX5_CAP_GEN(dev, log_max_uctx))
> +	if (!param->mask[0] && MLX5_CAP_GEN(dev, log_max_uctx))
>  		MLX5_SET(create_eq_in, in, uid,
> MLX5_SHARED_RESOURCE_UID);
>  
> -	MLX5_SET64(create_eq_in, in, event_bitmask, param->mask);
> +	for (i = 0; i < 4; i++)
> +		MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, i,
> +				 param->mask[i]);
>  
>  	eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
>  	MLX5_SET(eqc, eqc, log_eq_size, ilog2(eq->nent));
> @@ -507,10 +510,32 @@ static int cq_err_event_notifier(struct
> notifier_block *nb,
>  	return NOTIFY_OK;
>  }
>  
> -static u64 gather_async_events_mask(struct mlx5_core_dev *dev)
> +static void gather_async_events_from_cap(struct mlx5_core_dev *dev,
> +					 u64 mask[4])
> +{
> +	__be64 *user_unaffiliated_events;
> +	__be64 *user_affiliated_events;
> +	int i;
> +
> +	user_affiliated_events =
> +		MLX5_CAP_DEV_EVENT(dev, user_affiliated_events);
> +	user_unaffiliated_events =
> +		MLX5_CAP_DEV_EVENT(dev, user_unaffiliated_events);
> +
> +	for (i = 0; i < 4; i++)
> +		mask[i] = be64_to_cpu(user_affiliated_events[i] |
> +				      user_unaffiliated_events[i]);
> +}
> +
> +static void gather_async_events_mask(struct mlx5_core_dev *dev, u64
> mask[4])
>  {
>  	u64 async_event_mask = MLX5_ASYNC_EVENT_MASK;
>  
> +	if (MLX5_CAP_GEN(dev, event_cap)) {
> +		gather_async_events_from_cap(dev, mask);
> +		return;
> +	}
> +
>  	if (MLX5_VPORT_MANAGER(dev))
>  		async_event_mask |= (1ull <<
> MLX5_EVENT_TYPE_NIC_VPORT_CHANGE);
>  
> @@ -544,7 +569,7 @@ static u64 gather_async_events_mask(struct
> mlx5_core_dev *dev)
>  		async_event_mask |=
>  			(1ull <<
> MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED);
>  
> -	return async_event_mask;
> +	mask[0] = async_event_mask;
>  }
>  
>  static int create_async_eqs(struct mlx5_core_dev *dev)
> @@ -559,9 +584,11 @@ static int create_async_eqs(struct mlx5_core_dev
> *dev)
>  	table->cmd_eq.irq_nb.notifier_call = mlx5_eq_async_int;
>  	param = (struct mlx5_eq_param) {
>  		.irq_index = 0,
> -		.mask = 1ull << MLX5_EVENT_TYPE_CMD,
> +		.mask = {1ull << MLX5_EVENT_TYPE_CMD},
>  		.nent = MLX5_NUM_CMD_EQE,
>  	};
> +
> +	param.mask[0] = 1ull << MLX5_EVENT_TYPE_CMD;

No need to setup the mask twice, 
here and everywhere in this patch, pick one approach to set it.

>  	err = create_async_eq(dev, &table->cmd_eq.core, &param);
>  	if (err) {
>  		mlx5_core_warn(dev, "failed to create cmd EQ %d\n",
> err);
> @@ -577,9 +604,9 @@ static int create_async_eqs(struct mlx5_core_dev
> *dev)
>  	table->async_eq.irq_nb.notifier_call = mlx5_eq_async_int;
>  	param = (struct mlx5_eq_param) {
>  		.irq_index = 0,
> -		.mask = gather_async_events_mask(dev),
>  		.nent = MLX5_NUM_ASYNC_EQE,
>  	};
> +	gather_async_events_mask(dev, param.mask);
>  	err = create_async_eq(dev, &table->async_eq.core, &param);
>  	if (err) {
>  		mlx5_core_warn(dev, "failed to create async EQ %d\n",
> err);
> @@ -595,9 +622,11 @@ static int create_async_eqs(struct mlx5_core_dev
> *dev)
>  	table->pages_eq.irq_nb.notifier_call = mlx5_eq_async_int;
>  	param = (struct mlx5_eq_param) {
>  		.irq_index = 0,
> -		.mask =  1 << MLX5_EVENT_TYPE_PAGE_REQUEST,
> +		.mask = {1 << MLX5_EVENT_TYPE_PAGE_REQUEST},
>  		.nent = /* TODO: sriov max_vf + */ 1,
>  	};
> +
> +	param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST;
>  	err = create_async_eq(dev, &table->pages_eq.core, &param);
>  	if (err) {
>  		mlx5_core_warn(dev, "failed to create pages EQ %d\n",
> err);
> @@ -789,7 +818,7 @@ static int create_comp_eqs(struct mlx5_core_dev
> *dev)
>  		eq->irq_nb.notifier_call = mlx5_eq_comp_int;
>  		param = (struct mlx5_eq_param) {
>  			.irq_index = vecidx,
> -			.mask = 0,
> +			.mask = {0},
>  			.nent = nent,
>  		};
>  		err = create_map_eq(dev, &eq->core, &param);
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
> b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
> index 1ab6f7e3bec6..05367f15c3a7 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
> @@ -202,6 +202,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev
> *dev)
>  			return err;
>  	}
>  
> +	if (MLX5_CAP_GEN(dev, event_cap)) {
> +		err = mlx5_core_get_caps(dev, MLX5_CAP_DEV_EVENT);
> +		if (err)
> +			return err;
> +	}
> +
>  	return 0;
>  }
>  
> diff --git a/include/linux/mlx5/device.h
> b/include/linux/mlx5/device.h
> index 5e760067ac41..0d1abe097627 100644
> --- a/include/linux/mlx5/device.h
> +++ b/include/linux/mlx5/device.h
> @@ -351,7 +351,7 @@ enum mlx5_event {
>  
>  	MLX5_EVENT_TYPE_DEVICE_TRACER      = 0x26,
>  
> -	MLX5_EVENT_TYPE_MAX                =
> MLX5_EVENT_TYPE_DEVICE_TRACER + 1,
> +	MLX5_EVENT_TYPE_MAX                = 0x100,
>  };
>  
>  enum {
> @@ -1077,6 +1077,7 @@ enum mlx5_cap_type {
>  	MLX5_CAP_DEBUG,
>  	MLX5_CAP_RESERVED_14,
>  	MLX5_CAP_DEV_MEM,
> +	MLX5_CAP_DEV_EVENT = 0x14,
>  	/* NUM OF CAP Types */
>  	MLX5_CAP_NUM
>  };
> @@ -1255,6 +1256,9 @@ enum mlx5_qcam_feature_groups {
>  #define MLX5_CAP64_DEV_MEM(mdev, cap)\
>  	MLX5_GET64(device_mem_cap, mdev-
> >caps.hca_cur[MLX5_CAP_DEV_MEM], cap)
>  
> +#define MLX5_CAP_DEV_EVENT(mdev, cap)\
> +	MLX5_ADDR_OF(device_event_cap, (mdev)-
> >caps.hca_cur[MLX5_CAP_DEV_EVENT], cap)
> +
>  enum {
>  	MLX5_CMD_STAT_OK			= 0x0,
>  	MLX5_CMD_STAT_INT_ERR			= 0x1,
> diff --git a/include/linux/mlx5/eq.h b/include/linux/mlx5/eq.h
> index 70e16dcfb4c4..202df2e5fe8c 100644
> --- a/include/linux/mlx5/eq.h
> +++ b/include/linux/mlx5/eq.h
> @@ -15,7 +15,9 @@ struct mlx5_core_dev;
>  struct mlx5_eq_param {
>  	u8             irq_index;
>  	int            nent;
> -	u64            mask;
> +	u64            mask[4];
> +	void          *context;
> +	irq_handler_t  handler;
>  };
>  
>  struct mlx5_eq *
> diff --git a/include/linux/mlx5/mlx5_ifc.h
> b/include/linux/mlx5/mlx5_ifc.h
> index 16348528fef6..3ef716c054c2 100644
> --- a/include/linux/mlx5/mlx5_ifc.h
> +++ b/include/linux/mlx5/mlx5_ifc.h
> @@ -823,6 +823,12 @@ struct mlx5_ifc_device_mem_cap_bits {
>  	u8         reserved_at_180[0x680];
>  };
>  
> +struct mlx5_ifc_device_event_cap_bits {
> +	u8         user_affiliated_events[4][0x40];
> +
> +	u8         user_unaffiliated_events[4][0x40];
> +};
> +
>  enum {
>  	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_1_BYTE     = 0x0,
>  	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_2_BYTES    = 0x2,
> @@ -980,7 +986,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
>  
>  	u8         log_max_srq_sz[0x8];
>  	u8         log_max_qp_sz[0x8];
> -	u8         reserved_at_90[0x8];
> +	u8         event_cap[0x1];
> +	u8         reserved_at_91[0x7];
>  	u8         prio_tag_required[0x1];
>  	u8         reserved_at_99[0x2];
>  	u8         log_max_qp[0x5];
> @@ -7364,9 +7371,9 @@ struct mlx5_ifc_create_eq_in_bits {
>  
>  	u8         reserved_at_280[0x40];
>  
> -	u8         event_bitmask[0x40];
> +	u8         event_bitmask[4][0x40];
>  
> -	u8         reserved_at_300[0x580];
> +	u8         reserved_at_3c0[0x4c0];
>  
>  	u8         pas[0][0x40];
>  };

  reply index

Thread overview: 35+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-06-18 17:15 [PATCH rdma-next v1 00/12] DEVX asynchronous events Leon Romanovsky
2019-06-18 17:15 ` [PATCH mlx5-next v1 01/12] net/mlx5: Fix mlx5_core_destroy_cq() error flow Leon Romanovsky
2019-06-18 17:15 ` [PATCH mlx5-next v1 02/12] net/mlx5: Use event mask based on device capabilities Leon Romanovsky
2019-06-27  0:23   ` Saeed Mahameed [this message]
2019-06-18 17:15 ` [PATCH mlx5-next v1 03/12] net/mlx5: Expose the API to register for ANY event Leon Romanovsky
2019-06-18 17:15 ` [PATCH mlx5-next v1 04/12] net/mlx5: mlx5_core_create_cq() enhancements Leon Romanovsky
2019-06-18 17:15 ` [PATCH mlx5-next v1 05/12] net/mlx5: Report a CQ error event only when a handler was set Leon Romanovsky
2019-06-18 17:15 ` [PATCH mlx5-next v1 06/12] net/mlx5: Report EQE data upon CQ completion Leon Romanovsky
2019-06-18 17:15 ` [PATCH mlx5-next v1 07/12] net/mlx5: Expose device definitions for object events Leon Romanovsky
2019-06-18 17:15 ` [PATCH rdma-next v1 08/12] IB/mlx5: Introduce MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD Leon Romanovsky
2019-06-24 11:51   ` Jason Gunthorpe
2019-06-24 13:25     ` Yishai Hadas
2019-06-24 14:30       ` Jason Gunthorpe
2019-06-18 17:15 ` [PATCH rdma-next v1 09/12] IB/mlx5: Register DEVX with mlx5_core to get async events Leon Romanovsky
2019-06-24 11:52   ` Jason Gunthorpe
2019-06-24 13:36     ` Yishai Hadas
2019-06-24 14:30       ` Jason Gunthorpe
2019-06-18 17:15 ` [PATCH rdma-next v1 10/12] IB/mlx5: Enable subscription for device events over DEVX Leon Romanovsky
2019-06-24 11:57   ` Jason Gunthorpe
2019-06-24 16:13     ` Yishai Hadas
2019-06-24 17:56       ` Jason Gunthorpe
2019-06-18 17:15 ` [PATCH rdma-next v1 11/12] IB/mlx5: Implement DEVX dispatching event Leon Romanovsky
2019-06-24 12:03   ` Jason Gunthorpe
2019-06-24 16:55     ` Yishai Hadas
2019-06-24 18:06       ` Jason Gunthorpe
2019-06-25 14:41         ` Yishai Hadas
2019-06-25 20:23           ` Jason Gunthorpe
2019-06-18 17:15 ` [PATCH rdma-next v1 12/12] IB/mlx5: Add DEVX support for CQ events Leon Romanovsky
2019-06-24 12:04   ` Jason Gunthorpe
2019-06-24 17:03     ` Yishai Hadas
2019-06-24 18:06       ` Jason Gunthorpe
2019-06-18 18:51 ` [PATCH rdma-next v1 00/12] DEVX asynchronous events Saeed Mahameed
2019-06-19  4:45   ` Leon Romanovsky
2019-06-24 21:57     ` Saeed Mahameed
2019-06-30  8:53       ` Leon Romanovsky

Reply instructions:

You may reply publically to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=4ac3d24606b712514c0265d0073850f3f06ec9c3.camel@mellanox.com \
    --to=saeedm@mellanox.com \
    --cc=dledford@redhat.com \
    --cc=jgg@mellanox.com \
    --cc=leon@kernel.org \
    --cc=leonro@mellanox.com \
    --cc=linux-rdma@vger.kernel.org \
    --cc=netdev@vger.kernel.org \
    --cc=yishaih@mellanox.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Netdev Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/netdev/0 netdev/git/0.git
	git clone --mirror https://lore.kernel.org/netdev/1 netdev/git/1.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 netdev netdev/ https://lore.kernel.org/netdev \
		netdev@vger.kernel.org netdev@archiver.kernel.org
	public-inbox-index netdev


Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.netdev


AGPL code for this site: git clone https://public-inbox.org/ public-inbox