Netdev Archive on lore.kernel.org
 help / color / Atom feed
From: Saeed Mahameed <saeedm@mellanox.com>
To: Jason Gunthorpe <jgg@mellanox.com>,
	"leon@kernel.org" <leon@kernel.org>,
	"dledford@redhat.com" <dledford@redhat.com>
Cc: Mark Zhang <markz@mellanox.com>, Majd Dibbiny <majd@mellanox.com>,
	Leon Romanovsky <leonro@mellanox.com>,
	"linux-rdma@vger.kernel.org" <linux-rdma@vger.kernel.org>,
	"netdev@vger.kernel.org" <netdev@vger.kernel.org>
Subject: Re: [PATCH mlx5-next v4 07/17] IB/mlx5: Support set qp counter
Date: Tue, 18 Jun 2019 18:49:01 +0000
Message-ID: <29d643a5e342cc898b189be7f8f3f9889cf58e2c.camel@mellanox.com> (raw)
In-Reply-To: <20190618172625.13432-8-leon@kernel.org>

On Tue, 2019-06-18 at 20:26 +0300, Leon Romanovsky wrote:
> From: Mark Zhang <markz@mellanox.com>
> 
> Support bind a qp with counter. If counter is null then bind the qp
> to
> the default counter. Different QP state has different operation:
> - RESET: Set the counter field so that it will take effective
>   during RST2INIT change;
> - RTS: Issue an RTS2RTS change to update the QP counter;
> - Other: Set the counter field and mark the counter_pending flag,
>   when QP is moved to RTS state and this flag is set, then issue
>   an RTS2RTS modification to update the counter.
> 
> Signed-off-by: Mark Zhang <markz@mellanox.com>
> Reviewed-by: Majd Dibbiny <majd@mellanox.com>
> Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
> ---
>  drivers/infiniband/hw/mlx5/mlx5_ib.h |  6 +++
>  drivers/infiniband/hw/mlx5/qp.c      | 76
> +++++++++++++++++++++++++++-
>  include/linux/mlx5/qp.h              |  1 +
>  3 files changed, 81 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h
> b/drivers/infiniband/hw/mlx5/mlx5_ib.h
> index 35e2c8f5ae78..b7d36f4826c1 100644
> --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
> +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
> @@ -442,6 +442,10 @@ struct mlx5_ib_qp {
>  	u32			flags_en;
>  	/* storage for qp sub type when core qp type is IB_QPT_DRIVER
> */
>  	enum ib_qp_type		qp_sub_type;
> +	/* A flag to indicate if there's a new counter is configured
> +	 * but not take effective
> +	 */
> +	u32                     counter_pending;
>  };
> 
>  struct mlx5_ib_cq_buf {
> @@ -1442,4 +1446,6 @@ void mlx5_ib_put_xlt_emergency_page(void);
>  int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
>  			struct mlx5_bfreg_info *bfregi, u32 bfregn,
>  			bool dyn_bfreg);
> +
> +int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter
> *counter);
>  #endif /* MLX5_IB_H */
> diff --git a/drivers/infiniband/hw/mlx5/qp.c
> b/drivers/infiniband/hw/mlx5/qp.c
> index f6623c77443a..8dbbef843010 100644
> --- a/drivers/infiniband/hw/mlx5/qp.c
> +++ b/drivers/infiniband/hw/mlx5/qp.c
> @@ -34,6 +34,7 @@
>  #include <rdma/ib_umem.h>
>  #include <rdma/ib_cache.h>
>  #include <rdma/ib_user_verbs.h>
> +#include <rdma/rdma_counter.h>
>  #include <linux/mlx5/fs.h>
>  #include "mlx5_ib.h"
>  #include "ib_rep.h"
> @@ -3386,6 +3387,35 @@ static unsigned int get_tx_affinity(struct
> mlx5_ib_dev *dev,
>  	return tx_port_affinity;
>  }
> 
> +static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,
> +				    struct rdma_counter *counter)
> +{
> +	struct mlx5_ib_dev *dev = to_mdev(qp->device);
> +	struct mlx5_ib_qp *mqp = to_mqp(qp);
> +	struct mlx5_qp_context context = {};
> +	struct mlx5_ib_port *mibport = NULL;
> +	struct mlx5_ib_qp_base *base;
> +	u32 set_id;
> +
> +	if (!MLX5_CAP_GEN(dev->mdev, rts2rts_qp_counters_set_id))
> +		return 0;
> +
> +	if (counter) {
> +		set_id = counter->id;
> +	} else {
> +		mibport = &dev->port[mqp->port - 1];
> +		set_id = mibport->cnts.set_id;
> +	}
> +
> +	base = &mqp->trans_qp.base;
> +	context.qp_counter_set_usr_page &= cpu_to_be32(0xffffff);
> +	context.qp_counter_set_usr_page |= cpu_to_be32(set_id << 24);
> +	return mlx5_core_qp_modify(dev->mdev,
> +				   MLX5_CMD_OP_RTS2RTS_QP,
> +				   MLX5_QP_OPTPAR_COUNTER_SET_ID,
> +				   &context, &base->mqp);
> +}
> +
>  static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
>  			       const struct ib_qp_attr *attr, int
> attr_mask,
>  			       enum ib_qp_state cur_state,
> @@ -3439,6 +3469,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp
> *ibqp,
>  	struct mlx5_ib_port *mibport = NULL;
>  	enum mlx5_qp_state mlx5_cur, mlx5_new;
>  	enum mlx5_qp_optpar optpar;
> +	u32 set_id = 0;
>  	int mlx5_st;
>  	int err;
>  	u16 op;
> @@ -3601,8 +3632,12 @@ static int __mlx5_ib_modify_qp(struct ib_qp
> *ibqp,
>  			port_num = 0;
> 
>  		mibport = &dev->port[port_num];
> +		if (ibqp->counter)
> +			set_id = ibqp->counter->id;
> +		else
> +			set_id = mibport->cnts.set_id;
>  		context->qp_counter_set_usr_page |=
> -			cpu_to_be32((u32)(mibport->cnts.set_id) << 24);
> +			cpu_to_be32(set_id << 24);
>  	}
> 
>  	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state ==
> IB_QPS_INIT)
> @@ -3630,7 +3665,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp
> *ibqp,
> 
>  		raw_qp_param.operation = op;
>  		if (cur_state == IB_QPS_RESET && new_state ==
> IB_QPS_INIT) {
> -			raw_qp_param.rq_q_ctr_id = mibport-
> >cnts.set_id;
> +			raw_qp_param.rq_q_ctr_id = set_id;
>  			raw_qp_param.set_mask |=
> MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
>  		}
> 
> @@ -3707,6 +3742,12 @@ static int __mlx5_ib_modify_qp(struct ib_qp
> *ibqp,
>  		qp->db.db[MLX5_SND_DBR] = 0;
>  	}
> 
> +	if ((new_state == IB_QPS_RTS) && qp->counter_pending) {
> +		err = __mlx5_ib_qp_set_counter(ibqp, ibqp->counter);
> +		if (!err)
> +			qp->counter_pending = 0;
> +	}
> +
>  out:
>  	kfree(context);
>  	return err;
> @@ -6367,3 +6408,34 @@ void mlx5_ib_drain_rq(struct ib_qp *qp)
> 
>  	handle_drain_completion(cq, &rdrain, dev);
>  }
> +
> +/**
> + * Bind a qp to a counter. If @counter is NULL then bind the qp to
> + * the default counter
> + */
> +int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter
> *counter)
> +{
> +	struct mlx5_ib_qp *mqp = to_mqp(qp);
> +	int err = 0;
> +
> +	mutex_lock(&mqp->mutex);
> +	if (mqp->state == IB_QPS_RESET) {
> +		qp->counter = counter;
> +		goto out;
> +	}
> +
> +	if (mqp->state == IB_QPS_RTS) {
> +		err = __mlx5_ib_qp_set_counter(qp, counter);
> +		if (!err)
> +			qp->counter = counter;
> +
> +		goto out;
> +	}
> +
> +	mqp->counter_pending = 1;
> +	qp->counter = counter;
> +
> +out:
> +	mutex_unlock(&mqp->mutex);
> +	return err;
> +}
> diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h
> index 3ba4edbd17a6..787c5fabdc07 100644
> --- a/include/linux/mlx5/qp.h
> +++ b/include/linux/mlx5/qp.h
> @@ -70,6 +70,7 @@ enum mlx5_qp_optpar {
>  	MLX5_QP_OPTPAR_CQN_RCV			= 1 << 19,
>  	MLX5_QP_OPTPAR_DC_HS			= 1 << 20,
>  	MLX5_QP_OPTPAR_DC_KEY			= 1 << 21,
> +	MLX5_QP_OPTPAR_COUNTER_SET_ID		= 1 << 25,
>  };
> 

Acked-by: Saeed Mahameed <saeedm@mellanox.com>

>  enum mlx5_qp_state {
> --
> 2.20.1
> 

  reply index

Thread overview: 23+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2019-06-18 17:26 [PATCH rdma-next v4 00/17] Statistics counter support Leon Romanovsky
2019-06-18 17:26 ` [PATCH mlx5-next v4 01/17] net/mlx5: Add rts2rts_qp_counters_set_id field in hca cap Leon Romanovsky
2019-06-18 18:47   ` Saeed Mahameed
2019-06-18 17:26 ` [PATCH rdma-next v4 02/17] RDMA/restrack: Introduce statistic counter Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 03/17] RDMA/restrack: Add an API to attach a task to a resource Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 04/17] RDMA/restrack: Make is_visible_in_pid_ns() as an API Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 05/17] RDMA/counter: Add set/clear per-port auto mode support Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 06/17] RDMA/counter: Add "auto" configuration " Leon Romanovsky
2019-06-30  0:32   ` Jason Gunthorpe
2019-06-30  0:40   ` Jason Gunthorpe
2019-07-01  8:42     ` Leon Romanovsky
2019-06-18 17:26 ` [PATCH mlx5-next v4 07/17] IB/mlx5: Support set qp counter Leon Romanovsky
2019-06-18 18:49   ` Saeed Mahameed [this message]
2019-06-18 17:26 ` [PATCH rdma-next v4 08/17] IB/mlx5: Add counter set id as a parameter for mlx5_ib_query_q_counters() Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 09/17] IB/mlx5: Support statistic q counter configuration Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 10/17] RDMA/nldev: Allow counter auto mode configration through RDMA netlink Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 11/17] RDMA/netlink: Implement counter dumpit calback Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 12/17] IB/mlx5: Add counter_alloc_stats() and counter_update_stats() support Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 13/17] RDMA/core: Get sum value of all counters when perform a sysfs stat read Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 14/17] RDMA/counter: Allow manual mode configuration support Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 15/17] RDMA/nldev: Allow counter manual mode configration through RDMA netlink Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 16/17] RDMA/nldev: Allow get counter mode " Leon Romanovsky
2019-06-18 17:26 ` [PATCH rdma-next v4 17/17] RDMA/nldev: Allow get default counter statistics " Leon Romanovsky

Reply instructions:

You may reply publically to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=29d643a5e342cc898b189be7f8f3f9889cf58e2c.camel@mellanox.com \
    --to=saeedm@mellanox.com \
    --cc=dledford@redhat.com \
    --cc=jgg@mellanox.com \
    --cc=leon@kernel.org \
    --cc=leonro@mellanox.com \
    --cc=linux-rdma@vger.kernel.org \
    --cc=majd@mellanox.com \
    --cc=markz@mellanox.com \
    --cc=netdev@vger.kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Netdev Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/netdev/0 netdev/git/0.git
	git clone --mirror https://lore.kernel.org/netdev/1 netdev/git/1.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 netdev netdev/ https://lore.kernel.org/netdev \
		netdev@vger.kernel.org netdev@archiver.kernel.org
	public-inbox-index netdev


Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.netdev


AGPL code for this site: git clone https://public-inbox.org/ public-inbox