Linux-RDMA Archive on lore.kernel.org
 help / color / Atom feed
From: Zhu Yanjun <zyjzyj2000@gmail.com>
To: Bob Pearson <rpearsonhpe@gmail.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>,
	RDMA mailing list <linux-rdma@vger.kernel.org>,
	Bob Pearson <rpearson@hpe.com>
Subject: Re: [PATCH for-next v3 4/9] RDMA/rxe: Add ib_alloc_mw and ib_dealloc_mw verbs
Date: Wed, 21 Apr 2021 13:54:49 +0800
Message-ID: <CAD=hENe-Xw2L46=U2uLO4evmFfgNNMWJBLtSYid=SNrJ9W75Yw@mail.gmail.com> (raw)
In-Reply-To: <20210421052015.4546-5-rpearson@hpe.com>

On Wed, Apr 21, 2021 at 1:20 PM Bob Pearson <rpearsonhpe@gmail.com> wrote:
>
> Add ib_alloc_mw and ib_dealloc_mw verbs APIs.
>
> Added new file rxe_mw.c focused on MWs.
> Changed the 8 bit random key generator.
> Added a cleanup routine for MWs.
> Added verbs routines to ib_device_ops.
>
> Signed-off-by: Bob Pearson <rpearson@hpe.com>
> ---
>  drivers/infiniband/sw/rxe/Makefile    |  1 +
>  drivers/infiniband/sw/rxe/rxe_loc.h   |  6 +++
>  drivers/infiniband/sw/rxe/rxe_mr.c    | 20 +++++-----
>  drivers/infiniband/sw/rxe/rxe_mw.c    | 53 +++++++++++++++++++++++++++
>  drivers/infiniband/sw/rxe/rxe_pool.c  |  1 +
>  drivers/infiniband/sw/rxe/rxe_verbs.c |  3 ++
>  drivers/infiniband/sw/rxe/rxe_verbs.h |  2 +
>  7 files changed, 75 insertions(+), 11 deletions(-)
>  create mode 100644 drivers/infiniband/sw/rxe/rxe_mw.c
>
> diff --git a/drivers/infiniband/sw/rxe/Makefile b/drivers/infiniband/sw/rxe/Makefile
> index 66af72dca759..1e24673e9318 100644
> --- a/drivers/infiniband/sw/rxe/Makefile
> +++ b/drivers/infiniband/sw/rxe/Makefile
> @@ -15,6 +15,7 @@ rdma_rxe-y := \
>         rxe_qp.o \
>         rxe_cq.o \
>         rxe_mr.o \
> +       rxe_mw.o \
>         rxe_opcode.o \
>         rxe_mmap.o \
>         rxe_icrc.o \
> diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
> index ef8061d2fbe0..edf575930a98 100644
> --- a/drivers/infiniband/sw/rxe/rxe_loc.h
> +++ b/drivers/infiniband/sw/rxe/rxe_loc.h
> @@ -76,6 +76,7 @@ enum copy_direction {
>         from_mr_obj,
>  };
>
> +u8 rxe_get_next_key(u32 last_key);
>  void rxe_mr_init_dma(struct rxe_pd *pd, int access, struct rxe_mr *mr);
>
>  int rxe_mr_init_user(struct rxe_pd *pd, u64 start, u64 length, u64 iova,
> @@ -106,6 +107,11 @@ void rxe_mr_cleanup(struct rxe_pool_entry *arg);
>
>  int advance_dma_data(struct rxe_dma_info *dma, unsigned int length);
>
> +/* rxe_mw.c */
> +int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
> +int rxe_dealloc_mw(struct ib_mw *ibmw);
> +void rxe_mw_cleanup(struct rxe_pool_entry *arg);
> +
>  /* rxe_net.c */
>  void rxe_loopback(struct sk_buff *skb);
>  int rxe_send(struct rxe_pkt_info *pkt, struct sk_buff *skb);
> diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
> index 9f63947bab12..7f2cfc1ce659 100644
> --- a/drivers/infiniband/sw/rxe/rxe_mr.c
> +++ b/drivers/infiniband/sw/rxe/rxe_mr.c
> @@ -7,19 +7,17 @@
>  #include "rxe.h"
>  #include "rxe_loc.h"
>
> -/*
> - * lfsr (linear feedback shift register) with period 255
> +/* Return a random 8 bit key value that is
> + * different than the last_key. Set last_key to -1
> + * if this is the first key for an MR or MW
>   */
> -static u8 rxe_get_key(void)
> +u8 rxe_get_next_key(u32 last_key)
>  {
> -       static u32 key = 1;
> -
> -       key = key << 1;
> -
> -       key |= (0 != (key & 0x100)) ^ (0 != (key & 0x10))
> -               ^ (0 != (key & 0x80)) ^ (0 != (key & 0x40));
> +       u8 key;
>
> -       key &= 0xff;
> +       do {
> +               get_random_bytes(&key, 1);
> +       } while (key == last_key);
>
>         return key;
>  }
> @@ -47,7 +45,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
>
>  static void rxe_mr_init(int access, struct rxe_mr *mr)
>  {
> -       u32 lkey = mr->pelem.index << 8 | rxe_get_key();
> +       u32 lkey = mr->pelem.index << 8 | rxe_get_next_key(-1);
>         u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;
>
>         mr->ibmr.lkey = lkey;
> diff --git a/drivers/infiniband/sw/rxe/rxe_mw.c b/drivers/infiniband/sw/rxe/rxe_mw.c
> new file mode 100644
> index 000000000000..69128e298d44
> --- /dev/null
> +++ b/drivers/infiniband/sw/rxe/rxe_mw.c
> @@ -0,0 +1,53 @@
> +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
> +/*
> + * Copyright (c) 2020 Hewlett Packard Enterprise, Inc. All rights reserved.
> + */

The above is different with the following:
"
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
 */
"

One is Hewlett Packard Enterprise, Inc, the other is Mellanox Technologies Ltd.

I am not sure whether it is a legal problem.

Leave this problem to Jason Gunthorpe.

Zhu Yanjun

> +
> +#include "rxe.h"
> +
> +int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
> +{
> +       struct rxe_mw *mw = to_rmw(ibmw);
> +       struct rxe_pd *pd = to_rpd(ibmw->pd);
> +       struct rxe_dev *rxe = to_rdev(ibmw->device);
> +       int ret;
> +
> +       rxe_add_ref(pd);
> +
> +       ret = rxe_add_to_pool(&rxe->mw_pool, mw);
> +       if (ret) {
> +               rxe_drop_ref(pd);
> +               return ret;
> +       }
> +
> +       rxe_add_index(mw);
> +       ibmw->rkey = (mw->pelem.index << 8) | rxe_get_next_key(-1);
> +       mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?
> +                       RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;
> +       spin_lock_init(&mw->lock);
> +
> +       return 0;
> +}
> +
> +int rxe_dealloc_mw(struct ib_mw *ibmw)
> +{
> +       struct rxe_mw *mw = to_rmw(ibmw);
> +       struct rxe_pd *pd = to_rpd(ibmw->pd);
> +       unsigned long flags;
> +
> +       spin_lock_irqsave(&mw->lock, flags);
> +       mw->state = RXE_MW_STATE_INVALID;
> +       spin_unlock_irqrestore(&mw->lock, flags);
> +
> +       rxe_drop_ref(mw);
> +       rxe_drop_ref(pd);
> +
> +       return 0;
> +}
> +
> +void rxe_mw_cleanup(struct rxe_pool_entry *elem)
> +{
> +       struct rxe_mw *mw = container_of(elem, typeof(*mw), pelem);
> +
> +       rxe_drop_index(mw);
> +}
> diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
> index 2b795e2fc4b3..5b3277e8c35d 100644
> --- a/drivers/infiniband/sw/rxe/rxe_pool.c
> +++ b/drivers/infiniband/sw/rxe/rxe_pool.c
> @@ -65,6 +65,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
>                 .name           = "rxe-mw",
>                 .size           = sizeof(struct rxe_mw),
>                 .elem_offset    = offsetof(struct rxe_mw, pelem),
> +               .cleanup        = rxe_mw_cleanup,
>                 .flags          = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
>                 .max_index      = RXE_MAX_MW_INDEX,
>                 .min_index      = RXE_MIN_MW_INDEX,
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
> index aeb5e232c195..fff81bf78a86 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.c
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
> @@ -1060,6 +1060,7 @@ static const struct ib_device_ops rxe_dev_ops = {
>
>         .alloc_hw_stats = rxe_ib_alloc_hw_stats,
>         .alloc_mr = rxe_alloc_mr,
> +       .alloc_mw = rxe_alloc_mw,
>         .alloc_pd = rxe_alloc_pd,
>         .alloc_ucontext = rxe_alloc_ucontext,
>         .attach_mcast = rxe_attach_mcast,
> @@ -1069,6 +1070,7 @@ static const struct ib_device_ops rxe_dev_ops = {
>         .create_srq = rxe_create_srq,
>         .create_user_ah = rxe_create_ah,
>         .dealloc_driver = rxe_dealloc,
> +       .dealloc_mw = rxe_dealloc_mw,
>         .dealloc_pd = rxe_dealloc_pd,
>         .dealloc_ucontext = rxe_dealloc_ucontext,
>         .dereg_mr = rxe_dereg_mr,
> @@ -1106,6 +1108,7 @@ static const struct ib_device_ops rxe_dev_ops = {
>
>         INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
>         INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
> +       INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
>         INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
>         INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
>         INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
> index 8d32e3f50813..c8597ae8c833 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.h
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
> @@ -323,6 +323,8 @@ enum rxe_mw_state {
>  struct rxe_mw {
>         struct ib_mw ibmw;
>         struct rxe_pool_entry pelem;
> +       spinlock_t lock;
> +       enum rxe_mw_state state;
>  };
>
>  struct rxe_mc_grp {
> --
> 2.27.0
>

  reply index

Thread overview: 16+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2021-04-21  5:20 [PATCH for-next v3 0/9] RDMA/rxe: Implement memory windows Bob Pearson
2021-04-21  5:20 ` [PATCH for-next v3 1/9] RDMA/rxe: Add bind MW fields to rxe_send_wr Bob Pearson
2021-04-21  5:20 ` [PATCH for-next v3 2/9] RDMA/rxe: Return errors for add index and key Bob Pearson
2021-04-21  5:20 ` [PATCH for-next v3 3/9] RDMA/rxe: Enable MW object pool Bob Pearson
2021-04-21  5:20 ` [PATCH for-next v3 4/9] RDMA/rxe: Add ib_alloc_mw and ib_dealloc_mw verbs Bob Pearson
2021-04-21  5:54   ` Zhu Yanjun [this message]
2021-04-21  6:20     ` Leon Romanovsky
2021-04-21  6:26       ` Zhu Yanjun
2021-04-21  5:20 ` [PATCH for-next v3 5/9] RDMA/rxe: Replace WR_REG_MASK by WR_LOCAL_OP_MASK Bob Pearson
2021-04-21  5:20 ` [PATCH for-next v3 6/9] RDMA/rxe: Move local ops to subroutine Bob Pearson
2021-04-21  5:20 ` [PATCH for-next v3 7/9] RDMA/rxe: Add support for bind MW work requests Bob Pearson
2021-04-21  9:36   ` Zhu Yanjun
2021-04-21 15:21     ` Robert Pearson
2021-04-22  5:33     ` Bob Pearson
2021-04-21  5:20 ` [PATCH for-next v3 8/9] RDMA/rxe: Implement invalidate MW operations Bob Pearson
2021-04-21  5:20 ` [PATCH for-next v3 9/9] RDMA/rxe: Implement memory access through MWs Bob Pearson

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to='CAD=hENe-Xw2L46=U2uLO4evmFfgNNMWJBLtSYid=SNrJ9W75Yw@mail.gmail.com' \
    --to=zyjzyj2000@gmail.com \
    --cc=jgg@nvidia.com \
    --cc=linux-rdma@vger.kernel.org \
    --cc=rpearson@hpe.com \
    --cc=rpearsonhpe@gmail.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link

Linux-RDMA Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/linux-rdma/0 linux-rdma/git/0.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 linux-rdma linux-rdma/ https://lore.kernel.org/linux-rdma \
		linux-rdma@vger.kernel.org
	public-inbox-index linux-rdma

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.kernel.vger.linux-rdma


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git