RE: [PATCH rdma-core 1/5] RDMA-CORE/erdma: Add userspace verbs related header files.

[Devesh Sharma <devesh.s.sharma@oracle.com>]

> -----Original Message-----
> From: Cheng Xu <chengyou@linux.alibaba.com>
> Sent: Friday, December 24, 2021 12:25 PM
> To: leon@kernel.org
> Cc: dledford@redhat.com; jgg@mellanox.com; linux-rdma@vger.kernel.org;
> KaiShen@linux.alibaba.com; chengyou@linux.alibaba.com
> Subject: [PATCH rdma-core 1/5] RDMA-CORE/erdma: Add userspace verbs
> related header files.
> 
> Add the userspace verbs implementation related header files: 'erdma_hw.h'
> for hardware interface definitions, 'erdma_verbs.h' for verbs related
> definitions and 'erdma_db.h' for doorbell records related definitions.
> 
> Signed-off-by: Cheng Xu <chengyou@linux.alibaba.com>
> ---
>  providers/erdma/erdma_db.h    |  17 +++
>  providers/erdma/erdma_hw.h    | 206
> ++++++++++++++++++++++++++++++++++
>  providers/erdma/erdma_verbs.h | 134 ++++++++++++++++++++++
>  3 files changed, 357 insertions(+)
>  create mode 100644 providers/erdma/erdma_db.h  create mode 100644
> providers/erdma/erdma_hw.h  create mode 100644
> providers/erdma/erdma_verbs.h
> 
> diff --git a/providers/erdma/erdma_db.h b/providers/erdma/erdma_db.h
> new file mode 100644 index 00000000..c302cb7a
> --- /dev/null
> +++ b/providers/erdma/erdma_db.h
> @@ -0,0 +1,17 @@
> +/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING
> +file */
> +/*
> + * Authors: Cheng Xu <chengyou@linux.alibaba.com>
> + * Copyright (c) 2020-2021, Alibaba Group.
> + */
> +
> +#ifndef __ERDMA_DB_H__
> +#define __ERDMA_DB_H__
> +
> +#include <inttypes.h>
> +
> +#include "erdma.h"
> +
> +uint64_t *erdma_alloc_dbrecords(struct erdma_context *ctx); void
> +erdma_dealloc_dbrecords(struct erdma_context *ctx, uint64_t
> +*dbrecords);
> +
> +#endif
> diff --git a/providers/erdma/erdma_hw.h b/providers/erdma/erdma_hw.h
> new file mode 100644 index 00000000..f373f7f8
> --- /dev/null
> +++ b/providers/erdma/erdma_hw.h
> @@ -0,0 +1,206 @@
> +/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING
> +file */
> +/*
> + * Authors: Cheng Xu <chengyou@linux.alibaba.com>
> + * Copyright (c) 2020-2021, Alibaba Group.
> + */
> +
> +#ifndef __ERDMA_HW_H__
> +#define __ERDMA_HW_H__
> +
> +#include <stdint.h>
> +
> +#define ERDMA_SDB_PAGE     0
> +#define ERDMA_SDB_ENTRY    1
> +#define ERDMA_SDB_SHARED   2
> +
> +#define ERDMA_SQDB_SIZE 128
> +#define ERDMA_CQDB_SIZE 8
> +#define ERDMA_RQDB_SIZE 8
> +#define ERDMA_RQDB_SPACE_SIZE   32
> +
> +/* WQE related. */
> +#define EQE_SIZE 16
> +#define EQE_SHIFT 4
> +#define RQE_SIZE 32
> +#define RQE_SHIFT 5
> +#define CQE_SIZE 32
> +#define CQE_SHIFT 5
> +#define SQEBB_SIZE 32
> +#define SQEBB_SHIFT 5
> +#define SQEBB_MASK (~(SQEBB_SIZE - 1))
> +#define SQEBB_ALIGN(size) ((size + SQEBB_SIZE - 1) & SQEBB_MASK)
> +#define SQEBB_COUNT(size) (SQEBB_ALIGN(size) >> SQEBB_SHIFT)
> +
> +#define MAX_WQEBB_PER_SQE 4
> +
> +enum erdma_opcode {
> +	ERDMA_OP_WRITE           = 0,
> +	ERDMA_OP_READ            = 1,
> +	ERDMA_OP_SEND            = 2,
> +	ERDMA_OP_SEND_WITH_IMM   = 3,
> +
> +	ERDMA_OP_RECEIVE         = 4,
> +	ERDMA_OP_RECV_IMM        = 5,
> +	ERDMA_OP_RECV_INV        = 6,
> +
> +	ERDMA_OP_REQ_ERR         = 7,
> +	ERDNA_OP_READ_RESPONSE   = 8,
> +	ERDMA_OP_WRITE_WITH_IMM  = 9,
> +
> +	ERDMA_OP_RECV_ERR        = 10,
> +
> +	ERDMA_OP_INVALIDATE     = 11,
> +	ERDMA_OP_RSP_SEND_IMM   = 12,
> +	ERDMA_OP_SEND_WITH_INV  = 13,
> +
> +	ERDMA_OP_REG_MR         = 14,
> +	ERDMA_OP_LOCAL_INV      = 15,
> +	ERDMA_OP_READ_WITH_INV  = 16,
> +	ERDMA_NUM_OPCODES       = 17,
> +	ERDMA_OP_INVALID        = ERDMA_NUM_OPCODES + 1
> +};
> +
> +/*
> + * Inline data are kept within the work request itself occupying
> + * the space of sge[1] .. sge[n]. Therefore, inline data cannot be
> + * supported if ERDMA_MAX_SGE is below 2 elements.
> + */
> +#define ERDMA_MAX_INLINE	(sizeof(struct erdma_sge) *
> (ERDMA_MAX_SEND_SGE))
> +
> +enum erdma_wc_status {
> +	ERDMA_WC_SUCCESS = 0,
> +	ERDMA_WC_GENERAL_ERR = 1,
> +	ERDMA_WC_RECV_WQE_FORMAT_ERR = 2,
> +	ERDMA_WC_RECV_STAG_INVALID_ERR = 3,
> +	ERDMA_WC_RECV_ADDR_VIOLATION_ERR = 4,
> +	ERDMA_WC_RECV_RIGHT_VIOLATION_ERR = 5,
> +	ERDMA_WC_RECV_PDID_ERR = 6,
> +	ERDMA_WC_RECV_WARRPING_ERR = 7,
> +	ERDMA_WC_SEND_WQE_FORMAT_ERR = 8,
> +	ERDMA_WC_SEND_WQE_ORD_EXCEED = 9,
> +	ERDMA_WC_SEND_STAG_INVALID_ERR = 10,
> +	ERDMA_WC_SEND_ADDR_VIOLATION_ERR = 11,
> +	ERDMA_WC_SEND_RIGHT_VIOLATION_ERR = 12,
> +	ERDMA_WC_SEND_PDID_ERR = 13,
> +	ERDMA_WC_SEND_WARRPING_ERR = 14,
> +	ERDMA_WC_FLUSH_ERR = 15,
> +	ERDMA_WC_RETRY_EXC_ERR = 16,
> +	ERDMA_NUM_WC_STATUS
> +};
> +
> +enum erdma_vendor_err {
> +	ERDMA_WC_VENDOR_NO_ERR = 0,
> +	ERDMA_WC_VENDOR_INVALID_RQE = 1,
> +	ERDMA_WC_VENDOR_RQE_INVALID_STAG = 2,
> +	ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION = 3,
> +	ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR = 4,
> +	ERDMA_WC_VENDOR_RQE_INVALID_PD = 5,
> +	ERDMA_WC_VENDOR_RQE_WRAP_ERR = 6,
> +	ERDMA_WC_VENDOR_INVALID_SQE = 0x20,
> +	ERDMA_WC_VENDOR_ZERO_ORD = 0x21,
> +	ERDMA_WC_VENDOR_SQE_INVALID_STAG = 0x30,
> +	ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION = 0x31,
> +	ERDMA_WC_VENDOR_SQE_ACCESS_ERR = 0x32,
> +	ERDMA_WC_VENDOR_SQE_INVALID_PD = 0x33,
> +	ERDMA_WC_VENDOR_SQE_WARP_ERR = 0x34
> +};
> +
> +
> +/* Doorbell related. */
> +#define ERDMA_CQDB_EQN_MASK              GENMASK_ULL(63, 56)
> +#define ERDMA_CQDB_CQN_MASK              GENMASK_ULL(55, 32)
> +#define ERDMA_CQDB_ARM_MASK              BIT_ULL(31)
> +#define ERDMA_CQDB_SOL_MASK              BIT_ULL(30)
> +#define ERDMA_CQDB_CMDSN_MASK            GENMASK_ULL(29, 28)
> +#define ERDMA_CQDB_CI_MASK               GENMASK_ULL(23, 0)
> +
> +#define ERDMA_CQE_QTYPE_SQ    0
> +#define ERDMA_CQE_QTYPE_RQ    1
> +#define ERDMA_CQE_QTYPE_CMDQ  2
> +
> +/* CQE hdr */
> +#define ERDMA_CQE_HDR_OWNER_MASK         BIT(31)
> +#define ERDMA_CQE_HDR_OPCODE_MASK        GENMASK(23, 16)
> +#define ERDMA_CQE_HDR_QTYPE_MASK         GENMASK(15, 8)
> +#define ERDMA_CQE_HDR_SYNDROME_MASK      GENMASK(7, 0)
> +
> +#define ERDMA_CQE_QTYPE_SQ    0
> +#define ERDMA_CQE_QTYPE_RQ    1
> +#define ERDMA_CQE_QTYPE_CMDQ  2
> +
> +
> +struct erdma_cqe {
> +	__be32 hdr;
> +	__be32 qe_idx;
> +	__be32 qpn;
> +	__le32 imm_data;
> +	__be32 size;
> +	__be32 rsvd[3];
> +};
> +
> +struct erdma_sge {
> +	__aligned_le64 laddr;
> +	__le32 length;
> +	__le32 lkey;
> +};
> +
> +/* Receive Queue Element */
> +struct erdma_rqe {
> +	__le16 qe_idx;
> +	__le16 rsvd;
> +	__le32 qpn;
> +	__le32 rsvd2;
> +	__le32 rsvd3;
> +	__le64 to;
> +	__le32 length;
> +	__le32 stag;
> +};
> +
> +/* SQE */
> +#define ERDMA_SQE_HDR_SGL_LEN_MASK       GENMASK_ULL(63, 56)
> +#define ERDMA_SQE_HDR_WQEBB_CNT_MASK     GENMASK_ULL(54, 52)
> +#define ERDMA_SQE_HDR_QPN_MASK           GENMASK_ULL(51, 32)
> +#define ERDMA_SQE_HDR_OPCODE_MASK        GENMASK_ULL(31, 27)
> +#define ERDMA_SQE_HDR_DWQE_MASK          BIT_ULL(26)
> +#define ERDMA_SQE_HDR_INLINE_MASK        BIT_ULL(25)
> +#define ERDMA_SQE_HDR_FENCE_MASK         BIT_ULL(24)
> +#define ERDMA_SQE_HDR_SE_MASK            BIT_ULL(23)
> +#define ERDMA_SQE_HDR_CE_MASK            BIT_ULL(22)
> +#define ERDMA_SQE_HDR_WQEBB_INDEX_MASK   GENMASK_ULL(15, 0)
> +
> +struct erdma_write_sqe {
> +	__le64 hdr;
> +	__be32 imm_data;
> +	__le32 length;
> +
> +	__le32 sink_stag;
> +	/* avoid sink_to not 8-byte aligned. */
> +	__le32 sink_to_low;
> +	__le32 sink_to_high;
> +
> +	__le32 rsvd;
> +
> +	struct erdma_sge sgl[0];
> +};
> +
> +struct erdma_send_sqe {
> +	__le64 hdr;
> +	__be32 imm_data;
> +	__le32 length;
> +	struct erdma_sge sgl[0];
> +};
> +
> +struct erdma_readreq_sqe {
> +	__le64 hdr;
> +	__le32 invalid_stag;
> +	__le32 length;
> +	__le32 sink_stag;
> +	/* avoid sink_to not 8-byte aligned. */
> +	__le32 sink_to_low;
> +	__le32 sink_to_high;
> +	__le32 rsvd0;
> +	struct erdma_sge sgl;
> +};
> +
> +
> +#endif
> diff --git a/providers/erdma/erdma_verbs.h
> b/providers/erdma/erdma_verbs.h new file mode 100644 index
> 00000000..944d29f5
> --- /dev/null
> +++ b/providers/erdma/erdma_verbs.h
> @@ -0,0 +1,134 @@
> +/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING
> +file */
> +/*
> + * Authors: Cheng Xu <chengyou@linux.alibaba.com>
> + * Copyright (c) 2020-2021, Alibaba Group.
> + */
> +
> +#ifndef __ERDMA_VERBS_H__
> +#define __ERDMA_VERBS_H__
> +
> +#include <pthread.h>
> +#include <inttypes.h>
> +#include <stddef.h>
> +
> +#include "erdma.h"
> +#include "erdma_hw.h"
> +
> +#define ERDMA_MAX_SEND_SGE 6
> +#define ERDMA_MAX_RECV_SGE 1
> +
> +struct erdma_queue {
> +	void *qbuf;
> +	void *db;
> +
> +	uint16_t rsvd0;
> +	uint16_t depth;
> +	uint32_t size;
> +
> +	uint16_t pi;
> +	uint16_t ci;
> +
> +	uint32_t rsvd1;
> +	uint64_t *wr_tbl;
> +
> +	void *db_record;
> +};
> +
> +struct erdma_qp {
> +	struct ibv_qp base_qp;
> +	struct erdma_device *erdma_dev;
> +
> +	uint32_t id;  /* qpn */
> +
> +	pthread_spinlock_t sq_lock;
> +	pthread_spinlock_t rq_lock;
> +
> +	int sq_sig_all;
> +
> +	struct erdma_queue sq;
> +	struct erdma_queue rq;
> +
> +	void *qbuf;
> +	uint64_t *db_records;
> +};
> +
> +struct erdma_cq {
> +	struct ibv_cq base_cq;
> +	struct erdma_device *erdma_dev;
> +	uint32_t id;
> +
> +	uint32_t event_stats;
> +
> +	uint32_t depth;
> +	uint32_t ci;
> +	uint32_t owner;
> +	struct erdma_cqe *queue;
> +
> +	void *db;
> +	uint16_t db_offset;
> +
> +	void *db_record;
> +	uint32_t cmdsn;
> +	int comp_vector;
> +
> +	pthread_spinlock_t lock;
> +};
> +
> +static inline struct erdma_qp *to_eqp(struct ibv_qp *base) {
> +	return container_of(base, struct erdma_qp, base_qp); }
> +
> +static inline struct erdma_cq *to_ecq(struct ibv_cq *base) {
> +	return container_of(base, struct erdma_cq, base_cq); }
> +
> +static inline void *get_sq_wqebb(struct erdma_qp *qp, uint16_t idx) {
> +	idx &= (qp->sq.depth - 1);
> +	return qp->sq.qbuf + (idx << SQEBB_SHIFT); }
> +
> +static inline void __kick_sq_db(struct erdma_qp *qp, uint16_t pi) {
> +	uint64_t db_data;
> +
> +	db_data = FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, qp->id) |
> +		FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, pi);
> +
> +	*(__le64 *)qp->sq.db_record = htole64(db_data);
> +	udma_to_device_barrier();
> +	mmio_write64_le(qp->sq.db, htole64(db_data)); }
Standard function definition format
Func ()
{

}
Update all over the place.
> +
> +struct ibv_pd *erdma_alloc_pd(struct ibv_context *ctx); int
> +erdma_free_pd(struct ibv_pd *pd);
> +
> +int erdma_query_device(struct ibv_context *ctx,
> +		       const struct ibv_query_device_ex_input *input,
> +		       struct ibv_device_attr_ex *attr, size_t attr_size); int
> +erdma_query_port(struct ibv_context *ctx, uint8_t port, struct
> +ibv_port_attr *attr);
> +
> +struct ibv_mr *erdma_reg_mr(struct ibv_pd *pd, void *addr, size_t len,
> +			    uint64_t hca_va, int access);
> +int erdma_dereg_mr(struct verbs_mr *vmr);
> +
> +struct ibv_qp *erdma_create_qp(struct ibv_pd *pd, struct
> +ibv_qp_init_attr *attr); int erdma_modify_qp(struct ibv_qp *base_qp,
> +struct ibv_qp_attr *attr, int attr_mask); int erdma_query_qp(struct ibv_qp
> *qp, struct ibv_qp_attr *attr,
> +		   int attr_mask, struct ibv_qp_init_attr *init_attr); int
> +erdma_post_send(struct ibv_qp *base_qp, struct ibv_send_wr *wr, struct
> +ibv_send_wr **bad_wr); int erdma_post_recv(struct ibv_qp *base_qp,
> +struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr); int
> +erdma_destroy_qp(struct ibv_qp *base_qp);
> +
> +void erdma_free_context(struct ibv_context *ibv_ctx);
> +
> +struct ibv_cq *erdma_create_cq(struct ibv_context *ctx, int num_cqe,
> +			       struct ibv_comp_channel *channel, int
> comp_vector); int
> +erdma_destroy_cq(struct ibv_cq *base_cq); int erdma_notify_cq(struct
> +ibv_cq *ibcq, int solicited); void erdma_cq_event(struct ibv_cq *ibcq);
> +int erdma_poll_cq(struct ibv_cq *ibcq, int num_entries, struct ibv_wc
> +*wc);
> +
> +#endif
> --
> 2.27.0