Regarding using UMR feature from. rdma-core API

* Regarding using UMR feature from. rdma-core API
@ 2020-02-24 10:54 Umang Patel
  2020-02-24 11:13 ` Yishai Hadas
  2020-02-24 14:25 ` Umang Patel
  0 siblings, 2 replies; 4+ messages in thread
From: Umang Patel @ 2020-02-24 10:54 UTC (permalink / raw)
  To: linux-rdma

[-- Attachment #1: Type: text/plain, Size: 1219 bytes --]

Hello!

As you said, I have already read the mlx5dv_wr_post() manual but I am
still having trouble even after following the instructions given in
it. I have created a normal client-server code for transferring some
data from client to server using rdma. I am using the the rdma-core
repo for the code. For posting the send request with the UMR feature,
i use the function "mlx5dv_wr_mr_list" and to post the send request
without UMR feature, i use the function "ibv_wr_send". The two
functions that I mentioned are similar as mentioned in the manual. I
have written my code in such a way that it asks whether or not to use
the UMR feature and I have put an if else in the code which runs the
function according to the user's answer whether or not to use the
function and rest of the code is the does not change for with UMR and
without UMR. However, I am unable to receive the data with
mlx5dv_wr_mr_list function. Whereas, using ibv_wr_send, I am able to
receive the correct data.

Can you please look into it? Am I missing something in the code that i
have to add for the UMR feature or the server side requires something
else? what is the problem? I have attached the client-server code with
this email.

Thank You!

[-- Attachment #2: server.c --]
[-- Type: application/octet-stream, Size: 4694 bytes --]

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <getopt.h>
#include <netdb.h>
#include <sys/mman.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>
#include <mlx5dv.h>

#define LENGTH (256UL*1024*1024)
#define PROTECTION (PROT_READ | PROT_WRITE)
#define FLAGS (MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB)

enum {
        UMR_RECV_WRID = 1,
        UMR_SEND_WRID = 2,
};

int main()
{
	struct rdma_event_channel *cm_channel;
	struct rdma_cm_id *cm_id,*listen_id;
	cm_channel = rdma_create_event_channel();
	rdma_create_id(cm_channel, &listen_id, NULL, RDMA_PS_TCP);

	struct sockaddr_in sin;
	memset(&sin, 0, sizeof(sin));
	sin.sin_family = AF_INET;
	sin.sin_port        = htons(20079);
        sin.sin_addr.s_addr = INADDR_ANY;

	int ret = rdma_bind_addr(listen_id, (struct sockaddr *) &sin);
	if (ret) {
		printf("Error in rdma_bind_addr\n");
		goto clean_buffer;
	}

	printf("ANK Server listening...%d\n",errno);
	ret = rdma_listen(listen_id, 1);
	if (ret) {
		printf("Error in rdma_listen\n");
		goto clean_buffer;
	}
	printf("Listened%d\n",errno);
	struct rdma_cm_event *event;
	ret = rdma_get_cm_event(cm_channel, &event);
	if (ret) {
		printf("Error in rdma_get_cm_event\n");
		goto clean_buffer;
	}
	printf("Got the event%d\n",errno);
	if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
		printf("Error event is not CONNECT_REQUEST\n");
		goto clean_buffer;
	}
	cm_id = event->id;
	rdma_ack_cm_event(event);
	struct ibv_context *context;
	context = cm_id->verbs;
	struct ibv_pd *pd;
	pd = ibv_alloc_pd(context);
	if(pd==NULL) {
		printf("pd is null\n");
		exit(1);
	}
	printf("Allocated PD\n");
	struct ibv_cq *cq;
	cq = ibv_create_cq(context,501,NULL,NULL,0);
	if(cq==NULL) {
		printf("cq is NULL\n");
		exit(2);
	}
	printf("Allocated cq\n");
	struct ibv_qp_init_attr attr = {
		.send_cq = cq,
		.recv_cq = cq,
		.cap     = {
			.max_send_wr  = 1,
			.max_recv_wr  = 500,
			.max_send_sge = 2,
			.max_recv_sge = 2,
		},
		.qp_type = IBV_QPT_RC,
	};
	struct ibv_qp *qp = NULL;
	qp = ibv_create_qp(pd, &attr);
	if(qp == NULL) {
		printf("Could not create qp\n");
		exit(1);
	}

	cm_id->qp = qp;
	struct ibv_qp_attr nattr = {
                        .qp_state        = IBV_QPS_INIT,
                        .pkey_index      = 0,
                        .port_num        = 1,
                        .qp_access_flags = 0
                };

                if (ibv_modify_qp(qp, &nattr,
                                  IBV_QP_STATE              |
                                  IBV_QP_PKEY_INDEX         |
                                  IBV_QP_PORT               |
                                  IBV_QP_ACCESS_FLAGS)) {
                        fprintf(stderr, "Failed to modify QP to INIT\n");
                        goto clean_buffer;
                }

	struct rdma_conn_param conn_param = {};
	conn_param.responder_resources = 1;

	void *hbuf;
        void **buf;
        int num_mrs = 1;
        int size = 4*1024*1024;

	hbuf = mmap(NULL, LENGTH, PROTECTION, FLAGS, 0, 0);
	memset(hbuf, 0 , LENGTH);
        buf = calloc(num_mrs, sizeof(void *));
        int i;
        for (i = 0; i < num_mrs; i++) {
        	buf[i] = hbuf + (i * size);
        	memset(buf[i], 0, size);
        }

        struct ibv_mr *mr_arr;
        mr_arr = ibv_reg_mr(pd, hbuf, LENGTH,
                           IBV_ACCESS_LOCAL_WRITE);

	struct ibv_sge list = {
                .addr   = (uintptr_t) (uint64_t)(uintptr_t)mr_arr->addr,
                .length = mr_arr->length,
                .lkey   = mr_arr->lkey,
        };

        struct ibv_recv_wr wr = {
                .wr_id      = UMR_RECV_WRID,
                .sg_list    = &list,
                .num_sge    = 1,
        };

        struct ibv_recv_wr *bad_wr;
        ret = ibv_post_recv(qp, &wr, &bad_wr);
	if(ret) {
		printf("error in post_recv\n");
		exit(1);
	}

	ret = rdma_accept(cm_id, &conn_param);
        if (ret) {
		printf("Error in rdma_accept\n");
		goto close_umr;
	}

        ret = rdma_get_cm_event(cm_channel, &event);
        if (ret) {
		printf("Error in get event (established)\n");
		goto close_umr;
	}

        if (event->event != RDMA_CM_EVENT_ESTABLISHED) {
		printf("Error, event is not ESTABLISHED event = %d\n",event->event);
		exit(1);
		goto close_umr;
	}

        rdma_ack_cm_event(event);

	struct ibv_wc wc;
	ret = ibv_post_recv(qp, &wr, &bad_wr);
        if(ret) {
                printf("error in post_recv\n");
                exit(1);
        }
	while(ret==0)
		ret = ibv_poll_cq(cq, 1, &wc);
	printf("ret=%d status=%d opcode=%d\n",ret, wc.status, wc.opcode);
	printf("buf[0] = %x\n",*(int *)buf[0]);

	clean_buffer:
	close_umr:
	return 0;
}

[-- Attachment #3: client.c --]
[-- Type: application/octet-stream, Size: 6977 bytes --]

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <getopt.h>
#include <netdb.h>
#include <sys/mman.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>
#include <mlx5dv.h>
#define LENGTH (256UL*1024*1024)
#define PROTECTION (PROT_READ | PROT_WRITE)
#define FLAGS (MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB)

int main()
{
	int without_umr;
	printf("Enter 0 for UMR clinet and non-zero value for non-UMR client\n");
	scanf("%d",&without_umr);
	char *servername = "172.16.0.6";
	char *port ="20079";
	struct addrinfo hints = {
		.ai_family = AF_INET,
		.ai_socktype = SOCK_STREAM
	};
	struct rdma_event_channel *cm_channel = NULL;
	cm_channel = rdma_create_event_channel();
	if (!cm_channel) {
		printf("Error in rdma_create_event_channel\n");
		exit(1);
	}

	printf("created event channel\n");
	struct rdma_cm_id *cm_id = NULL;
        int ret = rdma_create_id(cm_channel, &cm_id, NULL, RDMA_PS_TCP);
        if (ret) {
		printf("Error in rdma_create_id\n");
		exit(1);
	}

	printf("created id\n");
	struct addrinfo *res=NULL, *t=NULL;
        int n = getaddrinfo(servername, port, &hints, &res);
        if (n < 0) {
		printf("Error in getaddrinfo\n");
		goto clean_buffer;
	}

	printf("got the address info\n");
        /* Resolve server address and route */
	ret = 1;
        for (t = res; t; t = t->ai_next) {
                ret = rdma_resolve_addr(cm_id, NULL, t->ai_addr,
                                        5000);
                if (!ret)
                        break;
        }

        if (ret) {
		printf("Error in rdma_resolve_addr\n");
		exit(1);
	}
	printf("Resolved Address\n");
	struct rdma_cm_event *event = NULL;
        ret = rdma_get_cm_event(cm_channel, &event);
        if (ret) {
		printf("Error in rdma_get_cm_event\n");
		exit(1);
	}

        if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
		printf("Error event is not addr_resolved\n");
		exit(1);
	}

	printf("GotCM event\n");
	rdma_ack_cm_event(event);

	ret = rdma_resolve_route(cm_id, 5000);
        if (ret) {
		printf("Error in rdma_resolve_route\n");
		exit(1);
	}

	printf("Resolved Route\n");
        ret = rdma_get_cm_event(cm_channel, &event);
        if (ret) {
		printf("Error in rdma_get_cm_event, resolve_route\n");
		exit(1);
	}

        if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
		printf("Error event is not ROUTE_RESOLVED\n");
		goto clean_buffer;
	}

        rdma_ack_cm_event(event);

	struct ibv_pd *pd=NULL;
	struct ibv_context *context=NULL;
	context = cm_id->verbs;
	struct mlx5dv_context_attr mlx5dv_cont_attr;
	memset(&mlx5dv_cont_attr, 0, sizeof(struct mlx5dv_context_attr));
	mlx5dv_cont_attr.flags = MLX5DV_CONTEXT_FLAGS_DEVX;
	if(!mlx5dv_is_supported(context->device))
		printf("device is not mlx5dv capable\n");
	context = mlx5dv_open_device(context->device, &mlx5dv_cont_attr);
	if(context == NULL) {
		printf("failed to open device for mlx5dv_context\n");
		exit(1);
	}
	pd = ibv_alloc_pd(context);
	if(pd == NULL) {
		printf("Could not allocate pd\n");
		exit(1);
	}
	struct ibv_cq *cq = ibv_create_cq(context, 501, NULL, NULL, 0);
	if(cq==NULL) {
		printf("Could not create cq\n");
		exit(1);
	}

	struct mlx5dv_qp_init_attr mlx5_qp_attr = {};
        struct ibv_qp_init_attr_ex init_attr_ex = {};
        struct ibv_qp *qp = NULL;
        struct ibv_qp_ex *qpx = NULL;
        struct mlx5dv_qp_ex *dv_qp = NULL;
        struct mlx5dv_mkey *dv_mkey = NULL;
        struct mlx5dv_mkey_init_attr mkey_init_attr = {};

        memset(&mlx5_qp_attr, 0, sizeof(mlx5_qp_attr));
        memset(&init_attr_ex, 0, sizeof(init_attr_ex));
	memset(&mkey_init_attr, 0, sizeof(mkey_init_attr));

        mlx5_qp_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS;
        mlx5_qp_attr.send_ops_flags = MLX5DV_QP_EX_WITH_MR_INTERLEAVED;

        init_attr_ex.pd = pd;
        init_attr_ex.send_cq = cq;
        init_attr_ex.recv_cq = cq;
        init_attr_ex.qp_type = IBV_QPT_RC;
        init_attr_ex.cap.max_inline_data = 128;
        init_attr_ex.send_ops_flags = IBV_QP_EX_WITH_SEND;
        init_attr_ex.cap.max_send_wr  = 1,
        init_attr_ex.cap.max_recv_wr  = 500,
        init_attr_ex.cap.max_send_sge = 2,
        init_attr_ex.cap.max_recv_sge = 2;
        init_attr_ex.comp_mask = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;	

	qp = mlx5dv_create_qp(context, &init_attr_ex, &mlx5_qp_attr);
	if(qp==NULL) {
		printf("Could not create qp\n");
		exit(1);
	}
	qpx = ibv_qp_to_qp_ex(qp);
	if(qpx==NULL) {
		printf("Could not create qpx\n");
		exit(1);
	}
        dv_qp = mlx5dv_qp_ex_from_ibv_qp_ex(qpx);
	if(dv_qp==NULL) {
		printf("Could not create dv_qp\n");
		exit(1);
	}

	int err;
	struct rdma_conn_param    conn_param = { };
	memset(&conn_param, 0, sizeof(conn_param));

	cm_id->qp = qp;

        err = rdma_connect(cm_id, &conn_param);
        if (err) {
		printf("error in rdma_connect\n");
		exit(1);
	}

        err = rdma_get_cm_event(cm_channel, &event);
        if (err) {
		printf("error in rdma_get_cm_event\n");
                exit(1);
	}

        if (event->event != RDMA_CM_EVENT_ESTABLISHED) {
		printf("event is not RDMA_CM_EVENT_ESTABLISHED\n");
                exit(1);
	}

        rdma_ack_cm_event(event);
	qpx->wr_flags |= IBV_SEND_INLINE | IBV_SEND_SIGNALED;

	void *hbuf;
	void **buf;
	int num_mrs = 1;
	int size = 4*1024*1024;

	hbuf = mmap(NULL, LENGTH, PROTECTION, FLAGS, 0, 0);
	buf = calloc(num_mrs, sizeof(void *));
	int i;
	for (i = 0; i < num_mrs; i++) {
		buf[i] = hbuf + (i * size);
		memset(buf[i], 0xaa, size);
	}

	struct ibv_mr *mr_arr;
	mr_arr = ibv_reg_mr(pd, hbuf, 4096,
                           IBV_ACCESS_LOCAL_WRITE |IBV_ACCESS_MW_BIND | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_HUGETLB);
	if(mr_arr==NULL) {
		printf("error in ibv_reg_mr\n");
		exit(1);
	}

    	struct ibv_sge sge;
	memset(&sge, 0, sizeof(sge));
	sge.addr   = (uintptr_t) mr_arr->addr;
        sge.length = 4096;
        sge.lkey   = mr_arr->lkey;

	mkey_init_attr.create_flags = MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT;
        mkey_init_attr.max_entries = 5;
        mkey_init_attr.pd = pd;
	dv_mkey = mlx5dv_create_mkey(&mkey_init_attr);

        if(dv_mkey == NULL) {
                printf("Could not create dv_mkey\n");
                exit(1);
        }

	ibv_wr_start(qpx);

	if(without_umr) {
		ibv_wr_send(qpx);
		ibv_wr_set_sge(qpx, sge.lkey, sge.addr, sge.length);
	}
	else
		mlx5dv_wr_mr_list(dv_qp, dv_mkey, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC, 1, &sge);

	ret = ibv_wr_complete(qpx);
	struct ibv_wc wc;
	memset(&wc, 0, sizeof(wc));
	ret = 0;
	while(ret == 0)
        	ret = ibv_poll_cq(cq, 1, &wc);
	printf("ret=%d status=%d %d opcode=%d\n", ret, wc.status, IBV_WC_SUCCESS, wc.opcode);
	printf("ANK, event established\n");

	clean_buffer:
	ibv_destroy_qp(qp);
	ibv_destroy_cq(cq);
	ibv_dereg_mr(mr_arr);
	ibv_dealloc_pd(pd);
	ibv_close_device(context);
	free(buf);
	munmap(hbuf,4096);
	return 0;
}

^ permalink raw reply	[flat|nested] 4+ messages in thread