RDMA transfer speed performance test

* RDMA transfer speed performance test
@ 2010-07-09  9:33 Andrea Gozzelino
       [not found] ` <8214320.1278667993311.SLOX.WebMail.wwwrun-XDIR3SKYeFbgKi2NxijLtw@public.gmane.org>
  0 siblings, 1 reply; 8+ messages in thread
From: Andrea Gozzelino @ 2010-07-09  9:33 UTC (permalink / raw)
  To: sean.hefty-ral2JQCrhuEAvxtiuMwx3w
  Cc: chien.tin.tung-ral2JQCrhuEAvxtiuMwx3w,
	dotanba-Re5JQEeQqe8AvxtiuMwx3w,
	linux-rdma-u79uwXL29TY76Z2rM5mHXA

[-- Attachment #1: Type: text/plain, Size: 1169 bytes --]

Hi Sean,

I have succefully done the latency test on NetEffect NE020 cards (see
attached xls file with graph).
I worked with librdmacm version 1.0.12 (of June 2010) and your new APIs
rdma_reg_msgs
rdma_post_recv
rdma_post_send

Is librdmacm version 1.0.12 stable?

The flag IBV_SEND_INLINE supports buffer size until 64 bytes (cards'
feature).

I know that RDMA is a protocol connected with 10 Gigabit/s Ethernet.
How can I see this trasfer speed? Can you suggest the "path"?
I attached the starting latency_RDMA.cpp code, which works with
librdmacm version 1.0.12 with BUF_SIZE < = 64 bytes (thanks to Chien).
I would like to explore transfer speed in range 1 bytes < BUF_SIZE < 4
Mbytes.

I try changing the flag IBV_SEND_INLINE - as you suggested in the mail
on July 1st - but the code does not work.

All contributions are welcome about RDMA transfer speed code.

Thank you very much.
Regards,
Andrea

Andrea Gozzelino

INFN - Laboratori Nazionali di Legnaro	(LNL)
Viale dell'Universita' 2 -I-35020 - Legnaro (PD)- ITALIA
Office: E-101
Tel: +39 049 8068346
Fax: +39 049 641925
Mail: andrea.gozzelino-PK20h7lG/Rc1GQ1Ptb7lUw@public.gmane.org
Cell: +39 3488245552				

[-- Attachment #2: latency_RDMA.cpp --]
[-- Type: text/plain, Size: 10466 bytes --]

/*
------------------------------------------------------------------
Program: RDMA wrapper2
Use: Reduce RDMA to socket model
Author: Damiano Bortolato - Andrea Gozzelino (INFN - LNL)
Date: June 2010
Compile:  g++ -c wrapper_RDMA2.cpp -lrdmacm
Note: use librdmacm-1.0.12 by Sean Hefty 
Note: help from Chien Tin Tung (Intel)
------------------------------------------------------------------
*/

//Include
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>
#include <infiniband/arch.h>
#include <netdb.h>
#include <errno.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <inttypes.h>
#include <unistd.h>
#include <time.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/times.h>
#include <iostream>
#include <fstream>
#include <assert.h>
#include <sys/param.h>
#include <iomanip> // format manipulation
#include <vector>

using namespace std;

// header file
#include "header_RDMA.h"

//******************************************************
//********************** RDMA SERVER *******************
//******************************************************

int asServer(TestMessage &TM){

	printf("\nRDMA server machine redigo-01 is starting. \n");

	// Information from TCP socket
	int BUF_SIZE = TM.data.message.tbuffer_size;
	printf("BUF_SIZE = %i \n", BUF_SIZE);

	int NTX=TM.data.message.loops;
	printf("NTX = %i \n", NTX);

	// Variables
	static char *port = "7471";
	static char *server = "100.168.0.56";
	struct rdma_cm_id *listen_id, *id;
	struct ibv_mr *mr;
	char *send_msg=new char[BUF_SIZE];
	char *recv_msg=new char[BUF_SIZE];
	struct rdma_addrinfo hints, *res;
	struct ibv_qp_init_attr attr;
	struct ibv_wc wc;
	int ret;// function return value
	int i; // loop counter

	// RDMA_getaddrinfo 
	memset(&hints, 0, sizeof hints);
	hints.ai_flags = RAI_PASSIVE;
	hints.ai_port_space = RDMA_PS_TCP;
	ret = rdma_getaddrinfo(NULL, port, &hints, &res);
	if (ret) {
		printf("rdma_getaddrinfo %d\n", errno);
		return ret;
		}
	else printf("OK rdma_getaddrinfo function. \n");

	// RDMA_create_ep
	memset(&attr, 0, sizeof attr);
	attr.cap.max_send_wr = attr.cap.max_recv_wr = 128;
	attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
	attr.cap.max_inline_data = BUF_SIZE;
	attr.sq_sig_all = 1;
	ret = rdma_create_ep(&listen_id, res, NULL, &attr);
	rdma_freeaddrinfo(res);
	if (ret) {
		printf("rdma_create_ep %d\n", errno);
		return ret;
		}
	else printf("OK rdma_create_ep funtion. \n");

	// RDMA_listen
	ret = rdma_listen(listen_id, 0);
	if (ret) {
		printf("rdma_listen %d\n", errno);
		return ret;
		}
	else printf("OK rdma_listen function. \n");

	// RDMA_get_request
	ret = rdma_get_request(listen_id, &id);
	if (ret) {
		printf("rdma_get_request %d\n", errno);
		return ret;
		}
	else printf("OK rdma_get_request function \n");

	// RDMA_reg_msgs
	mr = rdma_reg_msgs(id, recv_msg, BUF_SIZE);
	if (!mr) {
		printf("rdma_reg_msgs %d\n", errno);
		return ret;
		}
	else printf("OK rdma_reg_msgs function. \n");

	// Pre post receive: RDMA_post_recv
	ret = rdma_post_recv(id, NULL, recv_msg, BUF_SIZE, mr);
	if (ret) {
		printf("rdma_post_recv %d\n", errno);
		return ret;
		}
	else printf("OK PRE POST rdma_post_recv. \n");

	// RDMA_accept on server side
	ret = rdma_accept(id, NULL);
	if (ret) {
		printf("rdma_connect %d\n", errno);
		return ret;
		}
	else printf("OK rdma_connect: server accepts connection. \n");

	//---------------------------------------------------------
	//--------- SERVER RDMA: data movement --------------------
	//---------------------------------------------------------

	// Loop 
	for (i=0; i < NTX; i++){

		// ********** Server is receiving ***************

		// RDMA_post_recv
		ret = rdma_post_recv(id, NULL, recv_msg, BUF_SIZE, mr);
		if (ret) {
			printf("rdma_post_recv %d\n", errno);
			return ret;
		}
		//else printf("OK rdma_post_recv. \n");

		// RDMA_get_recv_comp
/*
		ret = rdma_get_recv_comp(id, &wc);
		if (ret <= 0) {
			printf("rdma_get_recv_comp %d\n", ret);
			return ret;
		}
*/
	        while (!ibv_poll_cq(id->recv_cq, 1, &wc))
			;
		//else printf("OK rdma_get_recv_comp %d\n", ret);

		// ************ Server is sending ****************

		// RDMA_post_send
		ret = rdma_post_send(id, NULL, send_msg, BUF_SIZE, NULL, IBV_SEND_INLINE);
		if (ret) {
			printf("rdma_post_send %d\n", errno);
			return ret;
		}
		//else printf("OK rdma_post_send \n");

		// RDMA_get_send_comp
/*
		ret = rdma_get_send_comp(id, &wc);
		if (ret <= 0) {
			printf("rdma_get_send_comp %d\n", ret);
			return ret;
		}
*/
	        while (!ibv_poll_cq(id->send_cq, 1, &wc))
			;
		//else printf("OK rdma_get_send_comp %d\n", ret);

	}// end loop

	printf("Loop number = %i \n", i);
	printf("RDMA server closes. \n \n");

	rdma_disconnect(id);
	rdma_dereg_mr(mr);
	rdma_destroy_ep(id);
	rdma_destroy_ep(listen_id);

}// end asServer

//******************************************************
//********************** CLIENT RDMA *******************
//******************************************************

int asClient(TestMessage &TM){

	printf("\nRDMA client machine redigo-02 is starting. \n");

	// Information from TCP socket
	int BUF_SIZE=TM.data.message.tbuffer_size;
	printf("BUF_SIZE = %i \n", BUF_SIZE);

	int NTX=TM.data.message.loops;
	printf("NTX = %i \n", NTX);

	// Variables
	static char *port = "7471";
	static char *server = "10.10.10.1";
	struct rdma_cm_id *listen_id, *id;
	struct ibv_mr *mr;
	char *send_msg=new char[BUF_SIZE];
	char *recv_msg=new char[BUF_SIZE];
	struct rdma_addrinfo hints, *res;
	struct ibv_qp_init_attr attr;
	struct ibv_wc wc;
	int ret; // function return value
	int i; // counter
	time_t start,end;// clock values
        double dif;// delta time

	// RDMA_getaddrinfo 
	memset(&hints, 0, sizeof hints);
	hints.ai_port_space = RDMA_PS_TCP;
	ret = rdma_getaddrinfo(server, port, &hints, &res);
	if (ret) {
		printf("rdma_getaddrinfo %d\n", errno);
		return ret;
		}
	else printf("OK rdma_getaddrinfo function. \n");

	// RDMA_create_ep
	memset(&attr, 0, sizeof attr);
	attr.cap.max_send_wr = attr.cap.max_recv_wr = 128;
	attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
	attr.cap.max_inline_data = BUF_SIZE;
	attr.qp_context = id;
	attr.sq_sig_all = 1;
	ret = rdma_create_ep(&id, res, NULL, &attr);
	rdma_freeaddrinfo(res);
	if (ret) {
		printf("rdma_create_ep %d\n", errno);
		return ret;
		}
	else printf("OK rdma_create_ep funtion. \n");

	// RDMA_reg_msgs 
	mr = rdma_reg_msgs(id, recv_msg, BUF_SIZE);
	if (!mr) {
		printf("rdma_reg_msgs %d\n", errno);
		return ret;
		}
	else printf("OK rdma_reg_msgs function. \n");

	// Pre post receive: RDMA_post_recv
	ret = rdma_post_recv(id, NULL, recv_msg, BUF_SIZE, mr);
  	if (ret) {
  		printf("rdma_post_recv %d\n", errno);
  		return ret;
  		}
  	else printf("OK PRE POST rdma_post_recv. \n");

	// RDMA_connect on client side
  	ret = rdma_connect(id, NULL);
  	if (ret) {
  		printf("rdma_connect %d\n", errno);
  		return ret;
  		}
  	else printf("OK rdma_connect: client is connected. \n");

  	//----------------------------------------------------------
	//--------- Clock starts -------------------------------------
	//----------------------------------------------------------
        time(&start);

        //----------------------------------------------------------
        //--------- CLIENT RDMA: data movement --------------------
        //----------------------------------------------------------
        // Loop
  	for(i = 0; i < NTX; i++){
  		// RDMA_post_recv
  		ret = rdma_post_recv(id, NULL, recv_msg, BUF_SIZE, mr);
  		if (ret) {
  			printf("rdma_post_recv %d\n", errno);
  			return ret;
  			}

  		//*************** Client is sending ***************************

  		// RDMA_post_send
  		ret = rdma_post_send(id, NULL, send_msg, BUF_SIZE, NULL, IBV_SEND_INLINE);
  		if (ret) {
  			printf("rdma_post_send %d\n", errno);
  			return ret;
  			}
  		//else printf("OK rdma_post_send \n");

  		// RDMA_get_send_comp
/*
  		ret = rdma_get_send_comp(id, &wc);
		if (ret <= 0) {
			printf("rdma_get_send_comp %d\n", ret);
			return ret;
		}
*/
	        while (!ibv_poll_cq(id->send_cq, 1, &wc))
			;
		//else printf("OK rdma_get_send_comp %d\n", ret);

  		//********************** Client is receiving *******************

  		//else printf("OK rdma_post_recv \n");

  		// RDMA_get_recv_comp
/*
  		ret = rdma_get_recv_comp(id, &wc);
  		if (ret <= 0) {
  			printf("rdma_get_recv_comp %d\n", ret);
  			return ret;
  		}
*/
	        while (!ibv_poll_cq(id->recv_cq, 1, &wc))
			;
  		//else printf("OK rdma_get_recv_comp \n");

  	}//end loop

  	//----------------------------------------------------------
	//--------- Clock stops -------------------------------------
	//----------------------------------------------------------
  	time(&end);

  	printf("Loop number = %i\n", i);
	printf("RDMA client closes. \n \n");

  	rdma_disconnect(id);
	rdma_dereg_mr(mr);
	rdma_destroy_ep(id);

	//------------------------------------------------------------------
	//------- Output txt file with data --------------------------------
	//------------------------------------------------------------------

	ofstream OutFile("Test_Latency.txt");
	// Open outfile test
	if (! OutFile){
		cout << "Error opening output txt file." << endl;
		return -1;
	}

	// Calculation and other variables
	dif = difftime(end,start); //delta time

	long double NTXld = (long double) NTX;
	long double BUF_SIZEld = (long double) BUF_SIZE;
	long double dif_ld = (long double) dif;
	long double speed = (NTXld*BUF_SIZEld)/(dif_ld);
	long double latency = dif_ld/NTXld;

	// Use setw(), setprecision(), scientific options if it is necessary
	cout.setf(ios::left);       // set option 'left justify output'

	// Outfile format
	OutFile << "#transfer|buffer_size(bytes)|total_time(s)|speed(bytes/s)|latency(s)| \n";
	OutFile << setw(9) << NTX << "|";
	OutFile << setw(18) << BUF_SIZE << "|";
	OutFile << setw(13) << dif << "|";
	OutFile << setw(14) << speed << "|";
	OutFile << setw(10) << latency << "|";
	OutFile << endl;

}// end asClient

//*******************************************************
//************* (fake) Main program *********************
//*******************************************************
// It does not work!
/*
int main(int narg, char **argv) {
  if (narg > 1)
    return asServer();
  else
    return asClient();
};
*/

[-- Attachment #3: REDIGO_Latency.xls --]
[-- Type: application/vnd.ms-excel, Size: 20992 bytes --]

^ permalink raw reply	[flat|nested] 8+ messages in thread