All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH] Replace FIO_HAVE_RUSAGE_THREAD with RUSAGE_THREAD
@ 2012-10-23  1:29 Yufei Ren
  2012-10-23  1:29 ` [PATCH] rdma ioengine improvement Yufei Ren
  2012-10-24  6:29 ` [PATCH] Replace FIO_HAVE_RUSAGE_THREAD with RUSAGE_THREAD Jens Axboe
  0 siblings, 2 replies; 4+ messages in thread
From: Yufei Ren @ 2012-10-23  1:29 UTC (permalink / raw)
  To: fio; +Cc: Yufei Ren

From: Yufei Ren <renyufei83@gmail.com>

RUSAGE_THREAD is a more clear macro to enable getrusage() thread support.
---
 os/os-linux.h |    5 -----
 stat.c        |    2 +-
 2 files changed, 1 insertions(+), 6 deletions(-)

diff --git a/os/os-linux.h b/os/os-linux.h
index 2b35f34..9b7ff29 100644
--- a/os/os-linux.h
+++ b/os/os-linux.h
@@ -14,7 +14,6 @@
 #include <linux/unistd.h>
 #include <linux/raw.h>
 #include <linux/major.h>
-#include <linux/version.h>
 #include <endian.h>
 
 #include "indirect.h"
@@ -63,10 +62,6 @@
 #define FIO_HAVE_FALLOC_ENG
 #endif
 
-#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26)
-#define FIO_HAVE_RUSAGE_THREAD
-#endif
-
 #ifdef SYNC_FILE_RANGE_WAIT_BEFORE
 #define FIO_HAVE_SYNC_FILE_RANGE
 #endif
diff --git a/stat.c b/stat.c
index af6e1f2..522901a 100644
--- a/stat.c
+++ b/stat.c
@@ -16,7 +16,7 @@ void update_rusage_stat(struct thread_data *td)
 {
 	struct thread_stat *ts = &td->ts;
 
-#ifdef FIO_HAVE_RUSAGE_THREAD
+#ifdef RUSAGE_THREAD
 	getrusage(RUSAGE_THREAD, &td->ru_end);
 #else
 	getrusage(RUSAGE_SELF, &td->ru_end);
-- 
1.7.2.3


^ permalink raw reply related	[flat|nested] 4+ messages in thread
* [PATCH] rdma ioengine improvement
  2012-10-23  1:29 [PATCH] Replace FIO_HAVE_RUSAGE_THREAD with RUSAGE_THREAD Yufei Ren
@ 2012-10-23  1:29 ` Yufei Ren
  2012-10-24  6:37   ` Jens Axboe
  2012-10-24  6:29 ` [PATCH] Replace FIO_HAVE_RUSAGE_THREAD with RUSAGE_THREAD Jens Axboe
  1 sibling, 1 reply; 4+ messages in thread
From: Yufei Ren @ 2012-10-23  1:29 UTC (permalink / raw)
  To: fio; +Cc: Yufei Ren

From: Yufei Ren <renyufei83@gmail.com>

1) Use fio shipped reentrant and thread-safe rand to replace buggy rand().
2) Add a pause time period before client start committing IOs.
	In SEND/RECV test, it's a good practice to setup the iodepth of
	of the RECV side deeper than that of the SEND side to
	avoid RNR (receiver not ready) error. The
	SEND side may send so many unsolicited message before
	RECV side commits sufficient recv buffers into recv queue.
	This may lead to RNR error. Here, SEND side pauses for a while
	during which RECV side commits sufficient recv buffers.
3) Fix server thread hanging bug.
	For RDMA WRITE/READ test, No IO bytes are accumulated in server side
	during test. Server thread indicates its task completion by changing
	its state into `done' as an alternitive. Add `td->done' checking in
	do_io().
4) Some comments revision.
	export EXTFLAGS and EXTLIBS with '+=".
	explanation on exchanging steps of RDMA ioengine control messages.

All these changes are tested in 40Gbps RoCE and 56Gbps InfiniBand testbeds.
---
 backend.c      |    2 +-
 engines/rdma.c |   46 +++++++++++++++++++++++++++++++++++++---------
 2 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/backend.c b/backend.c
index 3b42c4d..157fd52 100644
--- a/backend.c
+++ b/backend.c
@@ -591,7 +591,7 @@ static void do_io(struct thread_data *td)
 		int ret2, full;
 		enum fio_ddir ddir;
 
-		if (td->terminate)
+		if (td->terminate || td->done)
 			break;
 
 		update_tv_cache(td);
diff --git a/engines/rdma.c b/engines/rdma.c
index 79d72d2..2633432 100644
--- a/engines/rdma.c
+++ b/engines/rdma.c
@@ -7,14 +7,25 @@
  *
  * This I/O engine is disabled by default. To enable it, execute:
  *
- * $ export EXTFLAGS="-DFIO_HAVE_RDMA"
- * $ export EXTLIBS="-libverbs -lrdmacm"
+ * $ export EXTFLAGS+=" -DFIO_HAVE_RDMA "
+ * $ export EXTLIBS+=" -libverbs -lrdmacm "
  *
  * before running make. You will need the Linux RDMA software as well, either
  * from your Linux distributor or directly from openfabrics.org:
  *
  * http://www.openfabrics.org/downloads/OFED/
  *
+ * Exchanging steps of RDMA ioengine control messages:
+ *	1. client side sends test mode (RDMA_WRITE/RDMA_READ/SEND)
+ *	   to server side.
+ * 	2. server side parses test mode, and sends back confirmation
+ *	   to client side. In RDMA WRITE/READ test, this confirmation
+ * 	   includes memory information, such as rkey, address.
+ *	3. client side initiates test loop.
+ * 	4. In RDMA WRITE/READ test, client side sends a completion
+ *	   notification to server side. Server side updates its
+ * 	   td->done as true.
+ *
  */
 #include <stdio.h>
 #include <stdlib.h>
@@ -41,7 +52,7 @@
 #include <rdma/rdma_cma.h>
 #include <infiniband/arch.h>
 
-#define FIO_RDMA_MAX_IO_DEPTH    128
+#define FIO_RDMA_MAX_IO_DEPTH    512
 
 enum rdma_io_mode {
 	FIO_RDMA_UNKNOWN = 0,
@@ -591,7 +602,8 @@ static int fio_rdmaio_send(struct thread_data *td, struct io_u **io_us,
 	enum ibv_wc_opcode comp_opcode;
 	comp_opcode = IBV_WC_RDMA_WRITE;
 #endif
-	int i, index;
+	int i;
+	long index;
 	struct rdma_io_u_data *r_io_u_d;
 
 	r_io_u_d = NULL;
@@ -602,7 +614,10 @@ static int fio_rdmaio_send(struct thread_data *td, struct io_u **io_us,
 		case FIO_RDMA_MEM_WRITE:
 			/* compose work request */
 			r_io_u_d = io_us[i]->engine_data;
-			index = rand() % rd->rmt_nr;
+			if (td->o.use_os_rand)
+				index = os_random_long(&td->random_state) % rd->rmt_nr;
+			else
+				index = __rand(&td->__random_state) % rd->rmt_nr;
 			r_io_u_d->sq_wr.opcode = IBV_WR_RDMA_WRITE;
 			r_io_u_d->sq_wr.wr.rdma.rkey = rd->rmt_us[index].rkey;
 			r_io_u_d->sq_wr.wr.rdma.remote_addr = \
@@ -612,7 +627,10 @@ static int fio_rdmaio_send(struct thread_data *td, struct io_u **io_us,
 		case FIO_RDMA_MEM_READ:
 			/* compose work request */
 			r_io_u_d = io_us[i]->engine_data;
-			index = rand() % rd->rmt_nr;
+			if (td->o.use_os_rand)
+				index = os_random_long(&td->random_state) % rd->rmt_nr;
+			else
+				index = __rand(&td->__random_state) % rd->rmt_nr;
 			r_io_u_d->sq_wr.opcode = IBV_WR_RDMA_READ;
 			r_io_u_d->sq_wr.wr.rdma.rkey = rd->rmt_us[index].rkey;
 			r_io_u_d->sq_wr.wr.rdma.remote_addr = \
@@ -790,6 +808,16 @@ static int fio_rdmaio_connect(struct thread_data *td, struct fio_file *f)
 	/* wait for remote MR info from server side */
 	rdma_poll_wait(td, IBV_WC_RECV);
 
+	/* In SEND/RECV test, it's a good practice to setup the iodepth of
+	 * of the RECV side deeper than that of the SEND side to
+	 * avoid RNR (receiver not ready) error. The
+	 * SEND side may send so many unsolicited message before 
+	 * RECV side commits sufficient recv buffers into recv queue.
+	 * This may lead to RNR error. Here, SEND side pauses for a while
+	 * during which RECV side commits sufficient recv buffers.
+	 */
+	usleep(500000);
+
 	return 0;
 }
 
@@ -872,8 +900,8 @@ static int fio_rdmaio_close_file(struct thread_data *td, struct fio_file *f)
         return 1;
     }*/
 
-	ibv_destroy_qp(rd->qp);
 	ibv_destroy_cq(rd->cq);
+	ibv_destroy_qp(rd->qp);
 
 	if (rd->is_client == 1)
 		rdma_destroy_id(rd->cm_id);
@@ -1229,8 +1257,8 @@ static int fio_rdmaio_init(struct thread_data fio_unused * td)
 	log_err("     make sure OFED is installed,\n");
 	log_err("     $ ofed_info\n");
 	log_err("     then try to make fio as follows:\n");
-	log_err("     $ export EXTFLAGS=\"-DFIO_HAVE_RDMA\"\n");
-	log_err("     $ export EXTLIBS=\"-libverbs -lrdmacm\"\n");
+	log_err("     $ export EXTFLAGS+=\" -DFIO_HAVE_RDMA \"\n");
+	log_err("     $ export EXTLIBS+=\" -libverbs -lrdmacm \"\n");
 	log_err("     $ make clean && make\n");
 	return 1;
 }
-- 
1.7.2.3


^ permalink raw reply related	[flat|nested] 4+ messages in thread
* Re: [PATCH] Replace FIO_HAVE_RUSAGE_THREAD with RUSAGE_THREAD
  2012-10-23  1:29 [PATCH] Replace FIO_HAVE_RUSAGE_THREAD with RUSAGE_THREAD Yufei Ren
  2012-10-23  1:29 ` [PATCH] rdma ioengine improvement Yufei Ren
@ 2012-10-24  6:29 ` Jens Axboe
  1 sibling, 0 replies; 4+ messages in thread
From: Jens Axboe @ 2012-10-24  6:29 UTC (permalink / raw)
  To: Yufei Ren; +Cc: fio, Yufei Ren

On 2012-10-23 03:29, Yufei Ren wrote:
> From: Yufei Ren <renyufei83@gmail.com>
> 
> RUSAGE_THREAD is a more clear macro to enable getrusage() thread support.
> ---
>  os/os-linux.h |    5 -----
>  stat.c        |    2 +-
>  2 files changed, 1 insertions(+), 6 deletions(-)
> 
> diff --git a/os/os-linux.h b/os/os-linux.h
> index 2b35f34..9b7ff29 100644
> --- a/os/os-linux.h
> +++ b/os/os-linux.h
> @@ -14,7 +14,6 @@
>  #include <linux/unistd.h>
>  #include <linux/raw.h>
>  #include <linux/major.h>
> -#include <linux/version.h>
>  #include <endian.h>
>  
>  #include "indirect.h"
> @@ -63,10 +62,6 @@
>  #define FIO_HAVE_FALLOC_ENG
>  #endif
>  
> -#if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,26)
> -#define FIO_HAVE_RUSAGE_THREAD
> -#endif
> -
>  #ifdef SYNC_FILE_RANGE_WAIT_BEFORE
>  #define FIO_HAVE_SYNC_FILE_RANGE
>  #endif
> diff --git a/stat.c b/stat.c
> index af6e1f2..522901a 100644
> --- a/stat.c
> +++ b/stat.c
> @@ -16,7 +16,7 @@ void update_rusage_stat(struct thread_data *td)
>  {
>  	struct thread_stat *ts = &td->ts;
>  
> -#ifdef FIO_HAVE_RUSAGE_THREAD
> +#ifdef RUSAGE_THREAD
>  	getrusage(RUSAGE_THREAD, &td->ru_end);
>  #else
>  	getrusage(RUSAGE_SELF, &td->ru_end);
> 

Thanks, applied!

-- 
Jens Axboe


^ permalink raw reply	[flat|nested] 4+ messages in thread
* Re: [PATCH] rdma ioengine improvement
  2012-10-23  1:29 ` [PATCH] rdma ioengine improvement Yufei Ren
@ 2012-10-24  6:37   ` Jens Axboe
  0 siblings, 0 replies; 4+ messages in thread
From: Jens Axboe @ 2012-10-24  6:37 UTC (permalink / raw)
  To: Yufei Ren; +Cc: fio, Yufei Ren

On 2012-10-23 03:29, Yufei Ren wrote:
> From: Yufei Ren <renyufei83@gmail.com>
> 
> 1) Use fio shipped reentrant and thread-safe rand to replace buggy rand().
> 2) Add a pause time period before client start committing IOs.
> 	In SEND/RECV test, it's a good practice to setup the iodepth of
> 	of the RECV side deeper than that of the SEND side to
> 	avoid RNR (receiver not ready) error. The
> 	SEND side may send so many unsolicited message before
> 	RECV side commits sufficient recv buffers into recv queue.
> 	This may lead to RNR error. Here, SEND side pauses for a while
> 	during which RECV side commits sufficient recv buffers.
> 3) Fix server thread hanging bug.
> 	For RDMA WRITE/READ test, No IO bytes are accumulated in server side
> 	during test. Server thread indicates its task completion by changing
> 	its state into `done' as an alternitive. Add `td->done' checking in
> 	do_io().
> 4) Some comments revision.
> 	export EXTFLAGS and EXTLIBS with '+=".
> 	explanation on exchanging steps of RDMA ioengine control messages.

I applied the below on top. We can't reuse td->__random_state, as
that'll impact the generation of random numbers used for IO unit
offsets.

diff --git a/engines/rdma.c b/engines/rdma.c
index 2633432..87b061a 100644
--- a/engines/rdma.c
+++ b/engines/rdma.c
@@ -119,6 +119,8 @@ struct rdmaio_data {
 	int io_u_flight_nr;
 	struct io_u **io_us_completed;
 	int io_u_completed_nr;
+
+	struct frand_state rand_state;
 };
 
 static int client_recv(struct thread_data *td, struct ibv_wc *wc)
@@ -617,7 +619,7 @@ static int fio_rdmaio_send(struct thread_data *td, struct io_u **io_us,
 			if (td->o.use_os_rand)
 				index = os_random_long(&td->random_state) % rd->rmt_nr;
 			else
-				index = __rand(&td->__random_state) % rd->rmt_nr;
+				index = __rand(&rd->rand_state) % rd->rmt_nr;
 			r_io_u_d->sq_wr.opcode = IBV_WR_RDMA_WRITE;
 			r_io_u_d->sq_wr.wr.rdma.rkey = rd->rmt_us[index].rkey;
 			r_io_u_d->sq_wr.wr.rdma.remote_addr = \
@@ -630,7 +632,7 @@ static int fio_rdmaio_send(struct thread_data *td, struct io_u **io_us,
 			if (td->o.use_os_rand)
 				index = os_random_long(&td->random_state) % rd->rmt_nr;
 			else
-				index = __rand(&td->__random_state) % rd->rmt_nr;
+				index = __rand(&rd->rand_state) % rd->rmt_nr;
 			r_io_u_d->sq_wr.opcode = IBV_WR_RDMA_READ;
 			r_io_u_d->sq_wr.wr.rdma.rkey = rd->rmt_us[index].rkey;
 			r_io_u_d->sq_wr.wr.rdma.remote_addr = \
@@ -1210,6 +1212,7 @@ static int fio_rdmaio_setup(struct thread_data *td)
 		rd = malloc(sizeof(*rd));;
 
 		memset(rd, 0, sizeof(*rd));
+		init_rand_seed(&rd->rand_state, GOLDEN_RATIO_PRIME);
 		td->io_ops->data = rd;
 	}
 

-- 
Jens Axboe


^ permalink raw reply related	[flat|nested] 4+ messages in thread
end of thread, other threads:[~2012-10-24  6:37 UTC | newest]

Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2012-10-23  1:29 [PATCH] Replace FIO_HAVE_RUSAGE_THREAD with RUSAGE_THREAD Yufei Ren
2012-10-23  1:29 ` [PATCH] rdma ioengine improvement Yufei Ren
2012-10-24  6:37   ` Jens Axboe
2012-10-24  6:29 ` [PATCH] Replace FIO_HAVE_RUSAGE_THREAD with RUSAGE_THREAD Jens Axboe

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.