[PATCH v3 1/7] libibverbs: Infrastructure to support verbs extensions

[PATCH v3 1/7] libibverbs: Infrastructure to support verbs extensions

[Hefty, Sean]

Based on a patch from Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>

Infrastructure to support extended verbs capabilities in a forward/backward
manner.

Support for extensions is determined by the provider calling
verbs_register_driver in place of ibv_register_driver.  When
extensions are enabled, ibverbs sets the current alloc_context /
free_context device operations to NULL.  These are used to
indicate that the struct ibv_device may be cast to struct
verbs_device.

With extensions, ibverbs allocates the ibv_context structure
and calls into the provider to initialize it.  The init call
is part of the verbs_device struct.

Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
Tzahi, Yishai - I removed your signed off by lines because of the changes
from the original.  If you agree with the final patch, please add them back. :)

change from v2:
Added a has_comp_mask field to verbs_context

 include/infiniband/driver.h |    3 ++
 include/infiniband/verbs.h  |   50 ++++++++++++++++++++++++++++++++++++++++
 src/cmd.c                   |   41 +--------------------------------
 src/device.c                |   54 +++++++++++++++++++++++++++++++++----------
 src/init.c                  |   41 +++++++++++++++++++++++++++------
 src/libibverbs.map          |    1 +
 6 files changed, 130 insertions(+), 60 deletions(-)

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 9a81416..f22f287 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -55,8 +55,11 @@
 
 typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
 						   int abi_version);
+typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path,
+						       int abi_version);
 
 void ibv_register_driver(const char *name, ibv_driver_init_func init_func);
+void verbs_register_driver(const char *name, verbs_driver_init_func init_func);
 int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
 			size_t cmd_size, struct ibv_get_context_resp *resp,
 			size_t resp_size);
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 6acfc81..b196b83 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -38,6 +38,7 @@
 
 #include <stdint.h>
 #include <pthread.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 #  define BEGIN_C_DECLS extern "C" {
@@ -63,6 +64,19 @@ union ibv_gid {
 	} global;
 };
 
+#ifndef container_of
+/**
+  * container_of - cast a member of a structure out to the containing structure
+  * @ptr:        the pointer to the member.
+  * @type:       the type of the container struct this is embedded in.
+  * @member:     the name of the member within the struct.
+  *
+ */
+#define container_of(ptr, type, member) ({\
+	const typeof(((type *)0)->member) * __mptr = (ptr);\
+	(type *)((char *)__mptr - offsetof(type, member)); })
+#endif
+
 enum ibv_node_type {
 	IBV_NODE_UNKNOWN	= -1,
 	IBV_NODE_CA 		= 1,
@@ -634,6 +648,17 @@ struct ibv_device {
 	char			ibdev_path[IBV_SYSFS_PATH_MAX];
 };
 
+struct verbs_device {
+	struct ibv_device device; /* Must be first */
+	size_t	sz;
+	size_t	size_of_context;
+	int	(*init_context)(struct verbs_device *device,
+				struct ibv_context *ctx, int cmd_fd);
+	void	(*uninit_context)(struct verbs_device *device,
+				struct ibv_context *ctx);
+	/* future fields added here */
+};
+
 struct ibv_context_ops {
 	int			(*query_device)(struct ibv_context *context,
 					      struct ibv_device_attr *device_attr);
@@ -702,6 +727,31 @@ struct ibv_context {
 	void		       *abi_compat;
 };
 
+enum verbs_context_mask {
+	VERBS_CONTEXT_RESERVED = 1 << 0
+};
+
+struct verbs_context {
+	/*  "grows up" - new fields go here */
+	uint64_t has_comp_mask;
+	size_t   sz;	/* Must be immediately before struct ibv_context */
+	struct ibv_context context;/* Must be last field in the struct */
+};
+
+static inline struct verbs_context *verbs_get_ctx(
+					const struct ibv_context *ctx)
+{
+	return (ctx->abi_compat != ((uint8_t *) NULL) - 1) ?
+		NULL : container_of(ctx, struct verbs_context, context);
+}
+
+static inline struct verbs_device *verbs_get_device(
+					const struct ibv_device *dev)
+{
+	return (dev->ops.alloc_context) ?
+		NULL : container_of(dev, struct verbs_device, device);
+}
+
 /**
  * ibv_get_device_list - Get list of IB devices currently available
  * @num_devices: optional.  if non-NULL, set to the number of devices
diff --git a/src/cmd.c b/src/cmd.c
index 9789092..dab8930 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -45,52 +45,13 @@
 
 #include "ibverbs.h"
 
-static int ibv_cmd_get_context_v2(struct ibv_context *context,
-				  struct ibv_get_context *new_cmd,
-				  size_t new_cmd_size,
-				  struct ibv_get_context_resp *resp,
-				  size_t resp_size)
-{
-	struct ibv_abi_compat_v2 *t;
-	struct ibv_get_context_v2 *cmd;
-	size_t cmd_size;
-	uint32_t cq_fd;
-
-	t = malloc(sizeof *t);
-	if (!t)
-		return ENOMEM;
-	pthread_mutex_init(&t->in_use, NULL);
-
-	cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd;
-	cmd      = alloca(cmd_size);
-	memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd);
-
-	IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
-	cmd->cq_fd_tab = (uintptr_t) &cq_fd;
-
-	if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) {
-		free(t);
-		return errno;
-	}
-
-	(void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
-
-	context->async_fd         = resp->async_fd;
-	context->num_comp_vectors = 1;
-	t->channel.context        = context;
-	t->channel.fd		  = cq_fd;
-	t->channel.refcnt	  = 0;
-	context->abi_compat       = t;
-
-	return 0;
-}
 
 int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
 			size_t cmd_size, struct ibv_get_context_resp *resp,
 			size_t resp_size)
 {
 	if (abi_ver <= 2)
-		return ibv_cmd_get_context_v2(context, cmd, cmd_size, resp, resp_size);
+		return ENOSYS;
 
 	IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
 
diff --git a/src/device.c b/src/device.c
index 5798895..448a243 100644
--- a/src/device.c
+++ b/src/device.c
@@ -124,9 +124,11 @@ default_symver(__ibv_get_device_guid, ibv_get_device_guid);
 
 struct ibv_context *__ibv_open_device(struct ibv_device *device)
 {
+	struct verbs_device *verbs_device = verbs_get_device(device);
 	char *devpath;
-	int cmd_fd;
+	int cmd_fd, ret;
 	struct ibv_context *context;
+	struct verbs_context *context_ex;
 
 	if (asprintf(&devpath, "/dev/infiniband/%s", device->dev_name) < 0)
 		return NULL;
@@ -141,9 +143,33 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
 	if (cmd_fd < 0)
 		return NULL;
 
-	context = device->ops.alloc_context(device, cmd_fd);
-	if (!context)
-		goto err;
+	if (!verbs_device) {
+		context = device->ops.alloc_context(device, cmd_fd);
+		if (!context)
+			goto err;
+	} else {
+		/* Library now allocates the context */
+		context_ex = calloc(1, sizeof(*context_ex) +
+				       verbs_device->size_of_context);
+		if (!context_ex) {
+			errno = ENOMEM;
+			goto err;
+		}
+
+		context_ex->context.abi_compat  = ((uint8_t *) NULL) - 1;
+		context_ex->sz = sizeof(*context_ex);
+
+		context = &context_ex->context;
+		ret = verbs_device->init_context(verbs_device, context, cmd_fd);
+		if (ret)
+			goto verbs_err;
+
+		/* initialize *all* library ops to either lib calls or
+		 * directly to provider calls.
+		 * context_ex->lib_new_func1 = __verbs_new_func1;
+		 * context_ex->lib_new_func2 = __verbs_new_func2;
+		 */
+	}
 
 	context->device = device;
 	context->cmd_fd = cmd_fd;
@@ -151,9 +177,10 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
 
 	return context;
 
+verbs_err:
+	free(context_ex);
 err:
 	close(cmd_fd);
-
 	return NULL;
 }
 default_symver(__ibv_open_device, ibv_open_device);
@@ -163,14 +190,15 @@ int __ibv_close_device(struct ibv_context *context)
 	int async_fd = context->async_fd;
 	int cmd_fd   = context->cmd_fd;
 	int cq_fd    = -1;
-
-	if (abi_ver <= 2) {
-		struct ibv_abi_compat_v2 *t = context->abi_compat;
-		cq_fd = t->channel.fd;
-		free(context->abi_compat);
-	}
-
-	context->device->ops.free_context(context);
+	struct verbs_context *context_ex;
+
+	context_ex = verbs_get_ctx(context);
+	if (context_ex) {
+		struct verbs_device *verbs_device = verbs_get_device(context->device);
+		verbs_device->uninit_context(verbs_device, context);
+		free(context_ex);
+	} else
+		context->device->ops.free_context(context);
 
 	close(async_fd);
 	close(cmd_fd);
diff --git a/src/init.c b/src/init.c
index 8d6786e..d6cd84a 100644
--- a/src/init.c
+++ b/src/init.c
@@ -70,6 +70,7 @@ struct ibv_driver_name {
 struct ibv_driver {
 	const char	       *name;
 	ibv_driver_init_func	init_func;
+	verbs_driver_init_func	verbs_init_func;
 	struct ibv_driver      *next;
 };
 
@@ -153,7 +154,8 @@ static int find_sysfs_devs(void)
 	return ret;
 }
 
-void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
+static void register_driver(const char *name, ibv_driver_init_func init_func,
+			    verbs_driver_init_func verbs_init_func)
 {
 	struct ibv_driver *driver;
 
@@ -163,9 +165,10 @@ void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
 		return;
 	}
 
-	driver->name      = name;
-	driver->init_func = init_func;
-	driver->next      = NULL;
+	driver->name      	= name;
+	driver->init_func	= init_func;
+	driver->verbs_init_func = verbs_init_func;
+	driver->next      	= NULL;
 
 	if (tail_driver)
 		tail_driver->next = driver;
@@ -174,6 +177,19 @@ void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
 	tail_driver = driver;
 }
 
+void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
+{
+	register_driver(name, init_func, NULL);
+}
+
+/* New registration symbol with same functionality - used by providers to
+  * validate that library supports verbs extension.
+  */
+void verbs_register_driver(const char *name, verbs_driver_init_func init_func)
+{
+	register_driver(name, NULL, init_func);
+}
+
 static void load_driver(const char *name)
 {
 	char *so_name;
@@ -333,12 +349,23 @@ out:
 static struct ibv_device *try_driver(struct ibv_driver *driver,
 				     struct ibv_sysfs_dev *sysfs_dev)
 {
+	struct verbs_device *vdev;
 	struct ibv_device *dev;
 	char value[8];
 
-	dev = driver->init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver);
-	if (!dev)
-		return NULL;
+	if (driver->init_func) {
+		dev = driver->init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver);
+		if (!dev)
+			return NULL;
+	} else {
+		vdev = driver->verbs_init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver);
+		if (!vdev)
+			return NULL;
+
+		dev = &vdev->device;
+		dev->ops.alloc_context = NULL;
+		dev->ops.free_context = NULL;
+	}
 
 	if (ibv_read_sysfs_file(sysfs_dev->ibdev_path, "node_type", value, sizeof value) < 0) {
 		fprintf(stderr, PFX "Warning: no node_type attr under %s.\n",
diff --git a/src/libibverbs.map b/src/libibverbs.map
index 1827da0..ee9adea 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -91,6 +91,7 @@ IBVERBS_1.1 {
 		ibv_dontfork_range;
 		ibv_dofork_range;
 		ibv_register_driver;
+		verbs_register_driver;
 
 		ibv_node_type_str;
 		ibv_port_state_str;


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v3 1/7] libibverbs: Infrastructure to support verbs extensions

[Jason Gunthorpe]

On Fri, Sep 28, 2012 at 10:53:49PM +0000, Hefty, Sean wrote:

> change from v2:
> Added a has_comp_mask field to verbs_context

Did I miss something? It never gets used, and never gets set in the
mlx4 patches?

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH v3 1/7] libibverbs: Infrastructure to support verbs extensions

[Hefty, Sean]

> > change from v2:
> > Added a has_comp_mask field to verbs_context
> 
> Did I miss something? It never gets used, and never gets set in the
> mlx4 patches?

I think I sent a non-updated patch mlx4 patch 2 - it should be set there.  I'll resend that patch.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v3 1/7] libibverbs: Infrastructure to support verbs extensions

[Jason Gunthorpe]

On Mon, Oct 01, 2012 at 04:20:30PM +0000, Hefty, Sean wrote:
> > > change from v2:
> > > Added a has_comp_mask field to verbs_context
> > 
> > Did I miss something? It never gets used, and never gets set in the
> > mlx4 patches?
> 
> I think I sent a non-updated patch mlx4 patch 2 - it should be set
> there.  I'll resend that patch.

Do you have a git server you can put them up on? github?

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH v3 1/7] libibverbs: Infrastructure to support verbs extensions

[Hefty, Sean]

> > > Did I miss something? It never gets used, and never gets set in the
> > > mlx4 patches?
> >
> > I think I sent a non-updated patch mlx4 patch 2 - it should be set
> > there.  I'll resend that patch.
> 
> Do you have a git server you can put them up on? github?

git.openfabrics.org/~shefty/libmlx4.git master
and
git.openfabrics.org/~shefty/libibverbs.git xrc2
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v3 1/7] libibverbs: Infrastructure to support verbs extensions

[Or Gerlitz]

On 29/09/2012 00:53, Hefty, Sean wrote:
> [...] Infrastructure to support extended verbs capabilities in a forward/backward
> manner. [...]

Sean,

Just two nits re the patch set...

1. it would be easier to eventually pick these patches if the library 
name would be part of the text in brackets, e.g could you use libibverbs 
and libmlx4 with git format-patch --subject-prefix="PATCH LIBNAME" e.g 
--subject-prefix="PATCH libibverbs"

2. patch 0 (cover-letter) for both sub-series (libmlx4/libibverbs) 
stating the changes along the series versions will help to follow on the 
progress

thanks,

Or.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 1/7] Infrastructure to support verbs extensions

[sean.hefty-ral2JQCrhuEAvxtiuMwx3w]

From: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>

Infrastructure to support extended verbs capabilities in a forward/backward
manner.

Support for extensions is determeind by the provider calling
verbs_register_driver in place of ibv_register_driver.  When
extensions are enabled, ibverbs sets the current alloc_context /
free_context device operations to NULL.  These are used to
indicate that the struct ibv_device may be cast to struct
verbs_device.

With extensions, ibverbs allocates the ibv_context structure
and calls into the provider to initialize it.  The init call
is part of the verbs_device struct.

Signed-off-by: Yishai Hadas <yishaih-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Signed-off-by: Tzahi Oved <tzahio-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
 include/infiniband/driver.h |    3 ++
 include/infiniband/verbs.h  |   55 +++++++++++++++++++++++++++++++++++++++++++
 src/cmd.c                   |   41 +-------------------------------
 src/device.c                |   54 ++++++++++++++++++++++++++++++++----------
 src/init.c                  |   41 ++++++++++++++++++++++++++-----
 src/libibverbs.map          |    1 +
 6 files changed, 135 insertions(+), 60 deletions(-)

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 9a81416..f22f287 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -55,8 +55,11 @@
 
 typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
 						   int abi_version);
+typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path,
+						       int abi_version);
 
 void ibv_register_driver(const char *name, ibv_driver_init_func init_func);
+void verbs_register_driver(const char *name, verbs_driver_init_func init_func);
 int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
 			size_t cmd_size, struct ibv_get_context_resp *resp,
 			size_t resp_size);
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index 6acfc81..e4361aa 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -38,6 +38,7 @@
 
 #include <stdint.h>
 #include <pthread.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 #  define BEGIN_C_DECLS extern "C" {
@@ -63,6 +64,19 @@ union ibv_gid {
 	} global;
 };
 
+#ifndef container_of
+/**
+  * container_of - cast a member of a structure out to the containing structure
+  * @ptr:        the pointer to the member.
+  * @type:       the type of the container struct this is embedded in.
+  * @member:     the name of the member within the struct.
+  *
+ */
+#define container_of(ptr, type, member) ({\
+	const typeof(((type *)0)->member) * __mptr = (ptr);\
+	(type *)((char *)__mptr - offsetof(type, member)); })
+#endif
+
 enum ibv_node_type {
 	IBV_NODE_UNKNOWN	= -1,
 	IBV_NODE_CA 		= 1,
@@ -634,6 +648,17 @@ struct ibv_device {
 	char			ibdev_path[IBV_SYSFS_PATH_MAX];
 };
 
+struct verbs_device {
+	struct ibv_device device; /* Must be first */
+	size_t	sz;
+	size_t	size_of_context;
+	int	(*init_context)(struct verbs_device *device,
+				struct ibv_context *ctx, int cmd_fd);
+	void	(*uninit_context)(struct verbs_device *device,
+				struct ibv_context *ctx);
+	/* future fields added here */
+};
+
 struct ibv_context_ops {
 	int			(*query_device)(struct ibv_context *context,
 					      struct ibv_device_attr *device_attr);
@@ -702,6 +727,36 @@ struct ibv_context {
 	void		       *abi_compat;
 };
 
+enum verbs_context_mask {
+	VERBS_CONTEXT_RESERVED = 1 << 0
+};
+
+struct verbs_context {
+	/*  "grows up" - new fields go here */
+	uint64_t has_comp_mask;
+	size_t   sz;	/* Must be immediately before struct ibv_context */
+	struct ibv_context context;/* Must be last field in the struct */
+};
+
+static inline struct verbs_context *verbs_get_ctx(
+					const struct ibv_context *ctx)
+{
+	return (ctx->abi_compat != ((uint8_t *) NULL) - 1) ?
+		NULL : container_of(ctx, struct verbs_context, context);
+}
+
+#define verbs_get_ctx_op(ctx, op) ({ \
+	struct verbs_context *vctx = verbs_get_ctx(ctx); \
+	(!vctx || (vctx->sz < sizeof(*vctx) - offsetof(struct verbs_context, op)) || \
+	 !vctx->op) ? NULL : vctx; })
+
+static inline struct verbs_device *verbs_get_device(
+					const struct ibv_device *dev)
+{
+	return (dev->ops.alloc_context) ?
+		NULL : container_of(dev, struct verbs_device, device);
+}
+
 /**
  * ibv_get_device_list - Get list of IB devices currently available
  * @num_devices: optional.  if non-NULL, set to the number of devices
diff --git a/src/cmd.c b/src/cmd.c
index 9789092..dab8930 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -45,52 +45,13 @@
 
 #include "ibverbs.h"
 
-static int ibv_cmd_get_context_v2(struct ibv_context *context,
-				  struct ibv_get_context *new_cmd,
-				  size_t new_cmd_size,
-				  struct ibv_get_context_resp *resp,
-				  size_t resp_size)
-{
-	struct ibv_abi_compat_v2 *t;
-	struct ibv_get_context_v2 *cmd;
-	size_t cmd_size;
-	uint32_t cq_fd;
-
-	t = malloc(sizeof *t);
-	if (!t)
-		return ENOMEM;
-	pthread_mutex_init(&t->in_use, NULL);
-
-	cmd_size = sizeof *cmd + new_cmd_size - sizeof *new_cmd;
-	cmd      = alloca(cmd_size);
-	memcpy(cmd->driver_data, new_cmd->driver_data, new_cmd_size - sizeof *new_cmd);
-
-	IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
-	cmd->cq_fd_tab = (uintptr_t) &cq_fd;
-
-	if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) {
-		free(t);
-		return errno;
-	}
-
-	(void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
-
-	context->async_fd         = resp->async_fd;
-	context->num_comp_vectors = 1;
-	t->channel.context        = context;
-	t->channel.fd		  = cq_fd;
-	t->channel.refcnt	  = 0;
-	context->abi_compat       = t;
-
-	return 0;
-}
 
 int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd,
 			size_t cmd_size, struct ibv_get_context_resp *resp,
 			size_t resp_size)
 {
 	if (abi_ver <= 2)
-		return ibv_cmd_get_context_v2(context, cmd, cmd_size, resp, resp_size);
+		return ENOSYS;
 
 	IBV_INIT_CMD_RESP(cmd, cmd_size, GET_CONTEXT, resp, resp_size);
 
diff --git a/src/device.c b/src/device.c
index 5798895..448a243 100644
--- a/src/device.c
+++ b/src/device.c
@@ -124,9 +124,11 @@ default_symver(__ibv_get_device_guid, ibv_get_device_guid);
 
 struct ibv_context *__ibv_open_device(struct ibv_device *device)
 {
+	struct verbs_device *verbs_device = verbs_get_device(device);
 	char *devpath;
-	int cmd_fd;
+	int cmd_fd, ret;
 	struct ibv_context *context;
+	struct verbs_context *context_ex;
 
 	if (asprintf(&devpath, "/dev/infiniband/%s", device->dev_name) < 0)
 		return NULL;
@@ -141,9 +143,33 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
 	if (cmd_fd < 0)
 		return NULL;
 
-	context = device->ops.alloc_context(device, cmd_fd);
-	if (!context)
-		goto err;
+	if (!verbs_device) {
+		context = device->ops.alloc_context(device, cmd_fd);
+		if (!context)
+			goto err;
+	} else {
+		/* Library now allocates the context */
+		context_ex = calloc(1, sizeof(*context_ex) +
+				       verbs_device->size_of_context);
+		if (!context_ex) {
+			errno = ENOMEM;
+			goto err;
+		}
+
+		context_ex->context.abi_compat  = ((uint8_t *) NULL) - 1;
+		context_ex->sz = sizeof(*context_ex);
+
+		context = &context_ex->context;
+		ret = verbs_device->init_context(verbs_device, context, cmd_fd);
+		if (ret)
+			goto verbs_err;
+
+		/* initialize *all* library ops to either lib calls or
+		 * directly to provider calls.
+		 * context_ex->lib_new_func1 = __verbs_new_func1;
+		 * context_ex->lib_new_func2 = __verbs_new_func2;
+		 */
+	}
 
 	context->device = device;
 	context->cmd_fd = cmd_fd;
@@ -151,9 +177,10 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device)
 
 	return context;
 
+verbs_err:
+	free(context_ex);
 err:
 	close(cmd_fd);
-
 	return NULL;
 }
 default_symver(__ibv_open_device, ibv_open_device);
@@ -163,14 +190,15 @@ int __ibv_close_device(struct ibv_context *context)
 	int async_fd = context->async_fd;
 	int cmd_fd   = context->cmd_fd;
 	int cq_fd    = -1;
-
-	if (abi_ver <= 2) {
-		struct ibv_abi_compat_v2 *t = context->abi_compat;
-		cq_fd = t->channel.fd;
-		free(context->abi_compat);
-	}
-
-	context->device->ops.free_context(context);
+	struct verbs_context *context_ex;
+
+	context_ex = verbs_get_ctx(context);
+	if (context_ex) {
+		struct verbs_device *verbs_device = verbs_get_device(context->device);
+		verbs_device->uninit_context(verbs_device, context);
+		free(context_ex);
+	} else
+		context->device->ops.free_context(context);
 
 	close(async_fd);
 	close(cmd_fd);
diff --git a/src/init.c b/src/init.c
index 8d6786e..d6cd84a 100644
--- a/src/init.c
+++ b/src/init.c
@@ -70,6 +70,7 @@ struct ibv_driver_name {
 struct ibv_driver {
 	const char	       *name;
 	ibv_driver_init_func	init_func;
+	verbs_driver_init_func	verbs_init_func;
 	struct ibv_driver      *next;
 };
 
@@ -153,7 +154,8 @@ static int find_sysfs_devs(void)
 	return ret;
 }
 
-void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
+static void register_driver(const char *name, ibv_driver_init_func init_func,
+			    verbs_driver_init_func verbs_init_func)
 {
 	struct ibv_driver *driver;
 
@@ -163,9 +165,10 @@ void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
 		return;
 	}
 
-	driver->name      = name;
-	driver->init_func = init_func;
-	driver->next      = NULL;
+	driver->name      	= name;
+	driver->init_func	= init_func;
+	driver->verbs_init_func = verbs_init_func;
+	driver->next      	= NULL;
 
 	if (tail_driver)
 		tail_driver->next = driver;
@@ -174,6 +177,19 @@ void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
 	tail_driver = driver;
 }
 
+void ibv_register_driver(const char *name, ibv_driver_init_func init_func)
+{
+	register_driver(name, init_func, NULL);
+}
+
+/* New registration symbol with same functionality - used by providers to
+  * validate that library supports verbs extension.
+  */
+void verbs_register_driver(const char *name, verbs_driver_init_func init_func)
+{
+	register_driver(name, NULL, init_func);
+}
+
 static void load_driver(const char *name)
 {
 	char *so_name;
@@ -333,12 +349,23 @@ out:
 static struct ibv_device *try_driver(struct ibv_driver *driver,
 				     struct ibv_sysfs_dev *sysfs_dev)
 {
+	struct verbs_device *vdev;
 	struct ibv_device *dev;
 	char value[8];
 
-	dev = driver->init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver);
-	if (!dev)
-		return NULL;
+	if (driver->init_func) {
+		dev = driver->init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver);
+		if (!dev)
+			return NULL;
+	} else {
+		vdev = driver->verbs_init_func(sysfs_dev->sysfs_path, sysfs_dev->abi_ver);
+		if (!vdev)
+			return NULL;
+
+		dev = &vdev->device;
+		dev->ops.alloc_context = NULL;
+		dev->ops.free_context = NULL;
+	}
 
 	if (ibv_read_sysfs_file(sysfs_dev->ibdev_path, "node_type", value, sizeof value) < 0) {
 		fprintf(stderr, PFX "Warning: no node_type attr under %s.\n",
diff --git a/src/libibverbs.map b/src/libibverbs.map
index 1827da0..ee9adea 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -91,6 +91,7 @@ IBVERBS_1.1 {
 		ibv_dontfork_range;
 		ibv_dofork_range;
 		ibv_register_driver;
+		verbs_register_driver;
 
 		ibv_node_type_str;
 		ibv_port_state_str;
-- 
1.7.3

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 2/7] libibverbs: Introduce XRC domains

[sean.hefty-ral2JQCrhuEAvxtiuMwx3w]

From: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

XRC introduces several new concepts and structures, one of
which is the XRC domain.

XRC domains: xrcd's are a type of protection domain used to
associate shared receive queues with xrc queue pairs.  Since
xrcd are meant to be shared among multiple processes, we
introduce new APIs to open/close xrcd's.

The user to kernel ABI is extended to account for opening/
closing the xrcd.

Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
 include/infiniband/driver.h   |   16 +++++++++++++
 include/infiniband/kern-abi.h |   27 +++++++++++++++++++++-
 include/infiniband/verbs.h    |   50 ++++++++++++++++++++++++++++++++++++++--
 src/cmd.c                     |   41 +++++++++++++++++++++++++++++++++
 src/libibverbs.map            |    4 +++
 5 files changed, 134 insertions(+), 4 deletions(-)

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index f22f287..ce88442 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -53,6 +53,17 @@
  */
 #define IBV_DEVICE_LIBRARY_EXTENSION rdmav2
 
+enum verbs_xrcd_mask {
+	VERBS_XRCD_HANDLE	= 1 << 0,
+	VERBS_XRCD_RESERVED	= 1 << 1
+};
+
+struct verbs_xrcd {
+	struct ibv_xrcd		xrcd;
+	uint64_t		comp_mask;
+	uint32_t		handle;
+};
+
 typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
 						   int abi_version);
 typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path,
@@ -78,6 +89,11 @@ int ibv_cmd_alloc_pd(struct ibv_context *context, struct ibv_pd *pd,
 		     struct ibv_alloc_pd *cmd, size_t cmd_size,
 		     struct ibv_alloc_pd_resp *resp, size_t resp_size);
 int ibv_cmd_dealloc_pd(struct ibv_pd *pd);
+int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd,
+		      struct ibv_xrcd_init_attr *attr,
+		      struct ibv_open_xrcd *cmd, size_t cmd_size,
+		      struct ibv_open_xrcd_resp *resp, size_t resp_size);
+int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd);
 #define IBV_CMD_REG_MR_HAS_RESP_PARAMS
 int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
 		   uint64_t hca_va, int access,
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 619ea7e..d7c673f 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -85,7 +85,9 @@ enum {
 	IB_USER_VERBS_CMD_MODIFY_SRQ,
 	IB_USER_VERBS_CMD_QUERY_SRQ,
 	IB_USER_VERBS_CMD_DESTROY_SRQ,
-	IB_USER_VERBS_CMD_POST_SRQ_RECV
+	IB_USER_VERBS_CMD_POST_SRQ_RECV,
+	IB_USER_VERBS_CMD_OPEN_XRCD,
+	IB_USER_VERBS_CMD_CLOSE_XRCD,
 };
 
 /*
@@ -246,6 +248,27 @@ struct ibv_dealloc_pd {
 	__u32 pd_handle;
 };
 
+struct ibv_open_xrcd {
+	__u32 command;
+	__u16 in_words;
+	__u16 out_words;
+	__u64 response;
+	__u32 fd;
+	__u32 oflags;
+	__u64 driver_data[0];
+};
+
+struct ibv_open_xrcd_resp {
+	__u32 xrcd_handle;
+};
+
+struct ibv_close_xrcd {
+	__u32 command;
+	__u16 in_words;
+	__u16 out_words;
+	__u32 xrcd_handle;
+};
+
 struct ibv_reg_mr {
 	__u32 command;
 	__u16 in_words;
@@ -804,6 +827,8 @@ enum {
 	 * trick opcodes in IBV_INIT_CMD() doesn't break.
 	 */
 	IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
+	IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
+	IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
 };
 
 struct ibv_destroy_cq_v1 {
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index e4361aa..cf8299a 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
- * Copyright (c) 2004 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2004, 2011-2012 Intel Corporation.  All rights reserved.
  * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
  *
@@ -39,6 +39,7 @@
 #include <stdint.h>
 #include <pthread.h>
 #include <stddef.h>
+#include <errno.h>
 
 #ifdef __cplusplus
 #  define BEGIN_C_DECLS extern "C" {
@@ -312,6 +313,22 @@ struct ibv_pd {
 	uint32_t		handle;
 };
 
+enum ibv_xrcd_init_attr_mask {
+	IBV_XRCD_INIT_ATTR_FD	    = 1 << 0,
+	IBV_XRCD_INIT_ATTR_OFLAGS   = 1 << 1,
+	IBV_XRCD_INIT_ATTR_RESERVED = 1 << 2
+};
+
+struct ibv_xrcd_init_attr {
+	uint32_t comp_mask;
+	int	 fd;
+	int	 oflags;
+};
+
+struct ibv_xrcd {
+	struct ibv_context     *context;
+};
+
 enum ibv_rereg_mr_flags {
 	IBV_REREG_MR_CHANGE_TRANSLATION	= (1 << 0),
 	IBV_REREG_MR_CHANGE_PD		= (1 << 1),
@@ -728,11 +745,15 @@ struct ibv_context {
 };
 
 enum verbs_context_mask {
-	VERBS_CONTEXT_RESERVED = 1 << 0
+	VERBS_CONTEXT_XRCD	= 1 << 0,
+	VERBS_CONTEXT_RESERVED	= 1 << 1
 };
 
 struct verbs_context {
 	/*  "grows up" - new fields go here */
+	struct ibv_xrcd *	(*open_xrcd)(struct ibv_context *context,
+					     struct ibv_xrcd_init_attr *xrcd_init_attr);
+	int			(*close_xrcd)(struct ibv_xrcd *xrcd);
 	uint64_t has_comp_mask;
 	size_t   sz;	/* Must be immediately before struct ibv_context */
 	struct ibv_context context;/* Must be last field in the struct */
@@ -835,7 +856,7 @@ static inline int ___ibv_query_port(struct ibv_context *context,
 				    uint8_t port_num,
 				    struct ibv_port_attr *port_attr)
 {
-	/* For compatability when running with old libibverbs */
+	/* For compatibility when running with old libibverbs */
 	port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED;
 	port_attr->reserved   = 0;
 
@@ -868,6 +889,29 @@ struct ibv_pd *ibv_alloc_pd(struct ibv_context *context);
 int ibv_dealloc_pd(struct ibv_pd *pd);
 
 /**
+ * ibv_open_xrcd - Open an extended connection domain
+ */
+static inline struct ibv_xrcd *
+ibv_open_xrcd(struct ibv_context *context, struct ibv_xrcd_init_attr *xrcd_init_attr)
+{
+	struct verbs_context *vctx = verbs_get_ctx_op(context, open_xrcd);
+	if (!vctx) {
+		errno = ENOSYS;
+		return NULL;
+	}
+	return vctx->open_xrcd(context, xrcd_init_attr);
+}
+
+/**
+ * ibv_close_xrcd - Close an extended connection domain
+ */
+static inline int ibv_close_xrcd(struct ibv_xrcd *xrcd)
+{
+	struct verbs_context *vctx = verbs_get_ctx(xrcd->context);
+	return vctx->close_xrcd(xrcd);
+}
+
+/**
  * ibv_reg_mr - Register a memory region
  */
 struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
diff --git a/src/cmd.c b/src/cmd.c
index dab8930..f0fb1dc 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -194,6 +194,47 @@ int ibv_cmd_dealloc_pd(struct ibv_pd *pd)
 	return 0;
 }
 
+int ibv_cmd_open_xrcd(struct ibv_context *context, struct verbs_xrcd *xrcd,
+		      struct ibv_xrcd_init_attr *attr,
+		      struct ibv_open_xrcd *cmd, size_t cmd_size,
+		      struct ibv_open_xrcd_resp *resp, size_t resp_size)
+{
+	IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_XRCD, resp, resp_size);
+
+	if (attr->comp_mask >= IBV_XRCD_INIT_ATTR_RESERVED)
+		return ENOSYS;
+
+	if (!(attr->comp_mask & IBV_XRCD_INIT_ATTR_FD) ||
+	    !(attr->comp_mask & IBV_XRCD_INIT_ATTR_OFLAGS))
+		return EINVAL;
+
+	cmd->fd = attr->fd;
+	cmd->oflags = attr->oflags;
+	if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+		return errno;
+
+	VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+	xrcd->xrcd.context = context;
+	xrcd->comp_mask = VERBS_XRCD_HANDLE;
+	xrcd->handle  = resp->xrcd_handle;
+
+	return 0;
+}
+
+int ibv_cmd_close_xrcd(struct verbs_xrcd *xrcd)
+{
+	struct ibv_close_xrcd cmd;
+
+	IBV_INIT_CMD(&cmd, sizeof cmd, CLOSE_XRCD);
+	cmd.xrcd_handle = xrcd->handle;
+
+	if (write(xrcd->xrcd.context->cmd_fd, &cmd, sizeof cmd) != sizeof cmd)
+		return errno;
+
+	return 0;
+}
+
 int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
 		   uint64_t hca_va, int access,
 		   struct ibv_mr *mr, struct ibv_reg_mr *cmd,
diff --git a/src/libibverbs.map b/src/libibverbs.map
index ee9adea..9a15f3f 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -97,4 +97,8 @@ IBVERBS_1.1 {
 		ibv_port_state_str;
 		ibv_event_type_str;
 		ibv_wc_status_str;
+		
+		ibv_cmd_open_xrcd;
+		ibv_cmd_close_xrcd;
+		
 } IBVERBS_1.0;
-- 
1.7.3

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 3/7] livibverbs: Add support for XRC SRQs

[sean.hefty-ral2JQCrhuEAvxtiuMwx3w]

From: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

XRC support requires the use of a new type of SRQ.

XRC shared receive queues: xrc srq's are similar to normal
srq's, except that they are bound to an xrcd, rather
than to a protection domain.  Based on the current spec
and implementation, they are only usable with xrc qps.  To
support xrc srq's, we define a new srq_init_attr structure
to include an srq type and other needed information.

The kernel ABI is also updated to allow creating extended
SRQs.

Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
 include/infiniband/driver.h   |   27 ++++++++++++++++
 include/infiniband/kern-abi.h |   21 ++++++++++++-
 include/infiniband/verbs.h    |   61 +++++++++++++++++++++++++++++++++++-
 src/cmd.c                     |   69 +++++++++++++++++++++++++++++++++++++++++
 src/libibverbs.map            |    1 +
 5 files changed, 177 insertions(+), 2 deletions(-)

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index ce88442..05ff730 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -64,6 +64,23 @@ struct verbs_xrcd {
 	uint32_t		handle;
 };
 
+enum verbs_srq_mask {
+	VERBS_SRQ_TYPE		= 1 << 0,
+	VERBS_SRQ_XRCD		= 1 << 1,
+	VERBS_SRQ_CQ		= 1 << 2,
+	VERBS_SRQ_NUM		= 1 << 3,
+	VERBS_SRQ_RESERVED	= 1 << 4
+};
+
+struct verbs_srq {
+	struct ibv_srq		srq;
+	uint32_t		comp_mask;
+	enum ibv_srq_type	srq_type;
+	struct verbs_xrcd      *xrcd;
+	struct ibv_cq	       *cq;
+	uint32_t		srq_num;
+};
+
 typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
 						   int abi_version);
 typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path,
@@ -118,6 +135,10 @@ int ibv_cmd_create_srq(struct ibv_pd *pd,
 		       struct ibv_srq *srq, struct ibv_srq_init_attr *attr,
 		       struct ibv_create_srq *cmd, size_t cmd_size,
 		       struct ibv_create_srq_resp *resp, size_t resp_size);
+int ibv_cmd_create_srq_ex(struct ibv_context *context,
+			  struct verbs_srq *srq, struct ibv_srq_init_attr_ex *attr_ex,
+			  struct ibv_create_xsrq *cmd, size_t cmd_size,
+			  struct ibv_create_srq_resp *resp, size_t resp_size);
 int ibv_cmd_modify_srq(struct ibv_srq *srq,
 		       struct ibv_srq_attr *srq_attr,
 		       int srq_attr_mask,
@@ -162,4 +183,10 @@ const char *ibv_get_sysfs_path(void);
 int ibv_read_sysfs_file(const char *dir, const char *file,
 			char *buf, size_t size);
 
+static inline uint32_t verbs_get_srq_num(struct ibv_srq *srq)
+{
+	struct verbs_srq *vsrq = container_of(srq, struct verbs_srq, srq);
+	return (vsrq->comp_mask & VERBS_SRQ_NUM) ? vsrq->srq_num : 0;
+}
+
 #endif /* INFINIBAND_DRIVER_H */
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index d7c673f..3d72fa7 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -88,6 +88,7 @@ enum {
 	IB_USER_VERBS_CMD_POST_SRQ_RECV,
 	IB_USER_VERBS_CMD_OPEN_XRCD,
 	IB_USER_VERBS_CMD_CLOSE_XRCD,
+	IB_USER_VERBS_CMD_CREATE_XSRQ
 };
 
 /*
@@ -730,11 +731,28 @@ struct ibv_create_srq {
 	__u64 driver_data[0];
 };
 
+struct ibv_create_xsrq {
+	__u32 command;
+	__u16 in_words;
+	__u16 out_words;
+	__u64 response;
+	__u64 user_handle;
+	__u32 srq_type;
+	__u32 pd_handle;
+	__u32 max_wr;
+	__u32 max_sge;
+	__u32 srq_limit;
+	__u32 reserved;
+	__u32 xrcd_handle;
+	__u32 cq_handle;
+	__u64 driver_data[0];
+};
+
 struct ibv_create_srq_resp {
 	__u32 srq_handle;
 	__u32 max_wr;
 	__u32 max_sge;
-	__u32 reserved;
+	__u32 srqn;
 };
 
 struct ibv_modify_srq {
@@ -829,6 +847,7 @@ enum {
 	IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
 	IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
 	IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
+	IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1
 };
 
 struct ibv_destroy_cq_v1 {
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index cf8299a..d7dfcd3 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -427,6 +427,30 @@ struct ibv_srq_init_attr {
 	struct ibv_srq_attr	attr;
 };
 
+enum ibv_srq_type {
+	IBV_SRQT_BASIC,
+	IBV_SRQT_XRC
+};
+
+enum ibv_srq_init_attr_mask {
+	IBV_SRQ_INIT_ATTR_TYPE		= 1 << 0,
+	IBV_SRQ_INIT_ATTR_PD		= 1 << 1,
+	IBV_SRQ_INIT_ATTR_XRCD		= 1 << 2,
+	IBV_SRQ_INIT_ATTR_CQ		= 1 << 3,
+	IBV_SRQ_INIT_ATTR_RESERVED	= 1 << 4
+};
+
+struct ibv_srq_init_attr_ex {
+	void		       *srq_context;
+	struct ibv_srq_attr	attr;
+
+	uint32_t		comp_mask;
+	enum ibv_srq_type	srq_type;
+	struct ibv_pd	       *pd;
+	struct ibv_xrcd	       *xrcd;
+	struct ibv_cq	       *cq;
+};
+
 enum ibv_qp_type {
 	IBV_QPT_RC = 2,
 	IBV_QPT_UC,
@@ -746,11 +770,15 @@ struct ibv_context {
 
 enum verbs_context_mask {
 	VERBS_CONTEXT_XRCD	= 1 << 0,
-	VERBS_CONTEXT_RESERVED	= 1 << 1
+	VERBS_CONTEXT_SRQ	= 1 << 1,
+	VERBS_CONTEXT_RESERVED	= 1 << 2
 };
 
 struct verbs_context {
 	/*  "grows up" - new fields go here */
+	uint32_t		(*get_srq_num)(struct ibv_srq *srq);
+	struct ibv_srq *	(*create_srq_ex)(struct ibv_context *context,
+						 struct ibv_srq_init_attr_ex *srq_init_attr_ex);
 	struct ibv_xrcd *	(*open_xrcd)(struct ibv_context *context,
 					     struct ibv_xrcd_init_attr *xrcd_init_attr);
 	int			(*close_xrcd)(struct ibv_xrcd *xrcd);
@@ -1034,6 +1062,27 @@ static inline int ibv_req_notify_cq(struct ibv_cq *cq, int solicited_only)
 struct ibv_srq *ibv_create_srq(struct ibv_pd *pd,
 			       struct ibv_srq_init_attr *srq_init_attr);
 
+static inline struct ibv_srq *
+ibv_create_srq_ex(struct ibv_context *context,
+		  struct ibv_srq_init_attr_ex *srq_init_attr_ex)
+{
+	struct verbs_context *vctx;
+	uint32_t mask = srq_init_attr_ex->comp_mask;
+
+	if (!(mask & ~(IBV_SRQ_INIT_ATTR_PD | IBV_SRQ_INIT_ATTR_TYPE)) &&
+	    (mask & IBV_SRQ_INIT_ATTR_PD) &&
+	    (!(mask & IBV_SRQ_INIT_ATTR_TYPE) ||
+	     (srq_init_attr_ex->srq_type == IBV_SRQT_BASIC)))
+		return ibv_create_srq(srq_init_attr_ex->pd,
+				      (struct ibv_srq_init_attr *) srq_init_attr_ex);
+
+	if (!(vctx = verbs_get_ctx_op(context, create_srq_ex))) {
+		errno = ENOSYS;
+		return NULL;
+	}
+	return vctx->create_srq_ex(context, srq_init_attr_ex);
+}
+
 /**
  * ibv_modify_srq - Modifies the attributes for the specified SRQ.
  * @srq: The SRQ to modify.
@@ -1058,6 +1107,16 @@ int ibv_modify_srq(struct ibv_srq *srq,
  */
 int ibv_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr);
 
+static inline uint32_t ibv_get_srq_num(struct ibv_srq *srq)
+{
+	struct verbs_context *vctx = verbs_get_ctx_op(srq->context, get_srq_num);
+	if (!vctx) {
+		errno = ENOSYS;
+		return 0;
+	}
+	return vctx->get_srq_num(srq);
+}
+
 /**
  * ibv_destroy_srq - Destroys the specified SRQ.
  * @srq: The SRQ to destroy.
diff --git a/src/cmd.c b/src/cmd.c
index f0fb1dc..641ab28 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -488,6 +488,75 @@ int ibv_cmd_create_srq(struct ibv_pd *pd,
 	return 0;
 }
 
+int ibv_cmd_create_srq_ex(struct ibv_context *context,
+			  struct verbs_srq *srq, struct ibv_srq_init_attr_ex *attr_ex,
+			  struct ibv_create_xsrq *cmd, size_t cmd_size,
+			  struct ibv_create_srq_resp *resp, size_t resp_size)
+{
+	struct verbs_xrcd *vxrcd = NULL;
+
+	IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_XSRQ, resp, resp_size);
+
+	if (attr_ex->comp_mask >= IBV_SRQ_INIT_ATTR_RESERVED)
+		return ENOSYS;
+
+	if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_PD))
+		return EINVAL;
+
+	cmd->user_handle = (uintptr_t) srq;
+	cmd->pd_handle   = attr_ex->pd->handle;
+	cmd->max_wr      = attr_ex->attr.max_wr;
+	cmd->max_sge     = attr_ex->attr.max_sge;
+	cmd->srq_limit   = attr_ex->attr.srq_limit;
+
+	cmd->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ?
+			attr_ex->srq_type : IBV_SRQT_BASIC;
+	if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) {
+		if (!(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) ||
+		    !(attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ))
+			return EINVAL;
+
+		vxrcd = container_of(attr_ex->xrcd, struct verbs_xrcd, xrcd);
+		cmd->xrcd_handle = vxrcd->handle;
+		cmd->cq_handle   = attr_ex->cq->handle;
+	}
+
+	if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+		return errno;
+
+	VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+	srq->srq.handle 	  = resp->srq_handle;
+	srq->srq.context          = context;
+	srq->srq.srq_context      = attr_ex->srq_context;
+	srq->srq.pd               = attr_ex->pd;
+	srq->srq.events_completed = 0;
+	pthread_mutex_init(&srq->srq.mutex, NULL);
+	pthread_cond_init(&srq->srq.cond, NULL);
+
+	srq->comp_mask = IBV_SRQ_INIT_ATTR_TYPE;
+	srq->srq_type = (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_TYPE) ?
+			attr_ex->srq_type : IBV_SRQT_BASIC;
+	if (srq->srq_type == IBV_SRQT_XRC) {
+		srq->comp_mask |= VERBS_SRQ_NUM;
+		srq->srq_num = resp->srqn;
+	}
+	if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_XRCD) {
+		srq->comp_mask |= VERBS_SRQ_XRCD;
+		srq->xrcd = vxrcd;
+	}
+	if (attr_ex->comp_mask & IBV_SRQ_INIT_ATTR_CQ) {
+		srq->comp_mask |= VERBS_SRQ_CQ;
+		srq->cq = attr_ex->cq;
+	}
+
+	attr_ex->attr.max_wr = resp->max_wr;
+	attr_ex->attr.max_sge = resp->max_sge;
+
+	return 0;
+}
+
+
 static int ibv_cmd_modify_srq_v3(struct ibv_srq *srq,
 				 struct ibv_srq_attr *srq_attr,
 				 int srq_attr_mask,
diff --git a/src/libibverbs.map b/src/libibverbs.map
index 9a15f3f..c8ec645 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -100,5 +100,6 @@ IBVERBS_1.1 {
 		
 		ibv_cmd_open_xrcd;
 		ibv_cmd_close_xrcd;
+		ibv_cmd_create_srq_ex;
 		
 } IBVERBS_1.0;
-- 
1.7.3

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 4/7] libibverbs: Add support for XRC QPs

[sean.hefty-ral2JQCrhuEAvxtiuMwx3w]

From: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

XRC queue pairs: xrc defines two new types of QPs.  The
initiator, or send-side, xrc qp behaves similar to a send-
only RC qp.  xrc send qp's are managed through the existing
QP functions.  The send_wr structure is extended in a back-
wards compatible way to support posting sends on a send xrc
qp, which require specifying the remote xrc srq.

The target, or receive-side, xrc qp behaves differently
than other implemented qp's.  A recv xrc qp can be created,
modified, and destroyed like other qp's through the existing
calls.  The qp_init_attr structure is extended for xrc qp's.

Because xrc recv qp's are bound to an xrcd, rather than a pd,
it is intended to be used among multiple processes.  Any process
with access to an xrcd may allocate and connect an xrc recv qp.
The actual xrc recv qp is allocated and managed by the kernel.
If the owning process explicit destroys the xrc recv qp, it is
destroyed.  However, if the xrc recv qp is left open when the
user process exits or closes its device, then the lifetime of
the xrc recv qp is bound with the lifetime of the xrcd.

Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
 include/infiniband/driver.h   |   14 ++++++
 include/infiniband/kern-abi.h |    5 ++
 include/infiniband/verbs.h    |   54 ++++++++++++++++++++++-
 src/cmd.c                     |   94 ++++++++++++++++++++++++++++++++++++++++-
 src/libibverbs.map            |    1 +
 5 files changed, 164 insertions(+), 4 deletions(-)

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index 05ff730..d43ae07 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -81,6 +81,16 @@ struct verbs_srq {
 	uint32_t		srq_num;
 };
 
+enum verbs_qp_mask {
+	VERBS_QP_XRCD		= 1 << 0,
+	VERBS_QP_RESERVED	= 1 << 1
+};
+
+struct verbs_qp {
+	struct ibv_qp		qp;
+	uint32_t		comp_mask;
+	struct verbs_xrcd       *xrcd;
+};
 typedef struct ibv_device *(*ibv_driver_init_func)(const char *uverbs_sys_path,
 						   int abi_version);
 typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path,
@@ -152,6 +162,10 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
 		      struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
 		      struct ibv_create_qp *cmd, size_t cmd_size,
 		      struct ibv_create_qp_resp *resp, size_t resp_size);
+int ibv_cmd_create_qp_ex(struct ibv_context *context,
+			 struct verbs_qp *qp, struct ibv_qp_init_attr_ex *attr_ex,
+			 struct ibv_create_qp *cmd, size_t cmd_size,
+			 struct ibv_create_qp_resp *resp, size_t resp_size);
 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr,
 		     int attr_mask,
 		     struct ibv_qp_init_attr *qp_init_attr,
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index 3d72fa7..b6d5ce9 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -617,6 +617,11 @@ struct ibv_kern_send_wr {
 			__u32 remote_qkey;
 			__u32 reserved;
 		} ud;
+		struct {
+			__u64 reserved[3];
+			__u32 reserved2;
+			__u32 remote_srqn;
+		} xrc;
 	} wr;
 };
 
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index d7dfcd3..f321866 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -107,7 +107,8 @@ enum ibv_device_cap_flags {
 	IBV_DEVICE_SYS_IMAGE_GUID	= 1 << 11,
 	IBV_DEVICE_RC_RNR_NAK_GEN	= 1 << 12,
 	IBV_DEVICE_SRQ_RESIZE		= 1 << 13,
-	IBV_DEVICE_N_NOTIFY_CQ		= 1 << 14
+	IBV_DEVICE_N_NOTIFY_CQ		= 1 << 14,
+	IBV_DEVICE_XRC			= 1 << 20
 };
 
 enum ibv_atomic_cap {
@@ -454,7 +455,9 @@ struct ibv_srq_init_attr_ex {
 enum ibv_qp_type {
 	IBV_QPT_RC = 2,
 	IBV_QPT_UC,
-	IBV_QPT_UD
+	IBV_QPT_UD,
+	IBV_QPT_XRC_SEND = 9,
+	IBV_QPT_XRC_RECV
 };
 
 struct ibv_qp_cap {
@@ -475,6 +478,26 @@ struct ibv_qp_init_attr {
 	int			sq_sig_all;
 };
 
+enum ibv_qp_init_attr_mask {
+	IBV_QP_INIT_ATTR_PD		= 1 << 0,
+	IBV_QP_INIT_ATTR_XRCD		= 1 << 1,
+	IBV_QP_INIT_ATTR_RESERVED	= 1 << 2
+};
+
+struct ibv_qp_init_attr_ex {
+	void		       *qp_context;
+	struct ibv_cq	       *send_cq;
+	struct ibv_cq	       *recv_cq;
+	struct ibv_srq	       *srq;
+	struct ibv_qp_cap	cap;
+	enum ibv_qp_type	qp_type;
+	int			sq_sig_all;
+
+	uint32_t		comp_mask;
+	struct ibv_pd	       *pd;
+	struct ibv_xrcd	       *xrcd;
+};
+
 enum ibv_qp_attr_mask {
 	IBV_QP_STATE			= 1 << 	0,
 	IBV_QP_CUR_STATE		= 1 << 	1,
@@ -590,6 +613,11 @@ struct ibv_send_wr {
 			uint32_t	remote_qpn;
 			uint32_t	remote_qkey;
 		} ud;
+		struct {
+			uint64_t	reserved[3];
+			uint32_t	reserved2;
+			uint32_t	remote_srqn;
+		} xrc;
 	} wr;
 };
 
@@ -771,11 +799,14 @@ struct ibv_context {
 enum verbs_context_mask {
 	VERBS_CONTEXT_XRCD	= 1 << 0,
 	VERBS_CONTEXT_SRQ	= 1 << 1,
-	VERBS_CONTEXT_RESERVED	= 1 << 2
+	VERBS_CONTEXT_QP	= 1 << 2,
+	VERBS_CONTEXT_RESERVED	= 1 << 3
 };
 
 struct verbs_context {
 	/*  "grows up" - new fields go here */
+	struct ibv_qp *		(*create_qp_ex)(struct ibv_context *context,
+						struct ibv_qp_init_attr_ex *qp_init_attr_ex);
 	uint32_t		(*get_srq_num)(struct ibv_srq *srq);
 	struct ibv_srq *	(*create_srq_ex)(struct ibv_context *context,
 						 struct ibv_srq_init_attr_ex *srq_init_attr_ex);
@@ -1143,6 +1174,23 @@ static inline int ibv_post_srq_recv(struct ibv_srq *srq,
 struct ibv_qp *ibv_create_qp(struct ibv_pd *pd,
 			     struct ibv_qp_init_attr *qp_init_attr);
 
+static inline struct ibv_qp *
+ibv_create_qp_ex(struct ibv_context *context, struct ibv_qp_init_attr_ex *qp_init_attr_ex)
+{
+	struct verbs_context *vctx;
+	uint32_t mask = qp_init_attr_ex->comp_mask;
+
+	if (mask == IBV_QP_INIT_ATTR_PD)
+		return ibv_create_qp(qp_init_attr_ex->pd,
+				     (struct ibv_qp_init_attr *) qp_init_attr_ex);
+
+	if (!(vctx = verbs_get_ctx_op(context, create_qp_ex))) {
+		errno = ENOSYS;
+		return NULL;
+	}
+	return vctx->create_qp_ex(context, qp_init_attr_ex);
+}
+
 /**
  * ibv_modify_qp - Modify a queue pair.
  */
diff --git a/src/cmd.c b/src/cmd.c
index 641ab28..87ac062 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -666,6 +666,98 @@ int ibv_cmd_destroy_srq(struct ibv_srq *srq)
 	return 0;
 }
 
+int ibv_cmd_create_qp_ex(struct ibv_context *context,
+			 struct verbs_qp *qp, struct ibv_qp_init_attr_ex *attr_ex,
+			 struct ibv_create_qp *cmd, size_t cmd_size,
+			 struct ibv_create_qp_resp *resp, size_t resp_size)
+{
+	struct verbs_xrcd *vxrcd = NULL;
+
+	IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
+
+	if (attr_ex->comp_mask >= IBV_QP_INIT_ATTR_RESERVED)
+		return ENOSYS;
+
+	cmd->user_handle     = (uintptr_t) qp;
+
+	if (attr_ex->comp_mask & IBV_QP_INIT_ATTR_XRCD) {
+		vxrcd = container_of(attr_ex->xrcd, struct verbs_xrcd, xrcd);
+		cmd->pd_handle	= vxrcd->handle;
+	} else {
+		if (!(attr_ex->comp_mask & IBV_QP_INIT_ATTR_PD))
+			return EINVAL;
+
+		cmd->pd_handle	= attr_ex->pd->handle;
+		cmd->send_cq_handle = attr_ex->send_cq->handle;
+
+		if (attr_ex->qp_type != IBV_QPT_XRC_SEND) {
+			cmd->recv_cq_handle = attr_ex->recv_cq->handle;
+			cmd->srq_handle = attr_ex->srq ? attr_ex->srq->handle : 0;
+		}
+	}
+
+	cmd->max_send_wr     = attr_ex->cap.max_send_wr;
+	cmd->max_recv_wr     = attr_ex->cap.max_recv_wr;
+	cmd->max_send_sge    = attr_ex->cap.max_send_sge;
+	cmd->max_recv_sge    = attr_ex->cap.max_recv_sge;
+	cmd->max_inline_data = attr_ex->cap.max_inline_data;
+	cmd->sq_sig_all	     = attr_ex->sq_sig_all;
+	cmd->qp_type 	     = attr_ex->qp_type;
+	cmd->is_srq 	     = !!attr_ex->srq;
+	cmd->reserved	     = 0;
+
+	if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+		return errno;
+
+	(void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+	if (abi_ver > 3) {
+		attr_ex->cap.max_recv_sge    = resp->max_recv_sge;
+		attr_ex->cap.max_send_sge    = resp->max_send_sge;
+		attr_ex->cap.max_recv_wr     = resp->max_recv_wr;
+		attr_ex->cap.max_send_wr     = resp->max_send_wr;
+		attr_ex->cap.max_inline_data = resp->max_inline_data;
+	}
+
+	if (abi_ver == 4) {
+		struct ibv_create_qp_resp_v4 *resp_v4 =
+			(struct ibv_create_qp_resp_v4 *) resp;
+
+		memmove((void *) resp + sizeof *resp,
+			(void *) resp_v4 + sizeof *resp_v4,
+			resp_size - sizeof *resp);
+	} else if (abi_ver <= 3) {
+		struct ibv_create_qp_resp_v3 *resp_v3 =
+			(struct ibv_create_qp_resp_v3 *) resp;
+
+		memmove((void *) resp + sizeof *resp,
+			(void *) resp_v3 + sizeof *resp_v3,
+			resp_size - sizeof *resp);
+	}
+
+	qp->qp.handle		= resp->qp_handle;
+	qp->qp.qp_num		= resp->qpn;
+	qp->qp.context		= context;
+	qp->qp.qp_context	= attr_ex->qp_context;
+	qp->qp.pd		= attr_ex->pd;
+	qp->qp.send_cq		= attr_ex->send_cq;
+	qp->qp.recv_cq		= attr_ex->recv_cq;
+	qp->qp.srq		= attr_ex->srq;
+	qp->qp.qp_type		= attr_ex->qp_type;
+	qp->qp.state		= IBV_QPS_RESET;
+	qp->qp.events_completed = 0;
+	pthread_mutex_init(&qp->qp.mutex, NULL);
+	pthread_cond_init(&qp->qp.cond, NULL);
+
+	qp->comp_mask = 0;
+	if (attr_ex->comp_mask & IBV_QP_INIT_ATTR_XRCD) {
+		qp->comp_mask |= VERBS_QP_XRCD;
+		qp->xrcd = vxrcd;
+	}
+
+	return 0;
+}
+
 int ibv_cmd_create_qp(struct ibv_pd *pd,
 		      struct ibv_qp *qp, struct ibv_qp_init_attr *attr,
 		      struct ibv_create_qp *cmd, size_t cmd_size,
@@ -674,7 +766,7 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
 	IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_QP, resp, resp_size);
 
 	cmd->user_handle     = (uintptr_t) qp;
-	cmd->pd_handle 	     = pd->handle;
+	cmd->pd_handle       = pd->handle;
 	cmd->send_cq_handle  = attr->send_cq->handle;
 	cmd->recv_cq_handle  = attr->recv_cq->handle;
 	cmd->srq_handle      = attr->srq ? attr->srq->handle : 0;
diff --git a/src/libibverbs.map b/src/libibverbs.map
index c8ec645..4249793 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -101,5 +101,6 @@ IBVERBS_1.1 {
 		ibv_cmd_open_xrcd;
 		ibv_cmd_close_xrcd;
 		ibv_cmd_create_srq_ex;
+		ibv_cmd_create_qp_ex;
 		
 } IBVERBS_1.0;
-- 
1.7.3

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 5/7] libibverbs: Add ibv_open_qp

[sean.hefty-ral2JQCrhuEAvxtiuMwx3w]

From: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

XRC receive QPs are shareable across multiple processes.  Allow
any process with access to the xrc domain to open an existing
QP.  After opening the QP, the process will receive events
related to the QP and be able to modify the QP.

Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
 include/infiniband/driver.h   |    4 +++
 include/infiniband/kern-abi.h |   20 ++++++++++++++++-
 include/infiniband/verbs.h    |   35 +++++++++++++++++++++++++++++++-
 src/cmd.c                     |   45 +++++++++++++++++++++++++++++++++++++++++
 src/libibverbs.map            |    1 +
 5 files changed, 102 insertions(+), 3 deletions(-)

diff --git a/include/infiniband/driver.h b/include/infiniband/driver.h
index d43ae07..a63d1e9 100644
--- a/include/infiniband/driver.h
+++ b/include/infiniband/driver.h
@@ -166,6 +166,10 @@ int ibv_cmd_create_qp_ex(struct ibv_context *context,
 			 struct verbs_qp *qp, struct ibv_qp_init_attr_ex *attr_ex,
 			 struct ibv_create_qp *cmd, size_t cmd_size,
 			 struct ibv_create_qp_resp *resp, size_t resp_size);
+int ibv_cmd_open_qp(struct ibv_context *context,
+		    struct verbs_qp *qp, struct ibv_qp_open_attr *attr,
+		    struct ibv_open_qp *cmd, size_t cmd_size,
+		    struct ibv_create_qp_resp *resp, size_t resp_size);
 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *qp_attr,
 		     int attr_mask,
 		     struct ibv_qp_init_attr *qp_init_attr,
diff --git a/include/infiniband/kern-abi.h b/include/infiniband/kern-abi.h
index b6d5ce9..e24fa4f 100644
--- a/include/infiniband/kern-abi.h
+++ b/include/infiniband/kern-abi.h
@@ -88,7 +88,8 @@ enum {
 	IB_USER_VERBS_CMD_POST_SRQ_RECV,
 	IB_USER_VERBS_CMD_OPEN_XRCD,
 	IB_USER_VERBS_CMD_CLOSE_XRCD,
-	IB_USER_VERBS_CMD_CREATE_XSRQ
+	IB_USER_VERBS_CMD_CREATE_XSRQ,
+	IB_USER_VERBS_CMD_OPEN_QP
 };
 
 /*
@@ -476,6 +477,20 @@ struct ibv_create_qp {
 	__u64 driver_data[0];
 };
 
+struct ibv_open_qp {
+	__u32 command;
+	__u16 in_words;
+	__u16 out_words;
+	__u64 response;
+	__u64 user_handle;
+	__u32 pd_handle;
+	__u32 qpn;
+	__u8  qp_type;
+	__u8  reserved[7];
+	__u64 driver_data[0];
+};
+
+/* also used for open response */
 struct ibv_create_qp_resp {
 	__u32 qp_handle;
 	__u32 qpn;
@@ -852,7 +867,8 @@ enum {
 	IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
 	IB_USER_VERBS_CMD_OPEN_XRCD_V2 = -1,
 	IB_USER_VERBS_CMD_CLOSE_XRCD_V2 = -1,
-	IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1
+	IB_USER_VERBS_CMD_CREATE_XSRQ_V2 = -1,
+	IB_USER_VERBS_CMD_OPEN_QP_V2 = -1
 };
 
 struct ibv_destroy_cq_v1 {
diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
index f321866..a662ab4 100644
--- a/include/infiniband/verbs.h
+++ b/include/infiniband/verbs.h
@@ -498,6 +498,22 @@ struct ibv_qp_init_attr_ex {
 	struct ibv_xrcd	       *xrcd;
 };
 
+enum ibv_qp_open_attr_mask {
+	IBV_QP_OPEN_ATTR_NUM		= 1 << 0,
+	IBV_QP_OPEN_ATTR_CONTEXT	= 1 << 1,
+	IBV_QP_OPEN_ATTR_XRCD		= 1 << 2,
+	IBV_QP_OPEN_ATTR_TYPE		= 1 << 3,
+	IBV_QP_OPEN_ATTR_RESERVED	= 1 << 4
+ };
+
+struct ibv_qp_open_attr {
+	uint32_t		comp_mask;
+	uint32_t		qp_num;
+	struct ibv_xrcd        *xrcd;
+	void		       *qp_context;
+	enum ibv_qp_type	qp_type;
+};
+
 enum ibv_qp_attr_mask {
 	IBV_QP_STATE			= 1 << 	0,
 	IBV_QP_CUR_STATE		= 1 << 	1,
@@ -529,7 +545,8 @@ enum ibv_qp_state {
 	IBV_QPS_RTS,
 	IBV_QPS_SQD,
 	IBV_QPS_SQE,
-	IBV_QPS_ERR
+	IBV_QPS_ERR,
+	IBV_QPS_UNKNOWN
 };
 
 enum ibv_mig_state {
@@ -805,6 +822,8 @@ enum verbs_context_mask {
 
 struct verbs_context {
 	/*  "grows up" - new fields go here */
+	struct ibv_qp *		(*open_qp)(struct ibv_context *context,
+					   struct ibv_qp_open_attr *attr);
 	struct ibv_qp *		(*create_qp_ex)(struct ibv_context *context,
 						struct ibv_qp_init_attr_ex *qp_init_attr_ex);
 	uint32_t		(*get_srq_num)(struct ibv_srq *srq);
@@ -1192,6 +1211,20 @@ ibv_create_qp_ex(struct ibv_context *context, struct ibv_qp_init_attr_ex *qp_ini
 }
 
 /**
+ * ibv_open_qp - Open a shareable queue pair.
+ */
+static inline struct ibv_qp *
+ibv_open_qp(struct ibv_context *context, struct ibv_qp_open_attr *qp_open_attr)
+{
+	struct verbs_context *vctx = verbs_get_ctx_op(context, open_qp);
+	if (!vctx) {
+		errno = ENOSYS;
+		return NULL;
+	}
+	return vctx->open_qp(context, qp_open_attr);
+}
+
+/**
  * ibv_modify_qp - Modify a queue pair.
  */
 int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
diff --git a/src/cmd.c b/src/cmd.c
index 87ac062..da0d733 100644
--- a/src/cmd.c
+++ b/src/cmd.c
@@ -816,6 +816,51 @@ int ibv_cmd_create_qp(struct ibv_pd *pd,
 	return 0;
 }
 
+int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp,
+		    struct ibv_qp_open_attr *attr,
+		    struct ibv_open_qp *cmd, size_t cmd_size,
+		    struct ibv_create_qp_resp *resp, size_t resp_size)
+{
+	struct verbs_xrcd *xrcd;
+	IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_QP, resp, resp_size);
+
+	if (attr->comp_mask >= IBV_QP_OPEN_ATTR_RESERVED)
+		return ENOSYS;
+
+	if (!(attr->comp_mask & IBV_QP_OPEN_ATTR_XRCD) ||
+	    !(attr->comp_mask & IBV_QP_OPEN_ATTR_NUM) ||
+	    !(attr->comp_mask & IBV_QP_OPEN_ATTR_TYPE))
+		return EINVAL;
+
+	xrcd = container_of(attr->xrcd, struct verbs_xrcd, xrcd);
+	cmd->user_handle = (uintptr_t) qp;
+	cmd->pd_handle   = xrcd->handle;
+	cmd->qpn         = attr->qp_num;
+	cmd->qp_type     = attr->qp_type;
+
+	if (write(context->cmd_fd, cmd, cmd_size) != cmd_size)
+		return errno;
+
+	VALGRIND_MAKE_MEM_DEFINED(resp, resp_size);
+
+	qp->qp.handle     = resp->qp_handle;
+	qp->qp.context    = context;
+	qp->qp.qp_context = attr->qp_context;
+	qp->qp.pd	  = NULL;
+	qp->qp.send_cq	  = qp->qp.recv_cq = NULL;
+	qp->qp.srq	  = NULL;
+	qp->qp.qp_num	  = attr->qp_num;
+	qp->qp.qp_type	  = attr->qp_type;
+	qp->qp.state	  = IBV_QPS_UNKNOWN;
+	qp->qp.events_completed = 0;
+	pthread_mutex_init(&qp->qp.mutex, NULL);
+	pthread_cond_init(&qp->qp.cond, NULL);
+	qp->comp_mask	  = VERBS_QP_XRCD;
+	qp->xrcd	  = xrcd;
+
+	return 0;
+}
+
 int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 		     int attr_mask,
 		     struct ibv_qp_init_attr *init_attr,
diff --git a/src/libibverbs.map b/src/libibverbs.map
index 4249793..5ad8312 100644
--- a/src/libibverbs.map
+++ b/src/libibverbs.map
@@ -102,5 +102,6 @@ IBVERBS_1.1 {
 		ibv_cmd_close_xrcd;
 		ibv_cmd_create_srq_ex;
 		ibv_cmd_create_qp_ex;
+		ibv_cmd_open_qp;
 		
 } IBVERBS_1.0;
-- 
1.7.3

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 6/7] libibverbs: Add man page for ibv_open_qp

[sean.hefty-ral2JQCrhuEAvxtiuMwx3w]

From: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
 man/ibv_open_qp.3 |   50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 50 insertions(+), 0 deletions(-)
 create mode 100644 man/ibv_open_qp.3

diff --git a/man/ibv_open_qp.3 b/man/ibv_open_qp.3
new file mode 100644
index 0000000..0bc5647
--- /dev/null
+++ b/man/ibv_open_qp.3
@@ -0,0 +1,50 @@
+.\" -*- nroff -*-
+.\"
+.TH IBV_OPEN_QP 3 2011-08-12 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_open_qp \- open a shareable queue pair (QP)
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "struct ibv_qp *ibv_open_qp(struct ibv_xrcd " "*xrcd" ,
+.BI "                           struct ibv_qp_open_attr " "*qp_open_attr" );
+.fi
+.SH "DESCRIPTION"
+.B ibv_open_qp()
+opens an existing queue pair (QP) associated with the extended protection domain
+.I xrcd\fR.
+The argument
+.I qp_open_attr
+is an ibv_qp_open_attr struct, as defined in <infiniband/verbs.h>.
+.PP
+.nf
+struct ibv_qp_open_attr {
+.in +8
+uint32_t          comp_mask;  /* Identifies valid fields */
+uint32_t          qp_num;     /* QP number */
+void             *qp_context; /* Associated context of the QP */
+enum ibv_qp_type  qp_type;    /* QP transport service type */
+.fi
+.PP
+.B ibv_destroy_qp()
+closes the opened QP and destroys the underlying QP if it has no
+other references.
+.I qp\fR.
+.SH "RETURN VALUE"
+.B ibv_open_qp()
+returns a pointer to the opened QP, or NULL if the request fails.
+Check the QP number (\fBqp_num\fR) in the returned QP.
+.SH "NOTES"
+.B ibv_open_qp()
+will fail if a it is asked to open a QP that does not exist within
+the xrcd with the specified qp_num and qp_type.
+.SH "SEE ALSO"
+.BR ibv_alloc_pd (3),
+.BR ibv_create_qp (3),
+.BR ibv_create_qp_ex (3),
+.BR ibv_modify_qp (3),
+.BR ibv_query_qp (3)
+.SH "AUTHORS"
+.TP
+Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
-- 
1.7.3

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v4 7/7] libibverbs: Add XRC sample application

[sean.hefty-ral2JQCrhuEAvxtiuMwx3w]

From: Jay Sternberg <jay.e.sternberg-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>

Signed-off-by: Jay Sternberg <jay.e.sternberg-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
Signed-off-by: Sean Hefty <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org>
---
 Makefile.am              |    4 +-
 examples/xsrq_pingpong.c |  885 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 888 insertions(+), 1 deletions(-)
 create mode 100644 examples/xsrq_pingpong.c

diff --git a/Makefile.am b/Makefile.am
index cd00a65..4821ec6 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -18,7 +18,7 @@ src_libibverbs_la_DEPENDENCIES = $(srcdir)/src/libibverbs.map
 
 bin_PROGRAMS = examples/ibv_devices examples/ibv_devinfo \
     examples/ibv_asyncwatch examples/ibv_rc_pingpong examples/ibv_uc_pingpong \
-    examples/ibv_ud_pingpong examples/ibv_srq_pingpong
+    examples/ibv_ud_pingpong examples/ibv_srq_pingpong examples/ibv_xsrq_pingpong
 examples_ibv_devices_SOURCES = examples/device_list.c
 examples_ibv_devices_LDADD = $(top_builddir)/src/libibverbs.la
 examples_ibv_devinfo_SOURCES = examples/devinfo.c
@@ -31,6 +31,8 @@ examples_ibv_ud_pingpong_SOURCES = examples/ud_pingpong.c examples/pingpong.c
 examples_ibv_ud_pingpong_LDADD = $(top_builddir)/src/libibverbs.la
 examples_ibv_srq_pingpong_SOURCES = examples/srq_pingpong.c examples/pingpong.c
 examples_ibv_srq_pingpong_LDADD = $(top_builddir)/src/libibverbs.la
+examples_ibv_xsrq_pingpong_SOURCES = examples/xsrq_pingpong.c examples/pingpong.c
+examples_ibv_xsrq_pingpong_LDADD = $(top_builddir)/src/libibverbs.la
 examples_ibv_asyncwatch_SOURCES = examples/asyncwatch.c
 examples_ibv_asyncwatch_LDADD = $(top_builddir)/src/libibverbs.la
 
diff --git a/examples/xsrq_pingpong.c b/examples/xsrq_pingpong.c
new file mode 100644
index 0000000..55b7e6a
--- /dev/null
+++ b/examples/xsrq_pingpong.c
@@ -0,0 +1,885 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2011 Intel Corporation, Inc.  All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#if HAVE_CONFIG_H
+#  include <config.h>
+#endif /* HAVE_CONFIG_H */
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/time.h>
+#include <netdb.h>
+#include <malloc.h>
+#include <getopt.h>
+#include <arpa/inet.h>
+#include <time.h>
+
+#include "pingpong.h"
+
+#define MSG_FORMAT "%04x:%06x:%06x:%06x:%04x"
+#define MSG_SIZE   30
+#define MSG_SSCAN  "%x:%x:%x:%x:%x"
+#define ADDR_FORMAT \
+	"%8s: LID %04x, QPN RECV %06x SEND %06x, PSN %06x, SRQN %04x\n"
+static int page_size;
+
+struct pingpong_dest {
+	int lid;
+	int recv_qpn;
+	int send_qpn;
+	int recv_psn;
+	int send_psn;
+	int srqn;
+	int pp_cnt;
+};
+
+struct pingpong_context {
+	struct ibv_context	*context;
+	struct ibv_comp_channel *channel;
+	struct ibv_pd		*pd;
+	struct ibv_mr		*mr;
+	struct ibv_cq		*send_cq;
+	struct ibv_cq		*recv_cq;
+	struct ibv_srq		*srq;
+	struct ibv_xrcd		*xrcd;
+	struct ibv_qp		**recv_qp;
+	struct ibv_qp		**send_qp;
+	struct pingpong_dest	*rem_dest;
+	void			*buf;
+	int			 lid;
+	int			 sl;
+	enum ibv_mtu		 mtu;
+	int			 ib_port;
+	int			 fd;
+	int			 size;
+	int			 num_clients;
+	int			 num_tests;
+	int			 use_event;
+};
+
+struct pingpong_context ctx;
+
+
+static int open_device(char *ib_devname)
+{
+	struct ibv_device **dev_list;
+	int i = 0;
+
+	dev_list = ibv_get_device_list(NULL);
+	if (!dev_list) {
+		fprintf(stderr, "Failed to get IB devices list");
+		return -1;
+	}
+
+	if (ib_devname) {
+		for (; dev_list[i]; ++i) {
+			if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
+				break;
+		}
+	}
+	if (!dev_list[i]) {
+		fprintf(stderr, "IB device %s not found\n",
+			ib_devname ? ib_devname : "");
+		return -1;
+	}
+
+	ctx.context = ibv_open_device(dev_list[i]);
+	if (!ctx.context) {
+		fprintf(stderr, "Couldn't get context for %s\n",
+			ibv_get_device_name(dev_list[i]));
+		return -1;
+	}
+
+	ibv_free_device_list(dev_list);
+	return 0;
+}
+
+static int create_qps(void)
+{
+	struct ibv_qp_init_attr_ex init;
+	struct ibv_qp_attr mod;
+	int i;
+
+	for (i = 0; i < ctx.num_clients; ++i) {
+
+		memset(&init, 0, sizeof init);
+		init.qp_type = IBV_QPT_XRC_RECV;
+		init.comp_mask = IBV_QP_INIT_ATTR_XRCD;
+		init.xrcd = ctx.xrcd;
+
+		ctx.recv_qp[i] = ibv_create_qp_ex(ctx.context, &init);
+		if (!ctx.recv_qp[i])  {
+			fprintf(stderr, "Couldn't create recv QP[%d] errno %d\n",
+				i, errno);
+			return 1;
+		}
+
+		mod.qp_state        = IBV_QPS_INIT;
+		mod.pkey_index      = 0;
+		mod.port_num        = ctx.ib_port;
+		mod.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ;
+
+		if (ibv_modify_qp(ctx.recv_qp[i], &mod,
+				  IBV_QP_STATE | IBV_QP_PKEY_INDEX |
+				  IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) {
+			fprintf(stderr, "Failed to modify recv QP[%d] to INIT\n", i);
+			return 1;
+		}
+
+		memset(&init, 0, sizeof init);
+		init.qp_type	      = IBV_QPT_XRC_SEND;
+		init.send_cq	      = ctx.send_cq;
+		init.cap.max_send_wr  = ctx.num_clients * ctx.num_tests;
+		init.cap.max_send_sge = 1;
+		init.comp_mask	      = IBV_QP_INIT_ATTR_PD;
+		init.pd		      = ctx.pd;
+
+		ctx.send_qp[i] = ibv_create_qp_ex(ctx.context, &init);
+		if (!ctx.send_qp[i])  {
+			fprintf(stderr, "Couldn't create send QP[%d] errno %d\n",
+				i, errno);
+			return 1;
+		}
+
+		mod.qp_state        = IBV_QPS_INIT;
+		mod.pkey_index      = 0;
+		mod.port_num        = ctx.ib_port;
+		mod.qp_access_flags = 0;
+
+		if (ibv_modify_qp(ctx.send_qp[i], &mod,
+				  IBV_QP_STATE | IBV_QP_PKEY_INDEX |
+				  IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) {
+			fprintf(stderr, "Failed to modify send QP[%d] to INIT\n", i);
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static int pp_init_ctx(char *ib_devname)
+{
+	struct ibv_srq_init_attr_ex attr;
+	struct ibv_xrcd_init_attr xrcd_attr;
+
+	ctx.recv_qp = calloc(ctx.num_clients, sizeof *ctx.recv_qp);
+	ctx.send_qp = calloc(ctx.num_clients, sizeof *ctx.send_qp);
+	ctx.rem_dest = calloc(ctx.num_clients, sizeof *ctx.rem_dest);
+	if (!ctx.recv_qp || !ctx.send_qp || !ctx.rem_dest)
+		return 1;
+
+	if (open_device(ib_devname)) {
+		fprintf(stderr, "Failed to open device\n");
+		return 1;
+	}
+
+	ctx.lid = pp_get_local_lid(ctx.context, ctx.ib_port);
+	if (ctx.lid < 0) {
+		fprintf(stderr, "Failed to get SLID\n");
+		return 1;
+	}
+
+	ctx.buf = memalign(page_size, ctx.size);
+	if (!ctx.buf) {
+		fprintf(stderr, "Couldn't allocate work buf.\n");
+		return 1;
+	}
+
+	memset(ctx.buf, 0, ctx.size);
+
+	if (ctx.use_event) {
+		ctx.channel = ibv_create_comp_channel(ctx.context);
+		if (!ctx.channel) {
+			fprintf(stderr, "Couldn't create completion channel\n");
+			return 1;
+		}
+	}
+
+	ctx.pd = ibv_alloc_pd(ctx.context);
+	if (!ctx.pd) {
+		fprintf(stderr, "Couldn't allocate PD\n");
+		return 1;
+	}
+
+	ctx.mr = ibv_reg_mr(ctx.pd, ctx.buf, ctx.size, IBV_ACCESS_LOCAL_WRITE);
+	if (!ctx.mr) {
+		fprintf(stderr, "Couldn't register MR\n");
+		return 1;
+	}
+
+	ctx.fd = open("/tmp/xrc_domain", O_CREAT);
+	if (ctx.fd < 0) {
+		fprintf(stderr,
+			"Couldn't create the file for the XRC Domain "
+			"but not stopping %d\n", errno);
+		ctx.fd = -1;
+	}
+
+	memset(&xrcd_attr, 0, sizeof xrcd_attr);
+	xrcd_attr.comp_mask = IBV_XRCD_INIT_ATTR_FD | IBV_XRCD_INIT_ATTR_OFLAGS;
+	xrcd_attr.fd = ctx.fd;
+	xrcd_attr.oflags = O_CREAT;
+	ctx.xrcd = ibv_open_xrcd(ctx.context, &xrcd_attr);
+	if (!ctx.xrcd) {
+		fprintf(stderr, "Couldn't Open the XRC Domain %d\n", errno);
+		return 1;
+	}
+
+	ctx.recv_cq = ibv_create_cq(ctx.context, ctx.num_clients, &ctx.recv_cq,
+				    ctx.channel, 0);
+	if (!ctx.recv_cq) {
+		fprintf(stderr, "Couldn't create recv CQ\n");
+		return 1;
+	}
+
+	if (ctx.use_event) {
+		if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
+			fprintf(stderr, "Couldn't request CQ notification\n");
+			return 1;
+		}
+	}
+
+	ctx.send_cq = ibv_create_cq(ctx.context, ctx.num_clients, NULL, NULL, 0);
+	if (!ctx.send_cq) {
+		fprintf(stderr, "Couldn't create send CQ\n");
+		return 1;
+	}
+
+	memset(&attr, 0, sizeof attr);
+	attr.attr.max_wr = ctx.num_clients;
+	attr.attr.max_sge = 1;
+	attr.comp_mask = IBV_SRQ_INIT_ATTR_TYPE | IBV_SRQ_INIT_ATTR_XRCD |
+			 IBV_SRQ_INIT_ATTR_CQ | IBV_SRQ_INIT_ATTR_PD;
+	attr.srq_type = IBV_SRQT_XRC;
+	attr.xrcd = ctx.xrcd;
+	attr.cq = ctx.recv_cq;
+	attr.pd = ctx.pd;
+
+	ctx.srq = ibv_create_srq_ex(ctx.context, &attr);
+	if (!ctx.srq)  {
+		fprintf(stderr, "Couldn't create SRQ\n");
+		return 1;
+	}
+
+	if (create_qps())
+		return 1;
+
+	return 0;
+}
+
+static int send_local_dest(int sockfd, int index)
+{
+	char msg[MSG_SIZE];
+
+	ctx.rem_dest[index].recv_psn = lrand48() & 0xffffff;
+
+	printf(ADDR_FORMAT, "local", ctx.lid, ctx.recv_qp[index]->qp_num,
+		ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn,
+		ibv_get_srq_num(ctx.srq));
+
+	sprintf(msg, MSG_FORMAT, ctx.lid, ctx.recv_qp[index]->qp_num,
+		ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn,
+		ibv_get_srq_num(ctx.srq));
+
+	if (write(sockfd, msg, MSG_SIZE) != MSG_SIZE) {
+		fprintf(stderr, "Couldn't send local address\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int recv_remote_dest(int sockfd, int index)
+{
+	struct pingpong_dest *rem_dest;
+	char msg[MSG_SIZE];
+	int n = 0, r;
+
+	while (n < MSG_SIZE) {
+		r = read(sockfd, msg + n, MSG_SIZE - n);
+		if (r < 0) {
+			perror("client read");
+			fprintf(stderr,
+				"%d/%d: Couldn't read remote address [%d]\n",
+				n, MSG_SIZE, index);
+			return -1;
+		}
+		n += r;
+	}
+
+	rem_dest = &ctx.rem_dest[index];
+	sscanf(msg, MSG_SSCAN, &rem_dest->lid, &rem_dest->recv_qpn,
+	       &rem_dest->send_qpn, &rem_dest->send_psn, &rem_dest->srqn);
+
+	printf(ADDR_FORMAT, "remote", rem_dest->lid, rem_dest->recv_qpn,
+	       rem_dest->send_qpn, rem_dest->send_psn, rem_dest->srqn);
+
+	return 0;
+}
+
+static int connect_qps(int index)
+{
+	struct ibv_qp_attr attr;
+
+	memset(&attr, 0, sizeof attr);
+	attr.qp_state	      = IBV_QPS_RTR;
+	attr.dest_qp_num      = ctx.rem_dest[index].send_qpn;
+	attr.path_mtu	      = ctx.mtu;
+	attr.rq_psn	      = ctx.rem_dest[index].send_psn;
+	attr.min_rnr_timer    = 12;
+	attr.ah_attr.dlid     = ctx.rem_dest[index].lid;
+	attr.ah_attr.sl	      = ctx.sl;
+	attr.ah_attr.port_num = ctx.ib_port;
+
+	if (ibv_modify_qp(ctx.recv_qp[index], &attr,
+			  IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |
+			  IBV_QP_DEST_QPN | IBV_QP_RQ_PSN |
+			  IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) {
+		fprintf(stderr, "Failed to modify send QP[%d] to RTR\n", index);
+		return 1;
+	}
+
+	memset(&attr, 0, sizeof attr);
+	attr.qp_state = IBV_QPS_RTS;
+	attr.timeout = 14;
+	attr.sq_psn = ctx.rem_dest[index].recv_psn;
+
+	if (ibv_modify_qp(ctx.recv_qp[index], &attr,
+			  IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_SQ_PSN)) {
+		fprintf(stderr, "Failed to modify recv QP[%d] to RTS\n", index);
+		return 1;
+	}
+
+	memset(&attr, 0, sizeof attr);
+	attr.qp_state	      = IBV_QPS_RTR;
+	attr.dest_qp_num      = ctx.rem_dest[index].recv_qpn;
+	attr.path_mtu	      = ctx.mtu;
+	attr.rq_psn	      = ctx.rem_dest[index].send_psn;
+	attr.ah_attr.dlid     = ctx.rem_dest[index].lid;
+	attr.ah_attr.sl	      = ctx.sl;
+	attr.ah_attr.port_num = ctx.ib_port;
+
+	if (ibv_modify_qp(ctx.send_qp[index], &attr,
+			  IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |
+			  IBV_QP_DEST_QPN | IBV_QP_RQ_PSN)) {
+		fprintf(stderr, "Failed to modify send QP[%d] to RTR\n", index);
+		return 1;
+	}
+
+	memset(&attr, 0, sizeof attr);
+	attr.qp_state = IBV_QPS_RTS;
+	attr.timeout = 14;
+	attr.retry_cnt = 7;
+	attr.rnr_retry = 7;
+	attr.sq_psn = ctx.rem_dest[index].recv_psn;
+
+	if (ibv_modify_qp(ctx.send_qp[index], &attr,
+			  IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_SQ_PSN |
+			  IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC)) {
+		fprintf(stderr, "Failed to modify send QP[%d] to RTS\n", index);
+		return 1;
+	}
+
+	return 0;
+}
+
+static int pp_client_connect(const char *servername, int port)
+{
+	struct addrinfo *res, *t;
+	char *service;
+	int ret;
+	int sockfd = -1;
+	struct addrinfo hints = {
+		.ai_family   = AF_UNSPEC,
+		.ai_socktype = SOCK_STREAM
+	};
+
+	if (asprintf(&service, "%d", port) < 0)
+		return 1;
+
+	ret = getaddrinfo(servername, service, &hints, &res);
+	if (ret < 0) {
+		fprintf(stderr, "%s for %s:%d\n", gai_strerror(ret), servername, port);
+		free(service);
+		return 1;
+	}
+
+	for (t = res; t; t = t->ai_next) {
+		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
+		if (sockfd >= 0) {
+			if (!connect(sockfd, t->ai_addr, t->ai_addrlen))
+				break;
+			close(sockfd);
+			sockfd = -1;
+		}
+	}
+
+	freeaddrinfo(res);
+	free(service);
+
+	if (sockfd < 0) {
+		fprintf(stderr, "Couldn't connect to %s:%d\n", servername, port);
+		return 1;
+	}
+
+	if (send_local_dest(sockfd, 0))
+		return 1;
+
+	if (recv_remote_dest(sockfd, 0))
+		return 1;
+
+	if (connect_qps(0))
+		return 1;
+
+	shutdown(sockfd, SHUT_RDWR);
+	close(sockfd);
+	return 0;
+}
+
+static int pp_server_connect(int port)
+{
+	struct addrinfo *res, *t;
+	char *service;
+	int ret, i, n;
+	int sockfd = -1, connfd;
+	struct addrinfo hints = {
+		.ai_flags    = AI_PASSIVE,
+		.ai_family   = AF_UNSPEC,
+		.ai_socktype = SOCK_STREAM
+	};
+
+	if (asprintf(&service, "%d", port) < 0)
+		return 1;
+
+	ret = getaddrinfo(NULL, service, &hints, &res);
+	if (ret < 0) {
+		fprintf(stderr, "%s for port %d\n", gai_strerror(ret), port);
+		free(service);
+		return 1;
+	}
+
+	for (t = res; t; t = t->ai_next) {
+		sockfd = socket(t->ai_family, t->ai_socktype, t->ai_protocol);
+		if (sockfd >= 0) {
+			n = 1;
+			setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &n, sizeof n);
+			if (!bind(sockfd, t->ai_addr, t->ai_addrlen))
+				break;
+			close(sockfd);
+			sockfd = -1;
+		}
+	}
+
+	freeaddrinfo(res);
+	free(service);
+
+	if (sockfd < 0) {
+		fprintf(stderr, "Couldn't listen to port %d\n", port);
+		return 1;
+	}
+
+	listen(sockfd, ctx.num_clients);
+
+	for (i = 0; i < ctx.num_clients; i++) {
+		connfd = accept(sockfd, NULL, 0);
+		if (connfd < 0) {
+			fprintf(stderr, "accept() failed for client %d\n", i);
+			return 1;
+		}
+
+		if (recv_remote_dest(connfd, i))
+			return 1;
+
+		if (send_local_dest(connfd, i))
+			return 1;
+
+		if (connect_qps(i))
+			return 1;
+
+		shutdown(connfd, SHUT_RDWR);
+		close(connfd);
+	}
+
+	close(sockfd);
+	return 0;
+}
+
+
+static int pp_close_ctx(void)
+{
+	int i;
+
+	for (i = 0; i < ctx.num_clients; ++i) {
+		/*
+		 * Receive QPs may be used by other clients.  Leave them, they
+		 * will be cleaned up when the xrcd is destroyed.
+		 */
+		if (ibv_destroy_qp(ctx.send_qp[i])) {
+			fprintf(stderr, "Couldn't destroy INI QP[%d]\n", i);
+			return 1;
+		}
+	}
+
+	if (ibv_destroy_srq(ctx.srq)) {
+		fprintf(stderr, "Couldn't destroy SRQ\n");
+		return 1;
+	}
+
+	if (!ctx.xrcd && ibv_close_xrcd(ctx.xrcd)) {
+		fprintf(stderr, "Couldn't close the XRC Domain\n");
+		return 1;
+	}
+	if (ctx.fd >= 0 && close(ctx.fd)) {
+		fprintf(stderr, "Couldn't close the file for the XRC Domain\n");
+		return 1;
+	}
+
+	if (ibv_destroy_cq(ctx.send_cq)) {
+		fprintf(stderr, "Couldn't destroy send CQ\n");
+		return 1;
+	}
+
+	if (ibv_destroy_cq(ctx.recv_cq)) {
+		fprintf(stderr, "Couldn't destroy recv CQ\n");
+		return 1;
+	}
+
+	if (ibv_dereg_mr(ctx.mr)) {
+		fprintf(stderr, "Couldn't deregister MR\n");
+		return 1;
+	}
+
+	if (ibv_dealloc_pd(ctx.pd)) {
+		fprintf(stderr, "Couldn't deallocate PD\n");
+		return 1;
+	}
+
+	if (ctx.channel) {
+		if (ibv_destroy_comp_channel(ctx.channel)) {
+			fprintf(stderr,
+				"Couldn't destroy completion channel\n");
+			return 1;
+		}
+	}
+
+	if (ibv_close_device(ctx.context)) {
+		fprintf(stderr, "Couldn't release context\n");
+		return 1;
+	}
+
+	free(ctx.buf);
+	free(ctx.rem_dest);
+	free(ctx.send_qp);
+	free(ctx.recv_qp);
+	return 0;
+}
+
+static int pp_post_recv(int cnt)
+{
+	struct ibv_sge sge;
+	struct ibv_recv_wr wr, *bad_wr;
+
+	sge.addr = (uintptr_t) ctx.buf;
+	sge.length = ctx.size;
+	sge.lkey = ctx.mr->lkey;
+
+	wr.next       = NULL;
+	wr.wr_id      = (uintptr_t) &ctx;
+	wr.sg_list    = &sge;
+	wr.num_sge    = 1;
+
+	while (cnt--) {
+		if (ibv_post_srq_recv(ctx.srq, &wr, &bad_wr)) {
+			fprintf(stderr, "Failed to post receive to SRQ\n");
+			return 1;
+		}
+	}
+	return 0;
+}
+
+/*
+ * Send to each client round robin on each set of xrc send/recv qp.
+ * Generate a completion on the last send.
+ */
+static int pp_post_send(int index)
+{
+	struct ibv_sge sge;
+	struct ibv_send_wr wr, *bad_wr;
+	int qpi;
+
+	sge.addr = (uintptr_t) ctx.buf;
+	sge.length = ctx.size;
+	sge.lkey = ctx.mr->lkey;
+
+	wr.wr_id   = (uintptr_t) index;
+	wr.next    = NULL;
+	wr.sg_list = &sge;
+	wr.num_sge = 1;
+	wr.opcode  = IBV_WR_SEND;
+	wr.wr.xrc.remote_srqn = ctx.rem_dest[index].srqn;
+
+	qpi = (index + ctx.rem_dest[index].pp_cnt) % ctx.num_clients;
+	wr.send_flags = (++ctx.rem_dest[index].pp_cnt >= ctx.num_tests) ?
+			IBV_SEND_SIGNALED : 0;
+
+	return ibv_post_send(ctx.send_qp[qpi], &wr, &bad_wr);
+}
+
+static int find_qp(int qpn)
+{
+	int i;
+
+	if (ctx.num_clients == 1)
+		return 0;
+
+	for (i = 0; i < ctx.num_clients; ++i)
+		if (ctx.recv_qp[i]->qp_num == qpn)
+			return i;
+
+	fprintf(stderr, "Unable to find qp %x\n", qpn);
+	return 0;
+}
+
+static int get_cq_event(void)
+{
+	struct ibv_cq *ev_cq;
+	void          *ev_ctx;
+
+	if (ibv_get_cq_event(ctx.channel, &ev_cq, &ev_ctx)) {
+		fprintf(stderr, "Failed to get cq_event\n");
+		return 1;
+	}
+
+	if (ev_cq != ctx.recv_cq) {
+		fprintf(stderr, "CQ event for unknown CQ %p\n", ev_cq);
+		return 1;
+	}
+
+	if (ibv_req_notify_cq(ctx.recv_cq, 0)) {
+		fprintf(stderr, "Couldn't request CQ notification\n");
+		return 1;
+	}
+
+	return 0;
+}
+
+static void init(void)
+{
+	srand48(getpid() * time(NULL));
+
+	ctx.size = 4096;
+	ctx.ib_port = 1;
+	ctx.num_clients  = 1;
+	ctx.num_tests = 5;
+	ctx.mtu = IBV_MTU_2048;
+	ctx.sl = 0;
+}
+
+static void usage(const char *argv0)
+{
+	printf("Usage:\n");
+	printf("  %s            start a server and wait for connection\n", argv0);
+	printf("  %s <host>     connect to server at <host>\n", argv0);
+	printf("\n");
+	printf("Options:\n");
+	printf("  -p, --port=<port>      listen on/connect to port <port> (default 18515)\n");
+	printf("  -d, --ib-dev=<dev>     use IB device <dev> (default first device found)\n");
+	printf("  -i, --ib-port=<port>   use port <port> of IB device (default 1)\n");
+	printf("  -s, --size=<size>      size of message to exchange (default 4096)\n");
+	printf("  -m, --mtu=<size>       path MTU (default 2048)\n");
+	printf("  -c, --clients=<n>      number of clients (on server only, default 1)\n");
+	printf("  -n, --num_tests=<n>    number of tests per client (default 5)\n");
+	printf("  -l, --sl=<sl>          service level value\n");
+	printf("  -e, --events           sleep on CQ events (default poll)\n");
+}
+
+int main(int argc, char *argv[])
+{
+	char          *ib_devname = NULL;
+	char          *servername = NULL;
+	int           port = 18515;
+	int           i, total, cnt = 0;
+	int           ne, qpi, num_cq_events = 0;
+	struct ibv_wc wc;
+
+	init();
+	while (1) {
+		int c;
+
+		static struct option long_options[] = {
+			{ .name = "port",      .has_arg = 1, .val = 'p' },
+			{ .name = "ib-dev",    .has_arg = 1, .val = 'd' },
+			{ .name = "ib-port",   .has_arg = 1, .val = 'i' },
+			{ .name = "size",      .has_arg = 1, .val = 's' },
+			{ .name = "mtu",       .has_arg = 1, .val = 'm' },
+			{ .name = "clients",   .has_arg = 1, .val = 'c' },
+			{ .name = "num_tests", .has_arg = 1, .val = 'n' },
+			{ .name = "sl",        .has_arg = 1, .val = 'l' },
+			{ .name = "events",    .has_arg = 0, .val = 'e' },
+			{ 0 }
+		};
+
+		c = getopt_long(argc, argv, "p:d:i:s:m:c:n:l:e", long_options,
+				NULL);
+		if (c == -1)
+			break;
+
+		switch (c) {
+		case 'p':
+			port = strtol(optarg, NULL, 0);
+			if (port < 0 || port > 65535) {
+				usage(argv[0]);
+				return 1;
+			}
+			break;
+		case 'd':
+			ib_devname = strdupa(optarg);
+			break;
+		case 'i':
+			ctx.ib_port = strtol(optarg, NULL, 0);
+			if (ctx.ib_port < 0) {
+				usage(argv[0]);
+				return 1;
+			}
+			break;
+		case 's':
+			ctx.size = strtol(optarg, NULL, 0);
+			break;
+		case 'm':
+			ctx.mtu = pp_mtu_to_enum(strtol(optarg, NULL, 0));
+			if (ctx.mtu < 0) {
+				usage(argv[0]);
+				return 1;
+			}
+			break;
+		case 'c':
+			ctx.num_clients = strtol(optarg, NULL, 0);
+			break;
+		case 'n':
+			ctx.num_tests = strtol(optarg, NULL, 0);
+			break;
+		case 'l':
+			ctx.sl = strtol(optarg, NULL, 0);
+			break;
+		case 'e':
+			ctx.use_event = 1;
+			break;
+		default:
+			usage(argv[0]);
+			return 1;
+		}
+	}
+
+	if (optind == argc - 1) {
+		servername = strdupa(argv[optind]);
+		ctx.num_clients = 1;
+	} else if (optind < argc) {
+		usage(argv[0]);
+		return 1;
+	}
+
+	page_size = sysconf(_SC_PAGESIZE);
+
+	if (pp_init_ctx(ib_devname))
+		return 1;
+
+	if (pp_post_recv(ctx.num_clients)) {
+		fprintf(stderr, "Couldn't post receives\n");
+		return 1;
+	}
+
+	if (servername) {
+		if (pp_client_connect(servername, port))
+			return 1;
+	} else {
+		if (pp_server_connect(port))
+			return 1;
+
+		for (i = 0; i < ctx.num_clients; i++)
+			pp_post_send(i);
+	}
+
+	total = ctx.num_clients * ctx.num_tests;
+	while (cnt < total) {
+		if (ctx.use_event) {
+			if (get_cq_event())
+				return 1;
+
+			++num_cq_events;
+		}
+
+		do {
+			ne = ibv_poll_cq(ctx.recv_cq, 1, &wc);
+			if (ne < 0) {
+				fprintf(stderr, "Error polling cq %d\n", ne);
+				return 1;
+			} else if (ne == 0) {
+				break;
+			}
+
+			if (wc.status) {
+				fprintf(stderr, "Work completion error %d\n", wc.status);
+				return 1;
+			}
+
+			pp_post_recv(ne);
+			qpi = find_qp(wc.qp_num);
+			if (ctx.rem_dest[qpi].pp_cnt < ctx.num_tests)
+				pp_post_send(qpi);
+			cnt += ne;
+		} while (ne > 0);
+	}
+
+	for (cnt = 0; cnt < ctx.num_clients; cnt += ne) {
+		ne = ibv_poll_cq(ctx.send_cq, 1, &wc);
+		if (ne < 0) {
+			fprintf(stderr, "Error polling cq %d\n", ne);
+			return 1;
+		}
+	}
+
+	if (ctx.use_event)
+		ibv_ack_cq_events(ctx.recv_cq, num_cq_events);
+
+	if (pp_close_ctx())
+		return 1;
+
+	printf("success\n");
+	return 0;
+}
-- 
1.7.3

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v4 4/7] libibverbs: Add support for XRC QPs

[Or Gerlitz]

On 16/03/2013 02:38, sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org wrote:
>   
> diff --git a/include/infiniband/verbs.h b/include/infiniband/verbs.h
> index d7dfcd3..f321866 100644
> --- a/include/infiniband/verbs.h
> +++ b/include/infiniband/verbs.h
> @@ -475,6 +478,26 @@ struct ibv_qp_init_attr {
>   	int			sq_sig_all;
>   };
>   
> +enum ibv_qp_init_attr_mask {
> +	IBV_QP_INIT_ATTR_PD		= 1 << 0,
> +	IBV_QP_INIT_ATTR_XRCD		= 1 << 1,
> +	IBV_QP_INIT_ATTR_RESERVED	= 1 << 2
> +};
> +
> +struct ibv_qp_init_attr_ex {
> +	void		       *qp_context;
> +	struct ibv_cq	       *send_cq;
> +	struct ibv_cq	       *recv_cq;
> +	struct ibv_srq	       *srq;
> +	struct ibv_qp_cap	cap;
> +	enum ibv_qp_type	qp_type;
> +	int			sq_sig_all;
> +
> +	uint32_t		comp_mask;
> +	struct ibv_pd	       *pd;
> +	struct ibv_xrcd	       *xrcd;
> +};
> +
>   enum ibv_qp_attr_mask {
>   	IBV_QP_STATE			= 1 << 	0,
>   	IBV_QP_CUR_STATE		= 1 << 	1,
> @@ -590,6 +613,11 @@ struct ibv_send_wr {
>   			uint32_t	remote_qpn;
>   			uint32_t	remote_qkey;
>   		} ud;
> +		struct {
> +			uint64_t	reserved[3];
> +			uint32_t	reserved2;
> +			uint32_t	remote_srqn;
> +		} xrc;
>   	} wr;
>   };

these added reserved fields ofcourse don't enlarge the union size, 
correct? so why have it, maybe you
want to make sure no one will srink the union size in the future?

>   
> @@ -771,11 +799,14 @@ struct ibv_context {
>   enum verbs_context_mask {
>   	VERBS_CONTEXT_XRCD	= 1 << 0,
>   	VERBS_CONTEXT_SRQ	= 1 << 1,
> -	VERBS_CONTEXT_RESERVED	= 1 << 2
> +	VERBS_CONTEXT_QP	= 1 << 2,
> +	VERBS_CONTEXT_RESERVED	= 1 << 3
>   };
>   
>   struct verbs_context {
>   	/*  "grows up" - new fields go here */
> +	struct ibv_qp *		(*create_qp_ex)(struct ibv_context *context,
> +						struct ibv_qp_init_attr_ex *qp_init_attr_ex);
>   	

I easily think on 3-4 more use cases for a create_qp extended verb that 
allows the app to
specify QP creation flags e.g in the same manner we have them in the 
kernel, for example:

1. suport blocking of multicast self-loopback (kernel 
IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2. support TCP segementation (kernel IB_QP_CREATE_IPOIB_UD_LSO)
3. support some HPC optimizations whose patches are now under the works
4. etc more

Can we re-define this verb such that it can serve for both XRC QPs and 
non-XRC QPs for which creation flags are desired?
maybe add creation flags and IBV_QP_CREATE_XRC as a value that tells to 
create XRC QP, and we can add other flags on top of.

Or.


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v4 2/7] libibverbs: Introduce XRC domains

[Or Gerlitz]

Hi Sean,

I don't see cover-letter for V4 of libibverbs and V3 of libmlx4 patches 
you posted over the weekend, would be nice if you
can reply to one of your posting per series and send quick listing of 
the changes (if any) from previous versions.

Or.

Also a little nit... I think there's no need to have the library 
**name**  as part of the actual commit but rather
as part of the text in the brackets which is later dropped when applied 
into git.

e.g

[PATCH v3 1/2] libmlx4:  Infra-structure changes to support verbs extensions

needs to be

[PATCH libmlx4 v3 1/2] Infra-structure changes to support verbs extensions

and so on which you can do easily through git format-patch 
--subject-prefix="PATCH libmlx4 v3" etc


--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH v4 2/7] libibverbs: Introduce XRC domains

[Hefty, Sean]

> I don't see cover-letter for V4 of libibverbs and V3 of libmlx4 patches
> you posted over the weekend, would be nice if you
> can reply to one of your posting per series and send quick listing of
> the changes (if any) from previous versions.

There are no changes from v3.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH v4 4/7] libibverbs: Add support for XRC QPs

[Hefty, Sean]

> > @@ -590,6 +613,11 @@ struct ibv_send_wr {
> >   			uint32_t	remote_qpn;
> >   			uint32_t	remote_qkey;
> >   		} ud;
> > +		struct {
> > +			uint64_t	reserved[3];
> > +			uint32_t	reserved2;
> > +			uint32_t	remote_srqn;
> > +		} xrc;
> >   	} wr;
> >   };
> 
> these added reserved fields ofcourse don't enlarge the union size,
> correct? so why have it, maybe you
> want to make sure no one will srink the union size in the future?

The reserved fields are needed, so that remote_srqn doesn't fall over the top of an existing field.
 
> I easily think on 3-4 more use cases for a create_qp extended verb that
> allows the app to
> specify QP creation flags e.g in the same manner we have them in the
> kernel, for example:
> 
> 1. suport blocking of multicast self-loopback (kernel
> IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
> 2. support TCP segementation (kernel IB_QP_CREATE_IPOIB_UD_LSO)
> 3. support some HPC optimizations whose patches are now under the works
> 4. etc more
> 
> Can we re-define this verb such that it can serve for both XRC QPs and
> non-XRC QPs for which creation flags are desired?
> maybe add creation flags and IBV_QP_CREATE_XRC as a value that tells to
> create XRC QP, and we can add other flags on top of.

Honestly, I really don't care about any other verbs extensions or possible future changes until XRC is upstream.  The only thing we're doing is constantly delaying XRC because of continual background churn.  Version 3 of these patches were submitted 6 months ago.  The kernel support went in well over a year ago.  struct ibv_qp_init_attr_ex is already extensible.  If we need flags, they can be added to it when they're necessary.  But XRC doesn't need them.



--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH v4 2/7] libibverbs: Introduce XRC domains

[Christoph Lameter]

On Mon, 18 Mar 2013, Hefty, Sean wrote:

> > I don't see cover-letter for V4 of libibverbs and V3 of libmlx4 patches
> > you posted over the weekend, would be nice if you
> > can reply to one of your posting per series and send quick listing of
> > the changes (if any) from previous versions.
>
> There are no changes from v3.

Well the cover should still be included to give someone just reviewing the
patchset v4 the chance to understand what is going on.

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v4 4/7] libibverbs: Add support for XRC QPs

[Or Gerlitz]

On Mon, Mar 18, 2013 at 4:38 PM, Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org> wrote:
> Honestly, I really don't care about any other verbs extensions or possible future
> changes until XRC is upstream.  The only thing we're doing is constantly delaying XRC > because of continual background churn.  Version 3 of these patches were submitted 6 > months ago.  The kernel support went in well over a year ago.

no intent for background churn!  as for the delay welcome to join the
party... the last re-sping of the raw qp patches for
libibverbs/libmlx4 I submitted on Sep 2012, no progress there too.

> struct ibv_qp_init_attr_ex is already extensible.  If we need flags, they can be added to > it when they're necessary.  But XRC doesn't need them.

Got it good,  let me see if I follow what would it take to do that:

1. add  IBV_QP_INIT_ATTR_CREATE_FLAGS to enum ibv_qp_init_attr_mask
2. add uint32_t  create_flags to struct ibv_qp_init_attr_ex

and etc respective changes to the kernel/user structures, will that be all?

Or.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

RE: [PATCH v4 4/7] libibverbs: Add support for XRC QPs

[Hefty, Sean]

> Got it good,  let me see if I follow what would it take to do that:
> 
> 1. add  IBV_QP_INIT_ATTR_CREATE_FLAGS to enum ibv_qp_init_attr_mask
> 2. add uint32_t  create_flags to struct ibv_qp_init_attr_ex
> 
> and etc respective changes to the kernel/user structures, will that be all?

yes - that would be all that's needed to add the flags 
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v4 4/7] libibverbs: Add support for XRC QPs

[Or Gerlitz]

On Mon, Mar 18, 2013 at 8:01 PM, Hefty, Sean <sean.hefty-ral2JQCrhuEAvxtiuMwx3w@public.gmane.org> wrote:
>
> > Got it good,  let me see if I follow what would it take to do that:
> >
> > 1. add  IBV_QP_INIT_ATTR_CREATE_FLAGS to enum ibv_qp_init_attr_mask
> > 2. add uint32_t  create_flags to struct ibv_qp_init_attr_ex
> >
> > and etc respective changes to the kernel/user structures, will that be all?
>
> yes - that would be all that's needed to add the flags


I see, that would cause struct ibv_qp_init_attr_ex to grow up in size,
but generally speaking we claim to be OK with that, even if this
specific case could have been avoided if we used union with one entry
struct containing the XRC params and another the flags?
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html