[PATCH v2 iproute2-next 0/3] RDMA tool driver-specific resource tracking

[PATCH v2 iproute2-next 0/3] RDMA tool driver-specific resource tracking

[Steve Wise]

Hello,

This series enhances the iproute2 rdma tool to include displaying
driver-specific resource attributes.  It is the user-space part of the
kernel driver resource tracking series that has been accepted for merging
into linux-4.18 [1]

If there are no additional review comments, it can now be merged, I think.

Changes since v1:
- commit log editorial fixes
- cite kernel commits that updated rdma_netlink.h in the 
  iproute2 commit syncing this header
- reorder stack definitions ala "reverse christmas tree"
- correctly handle unknown driver attributes when printing

Changes since v0/rfc:
- changed "provider" to "driver" based on kernel side changes
- updated man pages
- removed "RFC" tag

Thanks,

Steve.

[1] https://www.spinics.net/lists/linux-rdma/msg64199.html

Steve Wise (3):
  rdma: update rdma_netlink.h to get new driver attributes
  rdma: print driver resource attributes
  rdma: update man pages

 man/man8/rdma-resource.8              |  29 ++++-
 man/man8/rdma.8                       |   2 +-
 rdma/include/uapi/rdma/rdma_netlink.h |  26 +++++
 rdma/rdma.c                           |   9 +-
 rdma/rdma.h                           |  11 ++
 rdma/res.c                            |  30 ++----
 rdma/utils.c                          | 196 ++++++++++++++++++++++++++++++++++
 7 files changed, 277 insertions(+), 26 deletions(-)

-- 
1.8.3.1

[PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[Steve Wise]

Pull in the rdma_netlink.h changes from kernel
commits:

25a0ad85156a ("RDMA/nldev: Add explicit pad attribute")
da5c85078215 ("RDMA/nldev: add driver-specific resource tracking)"

Signed-off-by: Steve Wise <swise@opengridcomputing.com>
---
 rdma/include/uapi/rdma/rdma_netlink.h | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h
index 60416ed..40be0d8 100644
--- a/rdma/include/uapi/rdma/rdma_netlink.h
+++ b/rdma/include/uapi/rdma/rdma_netlink.h
@@ -249,10 +249,22 @@ enum rdma_nldev_command {
 	RDMA_NLDEV_NUM_OPS
 };
 
+enum {
+	RDMA_NLDEV_ATTR_ENTRY_STRLEN = 16,
+};
+
+enum rdma_nldev_print_type {
+	RDMA_NLDEV_PRINT_TYPE_UNSPEC,
+	RDMA_NLDEV_PRINT_TYPE_HEX,
+};
+
 enum rdma_nldev_attr {
 	/* don't change the order or add anything between, this is ABI! */
 	RDMA_NLDEV_ATTR_UNSPEC,
 
+	/* Pad attribute for 64b alignment */
+	RDMA_NLDEV_ATTR_PAD = RDMA_NLDEV_ATTR_UNSPEC,
+
 	/* Identifier for ib_device */
 	RDMA_NLDEV_ATTR_DEV_INDEX,		/* u32 */
 
@@ -387,6 +399,20 @@ enum rdma_nldev_attr {
 	RDMA_NLDEV_ATTR_RES_PD_ENTRY,		/* nested table */
 	RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,	/* u32 */
 	RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,	/* u32 */
+	/*
+	 * driver-specific attributes.
+	 */
+	RDMA_NLDEV_ATTR_DRIVER,			/* nested table */
+	RDMA_NLDEV_ATTR_DRIVER_ENTRY,		/* nested table */
+	RDMA_NLDEV_ATTR_DRIVER_STRING,		/* string */
+	/*
+	 * u8 values from enum rdma_nldev_print_type
+	 */
+	RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE,	/* u8 */
+	RDMA_NLDEV_ATTR_DRIVER_S32,		/* s32 */
+	RDMA_NLDEV_ATTR_DRIVER_U32,		/* u32 */
+	RDMA_NLDEV_ATTR_DRIVER_S64,		/* s64 */
+	RDMA_NLDEV_ATTR_DRIVER_U64,		/* u64 */
 
 	/*
 	 * Provides logical name and index of netdevice which is
-- 
1.8.3.1

Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[David Ahern]

On 5/14/18 9:42 AM, Steve Wise wrote:
> diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h
> index 60416ed..40be0d8 100644
> --- a/rdma/include/uapi/rdma/rdma_netlink.h
> +++ b/rdma/include/uapi/rdma/rdma_netlink.h
> @@ -249,10 +249,22 @@ enum rdma_nldev_command {
>  	RDMA_NLDEV_NUM_OPS
>  };
>  
> +enum {
> +	RDMA_NLDEV_ATTR_ENTRY_STRLEN = 16,
> +};
> +
> +enum rdma_nldev_print_type {
> +	RDMA_NLDEV_PRINT_TYPE_UNSPEC,
> +	RDMA_NLDEV_PRINT_TYPE_HEX,
> +};
> +
>  enum rdma_nldev_attr {
>  	/* don't change the order or add anything between, this is ABI! */

I asked this before and did not get a response. As the comment above
states with an emphasis (!) ...

>  	RDMA_NLDEV_ATTR_UNSPEC,
>  
> +	/* Pad attribute for 64b alignment */
> +	RDMA_NLDEV_ATTR_PAD = RDMA_NLDEV_ATTR_UNSPEC,
> +

... are you really adding new attributes in the middle?

>  	/* Identifier for ib_device */
>  	RDMA_NLDEV_ATTR_DEV_INDEX,		/* u32 */
>  
> @@ -387,6 +399,20 @@ enum rdma_nldev_attr {
>  	RDMA_NLDEV_ATTR_RES_PD_ENTRY,		/* nested table */
>  	RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,	/* u32 */
>  	RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,	/* u32 */
> +	/*
> +	 * driver-specific attributes.
> +	 */
> +	RDMA_NLDEV_ATTR_DRIVER,			/* nested table */
> +	RDMA_NLDEV_ATTR_DRIVER_ENTRY,		/* nested table */
> +	RDMA_NLDEV_ATTR_DRIVER_STRING,		/* string */
> +	/*
> +	 * u8 values from enum rdma_nldev_print_type
> +	 */
> +	RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE,	/* u8 */
> +	RDMA_NLDEV_ATTR_DRIVER_S32,		/* s32 */
> +	RDMA_NLDEV_ATTR_DRIVER_U32,		/* u32 */
> +	RDMA_NLDEV_ATTR_DRIVER_S64,		/* s64 */
> +	RDMA_NLDEV_ATTR_DRIVER_U64,		/* u64 */

and again here.

>  
>  	/*
>  	 * Provides logical name and index of netdevice which is
> 

Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[Doug Ledford]

[-- Attachment #1: Type: text/plain, Size: 2540 bytes --]

On Tue, 2018-05-15 at 13:37 -0600, David Ahern wrote:
> On 5/14/18 9:42 AM, Steve Wise wrote:
> > diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h
> > index 60416ed..40be0d8 100644
> > --- a/rdma/include/uapi/rdma/rdma_netlink.h
> > +++ b/rdma/include/uapi/rdma/rdma_netlink.h
> > @@ -249,10 +249,22 @@ enum rdma_nldev_command {
> >  	RDMA_NLDEV_NUM_OPS
> >  };
> >  
> > +enum {
> > +	RDMA_NLDEV_ATTR_ENTRY_STRLEN = 16,
> > +};
> > +
> > +enum rdma_nldev_print_type {
> > +	RDMA_NLDEV_PRINT_TYPE_UNSPEC,
> > +	RDMA_NLDEV_PRINT_TYPE_HEX,
> > +};
> > +
> >  enum rdma_nldev_attr {
> >  	/* don't change the order or add anything between, this is ABI! */
> 
> I asked this before and did not get a response. As the comment above
> states with an emphasis (!) ...
> 
> >  	RDMA_NLDEV_ATTR_UNSPEC,
> >  
> > +	/* Pad attribute for 64b alignment */
> > +	RDMA_NLDEV_ATTR_PAD = RDMA_NLDEV_ATTR_UNSPEC,
> > +
> 
> ... are you really adding new attributes in the middle?

Not really.  The new item is being explicitly set to the same value as
the item above it.  It therefore becomes two entries with the same enum
value.  The rest of the enum is all unchanged.

> >  	/* Identifier for ib_device */
> >  	RDMA_NLDEV_ATTR_DEV_INDEX,		/* u32 */
> >  
> > @@ -387,6 +399,20 @@ enum rdma_nldev_attr {
> >  	RDMA_NLDEV_ATTR_RES_PD_ENTRY,		/* nested table */
> >  	RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,	/* u32 */
> >  	RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,	/* u32 */
> > +	/*
> > +	 * driver-specific attributes.
> > +	 */
> > +	RDMA_NLDEV_ATTR_DRIVER,			/* nested table */
> > +	RDMA_NLDEV_ATTR_DRIVER_ENTRY,		/* nested table */
> > +	RDMA_NLDEV_ATTR_DRIVER_STRING,		/* string */
> > +	/*
> > +	 * u8 values from enum rdma_nldev_print_type
> > +	 */
> > +	RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE,	/* u8 */
> > +	RDMA_NLDEV_ATTR_DRIVER_S32,		/* s32 */
> > +	RDMA_NLDEV_ATTR_DRIVER_U32,		/* u32 */
> > +	RDMA_NLDEV_ATTR_DRIVER_S64,		/* s64 */
> > +	RDMA_NLDEV_ATTR_DRIVER_U64,		/* u64 */
> 
> and again here.
> 
> >  
> >  	/*
> >  	 * Provides logical name and index of netdevice which is
> > 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

-- 
Doug Ledford <dledford@redhat.com>
    GPG KeyID: B826A3330E572FDD
    Key fingerprint = AE6B 1BDA 122B 23B4 265B  1274 B826 A333 0E57 2FDD

[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[David Ahern]

On 5/15/18 1:47 PM, Doug Ledford wrote:
> On Tue, 2018-05-15 at 13:37 -0600, David Ahern wrote:
>> On 5/14/18 9:42 AM, Steve Wise wrote:
>>> diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h
>>> index 60416ed..40be0d8 100644
>>> --- a/rdma/include/uapi/rdma/rdma_netlink.h
>>> +++ b/rdma/include/uapi/rdma/rdma_netlink.h
>>> @@ -249,10 +249,22 @@ enum rdma_nldev_command {
>>>  	RDMA_NLDEV_NUM_OPS
>>>  };
>>>  
>>> +enum {
>>> +	RDMA_NLDEV_ATTR_ENTRY_STRLEN = 16,
>>> +};
>>> +
>>> +enum rdma_nldev_print_type {
>>> +	RDMA_NLDEV_PRINT_TYPE_UNSPEC,
>>> +	RDMA_NLDEV_PRINT_TYPE_HEX,
>>> +};
>>> +
>>>  enum rdma_nldev_attr {
>>>  	/* don't change the order or add anything between, this is ABI! */
>>
>> I asked this before and did not get a response. As the comment above
>> states with an emphasis (!) ...
>>
>>>  	RDMA_NLDEV_ATTR_UNSPEC,
>>>  
>>> +	/* Pad attribute for 64b alignment */
>>> +	RDMA_NLDEV_ATTR_PAD = RDMA_NLDEV_ATTR_UNSPEC,
>>> +
>>
>> ... are you really adding new attributes in the middle?
> 
> Not really.  The new item is being explicitly set to the same value as
> the item above it.  It therefore becomes two entries with the same enum
> value.  The rest of the enum is all unchanged.

sure for the above, but not the ones below.

Before this patch, this program:
$ cat a.c
#include <uapi/rdma/rdma_netlink.h>
#include <stdio.h>

int main(void)
{
	printf("RDMA_NLDEV_ATTR_NDEV_INDEX = %d\n", RDMA_NLDEV_ATTR_NDEV_INDEX);

	return 0;
}

prints this:
$ ./a.out
RDMA_NLDEV_ATTR_NDEV_INDEX = 50

After this patch,
$ ./a.out
RDMA_NLDEV_ATTR_NDEV_INDEX = 58


> 
>>>  	/* Identifier for ib_device */
>>>  	RDMA_NLDEV_ATTR_DEV_INDEX,		/* u32 */
>>>  
>>> @@ -387,6 +399,20 @@ enum rdma_nldev_attr {
>>>  	RDMA_NLDEV_ATTR_RES_PD_ENTRY,		/* nested table */
>>>  	RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,	/* u32 */
>>>  	RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,	/* u32 */
>>> +	/*
>>> +	 * driver-specific attributes.
>>> +	 */
>>> +	RDMA_NLDEV_ATTR_DRIVER,			/* nested table */
>>> +	RDMA_NLDEV_ATTR_DRIVER_ENTRY,		/* nested table */
>>> +	RDMA_NLDEV_ATTR_DRIVER_STRING,		/* string */
>>> +	/*
>>> +	 * u8 values from enum rdma_nldev_print_type
>>> +	 */
>>> +	RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE,	/* u8 */
>>> +	RDMA_NLDEV_ATTR_DRIVER_S32,		/* s32 */
>>> +	RDMA_NLDEV_ATTR_DRIVER_U32,		/* u32 */
>>> +	RDMA_NLDEV_ATTR_DRIVER_S64,		/* s64 */
>>> +	RDMA_NLDEV_ATTR_DRIVER_U64,		/* u64 */
>>
>> and again here.
>>
>>>  
>>>  	/*
>>>  	 * Provides logical name and index of netdevice which is
>>>
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-rdma" icat n
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 

Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[Doug Ledford]

[-- Attachment #1: Type: text/plain, Size: 836 bytes --]

On Tue, 2018-05-15 at 13:53 -0600, David Ahern wrote:
> sure for the above, but not the ones below.
> 
> Before this patch, this program:
> $ cat a.c
> #include <uapi/rdma/rdma_netlink.h>
> #include <stdio.h>
> 
> int main(void)
> {
>         printf("RDMA_NLDEV_ATTR_NDEV_INDEX = %d\n", RDMA_NLDEV_ATTR_NDEV_INDEX);
> 
>         return 0;
> }
> 
> prints this:
> $ ./a.out
> RDMA_NLDEV_ATTR_NDEV_INDEX = 50
> 
> After this patch,
> $ ./a.out
> RDMA_NLDEV_ATTR_NDEV_INDEX = 58

I just sent an incremental fix to the list under separate cover.  You
can squash that fix into Steve's patch and it should resolve the issue. 
Or Steve can respin the set.  Either way.

-- 
Doug Ledford <dledford@redhat.com>
    GPG KeyID: B826A3330E572FDD
    Key fingerprint = AE6B 1BDA 122B 23B4 265B  1274 B826 A333 0E57 2FDD

[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[David Ahern]

On 5/15/18 2:00 PM, Doug Ledford wrote:
> I just sent an incremental fix to the list under separate cover.  You
> can squash that fix into Steve's patch and it should resolve the issue. 
> Or Steve can respin the set.  Either way.

Once the patch has been committed to the upstream repo, spin a new set
with the correct header update.

Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[Doug Ledford]

[-- Attachment #1: Type: text/plain, Size: 739 bytes --]

On Tue, 2018-05-15 at 14:01 -0600, David Ahern wrote:
> On 5/15/18 2:00 PM, Doug Ledford wrote:
> > I just sent an incremental fix to the list under separate cover.  You
> > can squash that fix into Steve's patch and it should resolve the issue. 
> > Or Steve can respin the set.  Either way.
> 
> Once the patch has been committed to the upstream repo, spin a new set
> with the correct header update.

Well, I wrote it, so it's in my repo now ;-).  I just sent it to the
list because that's what you do when you are writing fixup patches even
if you've already committed them.

-- 
Doug Ledford <dledford@redhat.com>
    GPG KeyID: B826A3330E572FDD
    Key fingerprint = AE6B 1BDA 122B 23B4 265B  1274 B826 A333 0E57 2FDD

[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

RE: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[Steve Wise]

> -----Original Message-----
> From: David Ahern <dsahern@gmail.com>
> Sent: Tuesday, May 15, 2018 3:02 PM
> To: Doug Ledford <dledford@redhat.com>; Steve Wise
> <swise@opengridcomputing.com>; leon@kernel.org
> Cc: stephen@networkplumber.org; netdev@vger.kernel.org; linux-
> rdma@vger.kernel.org
> Subject: Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to
> get new driver attributes
> 
> On 5/15/18 2:00 PM, Doug Ledford wrote:
> > I just sent an incremental fix to the list under separate cover.  You
> > can squash that fix into Steve's patch and it should resolve the issue.
> > Or Steve can respin the set.  Either way.
> 
> Once the patch has been committed to the upstream repo, spin a new set
> with the correct header update.

Will do!

Steve.

RE: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[Steve Wise]

> On Tue, 2018-05-15 at 13:37 -0600, David Ahern wrote:
> > On 5/14/18 9:42 AM, Steve Wise wrote:
> > > diff --git a/rdma/include/uapi/rdma/rdma_netlink.h
> b/rdma/include/uapi/rdma/rdma_netlink.h
> > > index 60416ed..40be0d8 100644
> > > --- a/rdma/include/uapi/rdma/rdma_netlink.h
> > > +++ b/rdma/include/uapi/rdma/rdma_netlink.h
> > > @@ -249,10 +249,22 @@ enum rdma_nldev_command {
> > >  	RDMA_NLDEV_NUM_OPS
> > >  };
> > >
> > > +enum {
> > > +	RDMA_NLDEV_ATTR_ENTRY_STRLEN = 16,
> > > +};
> > > +
> > > +enum rdma_nldev_print_type {
> > > +	RDMA_NLDEV_PRINT_TYPE_UNSPEC,
> > > +	RDMA_NLDEV_PRINT_TYPE_HEX,
> > > +};
> > > +
> > >  enum rdma_nldev_attr {
> > >  	/* don't change the order or add anything between, this is ABI! */
> >
> > I asked this before and did not get a response. As the comment above
> > states with an emphasis (!) ...

Sorry David, I missed your question previously. ☹

> >
> > >  	RDMA_NLDEV_ATTR_UNSPEC,
> > >
> > > +	/* Pad attribute for 64b alignment */
> > > +	RDMA_NLDEV_ATTR_PAD = RDMA_NLDEV_ATTR_UNSPEC,
> > > +
> >
> > ... are you really adding new attributes in the middle?
> 
> Not really.  The new item is being explicitly set to the same value as
> the item above it.  It therefore becomes two entries with the same enum
> value.  The rest of the enum is all unchanged.

Correct.

The reason this was done was because a kernel had already been released where 64b nlattrs were being padded with 0 instead of defining an explicit pad attribute.  Jason thought the kernel side should define an explicit PAD attribute and use it.  To preserve the ABI we defined it but set it to 0 (aka ATTR_UNSPEC aka the first in the enum).  


> 
> > >  	/* Identifier for ib_device */
> > >  	RDMA_NLDEV_ATTR_DEV_INDEX,		/* u32 */
> > >
> > > @@ -387,6 +399,20 @@ enum rdma_nldev_attr {
> > >  	RDMA_NLDEV_ATTR_RES_PD_ENTRY,		/* nested table */
> > >  	RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,	/* u32 */
> > >  	RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,	/* u32 */
> > > +	/*
> > > +	 * driver-specific attributes.
> > > +	 */
> > > +	RDMA_NLDEV_ATTR_DRIVER,			/* nested table */
> > > +	RDMA_NLDEV_ATTR_DRIVER_ENTRY,		/* nested table */
> > > +	RDMA_NLDEV_ATTR_DRIVER_STRING,		/* string */
> > > +	/*
> > > +	 * u8 values from enum rdma_nldev_print_type
> > > +	 */
> > > +	RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE,	/* u8 */
> > > +	RDMA_NLDEV_ATTR_DRIVER_S32,		/* s32 */
> > > +	RDMA_NLDEV_ATTR_DRIVER_U32,		/* u32 */
> > > +	RDMA_NLDEV_ATTR_DRIVER_S64,		/* s64 */
> > > +	RDMA_NLDEV_ATTR_DRIVER_U64,		/* u64 */
> >
> > and again here.
> >

Ugh, this looks like a mistake maybe due to me rebasing and not noticing this commit added the name/index attrs.

5b2cc79de878 leonro@mellanox.com RDMA/nldev: Provide netdevice name and index 

Both of these are in -next to be merged upstream together.  

Should I do anything?

Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[Doug Ledford]

[-- Attachment #1: Type: text/plain, Size: 1346 bytes --]

On Tue, 2018-05-15 at 13:37 -0600, David Ahern wrote:
> On 5/14/18 9:42 AM, Steve Wise wrote:
> > diff --git a/rdma/include/uapi/rdma/rdma_netlink.h b/rdma/include/uapi/rdma/rdma_netlink.h
> > index 60416ed..40be0d8 100644
> > --- a/rdma/include/uapi/rdma/rdma_netlink.h
> > +++ b/rdma/include/uapi/rdma/rdma_netlink.h
> > 
> > @@ -387,6 +399,20 @@ enum rdma_nldev_attr {
> >  	RDMA_NLDEV_ATTR_RES_PD_ENTRY,		/* nested table */
> >  	RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,	/* u32 */
> >  	RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,	/* u32 */
> > +	/*
> > +	 * driver-specific attributes.
> > +	 */
> > +	RDMA_NLDEV_ATTR_DRIVER,			/* nested table */
> > +	RDMA_NLDEV_ATTR_DRIVER_ENTRY,		/* nested table */
> > +	RDMA_NLDEV_ATTR_DRIVER_STRING,		/* string */
> > +	/*
> > +	 * u8 values from enum rdma_nldev_print_type
> > +	 */
> > +	RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE,	/* u8 */
> > +	RDMA_NLDEV_ATTR_DRIVER_S32,		/* s32 */
> > +	RDMA_NLDEV_ATTR_DRIVER_U32,		/* u32 */
> > +	RDMA_NLDEV_ATTR_DRIVER_S64,		/* s64 */
> > +	RDMA_NLDEV_ATTR_DRIVER_U64,		/* u64 */
> 
> and again here.

This chunk, however, is a problem.  We'll need to fix that in the kernel
and in this patch too.

-- 
Doug Ledford <dledford@redhat.com>
    GPG KeyID: B826A3330E572FDD
    Key fingerprint = AE6B 1BDA 122B 23B4 265B  1274 B826 A333 0E57 2FDD

[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

RE: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[Steve Wise]

> -----Original Message-----
> From: linux-rdma-owner@vger.kernel.org <linux-rdma-
> owner@vger.kernel.org> On Behalf Of Doug Ledford
> Sent: Tuesday, May 15, 2018 2:53 PM
> To: David Ahern <dsahern@gmail.com>; Steve Wise
> <swise@opengridcomputing.com>; leon@kernel.org
> Cc: stephen@networkplumber.org; netdev@vger.kernel.org; linux-
> rdma@vger.kernel.org
> Subject: Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to
> get new driver attributes
> 
> On Tue, 2018-05-15 at 13:37 -0600, David Ahern wrote:
> > On 5/14/18 9:42 AM, Steve Wise wrote:
> > > diff --git a/rdma/include/uapi/rdma/rdma_netlink.h
> b/rdma/include/uapi/rdma/rdma_netlink.h
> > > index 60416ed..40be0d8 100644
> > > --- a/rdma/include/uapi/rdma/rdma_netlink.h
> > > +++ b/rdma/include/uapi/rdma/rdma_netlink.h
> > >
> > > @@ -387,6 +399,20 @@ enum rdma_nldev_attr {
> > >  	RDMA_NLDEV_ATTR_RES_PD_ENTRY,		/* nested table */
> > >  	RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,	/* u32 */
> > >  	RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,	/* u32 */
> > > +	/*
> > > +	 * driver-specific attributes.
> > > +	 */
> > > +	RDMA_NLDEV_ATTR_DRIVER,			/* nested table */
> > > +	RDMA_NLDEV_ATTR_DRIVER_ENTRY,		/* nested table */
> > > +	RDMA_NLDEV_ATTR_DRIVER_STRING,		/* string */
> > > +	/*
> > > +	 * u8 values from enum rdma_nldev_print_type
> > > +	 */
> > > +	RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE,	/* u8 */
> > > +	RDMA_NLDEV_ATTR_DRIVER_S32,		/* s32 */
> > > +	RDMA_NLDEV_ATTR_DRIVER_U32,		/* u32 */
> > > +	RDMA_NLDEV_ATTR_DRIVER_S64,		/* s64 */
> > > +	RDMA_NLDEV_ATTR_DRIVER_U64,		/* u64 */
> >
> > and again here.
> 
> This chunk, however, is a problem.  We'll need to fix that in the kernel
> and in this patch too.

I'll fix this series once I fix the kernel side.  Doug, should I send a patch that basically moves the DRIVER attributes to the bottom?

Sorry about missing this!

Steve

Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to get new driver attributes

[Doug Ledford]

[-- Attachment #1: Type: text/plain, Size: 2390 bytes --]

On Tue, 2018-05-15 at 15:28 -0500, Steve Wise wrote:
> > -----Original Message-----
> > From: linux-rdma-owner@vger.kernel.org <linux-rdma-
> > owner@vger.kernel.org> On Behalf Of Doug Ledford
> > Sent: Tuesday, May 15, 2018 2:53 PM
> > To: David Ahern <dsahern@gmail.com>; Steve Wise
> > <swise@opengridcomputing.com>; leon@kernel.org
> > Cc: stephen@networkplumber.org; netdev@vger.kernel.org; linux-
> > rdma@vger.kernel.org
> > Subject: Re: [PATCH v2 iproute2-next 1/3] rdma: update rdma_netlink.h to
> > get new driver attributes
> > 
> > On Tue, 2018-05-15 at 13:37 -0600, David Ahern wrote:
> > > On 5/14/18 9:42 AM, Steve Wise wrote:
> > > > diff --git a/rdma/include/uapi/rdma/rdma_netlink.h
> > 
> > b/rdma/include/uapi/rdma/rdma_netlink.h
> > > > index 60416ed..40be0d8 100644
> > > > --- a/rdma/include/uapi/rdma/rdma_netlink.h
> > > > +++ b/rdma/include/uapi/rdma/rdma_netlink.h
> > > > 
> > > > @@ -387,6 +399,20 @@ enum rdma_nldev_attr {
> > > >  	RDMA_NLDEV_ATTR_RES_PD_ENTRY,		/* nested table */
> > > >  	RDMA_NLDEV_ATTR_RES_LOCAL_DMA_LKEY,	/* u32 */
> > > >  	RDMA_NLDEV_ATTR_RES_UNSAFE_GLOBAL_RKEY,	/* u32 */
> > > > +	/*
> > > > +	 * driver-specific attributes.
> > > > +	 */
> > > > +	RDMA_NLDEV_ATTR_DRIVER,			/* nested table */
> > > > +	RDMA_NLDEV_ATTR_DRIVER_ENTRY,		/* nested table */
> > > > +	RDMA_NLDEV_ATTR_DRIVER_STRING,		/* string */
> > > > +	/*
> > > > +	 * u8 values from enum rdma_nldev_print_type
> > > > +	 */
> > > > +	RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE,	/* u8 */
> > > > +	RDMA_NLDEV_ATTR_DRIVER_S32,		/* s32 */
> > > > +	RDMA_NLDEV_ATTR_DRIVER_U32,		/* u32 */
> > > > +	RDMA_NLDEV_ATTR_DRIVER_S64,		/* s64 */
> > > > +	RDMA_NLDEV_ATTR_DRIVER_U64,		/* u64 */
> > > 
> > > and again here.
> > 
> > This chunk, however, is a problem.  We'll need to fix that in the kernel
> > and in this patch too.
> 
> I'll fix this series once I fix the kernel side.  Doug, should I send a patch that basically moves the DRIVER attributes to the bottom?
> 
> Sorry about missing this!

I missed it too.  I knew about the first hunk, but thought this one was
at the end.  Anyway, I've already pushed the fix to my wip/dl-for-next
branch, so you can get it there now.

-- 
Doug Ledford <dledford@redhat.com>
    GPG KeyID: B826A3330E572FDD
    Key fingerprint = AE6B 1BDA 122B 23B4 265B  1274 B826 A333 0E57 2FDD

[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

[PATCH v2 iproute2-next 2/3] rdma: print driver resource attributes

[Steve Wise]

This enhancement allows printing rdma device-specific state, if provided
by the kernel. This is done in a generic manner, so rdma tool doesn't
need to know about the details of every type of rdma device.

Driver attributes for a rdma resource are in the form of <key,
[print_type], value> tuples, where the key is a string and the value can
be any supported driver attribute. The print_type attribute, if present,
provides a print format to use vs the standard print format for the type.
For example, the default print type for a PROVIDER_S32 value is "%d ",
but "0x%x " if the print_type of PRINT_TYPE_HEX is included inthe tuple.

Driver resources are only printed when the -dd flag is present.
If -p is present, then the output is formatted to not exceed 80 columns,
otherwise it is printed as a single row to be grep/awk friendly.

Example output:

# rdma resource show qp lqpn 1028 -dd -p
link cxgb4_0/- lqpn 1028 rqpn 0 type RC state RTS rq-psn 0 sq-psn 0 path-mig-state MIGRATED pid 0 comm [nvme_rdma]
    sqid 1028 flushed 0 memsize 123968 cidx 85 pidx 85 wq_pidx 106 flush_cidx 85 in_use 0
    size 386 flags 0x0 rqid 1029 memsize 16768 cidx 43 pidx 41 wq_pidx 171 msn 44 rqt_hwaddr 0x2a8a5d00
    rqt_size 256 in_use 128 size 130 idx 43 wr_id 0xffff881057c03408 idx 40 wr_id 0xffff881057c033f0

Signed-off-by: Steve Wise <swise@opengridcomputing.com>
---
 rdma/rdma.c  |   9 ++-
 rdma/rdma.h  |  11 ++++
 rdma/res.c   |  30 +++------
 rdma/utils.c | 196 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 224 insertions(+), 22 deletions(-)

diff --git a/rdma/rdma.c b/rdma/rdma.c
index b43e538..010e983 100644
--- a/rdma/rdma.c
+++ b/rdma/rdma.c
@@ -129,13 +129,14 @@ int main(int argc, char **argv)
 		{ "batch",		required_argument,	NULL, 'b' },
 		{ NULL, 0, NULL, 0 }
 	};
+	bool show_driver_details = false;
 	const char *batch_file = NULL;
 	bool pretty_output = false;
 	bool show_details = false;
 	bool json_output = false;
 	bool force = false;
-	char *filename;
 	struct rd rd = {};
+	char *filename;
 	int opt;
 	int err;
 
@@ -152,7 +153,10 @@ int main(int argc, char **argv)
 			pretty_output = true;
 			break;
 		case 'd':
-			show_details = true;
+			if (show_details)
+				show_driver_details = true;
+			else
+				show_details = true;
 			break;
 		case 'j':
 			json_output = true;
@@ -180,6 +184,7 @@ int main(int argc, char **argv)
 	argv += optind;
 
 	rd.show_details = show_details;
+	rd.show_driver_details = show_driver_details;
 	rd.json_output = json_output;
 	rd.pretty_output = pretty_output;
 
diff --git a/rdma/rdma.h b/rdma/rdma.h
index 1908fc4..fcaf9e6 100644
--- a/rdma/rdma.h
+++ b/rdma/rdma.h
@@ -55,6 +55,7 @@ struct rd {
 	char **argv;
 	char *filename;
 	bool show_details;
+	bool show_driver_details;
 	struct list_head dev_map_list;
 	uint32_t dev_idx;
 	uint32_t port_idx;
@@ -115,4 +116,14 @@ int rd_recv_msg(struct rd *rd, mnl_cb_t callback, void *data, uint32_t seq);
 void rd_prepare_msg(struct rd *rd, uint32_t cmd, uint32_t *seq, uint16_t flags);
 int rd_dev_init_cb(const struct nlmsghdr *nlh, void *data);
 int rd_attr_cb(const struct nlattr *attr, void *data);
+int rd_attr_check(const struct nlattr *attr, int *typep);
+
+/*
+ * Print helpers
+ */
+void print_driver_table(struct rd *rd, struct nlattr *tb);
+void newline(struct rd *rd);
+void newline_indent(struct rd *rd);
+#define MAX_LINE_LENGTH 80
+
 #endif /* _RDMA_TOOL_H_ */
diff --git a/rdma/res.c b/rdma/res.c
index 1a0aab6..074b992 100644
--- a/rdma/res.c
+++ b/rdma/res.c
@@ -439,10 +439,8 @@ static int res_qp_parse_cb(const struct nlmsghdr *nlh, void *data)
 		if (nla_line[RDMA_NLDEV_ATTR_RES_PID])
 			free(comm);
 
-		if (rd->json_output)
-			jsonw_end_array(rd->jw);
-		else
-			pr_out("\n");
+		print_driver_table(rd, nla_line[RDMA_NLDEV_ATTR_DRIVER]);
+		newline(rd);
 	}
 	return MNL_CB_OK;
 }
@@ -678,10 +676,8 @@ static int res_cm_id_parse_cb(const struct nlmsghdr *nlh, void *data)
 		if (nla_line[RDMA_NLDEV_ATTR_RES_PID])
 			free(comm);
 
-		if (rd->json_output)
-			jsonw_end_array(rd->jw);
-		else
-			pr_out("\n");
+		print_driver_table(rd, nla_line[RDMA_NLDEV_ATTR_DRIVER]);
+		newline(rd);
 	}
 	return MNL_CB_OK;
 }
@@ -804,10 +800,8 @@ static int res_cq_parse_cb(const struct nlmsghdr *nlh, void *data)
 		if (nla_line[RDMA_NLDEV_ATTR_RES_PID])
 			free(comm);
 
-		if (rd->json_output)
-			jsonw_end_array(rd->jw);
-		else
-			pr_out("\n");
+		print_driver_table(rd, nla_line[RDMA_NLDEV_ATTR_DRIVER]);
+		newline(rd);
 	}
 	return MNL_CB_OK;
 }
@@ -919,10 +913,8 @@ static int res_mr_parse_cb(const struct nlmsghdr *nlh, void *data)
 		if (nla_line[RDMA_NLDEV_ATTR_RES_PID])
 			free(comm);
 
-		if (rd->json_output)
-			jsonw_end_array(rd->jw);
-		else
-			pr_out("\n");
+		print_driver_table(rd, nla_line[RDMA_NLDEV_ATTR_DRIVER]);
+		newline(rd);
 	}
 	return MNL_CB_OK;
 }
@@ -1004,10 +996,8 @@ static int res_pd_parse_cb(const struct nlmsghdr *nlh, void *data)
 		if (nla_line[RDMA_NLDEV_ATTR_RES_PID])
 			free(comm);
 
-		if (rd->json_output)
-			jsonw_end_array(rd->jw);
-		else
-			pr_out("\n");
+		print_driver_table(rd, nla_line[RDMA_NLDEV_ATTR_DRIVER]);
+		newline(rd);
 	}
 	return MNL_CB_OK;
 }
diff --git a/rdma/utils.c b/rdma/utils.c
index 49c967f..79c3d9f 100644
--- a/rdma/utils.c
+++ b/rdma/utils.c
@@ -11,6 +11,7 @@
 
 #include "rdma.h"
 #include <ctype.h>
+#include <inttypes.h>
 
 int rd_argc(struct rd *rd)
 {
@@ -393,8 +394,32 @@ static const enum mnl_attr_data_type nldev_policy[RDMA_NLDEV_ATTR_MAX] = {
 	[RDMA_NLDEV_ATTR_RES_MRLEN] = MNL_TYPE_U64,
 	[RDMA_NLDEV_ATTR_NDEV_INDEX]		= MNL_TYPE_U32,
 	[RDMA_NLDEV_ATTR_NDEV_NAME]		= MNL_TYPE_NUL_STRING,
+	[RDMA_NLDEV_ATTR_DRIVER] = MNL_TYPE_NESTED,
+	[RDMA_NLDEV_ATTR_DRIVER_ENTRY] = MNL_TYPE_NESTED,
+	[RDMA_NLDEV_ATTR_DRIVER_STRING] = MNL_TYPE_NUL_STRING,
+	[RDMA_NLDEV_ATTR_DRIVER_PRINT_TYPE] = MNL_TYPE_U8,
+	[RDMA_NLDEV_ATTR_DRIVER_S32] = MNL_TYPE_U32,
+	[RDMA_NLDEV_ATTR_DRIVER_U32] = MNL_TYPE_U32,
+	[RDMA_NLDEV_ATTR_DRIVER_S64] = MNL_TYPE_U64,
+	[RDMA_NLDEV_ATTR_DRIVER_U64] = MNL_TYPE_U64,
 };
 
+int rd_attr_check(const struct nlattr *attr, int *typep)
+{
+	int type;
+
+	if (mnl_attr_type_valid(attr, RDMA_NLDEV_ATTR_MAX) < 0)
+		return MNL_CB_ERROR;
+
+	type = mnl_attr_get_type(attr);
+
+	if (mnl_attr_validate(attr, nldev_policy[type]) < 0)
+		return MNL_CB_ERROR;
+
+	*typep = nldev_policy[type];
+	return MNL_CB_OK;
+}
+
 int rd_attr_cb(const struct nlattr *attr, void *data)
 {
 	const struct nlattr **tb = data;
@@ -660,3 +685,174 @@ struct dev_map *dev_map_lookup(struct rd *rd, bool allow_port_index)
 	free(dev_name);
 	return dev_map;
 }
+
+#define nla_type(attr) ((attr)->nla_type & NLA_TYPE_MASK)
+
+void newline(struct rd *rd)
+{
+	if (rd->json_output)
+		jsonw_end_array(rd->jw);
+	else
+		pr_out("\n");
+}
+
+void newline_indent(struct rd *rd)
+{
+	newline(rd);
+	if (!rd->json_output)
+		pr_out("    ");
+}
+
+static int print_driver_string(struct rd *rd, const char *key_str,
+				 const char *val_str)
+{
+	if (rd->json_output) {
+		jsonw_string_field(rd->jw, key_str, val_str);
+		return 0;
+	} else {
+		return pr_out("%s %s ", key_str, val_str);
+	}
+}
+
+static int print_driver_s32(struct rd *rd, const char *key_str, int32_t val,
+			      enum rdma_nldev_print_type print_type)
+{
+	if (rd->json_output) {
+		jsonw_int_field(rd->jw, key_str, val);
+		return 0;
+	}
+	switch (print_type) {
+	case RDMA_NLDEV_PRINT_TYPE_UNSPEC:
+		return pr_out("%s %d ", key_str, val);
+	case RDMA_NLDEV_PRINT_TYPE_HEX:
+		return pr_out("%s 0x%x ", key_str, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int print_driver_u32(struct rd *rd, const char *key_str, uint32_t val,
+			      enum rdma_nldev_print_type print_type)
+{
+	if (rd->json_output) {
+		jsonw_int_field(rd->jw, key_str, val);
+		return 0;
+	}
+	switch (print_type) {
+	case RDMA_NLDEV_PRINT_TYPE_UNSPEC:
+		return pr_out("%s %u ", key_str, val);
+	case RDMA_NLDEV_PRINT_TYPE_HEX:
+		return pr_out("%s 0x%x ", key_str, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int print_driver_s64(struct rd *rd, const char *key_str, int64_t val,
+			      enum rdma_nldev_print_type print_type)
+{
+	if (rd->json_output) {
+		jsonw_int_field(rd->jw, key_str, val);
+		return 0;
+	}
+	switch (print_type) {
+	case RDMA_NLDEV_PRINT_TYPE_UNSPEC:
+		return pr_out("%s %" PRId64 " ", key_str, val);
+	case RDMA_NLDEV_PRINT_TYPE_HEX:
+		return pr_out("%s 0x%" PRIx64 " ", key_str, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int print_driver_u64(struct rd *rd, const char *key_str, uint64_t val,
+			      enum rdma_nldev_print_type print_type)
+{
+	if (rd->json_output) {
+		jsonw_int_field(rd->jw, key_str, val);
+		return 0;
+	}
+	switch (print_type) {
+	case RDMA_NLDEV_PRINT_TYPE_UNSPEC:
+		return pr_out("%s %" PRIu64 " ", key_str, val);
+	case RDMA_NLDEV_PRINT_TYPE_HEX:
+		return pr_out("%s 0x%" PRIx64 " ", key_str, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int print_driver_entry(struct rd *rd, struct nlattr *key_attr,
+				struct nlattr *val_attr,
+				enum rdma_nldev_print_type print_type)
+{
+	const char *key_str = mnl_attr_get_str(key_attr);
+	int attr_type = nla_type(val_attr);
+
+	switch (attr_type) {
+	case RDMA_NLDEV_ATTR_DRIVER_STRING:
+		return print_driver_string(rd, key_str,
+				mnl_attr_get_str(val_attr));
+	case RDMA_NLDEV_ATTR_DRIVER_S32:
+		return print_driver_s32(rd, key_str,
+				mnl_attr_get_u32(val_attr), print_type);
+	case RDMA_NLDEV_ATTR_DRIVER_U32:
+		return print_driver_u32(rd, key_str,
+				mnl_attr_get_u32(val_attr), print_type);
+	case RDMA_NLDEV_ATTR_DRIVER_S64:
+		return print_driver_s64(rd, key_str,
+				mnl_attr_get_u64(val_attr), print_type);
+	case RDMA_NLDEV_ATTR_DRIVER_U64:
+		return print_driver_u64(rd, key_str,
+				mnl_attr_get_u64(val_attr), print_type);
+	}
+	return -EINVAL;
+}
+
+void print_driver_table(struct rd *rd, struct nlattr *tb)
+{
+	int print_type = RDMA_NLDEV_PRINT_TYPE_UNSPEC;
+	struct nlattr *tb_entry, *key = NULL, *val;
+	int type, cc = 0;
+	int ret;
+
+	if (!rd->show_driver_details || !tb)
+		return;
+
+	if (rd->pretty_output)
+		newline_indent(rd);
+
+	/*
+	 * Driver attrs are tuples of {key, [print-type], value}.
+	 * The key must be a string.  If print-type is present, it 
+	 * defines an alternate printf format type vs the native format
+	 * for the attribute.  And the value can be any available
+	 * driver type.
+	 */
+	mnl_attr_for_each_nested(tb_entry, tb) {
+
+		if (cc > MAX_LINE_LENGTH) {
+			if (rd->pretty_output)
+				newline_indent(rd);
+			cc = 0;
+		}
+		if (rd_attr_check(tb_entry, &type) != MNL_CB_OK)
+			return;
+		if (!key) {
+			if (type != MNL_TYPE_NUL_STRING)
+				return;
+			key = tb_entry;
+		} else if (type == MNL_TYPE_U8) {
+			print_type = mnl_attr_get_u8(tb_entry);
+		} else {
+			val = tb_entry;
+			ret = print_driver_entry(rd, key, val, print_type);
+			if (ret < 0)
+				return;
+			cc += ret;
+			print_type = RDMA_NLDEV_PRINT_TYPE_UNSPEC;
+			key = NULL;
+		}
+	}
+	return;
+}
-- 
1.8.3.1

[PATCH v2 iproute2-next 3/3] rdma: update man pages

[Steve Wise]

Update the man pages for the resource attributes as well
as the driver-specific attributes.

Signed-off-by: Steve Wise <swise@opengridcomputing.com>
---
 man/man8/rdma-resource.8 | 29 ++++++++++++++++++++++++++---
 man/man8/rdma.8          |  2 +-
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/man/man8/rdma-resource.8 b/man/man8/rdma-resource.8
index ff5d25d..40b073d 100644
--- a/man/man8/rdma-resource.8
+++ b/man/man8/rdma-resource.8
@@ -7,13 +7,16 @@ rdma-resource \- rdma resource configuration
 .in +8
 .ti -8
 .B rdma
-.RI "[ " OPTIONS " ]"
-.B resource
-.RI  " { " COMMAND " | "
+.RI "[ " OPTIONS " ] " RESOURCE " { " COMMAND " | "
 .BR help " }"
 .sp
 
 .ti -8
+.IR RESOURCE " := { "
+.BR cm_id " | " cq " | " mr " | " pd " | " qp " }"
+.sp
+
+.ti -8
 .IR OPTIONS " := { "
 \fB\-j\fR[\fIson\fR] |
 \fB\-d\fR[\fIetails\fR] }
@@ -70,11 +73,31 @@ rdma res show qp link mlx5_4/- -d
 Detailed view.
 .RE
 .PP
+rdma res show qp link mlx5_4/- -dd
+.RS 4
+Detailed view including driver-specific details.
+.RE
+.PP
 rdma res show qp link mlx5_4/1 lqpn 0-6
 .RS 4
 Limit to specific Local QPNs.
 .RE
 .PP
+rdma resource show cm_id dst-port 7174
+.RS 4
+Show CM_IDs with destination ip port of 7174.
+.RE
+.PP
+rdma resource show cm_id src-addr 172.16.0.100
+.RS 4
+Show CM_IDs bound to local ip address 172.16.0.100
+.RE
+.PP
+rdma resource show cq pid 30489
+.RS 4
+Show CQs belonging to pid 30489
+.RE
+.PP
 
 .SH SEE ALSO
 .BR rdma (8),
diff --git a/man/man8/rdma.8 b/man/man8/rdma.8
index 6f88f37..12aa149 100644
--- a/man/man8/rdma.8
+++ b/man/man8/rdma.8
@@ -49,7 +49,7 @@ If there were any errors during execution of the commands, the application retur
 
 .TP
 .BR "\-d" , " --details"
-Output detailed information.
+Output detailed information.  Adding a second \-d includes driver-specific details.
 
 .TP
 .BR "\-p" , " --pretty"
-- 
1.8.3.1