[PATCH 0/2] nvme-cli: update NetApp plugin for ONTAP devices

[PATCH 0/2] nvme-cli: update NetApp plugin for ONTAP devices

[martinus.gpy]

From: Martin George <marting@netapp.com>

These patches update the current NetApp plugin to handle
NetApp ONTAP devices.

Martin George (2):
  nvme-cli: add netapp ontapdevices command
  nvme-cli: add netapp ontapdevices man page

 Documentation/nvme-netapp-ontapdevices.1    |  74 ++
 Documentation/nvme-netapp-ontapdevices.html | 814 ++++++++++++++++++++
 Documentation/nvme-netapp-ontapdevices.txt  |  35 +
 plugins/netapp/netapp-nvme.c                | 342 ++++++++
 plugins/netapp/netapp-nvme.h                |   1 +
 5 files changed, 1266 insertions(+)
 create mode 100644 Documentation/nvme-netapp-ontapdevices.1
 create mode 100644 Documentation/nvme-netapp-ontapdevices.html
 create mode 100644 Documentation/nvme-netapp-ontapdevices.txt

-- 
2.17.1

[PATCH 1/2] nvme-cli: add netapp ontapdevices command

[martinus.gpy]

From: Martin George <marting@netapp.com>

Add a new ontapdevices command to the NetApp plugin for
NetApp ONTAP devices.

Signed-off-by: Martin George <marting at netapp.com>
Reviewed-by: Steve Schremmer <sschremm at netapp.com>
---
 plugins/netapp/netapp-nvme.c | 342 +++++++++++++++++++++++++++++++++++
 plugins/netapp/netapp-nvme.h |   1 +
 2 files changed, 343 insertions(+)

diff --git a/plugins/netapp/netapp-nvme.c b/plugins/netapp/netapp-nvme.c
index 416e74e..2951311 100644
--- a/plugins/netapp/netapp-nvme.c
+++ b/plugins/netapp/netapp-nvme.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <string.h>
+#include <sys/ioctl.h>
 
 #include "nvme.h"
 #include "nvme-ioctl.h"
@@ -31,12 +32,28 @@
 #define CREATE_CMD
 #include "netapp-nvme.h"
 
+#define ONTAP_C2_LOG_ID		0xC2
+#define ONTAP_C2_LOG_SIZE	4096
+#define ONTAP_LABEL_LEN		260
+#define ONTAP_NS_PATHLEN	520
+
 enum {
 	NNORMAL,
 	NJSON,
 	NCOLUMN,
 };
 
+enum {
+	ONTAP_C2_LOG_SUPPORTED_LSP	= 0x0,
+	ONTAP_C2_LOG_NSINFO_LSP		= 0x1,
+};
+
+enum {
+	ONTAP_VSERVER_TLV		= 0x11,
+	ONTAP_VOLUME_TLV		= 0x12,
+	ONTAP_NS_TLV			= 0x13,
+};
+
 static const char *dev_path = "/dev/";
 
 struct smdevice_info {
@@ -46,6 +63,15 @@ struct smdevice_info {
 	char			dev[265];
 };
 
+struct ontapdevice_info {
+	int			nsid;
+	struct nvme_id_ctrl	ctrl;
+	struct nvme_id_ns	ns;
+	char			nsdesc[4096];
+	unsigned char		log_data[ONTAP_C2_LOG_SIZE];
+	char			dev[265];
+};
+
 #define ARRAY_LABEL_LEN		60
 #define VOLUME_LABEL_LEN	60
 
@@ -77,6 +103,100 @@ static void netapp_nguid_to_str(char *str, __u8 *nguid)
 		str += sprintf(str, "%02x", nguid[i]);
 }
 
+static void netapp_get_ns_size(char *size, long long *lba,
+		struct nvme_id_ns *ns)
+{
+	*lba = 1 << ns->lbaf[(ns->flbas & 0x0F)].ds;
+	double nsze = le64_to_cpu(ns->nsze) * (*lba);
+	const char *s_suffix = suffix_si_get(&nsze);
+
+	sprintf(size, "%.2f%sB", nsze, s_suffix);
+}
+
+static void netapp_uuid_to_str(char *str, void *data)
+{
+#ifdef LIBUUID
+	uuid_t uuid;
+	struct nvme_ns_id_desc *desc = data;
+
+	memcpy(uuid, data + sizeof(*desc), 16);
+	uuid_unparse_lower(uuid, str);
+#endif
+}
+
+static void ontap_labels_to_str(char *dst, char *src, int count)
+{
+	int i;
+
+	memset(dst, 0, ONTAP_LABEL_LEN);
+	for (i = 0; i < count; i++) {
+		if (src[i] >= '!' && src[i] <= '~')
+			dst[i] = src[i];
+		else
+			break;
+	}
+	dst[i] = '\0';
+}
+
+static void netapp_get_ontap_labels(char *vsname, char *nspath,
+		unsigned char *log_data)
+{
+	int lsp, tlv, label_len;
+	char *vserver_name, *volume_name, *namespace_name;
+	char vol_name[ONTAP_LABEL_LEN], ns_name[ONTAP_LABEL_LEN];
+	const char *ontap_vol = "/vol/";
+	int i, j;
+
+	/* get the lsp */
+	lsp = (*(__u8 *)&log_data[16]) & 0x0F;
+	if (lsp != ONTAP_C2_LOG_NSINFO_LSP)
+		/* lsp not related to nsinfo */
+		return;
+
+	/* get the vserver tlv and name */
+	tlv = *(__u8 *)&log_data[32];
+	if (tlv == ONTAP_VSERVER_TLV) {
+		label_len = (*(__u16 *)&log_data[34]) * 4;
+		vserver_name = (char *)&log_data[36];
+		ontap_labels_to_str(vsname, vserver_name, label_len);
+	} else {
+		/* not the expected vserver tlv */
+		fprintf(stderr, "Unable to fetch ONTAP vserver name\n");
+		return;
+	}
+
+	i = 36 + label_len;
+	j = i + 2;
+	/* get the volume tlv and name */
+	tlv = *(__u8 *)&log_data[i];
+	if (tlv == ONTAP_VOLUME_TLV) {
+		label_len = (*(__u16 *)&log_data[j]) * 4;
+		volume_name = (char *)&log_data[j + 2];
+		ontap_labels_to_str(vol_name, volume_name, label_len);
+	} else {
+		/* not the expected volume tlv */
+		fprintf(stderr, "Unable to fetch ONTAP volume name\n");
+		return;
+	}
+
+	i += 4 + label_len;
+	j += 4 + label_len;
+	/* get the namespace tlv and name */
+	tlv = *(__u8 *)&log_data[i];
+	if (tlv == ONTAP_NS_TLV) {
+		label_len = (*(__u16 *)&log_data[j]) * 4;
+		namespace_name = (char *)&log_data[j + 2];
+		ontap_labels_to_str(ns_name, namespace_name, label_len);
+	} else {
+		/* not the expected namespace tlv */
+		fprintf(stderr, "Unable to fetch ONTAP namespace name\n");
+		return;
+	}
+
+	snprintf(nspath, ONTAP_NS_PATHLEN, "%s%s%s%s", ontap_vol,
+			vol_name, "/", ns_name);
+}
+
 static void netapp_smdevice_json(struct json_array *devices, char *devname,
 		char *arrayname, char *volname, int nsid, char *nguid,
 		char *ctrl, char *astate, char *size, long long lba,
@@ -99,6 +219,25 @@ static void netapp_smdevice_json(struct json_array *devices, char *devname,
 	json_array_add_value_object(devices, device_attrs);
 }
 
+static void netapp_ontapdevice_json(struct json_array *devices, char *devname,
+		char *vsname, char *nspath, int nsid, char *uuid,
+		char *size, long long lba, long long nsze)
+{
+	struct json_object *device_attrs;
+
+	device_attrs = json_create_object();
+	json_object_add_value_string(device_attrs, "Device", devname);
+	json_object_add_value_string(device_attrs, "Vserver", vsname);
+	json_object_add_value_string(device_attrs, "Namespace_Path", nspath);
+	json_object_add_value_int(device_attrs, "NSID", nsid);
+	json_object_add_value_string(device_attrs, "UUID", uuid);
+	json_object_add_value_string(device_attrs, "Size", size);
+	json_object_add_value_int(device_attrs, "LBA_Data_Size", lba);
+	json_object_add_value_int(device_attrs, "Namespace_Size", nsze);
+
+	json_array_add_value_object(devices, device_attrs);
+}
+
 static void netapp_smdevices_print(struct smdevice_info *devices, int count, int format)
 {
 	struct json_object *root = NULL;
@@ -161,6 +300,94 @@ static void netapp_smdevices_print(struct smdevice_info *devices, int count, int
 	}
 }
 
+static void netapp_ontapdevices_print(struct ontapdevice_info *devices,
+		int count, int format)
+{
+	struct json_object *root = NULL;
+	struct json_array *json_devices = NULL;
+	char vsname[ONTAP_LABEL_LEN] = " ";
+	char nspath[ONTAP_NS_PATHLEN] = " ";
+	long long lba;
+	char size[128];
+	char uuid_str[37] = " ";
+	int i;
+
+	char basestr[] = "%s, Vserver %s, Namespace Path %s, NSID %d, UUID %s, %s\n";
+	char columnstr[] = "%-16s %-25s %-50s %-4d %-38s %-9s\n";
+
+	/* default to 'normal' output format */
+	char *formatstr = basestr;
+
+	if (format == NCOLUMN) {
+		/* change output string and print column headers */
+		formatstr = columnstr;
+		printf("%-16s %-25s %-50s %-4s %-38s %-9s\n",
+				"Device", "Vserver", "Namespace Path",
+				"NSID", "UUID", "Size");
+		printf("%-16s %-25s %-50s %-4s %-38s %-9s\n",
+				"----------------", "-------------------------",
+				"--------------------------------------------------",
+				"----", "--------------------------------------",
+				"---------");
+	} else if (format == NJSON) {
+		/* prepare for json output */
+		root = json_create_object();
+		json_devices = json_create_array();
+	}
+
+	for (i = 0; i < count; i++) {
+
+		netapp_get_ns_size(size, &lba, &devices[i].ns);
+		netapp_uuid_to_str(uuid_str, devices[i].nsdesc);
+		netapp_get_ontap_labels(vsname, nspath, devices[i].log_data);
+
+		if (format == NJSON) {
+			netapp_ontapdevice_json(json_devices, devices[i].dev,
+					vsname, nspath, devices[i].nsid,
+					uuid_str, size, lba,
+					le64_to_cpu(devices[i].ns.nsze));
+		} else
+			printf(formatstr, devices[i].dev, vsname, nspath,
+					devices[i].nsid, uuid_str, size);
+	}
+
+	if (format == NJSON) {
+		/* complete the json output */
+		json_object_add_value_array(root, "ONTAPdevices", json_devices);
+		json_print_object(root, NULL);
+	}
+}
+
+static int nvme_get_ontap_c2_log(int fd, __u32 nsid, void *buf, __u32 buflen)
+{
+	struct nvme_admin_cmd get_log;
+	int err;
+
+	memset(buf, 0, buflen);
+	memset(&get_log, 0, sizeof(struct nvme_admin_cmd));
+
+	get_log.opcode = nvme_admin_get_log_page;
+	get_log.nsid = nsid;
+	get_log.addr = (__u64)(uintptr_t)buf;
+	get_log.data_len = buflen;
+
+	__u32 numd = (get_log.data_len >> 2) - 1;
+	__u32 numdu = numd >> 16;
+	__u32 numdl = numd & 0xFFFF;
+
+	get_log.cdw10 = ONTAP_C2_LOG_ID | (numdl << 16);
+	get_log.cdw10 |= ONTAP_C2_LOG_NSINFO_LSP << 8;
+	get_log.cdw11 = numdu;
+
+	err = nvme_submit_passthru(fd, NVME_IOCTL_ADMIN_CMD, &get_log);
+	if (err) {
+		fprintf(stderr, "ioctl error %0x\n", err);
+		return 1;
+	}
+
+	return 0;
+}
+
 static int netapp_smdevices_get_info(int fd, struct smdevice_info *item,
 				     const char *dev)
 {
@@ -188,6 +415,50 @@ static int netapp_smdevices_get_info(int fd, struct smdevice_info *item,
 	return 1;
 }
 
+static int netapp_ontapdevices_get_info(int fd, struct ontapdevice_info *item,
+		const char *dev)
+{
+	int err;
+
+	err = nvme_identify_ctrl(fd, &item->ctrl);
+	if (err) {
+		fprintf(stderr, "Identify Controller failed to %s (%s)\n",
+				dev, strerror(err));
+		return 0;
+	}
+
+	if (strncmp("NetApp ONTAP Controller", item->ctrl.mn, 23) != 0)
+		/* not the right controller model */
+		return 0;
+
+	item->nsid = nvme_get_nsid(fd);
+
+	err = nvme_identify_ns(fd, item->nsid, 0, &item->ns);
+	if (err) {
+		fprintf(stderr, "Unable to identify namespace for %s (%s)\n",
+				dev, strerror(err));
+		return 0;
+	}
+
+	err = nvme_identify_ns_descs(fd, item->nsid, item->nsdesc);
+	if (err) {
+		fprintf(stderr, "Unable to identify namespace descriptor for %s (%s)\n",
+				dev, strerror(err));
+		return 0;
+	}
+
+	err = nvme_get_ontap_c2_log(fd, item->nsid, item->log_data, ONTAP_C2_LOG_SIZE);
+	if (err) {
+		fprintf(stderr, "Unable to get log page data for %s (%s)\n",
+				dev, strerror(err));
+		return 0;
+	}
+
+	strncpy(item->dev, dev, sizeof(item->dev));
+
+	return 1;
+}
+
 static int netapp_nvme_filter(const struct dirent *d)
 {
 	char path[264];
@@ -294,3 +565,74 @@ static int netapp_smdevices(int argc, char **argv, struct command *command,
 	free(smdevices);
 	return 0;
 }
+
+/* handler for 'nvme netapp ontapdevices' */
+static int netapp_ontapdevices(int argc, char **argv, struct command *command,
+		struct plugin *plugin)
+{
+	const char *desc = "Display information about ONTAP devices.";
+	struct config {
+		char *output_format;
+	};
+	struct config cfg = {
+		.output_format = "normal",
+	};
+	struct dirent **devices;
+	int num, i, fd, ret, fmt;
+	struct ontapdevice_info *ontapdevices;
+	char path[264];
+	int num_ontapdevices = 0;
+
+	const struct argconfig_commandline_options opts[] = {
+		{"output-format", 'o', "FMT", CFG_STRING, &cfg.output_format,
+			required_argument, "Output Format: normal|json|column"},
+		{NULL}
+	};
+
+	ret = argconfig_parse(argc, argv, desc, opts, &cfg, sizeof(cfg));
+	if (ret < 0)
+		return ret;
+
+	fmt = netapp_output_format(cfg.output_format);
+	if (fmt != NNORMAL && fmt != NCOLUMN && fmt != NJSON) {
+		fprintf(stderr, "Unrecognized output format: %s\n", cfg.output_format);
+		return -EINVAL;
+	}
+
+	num = scandir(dev_path, &devices, netapp_nvme_filter, alphasort);
+	if (num <= 0) {
+		fprintf(stderr, "No NVMe devices detected.\n");
+		return num;
+	}
+
+	ontapdevices = calloc(num, sizeof(*ontapdevices));
+	if (!ontapdevices) {
+		fprintf(stderr, "Unable to allocate memory for devices.\n");
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < num; i++) {
+		snprintf(path, sizeof(path), "%s%s", dev_path,
+				devices[i]->d_name);
+		fd = open(path, O_RDONLY);
+		if (fd < 0) {
+			fprintf(stderr, "Unable to open %s: %s\n", path,
+					strerror(errno));
+			continue;
+		}
+
+		num_ontapdevices += netapp_ontapdevices_get_info(fd,
+				&ontapdevices[num_ontapdevices], path);
+
+		close(fd);
+	}
+
+	if (num_ontapdevices)
+		netapp_ontapdevices_print(ontapdevices, num_ontapdevices, fmt);
+
+	for (i = 0; i < num; i++)
+		free(devices[i]);
+	free(devices);
+	free(ontapdevices);
+	return 0;
+}
diff --git a/plugins/netapp/netapp-nvme.h b/plugins/netapp/netapp-nvme.h
index 3dd019e..d4eebd6 100644
--- a/plugins/netapp/netapp-nvme.h
+++ b/plugins/netapp/netapp-nvme.h
@@ -9,6 +9,7 @@
 PLUGIN(NAME("netapp", "NetApp vendor specific extensions"),
 	COMMAND_LIST(
 		ENTRY("smdevices", "NetApp SMdevices", netapp_smdevices)
+		ENTRY("ontapdevices", "NetApp ONTAPdevices", netapp_ontapdevices)
 	)
 );
 
-- 
2.17.1

[PATCH 2/2] nvme-cli: add netapp ontapdevices man page

[martinus.gpy]

From: Martin George <marting@netapp.com>

Signed-off-by: Martin George <marting at netapp.com>
Reviewed-by: Steve Schremmer <sschremm at netapp.com>
---
 Documentation/nvme-netapp-ontapdevices.1    |  74 ++
 Documentation/nvme-netapp-ontapdevices.html | 814 ++++++++++++++++++++
 Documentation/nvme-netapp-ontapdevices.txt  |  35 +
 3 files changed, 923 insertions(+)
 create mode 100644 Documentation/nvme-netapp-ontapdevices.1
 create mode 100644 Documentation/nvme-netapp-ontapdevices.html
 create mode 100644 Documentation/nvme-netapp-ontapdevices.txt

diff --git a/Documentation/nvme-netapp-ontapdevices.1 b/Documentation/nvme-netapp-ontapdevices.1
new file mode 100644
index 0000000..0669457
--- /dev/null
+++ b/Documentation/nvme-netapp-ontapdevices.1
@@ -0,0 +1,74 @@
+'\" t
+.\"     Title: nvme-netapp-ontapdevices
+.\"    Author: [FIXME: author] [see http://www.docbook.org/tdg5/en/html/author]
+.\" Generator: DocBook XSL Stylesheets vsnapshot <http://docbook.sf.net/>
+.\"      Date: 01/08/2019
+.\"    Manual: NVMe Manual
+.\"    Source: NVMe
+.\"  Language: English
+.\"
+.TH "NVME\-NETAPP\-ONTAPDEV" "1" "01/08/2019" "NVMe" "NVMe Manual"
+.\" -----------------------------------------------------------------
+.\" * Define some portability stuff
+.\" -----------------------------------------------------------------
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.\" http://bugs.debian.org/507673
+.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.ie \n(.g .ds Aq \(aq
+.el       .ds Aq '
+.\" -----------------------------------------------------------------
+.\" * set default formatting
+.\" -----------------------------------------------------------------
+.\" disable hyphenation
+.nh
+.\" disable justification (adjust text to left margin only)
+.ad l
+.\" -----------------------------------------------------------------
+.\" * MAIN CONTENT STARTS HERE *
+.\" -----------------------------------------------------------------
+.SH "NAME"
+nvme-netapp-ontapdevices \- Display information about ONTAP devices
+.SH "SYNOPSIS"
+.sp
+.nf
+\fInvme netapp ontapdevices\fR [\-o <fmt> | \-\-output\-format=<fmt>]
+.fi
+.SH "DESCRIPTION"
+.sp
+Display information about ONTAP devices on the host\&. The ONTAP devices are identified using the Identify Controller data\&.
+.SH "OPTIONS"
+.PP
+\-o <fmt>, \-\-output\-format=<fmt>
+.RS 4
+Set the reporting format to
+\fInormal\fR
+(default),
+\fIcolumn\fR, or
+\fIjson\fR\&. Only one output format can be used at a time\&.
+.RE
+.SH "EXAMPLES"
+.sp
+.RS 4
+.ie n \{\
+\h'-04'\(bu\h'+03'\c
+.\}
+.el \{\
+.sp -1
+.IP \(bu 2.3
+.\}
+Display information, in a column\-based format, for ONTAP devices\&.
+.sp
+.if n \{\
+.RS 4
+.\}
+.nf
+# nvme netapp ontapdevices \-o column
+.fi
+.if n \{\
+.RE
+.\}
+.RE
+.SH "NVME"
+.sp
+Part of the nvme\-user suite
diff --git a/Documentation/nvme-netapp-ontapdevices.html b/Documentation/nvme-netapp-ontapdevices.html
new file mode 100644
index 0000000..792801e
--- /dev/null
+++ b/Documentation/nvme-netapp-ontapdevices.html
@@ -0,0 +1,814 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
+    "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+<head>
+<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=UTF-8" />
+<meta name="generator" content="AsciiDoc 8.6.10" />
+<title>nvme-netapp-ontapdevices(1)</title>
+<style type="text/css">
+/* Shared CSS for AsciiDoc xhtml11 and html5 backends */
+
+/* Default font. */
+body {
+  font-family: Georgia,serif;
+}
+
+/* Title font. */
+h1, h2, h3, h4, h5, h6,
+div.title, caption.title,
+thead, p.table.header,
+#toctitle,
+#author, #revnumber, #revdate, #revremark,
+#footer {
+  font-family: Arial,Helvetica,sans-serif;
+}
+
+body {
+  margin: 1em 5% 1em 5%;
+}
+
+a {
+  color: blue;
+  text-decoration: underline;
+}
+a:visited {
+  color: fuchsia;
+}
+
+em {
+  font-style: italic;
+  color: navy;
+}
+
+strong {
+  font-weight: bold;
+  color: #083194;
+}
+
+h1, h2, h3, h4, h5, h6 {
+  color: #527bbd;
+  margin-top: 1.2em;
+  margin-bottom: 0.5em;
+  line-height: 1.3;
+}
+
+h1, h2, h3 {
+  border-bottom: 2px solid silver;
+}
+h2 {
+  padding-top: 0.5em;
+}
+h3 {
+  float: left;
+}
+h3 + * {
+  clear: left;
+}
+h5 {
+  font-size: 1.0em;
+}
+
+div.sectionbody {
+  margin-left: 0;
+}
+
+hr {
+  border: 1px solid silver;
+}
+
+p {
+  margin-top: 0.5em;
+  margin-bottom: 0.5em;
+}
+
+ul, ol, li > p {
+  margin-top: 0;
+}
+ul > li     { color: #aaa; }
+ul > li > * { color: black; }
+
+.monospaced, code, pre {
+  font-family: "Courier New", Courier, monospace;
+  font-size: inherit;
+  color: navy;
+  padding: 0;
+  margin: 0;
+}
+pre {
+  white-space: pre-wrap;
+}
+
+#author {
+  color: #527bbd;
+  font-weight: bold;
+  font-size: 1.1em;
+}
+#email {
+}
+#revnumber, #revdate, #revremark {
+}
+
+#footer {
+  font-size: small;
+  border-top: 2px solid silver;
+  padding-top: 0.5em;
+  margin-top: 4.0em;
+}
+#footer-text {
+  float: left;
+  padding-bottom: 0.5em;
+}
+#footer-badges {
+  float: right;
+  padding-bottom: 0.5em;
+}
+
+#preamble {
+  margin-top: 1.5em;
+  margin-bottom: 1.5em;
+}
+div.imageblock, div.exampleblock, div.verseblock,
+div.quoteblock, div.literalblock, div.listingblock, div.sidebarblock,
+div.admonitionblock {
+  margin-top: 1.0em;
+  margin-bottom: 1.5em;
+}
+div.admonitionblock {
+  margin-top: 2.0em;
+  margin-bottom: 2.0em;
+  margin-right: 10%;
+  color: #606060;
+}
+
+div.content { /* Block element content. */
+  padding: 0;
+}
+
+/* Block element titles. */
+div.title, caption.title {
+  color: #527bbd;
+  font-weight: bold;
+  text-align: left;
+  margin-top: 1.0em;
+  margin-bottom: 0.5em;
+}
+div.title + * {
+  margin-top: 0;
+}
+
+td div.title:first-child {
+  margin-top: 0.0em;
+}
+div.content div.title:first-child {
+  margin-top: 0.0em;
+}
+div.content + div.title {
+  margin-top: 0.0em;
+}
+
+div.sidebarblock > div.content {
+  background: #ffffee;
+  border: 1px solid #dddddd;
+  border-left: 4px solid #f0f0f0;
+  padding: 0.5em;
+}
+
+div.listingblock > div.content {
+  border: 1px solid #dddddd;
+  border-left: 5px solid #f0f0f0;
+  background: #f8f8f8;
+  padding: 0.5em;
+}
+
+div.quoteblock, div.verseblock {
+  padding-left: 1.0em;
+  margin-left: 1.0em;
+  margin-right: 10%;
+  border-left: 5px solid #f0f0f0;
+  color: #888;
+}
+
+div.quoteblock > div.attribution {
+  padding-top: 0.5em;
+  text-align: right;
+}
+
+div.verseblock > pre.content {
+  font-family: inherit;
+  font-size: inherit;
+}
+div.verseblock > div.attribution {
+  padding-top: 0.75em;
+  text-align: left;
+}
+/* DEPRECATED: Pre version 8.2.7 verse style literal block. */
+div.verseblock + div.attribution {
+  text-align: left;
+}
+
+div.admonitionblock .icon {
+  vertical-align: top;
+  font-size: 1.1em;
+  font-weight: bold;
+  text-decoration: underline;
+  color: #527bbd;
+  padding-right: 0.5em;
+}
+div.admonitionblock td.content {
+  padding-left: 0.5em;
+  border-left: 3px solid #dddddd;
+}
+
+div.exampleblock > div.content {
+  border-left: 3px solid #dddddd;
+  padding-left: 0.5em;
+}
+
+div.imageblock div.content { padding-left: 0; }
+span.image img { border-style: none; vertical-align: text-bottom; }
+a.image:visited { color: white; }
+
+dl {
+  margin-top: 0.8em;
+  margin-bottom: 0.8em;
+}
+dt {
+  margin-top: 0.5em;
+  margin-bottom: 0;
+  font-style: normal;
+  color: navy;
+}
+dd > *:first-child {
+  margin-top: 0.1em;
+}
+
+ul, ol {
+    list-style-position: outside;
+}
+ol.arabic {
+  list-style-type: decimal;
+}
+ol.loweralpha {
+  list-style-type: lower-alpha;
+}
+ol.upperalpha {
+  list-style-type: upper-alpha;
+}
+ol.lowerroman {
+  list-style-type: lower-roman;
+}
+ol.upperroman {
+  list-style-type: upper-roman;
+}
+
+div.compact ul, div.compact ol,
+div.compact p, div.compact p,
+div.compact div, div.compact div {
+  margin-top: 0.1em;
+  margin-bottom: 0.1em;
+}
+
+tfoot {
+  font-weight: bold;
+}
+td > div.verse {
+  white-space: pre;
+}
+
+div.hdlist {
+  margin-top: 0.8em;
+  margin-bottom: 0.8em;
+}
+div.hdlist tr {
+  padding-bottom: 15px;
+}
+dt.hdlist1.strong, td.hdlist1.strong {
+  font-weight: bold;
+}
+td.hdlist1 {
+  vertical-align: top;
+  font-style: normal;
+  padding-right: 0.8em;
+  color: navy;
+}
+td.hdlist2 {
+  vertical-align: top;
+}
+div.hdlist.compact tr {
+  margin: 0;
+  padding-bottom: 0;
+}
+
+.comment {
+  background: yellow;
+}
+
+.footnote, .footnoteref {
+  font-size: 0.8em;
+}
+
+span.footnote, span.footnoteref {
+  vertical-align: super;
+}
+
+#footnotes {
+  margin: 20px 0 20px 0;
+  padding: 7px 0 0 0;
+}
+
+#footnotes div.footnote {
+  margin: 0 0 5px 0;
+}
+
+#footnotes hr {
+  border: none;
+  border-top: 1px solid silver;
+  height: 1px;
+  text-align: left;
+  margin-left: 0;
+  width: 20%;
+  min-width: 100px;
+}
+
+div.colist td {
+  padding-right: 0.5em;
+  padding-bottom: 0.3em;
+  vertical-align: top;
+}
+div.colist td img {
+  margin-top: 0.3em;
+}
+
+ at media print {
+  #footer-badges { display: none; }
+}
+
+#toc {
+  margin-bottom: 2.5em;
+}
+
+#toctitle {
+  color: #527bbd;
+  font-size: 1.1em;
+  font-weight: bold;
+  margin-top: 1.0em;
+  margin-bottom: 0.1em;
+}
+
+div.toclevel0, div.toclevel1, div.toclevel2, div.toclevel3, div.toclevel4 {
+  margin-top: 0;
+  margin-bottom: 0;
+}
+div.toclevel2 {
+  margin-left: 2em;
+  font-size: 0.9em;
+}
+div.toclevel3 {
+  margin-left: 4em;
+  font-size: 0.9em;
+}
+div.toclevel4 {
+  margin-left: 6em;
+  font-size: 0.9em;
+}
+
+span.aqua { color: aqua; }
+span.black { color: black; }
+span.blue { color: blue; }
+span.fuchsia { color: fuchsia; }
+span.gray { color: gray; }
+span.green { color: green; }
+span.lime { color: lime; }
+span.maroon { color: maroon; }
+span.navy { color: navy; }
+span.olive { color: olive; }
+span.purple { color: purple; }
+span.red { color: red; }
+span.silver { color: silver; }
+span.teal { color: teal; }
+span.white { color: white; }
+span.yellow { color: yellow; }
+
+span.aqua-background { background: aqua; }
+span.black-background { background: black; }
+span.blue-background { background: blue; }
+span.fuchsia-background { background: fuchsia; }
+span.gray-background { background: gray; }
+span.green-background { background: green; }
+span.lime-background { background: lime; }
+span.maroon-background { background: maroon; }
+span.navy-background { background: navy; }
+span.olive-background { background: olive; }
+span.purple-background { background: purple; }
+span.red-background { background: red; }
+span.silver-background { background: silver; }
+span.teal-background { background: teal; }
+span.white-background { background: white; }
+span.yellow-background { background: yellow; }
+
+span.big { font-size: 2em; }
+span.small { font-size: 0.6em; }
+
+span.underline { text-decoration: underline; }
+span.overline { text-decoration: overline; }
+span.line-through { text-decoration: line-through; }
+
+div.unbreakable { page-break-inside: avoid; }
+
+
+/*
+ * xhtml11 specific
+ *
+ * */
+
+div.tableblock {
+  margin-top: 1.0em;
+  margin-bottom: 1.5em;
+}
+div.tableblock > table {
+  border: 3px solid #527bbd;
+}
+thead, p.table.header {
+  font-weight: bold;
+  color: #527bbd;
+}
+p.table {
+  margin-top: 0;
+}
+/* Because the table frame attribute is overriden by CSS in most browsers. */
+div.tableblock > table[frame="void"] {
+  border-style: none;
+}
+div.tableblock > table[frame="hsides"] {
+  border-left-style: none;
+  border-right-style: none;
+}
+div.tableblock > table[frame="vsides"] {
+  border-top-style: none;
+  border-bottom-style: none;
+}
+
+
+/*
+ * html5 specific
+ *
+ * */
+
+table.tableblock {
+  margin-top: 1.0em;
+  margin-bottom: 1.5em;
+}
+thead, p.tableblock.header {
+  font-weight: bold;
+  color: #527bbd;
+}
+p.tableblock {
+  margin-top: 0;
+}
+table.tableblock {
+  border-width: 3px;
+  border-spacing: 0px;
+  border-style: solid;
+  border-color: #527bbd;
+  border-collapse: collapse;
+}
+th.tableblock, td.tableblock {
+  border-width: 1px;
+  padding: 4px;
+  border-style: solid;
+  border-color: #527bbd;
+}
+
+table.tableblock.frame-topbot {
+  border-left-style: hidden;
+  border-right-style: hidden;
+}
+table.tableblock.frame-sides {
+  border-top-style: hidden;
+  border-bottom-style: hidden;
+}
+table.tableblock.frame-none {
+  border-style: hidden;
+}
+
+th.tableblock.halign-left, td.tableblock.halign-left {
+  text-align: left;
+}
+th.tableblock.halign-center, td.tableblock.halign-center {
+  text-align: center;
+}
+th.tableblock.halign-right, td.tableblock.halign-right {
+  text-align: right;
+}
+
+th.tableblock.valign-top, td.tableblock.valign-top {
+  vertical-align: top;
+}
+th.tableblock.valign-middle, td.tableblock.valign-middle {
+  vertical-align: middle;
+}
+th.tableblock.valign-bottom, td.tableblock.valign-bottom {
+  vertical-align: bottom;
+}
+
+
+/*
+ * manpage specific
+ *
+ * */
+
+body.manpage h1 {
+  padding-top: 0.5em;
+  padding-bottom: 0.5em;
+  border-top: 2px solid silver;
+  border-bottom: 2px solid silver;
+}
+body.manpage h2 {
+  border-style: none;
+}
+body.manpage div.sectionbody {
+  margin-left: 3em;
+}
+
+ at media print {
+  body.manpage div#toc { display: none; }
+}
+
+
+</style>
+<script type="text/javascript">
+/*<![CDATA[*/
+var asciidoc = {  // Namespace.
+
+/////////////////////////////////////////////////////////////////////
+// Table Of Contents generator
+/////////////////////////////////////////////////////////////////////
+
+/* Author: Mihai Bazon, September 2002
+ * http://students.infoiasi.ro/~mishoo
+ *
+ * Table Of Content generator
+ * Version: 0.4
+ *
+ * Feel free to use this script under the terms of the GNU General Public
+ * License, as long as you do not remove or alter this notice.
+ */
+
+ /* modified by Troy D. Hanson, September 2006. License: GPL */
+ /* modified by Stuart Rackham, 2006, 2009. License: GPL */
+
+// toclevels = 1..4.
+toc: function (toclevels) {
+
+  function getText(el) {
+    var text = "";
+    for (var i = el.firstChild; i != null; i = i.nextSibling) {
+      if (i.nodeType == 3 /* Node.TEXT_NODE */) // IE doesn't speak constants.
+        text += i.data;
+      else if (i.firstChild != null)
+        text += getText(i);
+    }
+    return text;
+  }
+
+  function TocEntry(el, text, toclevel) {
+    this.element = el;
+    this.text = text;
+    this.toclevel = toclevel;
+  }
+
+  function tocEntries(el, toclevels) {
+    var result = new Array;
+    var re = new RegExp('[hH]([1-'+(toclevels+1)+'])');
+    // Function that scans the DOM tree for header elements (the DOM2
+    // nodeIterator API would be a better technique but not supported by all
+    // browsers).
+    var iterate = function (el) {
+      for (var i = el.firstChild; i != null; i = i.nextSibling) {
+        if (i.nodeType == 1 /* Node.ELEMENT_NODE */) {
+          var mo = re.exec(i.tagName);
+          if (mo && (i.getAttribute("class") || i.getAttribute("className")) != "float") {
+            result[result.length] = new TocEntry(i, getText(i), mo[1]-1);
+          }
+          iterate(i);
+        }
+      }
+    }
+    iterate(el);
+    return result;
+  }
+
+  var toc = document.getElementById("toc");
+  if (!toc) {
+    return;
+  }
+
+  // Delete existing TOC entries in case we're reloading the TOC.
+  var tocEntriesToRemove = [];
+  var i;
+  for (i = 0; i < toc.childNodes.length; i++) {
+    var entry = toc.childNodes[i];
+    if (entry.nodeName.toLowerCase() == 'div'
+     && entry.getAttribute("class")
+     && entry.getAttribute("class").match(/^toclevel/))
+      tocEntriesToRemove.push(entry);
+  }
+  for (i = 0; i < tocEntriesToRemove.length; i++) {
+    toc.removeChild(tocEntriesToRemove[i]);
+  }
+
+  // Rebuild TOC entries.
+  var entries = tocEntries(document.getElementById("content"), toclevels);
+  for (var i = 0; i < entries.length; ++i) {
+    var entry = entries[i];
+    if (entry.element.id == "")
+      entry.element.id = "_toc_" + i;
+    var a = document.createElement("a");
+    a.href = "#" + entry.element.id;
+    a.appendChild(document.createTextNode(entry.text));
+    var div = document.createElement("div");
+    div.appendChild(a);
+    div.className = "toclevel" + entry.toclevel;
+    toc.appendChild(div);
+  }
+  if (entries.length == 0)
+    toc.parentNode.removeChild(toc);
+},
+
+
+/////////////////////////////////////////////////////////////////////
+// Footnotes generator
+/////////////////////////////////////////////////////////////////////
+
+/* Based on footnote generation code from:
+ * http://www.brandspankingnew.net/archive/2005/07/format_footnote.html
+ */
+
+footnotes: function () {
+  // Delete existing footnote entries in case we're reloading the footnodes.
+  var i;
+  var noteholder = document.getElementById("footnotes");
+  if (!noteholder) {
+    return;
+  }
+  var entriesToRemove = [];
+  for (i = 0; i < noteholder.childNodes.length; i++) {
+    var entry = noteholder.childNodes[i];
+    if (entry.nodeName.toLowerCase() == 'div' && entry.getAttribute("class") == "footnote")
+      entriesToRemove.push(entry);
+  }
+  for (i = 0; i < entriesToRemove.length; i++) {
+    noteholder.removeChild(entriesToRemove[i]);
+  }
+
+  // Rebuild footnote entries.
+  var cont = document.getElementById("content");
+  var spans = cont.getElementsByTagName("span");
+  var refs = {};
+  var n = 0;
+  for (i=0; i<spans.length; i++) {
+    if (spans[i].className == "footnote") {
+      n++;
+      var note = spans[i].getAttribute("data-note");
+      if (!note) {
+        // Use [\s\S] in place of . so multi-line matches work.
+        // Because JavaScript has no s (dotall) regex flag.
+        note = spans[i].innerHTML.match(/\s*\[([\s\S]*)]\s*/)[1];
+        spans[i].innerHTML =
+          "[<a id='_footnoteref_" + n + "' href='#_footnote_" + n +
+          "' title='View footnote' class='footnote'>" + n + "</a>]";
+        spans[i].setAttribute("data-note", note);
+      }
+      noteholder.innerHTML +=
+        "<div class='footnote' id='_footnote_" + n + "'>" +
+        "<a href='#_footnoteref_" + n + "' title='Return to text'>" +
+        n + "</a>. " + note + "</div>";
+      var id =spans[i].getAttribute("id");
+      if (id != null) refs["#"+id] = n;
+    }
+  }
+  if (n == 0)
+    noteholder.parentNode.removeChild(noteholder);
+  else {
+    // Process footnoterefs.
+    for (i=0; i<spans.length; i++) {
+      if (spans[i].className == "footnoteref") {
+        var href = spans[i].getElementsByTagName("a")[0].getAttribute("href");
+        href = href.match(/#.*/)[0];  // Because IE return full URL.
+        n = refs[href];
+        spans[i].innerHTML =
+          "[<a href='#_footnote_" + n +
+          "' title='View footnote' class='footnote'>" + n + "</a>]";
+      }
+    }
+  }
+},
+
+install: function(toclevels) {
+  var timerId;
+
+  function reinstall() {
+    asciidoc.footnotes();
+    if (toclevels) {
+      asciidoc.toc(toclevels);
+    }
+  }
+
+  function reinstallAndRemoveTimer() {
+    clearInterval(timerId);
+    reinstall();
+  }
+
+  timerId = setInterval(reinstall, 500);
+  if (document.addEventListener)
+    document.addEventListener("DOMContentLoaded", reinstallAndRemoveTimer, false);
+  else
+    window.onload = reinstallAndRemoveTimer;
+}
+
+}
+asciidoc.install();
+/*]]>*/
+</script>
+</head>
+<body class="manpage">
+<div id="header">
+<h1>
+nvme-netapp-ontapdevices(1) Manual Page
+</h1>
+<h2>NAME</h2>
+<div class="sectionbody">
+<p>nvme-netapp-ontapdevices -
+   Display information about ONTAP devices
+</p>
+</div>
+</div>
+<div id="content">
+<div class="sect1">
+<h2 id="_synopsis">SYNOPSIS</h2>
+<div class="sectionbody">
+<div class="verseblock">
+<pre class="content"><em>nvme netapp ontapdevices</em> [-o &lt;fmt&gt; | --output-format=&lt;fmt&gt;]</pre>
+<div class="attribution">
+</div></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_description">DESCRIPTION</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>Display information about ONTAP devices on the host.
+The ONTAP devices are identified using the Identify Controller data.</p></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_options">OPTIONS</h2>
+<div class="sectionbody">
+<div class="dlist"><dl>
+<dt class="hdlist1">
+-o &lt;fmt&gt;
+</dt>
+<dt class="hdlist1">
+--output-format=&lt;fmt&gt;
+</dt>
+<dd>
+<p>
+              Set the reporting format to <em>normal</em> (default), <em>column</em>, or
+              <em>json</em>. Only one output format can be used at a time.
+</p>
+</dd>
+</dl></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_examples">EXAMPLES</h2>
+<div class="sectionbody">
+<div class="ulist"><ul>
+<li>
+<p>
+Display information, in a column-based format, for ONTAP devices.
+</p>
+<div class="listingblock">
+<div class="content">
+<pre><code># nvme netapp ontapdevices -o column</code></pre>
+</div></div>
+</li>
+</ul></div>
+</div>
+</div>
+<div class="sect1">
+<h2 id="_nvme">NVME</h2>
+<div class="sectionbody">
+<div class="paragraph"><p>Part of the nvme-user suite</p></div>
+</div>
+</div>
+</div>
+<div id="footnotes"><hr /></div>
+<div id="footer">
+<div id="footer-text">
+Last updated
+ 2018-11-29 13:36:56 GMT
+</div>
+</div>
+</body>
+</html>
diff --git a/Documentation/nvme-netapp-ontapdevices.txt b/Documentation/nvme-netapp-ontapdevices.txt
new file mode 100644
index 0000000..09369f1
--- /dev/null
+++ b/Documentation/nvme-netapp-ontapdevices.txt
@@ -0,0 +1,35 @@
+nvme-netapp-ontapdevices(1)
+========================
+
+NAME
+----
+nvme-netapp-ontapdevices - Display information about ONTAP devices
+
+SYNOPSIS
+--------
+[verse]
+'nvme netapp ontapdevices' [-o <fmt> | --output-format=<fmt>]
+
+DESCRIPTION
+-----------
+Display information about ONTAP devices on the host. The ONTAP devices are
+are identified using the Identify Controller data.
+
+OPTIONS
+-------
+-o <fmt>::
+--output-format=<fmt>::
+              Set the reporting format to 'normal' (default), 'column', or
+              'json'. Only one output format can be used at a time.
+
+EXAMPLES
+--------
+* Display information, in a column-based format, for ONTAP devices.
++
+------------
+# nvme netapp ontapdevices -o column
+------------
+
+NVME
+----
+Part of the nvme-user suite
-- 
2.17.1

[PATCH 1/2] nvme-cli: add netapp ontapdevices command

[Schremmer, Steven]

> From: Martin George <marting at netapp.com>
> 
> Add a new ontapdevices command to the NetApp plugin for
> NetApp ONTAP devices.
> 
> Signed-off-by: Martin George <marting at netapp.com>
> Reviewed-by: Steve Schremmer <sschremm at netapp.com>
> ---

If it matters for the Reviewed-by, I'm actually subscribed to the mailing list
using steven.schremmer at netapp.com I believe.

[PATCH 0/2] nvme-cli: update NetApp plugin for ONTAP devices

[Keith Busch]

On Mon, Apr 01, 2019@09:39:55AM -0700, martinus.gpy@gmail.com wrote:
> From: Martin George <marting at netapp.com>
> 
> These patches update the current NetApp plugin to handle
> NetApp ONTAP devices.

Applied, thank you!

[PATCH 0/2] nvme-cli: update NetApp plugin for ONTAP devices

[George, Martin]

On Thu, 2019-04-04@13:17 -0600, Keith Busch wrote:
> > From: Martin George <marting at netapp.com>
> > 
> > These patches update the current NetApp plugin to handle
> > NetApp ONTAP devices.
> 
> Applied, thank you!

Keith,

I don't see these patches in github. Could you please confirm that it
got applied?

-Martin

[PATCH 0/2] nvme-cli: update NetApp plugin for ONTAP devices

[Keith Busch]

On Mon, Apr 08, 2019@01:53:29AM -0700, George, Martin wrote:
> I don't see these patches in github. Could you please confirm that it
> got applied?

Ah, the previous push failed. Should be good now.