[NDCTL PATCH v3 2/3] ndctl: cxl: Add QoS class support for the memory device

[Dave Jiang <dave.jiang@intel.com>]

Add libcxl API to retrieve the QoS class tokens for the memory
devices. Two API calls are added. One for 'ram' or 'volatile'
mode and another for 'pmem' or 'persistent' mode. Support also added
for displaying the QoS class tokens through the 'cxl list' command.
There can be 1 or more QoS class tokens for the memory device if
they are valid. The qos_class tokens are displayed behind -vvv
verbose level.

Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
v3:
- Rebase to pending branch
- Skip from failing if no qos_class sysfs attrib found
---
 cxl/json.c         |   36 +++++++++++++++++++++++++++++++++++-
 cxl/lib/libcxl.c   |   48 ++++++++++++++++++++++++++++++++++++++++++++++++
 cxl/lib/libcxl.sym |    2 ++
 cxl/lib/private.h  |    2 ++
 cxl/libcxl.h       |    7 +++++++
 5 files changed, 94 insertions(+), 1 deletion(-)

diff --git a/cxl/json.c b/cxl/json.c
index 48a43ddf14b0..dcbac8c14f03 100644
--- a/cxl/json.c
+++ b/cxl/json.c
@@ -770,12 +770,32 @@ err_free:
 	return jpoison;
 }
 
+static struct json_object *get_qos_json_object(struct json_object *jdev,
+					       struct qos_class *qos_class)
+{
+	struct json_object *jqos_array = json_object_new_array();
+	struct json_object *jobj;
+	int i;
+
+	if (!jqos_array)
+		return NULL;
+
+	for (i = 0; i < qos_class->nr; i++) {
+		jobj = json_object_new_int(qos_class->qos[i]);
+		if (jobj)
+			json_object_array_add(jqos_array, jobj);
+	}
+
+	return jqos_array;
+}
+
 struct json_object *util_cxl_memdev_to_json(struct cxl_memdev *memdev,
 		unsigned long flags)
 {
 	const char *devname = cxl_memdev_get_devname(memdev);
-	struct json_object *jdev, *jobj;
+	struct json_object *jdev, *jobj, *jqos;
 	unsigned long long serial, size;
+	struct qos_class *qos_class;
 	int numa_node;
 
 	jdev = json_object_new_object();
@@ -791,6 +811,13 @@ struct json_object *util_cxl_memdev_to_json(struct cxl_memdev *memdev,
 		jobj = util_json_object_size(size, flags);
 		if (jobj)
 			json_object_object_add(jdev, "pmem_size", jobj);
+
+		if (flags & UTIL_JSON_QOS_CLASS) {
+			qos_class = cxl_memdev_get_pmem_qos_class(memdev);
+			jqos = get_qos_json_object(jdev, qos_class);
+			if (jqos)
+				json_object_object_add(jdev, "pmem_qos_class", jqos);
+		}
 	}
 
 	size = cxl_memdev_get_ram_size(memdev);
@@ -798,6 +825,13 @@ struct json_object *util_cxl_memdev_to_json(struct cxl_memdev *memdev,
 		jobj = util_json_object_size(size, flags);
 		if (jobj)
 			json_object_object_add(jdev, "ram_size", jobj);
+
+		if (flags & UTIL_JSON_QOS_CLASS) {
+			qos_class = cxl_memdev_get_ram_qos_class(memdev);
+			jqos = get_qos_json_object(jdev, qos_class);
+			if (jqos)
+				json_object_object_add(jdev, "ram_qos_class", jqos);
+		}
 	}
 
 	if (flags & UTIL_JSON_HEALTH) {
diff --git a/cxl/lib/libcxl.c b/cxl/lib/libcxl.c
index 9a1ac7001803..c69a18c4d237 100644
--- a/cxl/lib/libcxl.c
+++ b/cxl/lib/libcxl.c
@@ -88,6 +88,10 @@ static void free_memdev(struct cxl_memdev *memdev, struct list_head *head)
 	free(memdev->dev_buf);
 	free(memdev->dev_path);
 	free(memdev->host_path);
+	if (memdev->ram_qos_class.nr)
+		free(memdev->ram_qos_class.qos);
+	if (memdev->pmem_qos_class.nr)
+		free(memdev->pmem_qos_class.qos);
 	free(memdev);
 }
 
@@ -1224,6 +1228,27 @@ static int add_cxl_memdev_fwl(struct cxl_memdev *memdev,
 	return -ENOMEM;
 }
 
+static int *get_qos_class(struct cxl_ctx *ctx, char *buf, int *entries)
+{
+	int *varray = NULL;
+	int i = 0;
+	char *p;
+
+	p = strtok(buf, ",");
+	while (p != NULL) {
+		int val = atoi(p);
+
+		varray = reallocarray(varray, i + 1, sizeof(int));
+		varray[i] = val;
+		p = strtok(NULL, ",");
+		i++;
+	}
+
+	*entries = i;
+
+	return varray;
+}
+
 static void *add_cxl_memdev(void *parent, int id, const char *cxlmem_base)
 {
 	const char *devname = devpath_to_devname(cxlmem_base);
@@ -1233,6 +1258,7 @@ static void *add_cxl_memdev(void *parent, int id, const char *cxlmem_base)
 	char buf[SYSFS_ATTR_SIZE];
 	struct stat st;
 	char *host;
+	int qnr;
 
 	if (!path)
 		return NULL;
@@ -1260,6 +1286,18 @@ static void *add_cxl_memdev(void *parent, int id, const char *cxlmem_base)
 		goto err_read;
 	memdev->ram_size = strtoull(buf, NULL, 0);
 
+	sprintf(path, "%s/pmem/qos_class", cxlmem_base);
+	if (sysfs_read_attr(ctx, path, buf) == 0) {
+		memdev->pmem_qos_class.qos = get_qos_class(ctx, buf, &qnr);
+		memdev->pmem_qos_class.nr = qnr;
+	}
+
+	sprintf(path, "%s/ram/qos_class", cxlmem_base);
+	if (sysfs_read_attr(ctx, path, buf) == 0) {
+		memdev->ram_qos_class.qos = get_qos_class(ctx, buf, &qnr);
+		memdev->ram_qos_class.nr = qnr;
+	}
+
 	sprintf(path, "%s/payload_max", cxlmem_base);
 	if (sysfs_read_attr(ctx, path, buf) < 0)
 		goto err_read;
@@ -1483,6 +1521,16 @@ CXL_EXPORT unsigned long long cxl_memdev_get_ram_size(struct cxl_memdev *memdev)
 	return memdev->ram_size;
 }
 
+CXL_EXPORT struct qos_class *cxl_memdev_get_pmem_qos_class(struct cxl_memdev *memdev)
+{
+	return &memdev->pmem_qos_class;
+}
+
+CXL_EXPORT struct qos_class *cxl_memdev_get_ram_qos_class(struct cxl_memdev *memdev)
+{
+	return &memdev->ram_qos_class;
+}
+
 CXL_EXPORT const char *cxl_memdev_get_firmware_verison(struct cxl_memdev *memdev)
 {
 	return memdev->firmware_version;
diff --git a/cxl/lib/libcxl.sym b/cxl/lib/libcxl.sym
index 384fea2c25e3..465c78dc6c70 100644
--- a/cxl/lib/libcxl.sym
+++ b/cxl/lib/libcxl.sym
@@ -283,4 +283,6 @@ LIBCXL_8 {
 global:
 	cxl_memdev_wait_sanitize;
 	cxl_root_decoder_get_qos_class;
+	cxl_memdev_get_pmem_qos_class;
+	cxl_memdev_get_ram_qos_class;
 } LIBCXL_7;
diff --git a/cxl/lib/private.h b/cxl/lib/private.h
index 4847ff448f71..1fe3654bc7ff 100644
--- a/cxl/lib/private.h
+++ b/cxl/lib/private.h
@@ -47,6 +47,8 @@ struct cxl_memdev {
 	struct list_node list;
 	unsigned long long pmem_size;
 	unsigned long long ram_size;
+	struct qos_class ram_qos_class;
+	struct qos_class pmem_qos_class;
 	int payload_max;
 	size_t lsa_size;
 	struct kmod_module *module;
diff --git a/cxl/libcxl.h b/cxl/libcxl.h
index e5c08da77f77..84d1683f234c 100644
--- a/cxl/libcxl.h
+++ b/cxl/libcxl.h
@@ -58,6 +58,11 @@ static inline enum cxl_fwl_status cxl_fwl_status_from_ident(char *status)
 	return CXL_FWL_STATUS_UNKNOWN;
 }
 
+struct qos_class {
+	int nr;
+	int *qos;
+};
+
 struct cxl_memdev;
 struct cxl_memdev *cxl_memdev_get_first(struct cxl_ctx *ctx);
 struct cxl_memdev *cxl_memdev_get_next(struct cxl_memdev *memdev);
@@ -72,6 +77,8 @@ int cxl_memdev_get_minor(struct cxl_memdev *memdev);
 struct cxl_ctx *cxl_memdev_get_ctx(struct cxl_memdev *memdev);
 unsigned long long cxl_memdev_get_pmem_size(struct cxl_memdev *memdev);
 unsigned long long cxl_memdev_get_ram_size(struct cxl_memdev *memdev);
+struct qos_class *cxl_memdev_get_pmem_qos_class(struct cxl_memdev *memdev);
+struct qos_class *cxl_memdev_get_ram_qos_class(struct cxl_memdev *memdev);
 const char *cxl_memdev_get_firmware_verison(struct cxl_memdev *memdev);
 bool cxl_memdev_fw_update_in_progress(struct cxl_memdev *memdev);
 size_t cxl_memdev_fw_update_get_remaining(struct cxl_memdev *memdev);