[PATCH 0/9] Add CXL 2.0 DVSEC Decoding

[PATCH 0/9] Add CXL 2.0 DVSEC Decoding

[Ben Widawsky]

This series improves decoding of CXL 2.0 DVSEC registers by adding more DVSEC
identifiers and adding fields for the existing decoded identifier. Not all DVSEC
fields from 2.0 spec are enabled here, only enough for what we needed in driver
bring-up. The restructuring of the code does make it easy to add support for the
remaining fields. I submitted a PR for this on github a few months ago [1]. The
spec is available for download [2].

Breakdown of patches:
1-5: Rework existing decoding to support more DVSEC IDs
6: Improve CXL Device Decoding (8.1.3 from CXL 2.0 spec)
7: Add port capabilities (8.1.5 from CXL 2.0 spec)
8: Add register locator (8.1.9 from CXL 2.0 spec)
9: Report undecoded DVSECs

Here is an example decoded output of a 5.12 based kernel running in QEMU
emulation [3]

36:00.0 Memory controller [0502]: Intel Corporation Device 0d93 (rev 01) (prog-if 10)
	Subsystem: Red Hat, Inc. Device 1100

	<...>

	Capabilities: [100 v1] Designated Vendor-Specific: Vendor=1e98 ID=0000 Rev=1 Len=56: CXL
		CXLCap:	Cache- IO+ Mem+ Mem HW Init+ HDMCount 1 Viral-
		CXLCtl:	Cache- IO+ Mem+ Cache SF Cov 0 Cache SF Gran 0 Cache Clean- Viral-
		CXLSta:	Viral-
		CXLSta2:	ResetComplete+ ResetError- PMComplete-
		Cache Size Not Reported
		Range1: 10000000-fffffff
			Valid+ Active+ Type=CDAT Class=CDAT interleave=0 timeout=1s
		Range2: 0-ffffffffffffffff
			Valid- Active- Type=Volatile Class=DRAM interleave=0 timeout=1s
	Capabilities: [138 v1] Designated Vendor-Specific: Vendor=1e98 ID=0008 Rev=0 Len=36: CXL
		Block2	BIR: bar0	ID: component registers
			RegisterOffset: 0000000000000000
		Block3	BIR: bar2	ID: CXL device registers
			RegisterOffset: 0000000000000000
	Kernel driver in use: cxl_pci

Ben Widawsky (9):
  cxl: Rename variable to match other code
  cxl: Make id check more explicit
  cxl: Collect all DVSEC Device fields
  cxl: Rework caps to new function
  cxl: Rename caps to be device caps
  cxl: Implement more device DVSEC decoding
  cxl: Add support for DVSEC port cap
  cxl: Add DVSEC Register Locator
  cxl: Add placeholder for undecoded DVSECs

 lib/header.h |  78 ++++++++++++++-----
 ls-ecaps.c   | 206 ++++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 249 insertions(+), 35 deletions(-)

[1]: https://github.com/pciutils/pciutils/pull/59
[2]: https://www.computeexpresslink.org/download-the-specification
[3]: https://gitlab.com/bwidawsk/qemu

-- 
2.31.1


-- 
2.31.1

[PATCH 1/9] cxl: Rename variable to match other code

[Ben Widawsky]

The current variable is word sized, and so this makes the CXL code match
the rest of the code.
---
 ls-ecaps.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/ls-ecaps.c b/ls-ecaps.c
index 99c55ff..edb4401 100644
--- a/ls-ecaps.c
+++ b/ls-ecaps.c
@@ -692,7 +692,7 @@ cap_rcec(struct device *d, int where)
 static void
 cap_dvsec_cxl(struct device *d, int where)
 {
-  u16 l;
+  u16 w;
 
   printf(": CXL\n");
   if (verbose < 2)
@@ -701,19 +701,19 @@ cap_dvsec_cxl(struct device *d, int where)
   if (!config_fetch(d, where + PCI_CXL_CAP, 12))
     return;
 
-  l = get_conf_word(d, where + PCI_CXL_CAP);
+  w = get_conf_word(d, where + PCI_CXL_CAP);
   printf("\t\tCXLCap:\tCache%c IO%c Mem%c Mem HW Init%c HDMCount %d Viral%c\n",
-    FLAG(l, PCI_CXL_CAP_CACHE), FLAG(l, PCI_CXL_CAP_IO), FLAG(l, PCI_CXL_CAP_MEM),
-    FLAG(l, PCI_CXL_CAP_MEM_HWINIT), PCI_CXL_CAP_HDM_CNT(l), FLAG(l, PCI_CXL_CAP_VIRAL));
+    FLAG(w, PCI_CXL_CAP_CACHE), FLAG(w, PCI_CXL_CAP_IO), FLAG(w, PCI_CXL_CAP_MEM),
+    FLAG(w, PCI_CXL_CAP_MEM_HWINIT), PCI_CXL_CAP_HDM_CNT(w), FLAG(w, PCI_CXL_CAP_VIRAL));
 
-  l = get_conf_word(d, where + PCI_CXL_CTRL);
+  w = get_conf_word(d, where + PCI_CXL_CTRL);
   printf("\t\tCXLCtl:\tCache%c IO%c Mem%c Cache SF Cov %d Cache SF Gran %d Cache Clean%c Viral%c\n",
-    FLAG(l, PCI_CXL_CTRL_CACHE), FLAG(l, PCI_CXL_CTRL_IO), FLAG(l, PCI_CXL_CTRL_MEM),
-    PCI_CXL_CTRL_CACHE_SF_COV(l), PCI_CXL_CTRL_CACHE_SF_GRAN(l), FLAG(l, PCI_CXL_CTRL_CACHE_CLN),
-    FLAG(l, PCI_CXL_CTRL_VIRAL));
+    FLAG(w, PCI_CXL_CTRL_CACHE), FLAG(w, PCI_CXL_CTRL_IO), FLAG(w, PCI_CXL_CTRL_MEM),
+    PCI_CXL_CTRL_CACHE_SF_COV(w), PCI_CXL_CTRL_CACHE_SF_GRAN(w), FLAG(w, PCI_CXL_CTRL_CACHE_CLN),
+    FLAG(w, PCI_CXL_CTRL_VIRAL));
 
-  l = get_conf_word(d, where + PCI_CXL_STATUS);
-  printf("\t\tCXLSta:\tViral%c\n", FLAG(l, PCI_CXL_STATUS_VIRAL));
+  w = get_conf_word(d, where + PCI_CXL_STATUS);
+  printf("\t\tCXLSta:\tViral%c\n", FLAG(w, PCI_CXL_STATUS_VIRAL));
 }
 
 static void
-- 
2.31.1

[PATCH 2/9] cxl: Make id check more explicit

[Ben Widawsky]

Currently only type 0 DVSEC caps are handled. Moving this check will
allow more robust type handling in the future.

Should be no functional change.

Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
---
 ls-ecaps.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/ls-ecaps.c b/ls-ecaps.c
index edb4401..83ca93e 100644
--- a/ls-ecaps.c
+++ b/ls-ecaps.c
@@ -690,7 +690,7 @@ cap_rcec(struct device *d, int where)
 }
 
 static void
-cap_dvsec_cxl(struct device *d, int where)
+cap_dvsec_cxl(struct device *d, int id, int where)
 {
   u16 w;
 
@@ -698,6 +698,9 @@ cap_dvsec_cxl(struct device *d, int where)
   if (verbose < 2)
     return;
 
+  if (id != 0)
+    return;
+
   if (!config_fetch(d, where + PCI_CXL_CAP, 12))
     return;
 
@@ -734,8 +737,8 @@ cap_dvsec(struct device *d, int where)
   u16 id = get_conf_long(d, where + PCI_DVSEC_HEADER2);
 
   printf("Vendor=%04x ID=%04x Rev=%d Len=%d", vendor, id, rev, len);
-  if (vendor == PCI_DVSEC_VENDOR_ID_CXL && id == PCI_DVSEC_ID_CXL && len >= 16)
-    cap_dvsec_cxl(d, where);
+  if (vendor == PCI_DVSEC_VENDOR_ID_CXL && len >= 16)
+    cap_dvsec_cxl(d, id, where);
   else
     printf(" <?>\n");
 }
-- 
2.31.1

[PATCH 3/9] cxl: Collect all DVSEC Device fields

[Ben Widawsky]

Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
---
 ls-ecaps.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ls-ecaps.c b/ls-ecaps.c
index 83ca93e..2b3f0f9 100644
--- a/ls-ecaps.c
+++ b/ls-ecaps.c
@@ -701,7 +701,7 @@ cap_dvsec_cxl(struct device *d, int id, int where)
   if (id != 0)
     return;
 
-  if (!config_fetch(d, where + PCI_CXL_CAP, 12))
+  if (!config_fetch(d, where + PCI_CXL_CAP, 0x38 - 0xa))
     return;
 
   w = get_conf_word(d, where + PCI_CXL_CAP);
-- 
2.31.1

[PATCH 4/9] cxl: Rework caps to new function

[Ben Widawsky]

This will help upcoming caps

Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
---
 ls-ecaps.c | 38 ++++++++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/ls-ecaps.c b/ls-ecaps.c
index 2b3f0f9..c2a13d5 100644
--- a/ls-ecaps.c
+++ b/ls-ecaps.c
@@ -690,35 +690,49 @@ cap_rcec(struct device *d, int where)
 }
 
 static void
-cap_dvsec_cxl(struct device *d, int id, int where)
+dvsec_cxl_device(uint8_t *data, int rev)
 {
   u16 w;
 
-  printf(": CXL\n");
-  if (verbose < 2)
-    return;
-
-  if (id != 0)
-    return;
-
-  if (!config_fetch(d, where + PCI_CXL_CAP, 0x38 - 0xa))
+  /* Legacy 1.1 revs aren't handled */
+  if (rev != 1)
     return;
 
-  w = get_conf_word(d, where + PCI_CXL_CAP);
+  w = *(u16 *)(data + PCI_CXL_CAP);
   printf("\t\tCXLCap:\tCache%c IO%c Mem%c Mem HW Init%c HDMCount %d Viral%c\n",
     FLAG(w, PCI_CXL_CAP_CACHE), FLAG(w, PCI_CXL_CAP_IO), FLAG(w, PCI_CXL_CAP_MEM),
     FLAG(w, PCI_CXL_CAP_MEM_HWINIT), PCI_CXL_CAP_HDM_CNT(w), FLAG(w, PCI_CXL_CAP_VIRAL));
 
-  w = get_conf_word(d, where + PCI_CXL_CTRL);
+  w = *(u16 *)(data + PCI_CXL_CTRL);
   printf("\t\tCXLCtl:\tCache%c IO%c Mem%c Cache SF Cov %d Cache SF Gran %d Cache Clean%c Viral%c\n",
     FLAG(w, PCI_CXL_CTRL_CACHE), FLAG(w, PCI_CXL_CTRL_IO), FLAG(w, PCI_CXL_CTRL_MEM),
     PCI_CXL_CTRL_CACHE_SF_COV(w), PCI_CXL_CTRL_CACHE_SF_GRAN(w), FLAG(w, PCI_CXL_CTRL_CACHE_CLN),
     FLAG(w, PCI_CXL_CTRL_VIRAL));
 
-  w = get_conf_word(d, where + PCI_CXL_STATUS);
+  w = *(u16 *)(data + PCI_CXL_STATUS);
   printf("\t\tCXLSta:\tViral%c\n", FLAG(w, PCI_CXL_STATUS_VIRAL));
 }
 
+static void
+cap_dvsec_cxl(struct device *d, int id, int where)
+{
+  u8 rev;
+
+  printf(": CXL\n");
+  if (verbose < 2)
+    return;
+
+  if (id != 0)
+    return;
+
+  rev = BITS(get_conf_byte(d, where + 0x6), 0, 4);
+
+  if (!config_fetch(d, where, 0x38))
+    return;
+
+  dvsec_cxl_device(d->config + where, rev);
+}
+
 static void
 cap_dvsec(struct device *d, int where)
 {
-- 
2.31.1

[PATCH 5/9] cxl: Rename caps to be device caps

[Ben Widawsky]

Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
---
 lib/header.h | 36 ++++++++++++++++++------------------
 ls-ecaps.c   | 18 +++++++++---------
 2 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/lib/header.h b/lib/header.h
index 170e5c1..3ff514a 100644
--- a/lib/header.h
+++ b/lib/header.h
@@ -1067,24 +1067,24 @@
 #define PCI_DVSEC_VENDOR_ID_CXL	0x1e98	/* Designated Vendor-Specific Vendor ID for CXL */
 #define PCI_DVSEC_ID_CXL	0	/* Designated Vendor-Specific ID for Intel CXL */
 
-/* PCIe CXL Designated Vendor-Specific Capabilities, Control, Status */
-#define PCI_CXL_CAP		0x0a	/* CXL Capability Register */
-#define  PCI_CXL_CAP_CACHE	0x0001	/* CXL.cache Protocol Support */
-#define  PCI_CXL_CAP_IO		0x0002	/* CXL.io Protocol Support */
-#define  PCI_CXL_CAP_MEM	0x0004	/* CXL.mem Protocol Support */
-#define  PCI_CXL_CAP_MEM_HWINIT	0x0008	/* CXL.mem Initializes with HW/FW Support */
-#define  PCI_CXL_CAP_HDM_CNT(x)	(((x) & (3 << 4)) >> 4)	/* CXL Number of HDM ranges */
-#define  PCI_CXL_CAP_VIRAL	0x4000	/* CXL Viral Handling Support */
-#define PCI_CXL_CTRL		0x0c	/* CXL Control Register */
-#define  PCI_CXL_CTRL_CACHE	0x0001	/* CXL.cache Protocol Enable */
-#define  PCI_CXL_CTRL_IO	0x0002	/* CXL.io Protocol Enable */
-#define  PCI_CXL_CTRL_MEM	0x0004	/* CXL.mem Protocol Enable */
-#define  PCI_CXL_CTRL_CACHE_SF_COV(x)	(((x) & (0x1f << 3)) >> 3) /* Snoop Filter Coverage */
-#define  PCI_CXL_CTRL_CACHE_SF_GRAN(x)	(((x) & (0x7 << 8)) >> 8) /* Snoop Filter Granularity */
-#define  PCI_CXL_CTRL_CACHE_CLN	0x0800	/* CXL.cache Performance Hint on Clean Evictions */
-#define  PCI_CXL_CTRL_VIRAL	0x4000	/* CXL Viral Handling Enable */
-#define PCI_CXL_STATUS		0x0e	/* CXL Status Register */
-#define  PCI_CXL_STATUS_VIRAL	0x4000	/* CXL Viral Handling Status */
+/* PCIe CXL Designated Vendor-Specific Capabilities for Devices: Control, Status */
+#define PCI_CXL_DEV_CAP			0x0a	/* CXL Capability Register */
+#define  PCI_CXL_DEV_CAP_CACHE		0x0001	/* CXL.cache Protocol Support */
+#define  PCI_CXL_DEV_CAP_IO		0x0002	/* CXL.io Protocol Support */
+#define  PCI_CXL_DEV_CAP_MEM		0x0004	/* CXL.mem Protocol Support */
+#define  PCI_CXL_DEV_CAP_MEM_HWINIT	0x0008	/* CXL.mem Initializes with HW/FW Support */
+#define  PCI_CXL_DEV_CAP_HDM_CNT(x)	(((x) & (3 << 4)) >> 4)	/* CXL Number of HDM ranges */
+#define  PCI_CXL_DEV_CAP_VIRAL		0x4000	/* CXL Viral Handling Support */
+#define PCI_CXL_DEV_CTRL		0x0c	/* CXL Control Register */
+#define  PCI_CXL_DEV_CTRL_CACHE		0x0001	/* CXL.cache Protocol Enable */
+#define  PCI_CXL_DEV_CTRL_IO		0x0002	/* CXL.io Protocol Enable */
+#define  PCI_CXL_DEV_CTRL_MEM		0x0004	/* CXL.mem Protocol Enable */
+#define  PCI_CXL_DEV_CTRL_CACHE_SF_COV(x) (((x) & (0x1f << 3)) >> 3) /* Snoop Filter Coverage */
+#define  PCI_CXL_DEV_CTRL_CACHE_SF_GRAN(x) (((x) & (0x7 << 8)) >> 8) /* Snoop Filter Granularity */
+#define  PCI_CXL_DEV_CTRL_CACHE_CLN	0x0800	/* CXL.cache Performance Hint on Clean Evictions */
+#define  PCI_CXL_DEV_CTRL_VIRAL		0x4000	/* CXL Viral Handling Enable */
+#define PCI_CXL_DEV_STATUS		0x0e	/* CXL Status Register */
+#define  PCI_CXL_DEV_STATUS_VIRAL	0x4000	/* CXL Viral Handling Status */
 
 /* Access Control Services */
 #define PCI_ACS_CAP		0x04	/* ACS Capability Register */
diff --git a/ls-ecaps.c b/ls-ecaps.c
index c2a13d5..443d11d 100644
--- a/ls-ecaps.c
+++ b/ls-ecaps.c
@@ -698,19 +698,19 @@ dvsec_cxl_device(uint8_t *data, int rev)
   if (rev != 1)
     return;
 
-  w = *(u16 *)(data + PCI_CXL_CAP);
+  w = *(u16 *)(data + PCI_CXL_DEV_CAP);
   printf("\t\tCXLCap:\tCache%c IO%c Mem%c Mem HW Init%c HDMCount %d Viral%c\n",
-    FLAG(w, PCI_CXL_CAP_CACHE), FLAG(w, PCI_CXL_CAP_IO), FLAG(w, PCI_CXL_CAP_MEM),
-    FLAG(w, PCI_CXL_CAP_MEM_HWINIT), PCI_CXL_CAP_HDM_CNT(w), FLAG(w, PCI_CXL_CAP_VIRAL));
+    FLAG(w, PCI_CXL_DEV_CAP_CACHE), FLAG(w, PCI_CXL_DEV_CAP_IO), FLAG(w, PCI_CXL_DEV_CAP_MEM),
+    FLAG(w, PCI_CXL_DEV_CAP_MEM_HWINIT), PCI_CXL_DEV_CAP_HDM_CNT(w), FLAG(w, PCI_CXL_DEV_CAP_VIRAL));
 
-  w = *(u16 *)(data + PCI_CXL_CTRL);
+  w = *(u16 *)(data + PCI_CXL_DEV_CTRL);
   printf("\t\tCXLCtl:\tCache%c IO%c Mem%c Cache SF Cov %d Cache SF Gran %d Cache Clean%c Viral%c\n",
-    FLAG(w, PCI_CXL_CTRL_CACHE), FLAG(w, PCI_CXL_CTRL_IO), FLAG(w, PCI_CXL_CTRL_MEM),
-    PCI_CXL_CTRL_CACHE_SF_COV(w), PCI_CXL_CTRL_CACHE_SF_GRAN(w), FLAG(w, PCI_CXL_CTRL_CACHE_CLN),
-    FLAG(w, PCI_CXL_CTRL_VIRAL));
+    FLAG(w, PCI_CXL_DEV_CTRL_CACHE), FLAG(w, PCI_CXL_DEV_CTRL_IO), FLAG(w, PCI_CXL_DEV_CTRL_MEM),
+    PCI_CXL_DEV_CTRL_CACHE_SF_COV(w), PCI_CXL_DEV_CTRL_CACHE_SF_GRAN(w), FLAG(w, PCI_CXL_DEV_CTRL_CACHE_CLN),
+    FLAG(w, PCI_CXL_DEV_CTRL_VIRAL));
 
-  w = *(u16 *)(data + PCI_CXL_STATUS);
-  printf("\t\tCXLSta:\tViral%c\n", FLAG(w, PCI_CXL_STATUS_VIRAL));
+  w = *(u16 *)(data + PCI_CXL_DEV_STATUS);
+  printf("\t\tCXLSta:\tViral%c\n", FLAG(w, PCI_CXL_DEV_STATUS_VIRAL));
 }
 
 static void
-- 
2.31.1

[PATCH 6/9] cxl: Implement more device DVSEC decoding

[Ben Widawsky]

---
 lib/header.h | 23 +++++++++++++++++++
 ls-ecaps.c   | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+)

diff --git a/lib/header.h b/lib/header.h
index 3ff514a..c346548 100644
--- a/lib/header.h
+++ b/lib/header.h
@@ -1085,6 +1085,29 @@
 #define  PCI_CXL_DEV_CTRL_VIRAL		0x4000	/* CXL Viral Handling Enable */
 #define PCI_CXL_DEV_STATUS		0x0e	/* CXL Status Register */
 #define  PCI_CXL_DEV_STATUS_VIRAL	0x4000	/* CXL Viral Handling Status */
+#define PCI_CXL_DEV_STATUS2		0x12
+#define  PCI_CXL_DEV_STATUS_CACHE_INV	0x0001
+#define  PCI_CXL_DEV_STATUS_RC		0x0002  /* Device Reset Complete */
+#define  PCI_CXL_DEV_STATUS_RE		0x0004  /* Device Reset Error */
+#define  PCI_CXL_DEV_STATUS_PMC		0x8000  /* Power Management Init Complete */
+#define PCI_CXL_DEV_CAP2		0x16
+#define  PCI_CXL_DEV_CAP2_CACHE_UNK	0x0000	/* Cache Size Isn't Reported */
+#define  PCI_CXL_DEV_CAP2_CACHE_64K	0x0001  /* Unit Size 64K */
+#define  PCI_CXL_DEV_CAP2_CACHE_1M	0x0002  /* Unit Size 1M */
+#define PCI_CXL_DEV_RANGE1_SIZE_HI	0x18
+#define PCI_CXL_DEV_RANGE1_SIZE_LO	0x1c
+#define  PCI_CXL_RANGE_VALID		0x0001
+#define  PCI_CXL_RANGE_ACTIVE		0x0002
+#define  PCI_CXL_RANGE_TYPE(x)		(((x) >> 2) & 0x7)
+#define  PCI_CXL_RANGE_CLASS(x)		(((x) >> 5) & 0x7)
+#define  PCI_CXL_RANGE_INTERLEAVE(x)	(((x) >> 8) & 0x1f)
+#define  PCI_CXL_RANGE_TIMEOUT(x)	(((x) >> 13) & 0x7)
+#define PCI_CXL_DEV_RANGE1_BASE_HI	0x20
+#define PCI_CXL_DEV_RANGE1_BASE_LO	0x24
+#define PCI_CXL_DEV_RANGE2_SIZE_HI	0x28
+#define PCI_CXL_DEV_RANGE2_SIZE_LO	0x2c
+#define PCI_CXL_DEV_RANGE2_BASE_HI	0x30
+#define PCI_CXL_DEV_RANGE2_BASE_LO	0x34
 
 /* Access Control Services */
 #define PCI_ACS_CAP		0x04	/* ACS Capability Register */
diff --git a/ls-ecaps.c b/ls-ecaps.c
index 443d11d..8072bbe 100644
--- a/ls-ecaps.c
+++ b/ls-ecaps.c
@@ -689,9 +689,31 @@ cap_rcec(struct device *d, int where)
     printf("\t\tAssociatedBusNumbers: %02x-%02x\n", nextbusn, lastbusn );
 }
 
+static void
+cxl_range(u64 size, u64 base, int n)
+{
+  u32 interleave[] = { 0, 256, 4096, 512, 1024, 2048, 8192, 16384 };
+  const char *type[] = { "Volatile", "Non-volatile", "CDAT" };
+  const char *class[] = { "DRAM", "Storage", "CDAT" };
+  u16 w;
+
+  w = (u16) base;
+
+  base &= ~0x0fffffffULL;
+
+  printf("\t\tRange%d: %"PRIx64"-%"PRIx64"\n", n, base, base + size - 1);
+  printf("\t\t\tValid%c Active%c Type=%s Class=%s interleave=%d timeout=%ds\n",
+    FLAG(w, PCI_CXL_RANGE_VALID), FLAG(w, PCI_CXL_RANGE_ACTIVE),
+    type[PCI_CXL_RANGE_TYPE(w)], class[PCI_CXL_RANGE_CLASS(w)],
+    interleave[PCI_CXL_RANGE_INTERLEAVE(w)],
+    1 << (PCI_CXL_RANGE_TIMEOUT(w) * 2));
+}
+
 static void
 dvsec_cxl_device(uint8_t *data, int rev)
 {
+  u32 cache_size, cache_unit_size, l;
+  u64 range_base, range_size;
   u16 w;
 
   /* Legacy 1.1 revs aren't handled */
@@ -711,6 +733,49 @@ dvsec_cxl_device(uint8_t *data, int rev)
 
   w = *(u16 *)(data + PCI_CXL_DEV_STATUS);
   printf("\t\tCXLSta:\tViral%c\n", FLAG(w, PCI_CXL_DEV_STATUS_VIRAL));
+
+  w = *(u16 *)(data + PCI_CXL_DEV_STATUS2);
+  printf("\t\tCXLSta2:\tResetComplete%c ResetError%c PMComplete%c\n",
+    FLAG(w, PCI_CXL_DEV_STATUS_RC), FLAG(w,PCI_CXL_DEV_STATUS_RE), FLAG(w, PCI_CXL_DEV_STATUS_PMC));
+
+  w = *(u16 *)(data + PCI_CXL_DEV_CAP2);
+  cache_unit_size = BITS(w, 0, 4);
+  cache_size = BITS(w, 8, 8);
+  switch (cache_unit_size)
+    {
+      case PCI_CXL_DEV_CAP2_CACHE_1M:
+        printf("\t\tCache Size: %08x\n", cache_size * (1<<20));
+	break;
+      case PCI_CXL_DEV_CAP2_CACHE_64K:
+        printf("\t\tCache Size: %08x\n", cache_size * (64<<10));
+	break;
+      case PCI_CXL_DEV_CAP2_CACHE_UNK:
+        printf("\t\tCache Size Not Reported\n");
+	break;
+      default:
+        printf("\t\tCache Size: %d of unknown unit size (%d)\n", cache_size, cache_unit_size);
+	break;
+    }
+
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE1_SIZE_HI);
+  range_size = (u64) l << 32;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE1_SIZE_LO);
+  range_size |= l;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE1_BASE_HI);
+  range_base = (u64) l << 32;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE1_BASE_LO);
+  range_base |= l;
+  cxl_range(range_base, range_size, 1);
+
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE2_SIZE_HI);
+  range_size = (u64) l << 32;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE2_SIZE_LO);
+  range_size |= l;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE2_BASE_HI);
+  range_base = (u64) l << 32;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE2_BASE_LO);
+  range_base |= l;
+  cxl_range(range_base, range_size, 2);
 }
 
 static void
-- 
2.31.1

[PATCH v2 6/9] cxl: Implement more device DVSEC decoding

[Ben Widawsky]

V2: - base and size were swapped for range decoding

---
 lib/header.h | 23 +++++++++++++++++++
 ls-ecaps.c   | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+)

diff --git a/lib/header.h b/lib/header.h
index 3ff514a..c346548 100644
--- a/lib/header.h
+++ b/lib/header.h
@@ -1085,6 +1085,29 @@
 #define  PCI_CXL_DEV_CTRL_VIRAL		0x4000	/* CXL Viral Handling Enable */
 #define PCI_CXL_DEV_STATUS		0x0e	/* CXL Status Register */
 #define  PCI_CXL_DEV_STATUS_VIRAL	0x4000	/* CXL Viral Handling Status */
+#define PCI_CXL_DEV_STATUS2		0x12
+#define  PCI_CXL_DEV_STATUS_CACHE_INV	0x0001
+#define  PCI_CXL_DEV_STATUS_RC		0x0002  /* Device Reset Complete */
+#define  PCI_CXL_DEV_STATUS_RE		0x0004  /* Device Reset Error */
+#define  PCI_CXL_DEV_STATUS_PMC		0x8000  /* Power Management Init Complete */
+#define PCI_CXL_DEV_CAP2		0x16
+#define  PCI_CXL_DEV_CAP2_CACHE_UNK	0x0000	/* Cache Size Isn't Reported */
+#define  PCI_CXL_DEV_CAP2_CACHE_64K	0x0001  /* Unit Size 64K */
+#define  PCI_CXL_DEV_CAP2_CACHE_1M	0x0002  /* Unit Size 1M */
+#define PCI_CXL_DEV_RANGE1_SIZE_HI	0x18
+#define PCI_CXL_DEV_RANGE1_SIZE_LO	0x1c
+#define  PCI_CXL_RANGE_VALID		0x0001
+#define  PCI_CXL_RANGE_ACTIVE		0x0002
+#define  PCI_CXL_RANGE_TYPE(x)		(((x) >> 2) & 0x7)
+#define  PCI_CXL_RANGE_CLASS(x)		(((x) >> 5) & 0x7)
+#define  PCI_CXL_RANGE_INTERLEAVE(x)	(((x) >> 8) & 0x1f)
+#define  PCI_CXL_RANGE_TIMEOUT(x)	(((x) >> 13) & 0x7)
+#define PCI_CXL_DEV_RANGE1_BASE_HI	0x20
+#define PCI_CXL_DEV_RANGE1_BASE_LO	0x24
+#define PCI_CXL_DEV_RANGE2_SIZE_HI	0x28
+#define PCI_CXL_DEV_RANGE2_SIZE_LO	0x2c
+#define PCI_CXL_DEV_RANGE2_BASE_HI	0x30
+#define PCI_CXL_DEV_RANGE2_BASE_LO	0x34
 
 /* Access Control Services */
 #define PCI_ACS_CAP		0x04	/* ACS Capability Register */
diff --git a/ls-ecaps.c b/ls-ecaps.c
index 443d11d..7d4225b 100644
--- a/ls-ecaps.c
+++ b/ls-ecaps.c
@@ -689,9 +689,31 @@ cap_rcec(struct device *d, int where)
     printf("\t\tAssociatedBusNumbers: %02x-%02x\n", nextbusn, lastbusn );
 }
 
+static void
+cxl_range(u64 base, u64 size, int n)
+{
+  u32 interleave[] = { 0, 256, 4096, 512, 1024, 2048, 8192, 16384 };
+  const char *type[] = { "Volatile", "Non-volatile", "CDAT" };
+  const char *class[] = { "DRAM", "Storage", "CDAT" };
+  u16 w;
+
+  w = (u16) size;
+
+  size &= ~0x0fffffffULL;
+
+  printf("\t\tRange%d: %"PRIx64"-%"PRIx64"\n", n, base, base + size - 1);
+  printf("\t\t\tValid%c Active%c Type=%s Class=%s interleave=%d timeout=%ds\n",
+    FLAG(w, PCI_CXL_RANGE_VALID), FLAG(w, PCI_CXL_RANGE_ACTIVE),
+    type[PCI_CXL_RANGE_TYPE(w)], class[PCI_CXL_RANGE_CLASS(w)],
+    interleave[PCI_CXL_RANGE_INTERLEAVE(w)],
+    1 << (PCI_CXL_RANGE_TIMEOUT(w) * 2));
+}
+
 static void
 dvsec_cxl_device(uint8_t *data, int rev)
 {
+  u32 cache_size, cache_unit_size, l;
+  u64 range_base, range_size;
   u16 w;
 
   /* Legacy 1.1 revs aren't handled */
@@ -711,6 +733,49 @@ dvsec_cxl_device(uint8_t *data, int rev)
 
   w = *(u16 *)(data + PCI_CXL_DEV_STATUS);
   printf("\t\tCXLSta:\tViral%c\n", FLAG(w, PCI_CXL_DEV_STATUS_VIRAL));
+
+  w = *(u16 *)(data + PCI_CXL_DEV_STATUS2);
+  printf("\t\tCXLSta2:\tResetComplete%c ResetError%c PMComplete%c\n",
+    FLAG(w, PCI_CXL_DEV_STATUS_RC), FLAG(w,PCI_CXL_DEV_STATUS_RE), FLAG(w, PCI_CXL_DEV_STATUS_PMC));
+
+  w = *(u16 *)(data + PCI_CXL_DEV_CAP2);
+  cache_unit_size = BITS(w, 0, 4);
+  cache_size = BITS(w, 8, 8);
+  switch (cache_unit_size)
+    {
+      case PCI_CXL_DEV_CAP2_CACHE_1M:
+        printf("\t\tCache Size: %08x\n", cache_size * (1<<20));
+	break;
+      case PCI_CXL_DEV_CAP2_CACHE_64K:
+        printf("\t\tCache Size: %08x\n", cache_size * (64<<10));
+	break;
+      case PCI_CXL_DEV_CAP2_CACHE_UNK:
+        printf("\t\tCache Size Not Reported\n");
+	break;
+      default:
+        printf("\t\tCache Size: %d of unknown unit size (%d)\n", cache_size, cache_unit_size);
+	break;
+    }
+
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE1_SIZE_HI);
+  range_size = (u64) l << 32;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE1_SIZE_LO);
+  range_size |= l;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE1_BASE_HI);
+  range_base = (u64) l << 32;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE1_BASE_LO);
+  range_base |= l;
+  cxl_range(range_base, range_size, 1);
+
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE2_SIZE_HI);
+  range_size = (u64) l << 32;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE2_SIZE_LO);
+  range_size |= l;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE2_BASE_HI);
+  range_base = (u64) l << 32;
+  l = *(u32 *)(data + PCI_CXL_DEV_RANGE2_BASE_LO);
+  range_base |= l;
+  cxl_range(range_base, range_size, 2);
 }
 
 static void
-- 
2.31.1

[PATCH 7/9] cxl: Add support for DVSEC port cap

[Ben Widawsky]

Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
---
 lib/header.h | 11 +++++++++++
 ls-ecaps.c   | 43 +++++++++++++++++++++++++++++++++++++------
 2 files changed, 48 insertions(+), 6 deletions(-)

diff --git a/lib/header.h b/lib/header.h
index c346548..8141e13 100644
--- a/lib/header.h
+++ b/lib/header.h
@@ -1109,6 +1109,17 @@
 #define PCI_CXL_DEV_RANGE2_BASE_HI	0x30
 #define PCI_CXL_DEV_RANGE2_BASE_LO	0x34
 
+/* PCIe CXL 2.0 Designated Vendor-Specific Capabilities for Ports */
+#define PCI_CXL_PORT_EXT_STATUS 0x0a		/* Port Extension Status */
+#define  PCI_CXL_PORT_PM_INIT_COMPLETE 0x1	/* Port Power Management Initialization Complete */
+#define PCI_CXL_PORT_CTRL 0x0c			/* Port Control Override */
+#define  PCI_CXL_PORT_UNMASK_SBR 0x0001		/* Unmask SBR */
+#define  PCI_CXL_PORT_UNMASK_LINK 0x0002	/* Unmask Link Disable */
+#define  PCI_CXL_PORT_ALT_MEMORY 0x0004		/* Alt Memory and ID Space Enable */
+#define  PCI_CXL_PORT_ALT_BME 0x0008		/* Alt BME */
+#define PCI_CXL_PORT_ALT_BUS_BASE 0xe
+#define PCI_CXL_PORT_ALT_BUS_LIMIT 0xf
+
 /* Access Control Services */
 #define PCI_ACS_CAP		0x04	/* ACS Capability Register */
 #define PCI_ACS_CAP_VALID	0x0001	/* ACS Source Validation */
diff --git a/ls-ecaps.c b/ls-ecaps.c
index 8072bbe..b11d5a9 100644
--- a/ls-ecaps.c
+++ b/ls-ecaps.c
@@ -778,6 +778,28 @@ dvsec_cxl_device(uint8_t *data, int rev)
   cxl_range(range_base, range_size, 2);
 }
 
+static void
+dvsec_cxl_port(uint8_t* data, int rev)
+{
+  u16 w;
+  u8 b1, b2;
+
+  if (rev != 0)
+    return;
+
+  w = *(u16 *)(data + PCI_CXL_PORT_EXT_STATUS);
+  printf("\t\tCXLPortSta:\tPMComplete%c\n", FLAG(w, PCI_CXL_PORT_EXT_STATUS));
+
+  w = *(u16 *)(data + PCI_CXL_PORT_CTRL);
+  printf("\t\tCXLPortCtl:\tUnmaskSBR%c UnmaskLinkDisable%c AltMem%c AltBME%c\n",
+    FLAG(w, PCI_CXL_PORT_UNMASK_SBR), FLAG(w, PCI_CXL_PORT_UNMASK_LINK),
+    FLAG(w, PCI_CXL_PORT_ALT_MEMORY), FLAG(w, PCI_CXL_PORT_ALT_BME));
+
+  b1 = *(u8 *)(data + PCI_CXL_PORT_ALT_BUS_BASE);
+  b2 = *(u8 *)(data + PCI_CXL_PORT_ALT_BUS_LIMIT);
+  printf("\t\tAlternateBus: %02x-%02x\n", b1, b2);
+}
+
 static void
 cap_dvsec_cxl(struct device *d, int id, int where)
 {
@@ -787,15 +809,24 @@ cap_dvsec_cxl(struct device *d, int id, int where)
   if (verbose < 2)
     return;
 
-  if (id != 0)
-    return;
-
   rev = BITS(get_conf_byte(d, where + 0x6), 0, 4);
 
-  if (!config_fetch(d, where, 0x38))
-    return;
+  switch (id) {
+    case 0:
+      if (!config_fetch(d, where, 0x38))
+        return;
+
+      dvsec_cxl_device(d->config + where, rev);
+      break;
+    case 3:
+      if (!config_fetch(d, where, 0x28))
+        return;
 
-  dvsec_cxl_device(d->config + where, rev);
+      dvsec_cxl_port(d->config + where, rev);
+      break;
+    default:
+      break;
+  }
 }
 
 static void
-- 
2.31.1

[PATCH 8/9] cxl: Add DVSEC Register Locator

[Ben Widawsky]

Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
---
 lib/header.h |  8 ++++++++
 ls-ecaps.c   | 44 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/lib/header.h b/lib/header.h
index 8141e13..b77a611 100644
--- a/lib/header.h
+++ b/lib/header.h
@@ -1120,6 +1120,14 @@
 #define PCI_CXL_PORT_ALT_BUS_BASE 0xe
 #define PCI_CXL_PORT_ALT_BUS_LIMIT 0xf
 
+/* PCIe CXL 2.0 Designated Vendor-Specific Capabilities for Register Locator */
+#define PCI_CXL_RL_BASE0_LO 0x0c
+#define PCI_CXL_RL_BASE0_HI 0x10
+#define PCI_CXL_RL_BASE1_LO 0x14
+#define PCI_CXL_RL_BASE1_HI 0x18
+#define PCI_CXL_RL_BASE2_LO 0x1c
+#define PCI_CXL_RL_BASE2_HI 0x20
+
 /* Access Control Services */
 #define PCI_ACS_CAP		0x04	/* ACS Capability Register */
 #define PCI_ACS_CAP_VALID	0x0001	/* ACS Source Validation */
diff --git a/ls-ecaps.c b/ls-ecaps.c
index b11d5a9..a0ef83d 100644
--- a/ls-ecaps.c
+++ b/ls-ecaps.c
@@ -800,9 +800,46 @@ dvsec_cxl_port(uint8_t* data, int rev)
   printf("\t\tAlternateBus: %02x-%02x\n", b1, b2);
 }
 
+static const char *id[] = {
+  "empty",
+  "component registers",
+  "BAR virtualization",
+  "CXL device registers"};
+
+static inline void
+dvsec_decode_block(uint32_t lo, uint32_t hi, char which)
+{
+  u64 base_hi = hi, base_lo;
+  u8 bir, block_id;
+
+  bir = BITS(lo, 0, 3);
+  block_id = BITS(lo, 8, 8);
+  base_lo = BITS(lo, 16, 16);
+
+  if (!block_id)
+    return;
+
+  printf("\t\tBlock%c\tBIR: bar%d\tID: %s\n", which, bir, id[block_id]);
+  printf("\t\t\tRegisterOffset: %016" PCI_U64_FMT_X "\n", (base_hi << 32ULL) | base_lo << 16);
+}
+
+static void
+dvsec_cxl_register_locator(uint8_t* data, int len, int rev)
+{
+  int i, j;
+
+  if (rev != 0)
+    return;
+
+  for (i = 0xc, j = 1; i < len; i += 8, j++) {
+    dvsec_decode_block(*(u32 *)(data + i), *(u32 *)(data + i + 4), j + 0x31);
+  }
+}
+
 static void
 cap_dvsec_cxl(struct device *d, int id, int where)
 {
+  u16 len;
   u8 rev;
 
   printf(": CXL\n");
@@ -824,6 +861,13 @@ cap_dvsec_cxl(struct device *d, int id, int where)
 
       dvsec_cxl_port(d->config + where, rev);
       break;
+    case 8:
+      len = BITS(get_conf_word(d, where + 0x6), 4, 12);
+      if (!config_fetch(d, where, len))
+        return;
+
+      dvsec_cxl_register_locator(d->config + where, len, rev);
+      break;
     default:
       break;
   }
-- 
2.31.1

[PATCH 9/9] cxl: Add placeholder for undecoded DVSECs

[Ben Widawsky]

Signed-off-by: Ben Widawsky <ben.widawsky@intel.com>
---
 ls-ecaps.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ls-ecaps.c b/ls-ecaps.c
index a0ef83d..2d064be 100644
--- a/ls-ecaps.c
+++ b/ls-ecaps.c
@@ -868,6 +868,21 @@ cap_dvsec_cxl(struct device *d, int id, int where)
 
       dvsec_cxl_register_locator(d->config + where, len, rev);
       break;
+    case 2:
+      printf("\t\tNon-CXL Function Map DVSEC\n");
+      break;
+    case 4:
+      printf("\t\tGPF DVSEC for Port\n");
+      break;
+    case 5:
+      printf("\t\tGPF DVSEC for Device\n");
+      break;
+    case 7:
+      printf("\t\tPCIe DVSEC Flex Bus Port\n");
+      break;
+    case 9:
+      printf("\t\tMLD DVSEC\n");
+      break;
     default:
       break;
   }
-- 
2.31.1