From: Dave Jiang <dave.jiang@intel.com>
To: linux-cxl@vger.kernel.org
Cc: Jonathan Cameron <Jonathan.Cameron@huawei.com>,
dan.j.williams@intel.com, ira.weiny@intel.com,
vishal.l.verma@intel.com, alison.schofield@intel.com,
Jonathan.Cameron@huawei.com, dave@stgolabs.net
Subject: [PATCH v11 16/22] cxl: Calculate and store PCI link latency for the downstream ports
Date: Thu, 12 Oct 2023 11:55:06 -0700 [thread overview]
Message-ID: <169713690650.2205276.6608428147494650680.stgit@djiang5-mobl3> (raw)
In-Reply-To: <169713674328.2205276.10184241477215488339.stgit@djiang5-mobl3>
The latency is calculated by dividing the flit size over the bandwidth. Add
support to retrieve the flit size for the CXL switch device and calculate
the latency of the PCIe link. Cache the latency number with cxl_dport.
Reviewed-by: Jonathan Cameron <Jonathan.Cameron@huawei.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
---
v11:
- Fix 0-day report on cxl_pci_get_latency comment header. '-' to ':' for
@pdev
---
drivers/cxl/core/pci.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++
drivers/cxl/core/port.c | 6 ++++
drivers/cxl/cxl.h | 4 +++
drivers/cxl/cxlpci.h | 15 ++++++++++
4 files changed, 98 insertions(+)
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index c86bcc181a5d..2ba6655dcfd1 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
+#include <linux/units.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/device.h>
#include <linux/delay.h>
@@ -783,3 +784,75 @@ pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
return PCI_ERS_RESULT_NEED_RESET;
}
EXPORT_SYMBOL_NS_GPL(cxl_error_detected, CXL);
+
+extern const unsigned char pcie_link_speed[];
+
+static enum pci_bus_speed get_link_speed(struct pci_dev *pdev)
+{
+ u16 linkstat;
+ int err;
+
+ err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &linkstat);
+ if (err)
+ return -EINVAL;
+
+ return pcie_link_speed[linkstat & PCI_EXP_LNKSTA_CLS];
+}
+
+static int pci_bus_speed_to_mbps(enum pci_bus_speed speed)
+{
+ switch (speed) {
+ case PCIE_SPEED_2_5GT:
+ return 2500;
+ case PCIE_SPEED_5_0GT:
+ return 5000;
+ case PCIE_SPEED_8_0GT:
+ return 8000;
+ case PCIE_SPEED_16_0GT:
+ return 16000;
+ case PCIE_SPEED_32_0GT:
+ return 32000;
+ case PCIE_SPEED_64_0GT:
+ return 64000;
+ default:
+ break;
+ }
+
+ return -EINVAL;
+}
+
+static int cxl_flit_size(struct pci_dev *pdev)
+{
+ if (cxl_pci_flit_256(pdev))
+ return 256;
+
+ return 68;
+}
+
+/**
+ * cxl_pci_get_latency - calculate the link latency for the PCIe link
+ * @pdev: PCI device
+ *
+ * return: calculated latency or 0 for no latency
+ *
+ * CXL Memory Device SW Guide v1.0 2.11.4 Link latency calculation
+ * Link latency = LinkPropagationLatency + FlitLatency + RetimerLatency
+ * LinkProgationLatency is negligible, so 0 will be used
+ * RetimerLatency is assumed to be negligible and 0 will be used
+ * FlitLatency = FlitSize / LinkBandwidth
+ * FlitSize is defined by spec. CXL rev3.0 4.2.1.
+ * 68B flit is used up to 32GT/s. >32GT/s, 256B flit size is used.
+ * The FlitLatency is converted to picoseconds.
+ */
+long cxl_pci_get_latency(struct pci_dev *pdev)
+{
+ long bw;
+
+ bw = pci_bus_speed_to_mbps(get_link_speed(pdev));
+ if (bw < 0)
+ return 0;
+ bw /= BITS_PER_BYTE;
+
+ return cxl_flit_size(pdev) * MEGA / bw;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_pci_get_latency, CXL);
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index 23403538505b..3b45da2b425e 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -793,6 +793,9 @@ static struct cxl_port *__devm_cxl_add_port(struct device *host,
if (rc)
return ERR_PTR(rc);
+ if (parent_dport && dev_is_pci(uport_dev))
+ port->pci_latency = cxl_pci_get_latency(to_pci_dev(uport_dev));
+
return port;
err:
@@ -1067,6 +1070,9 @@ __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
if (rc)
return ERR_PTR(rc);
+ if (dev_is_pci(dport_dev))
+ dport->link_latency = cxl_pci_get_latency(to_pci_dev(dport_dev));
+
return dport;
}
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 587677fc597c..f1c43f02d65e 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -586,6 +586,7 @@ struct cxl_dax_region {
* @depth: How deep this port is relative to the root. depth 0 is the root.
* @cdat: Cached CDAT data
* @cdat_available: Should a CDAT attribute be available in sysfs
+ * @pci_latency: Upstream latency in picoseconds
*/
struct cxl_port {
struct device dev;
@@ -609,6 +610,7 @@ struct cxl_port {
size_t length;
} cdat;
bool cdat_available;
+ long pci_latency;
};
struct cxl_root_ops {
@@ -654,6 +656,7 @@ struct cxl_rcrb_info {
* @rch: Indicate whether this dport was enumerated in RCH or VH mode
* @port: reference to cxl_port that contains this downstream port
* @coord: access coordinates (performance) for switch from CDAT
+ * @link_latency: calculated PCIe downstream latency
*/
struct cxl_dport {
struct device *dport_dev;
@@ -663,6 +666,7 @@ struct cxl_dport {
bool rch;
struct cxl_port *port;
struct access_coordinate coord;
+ long link_latency;
};
/**
diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
index 0fa4799ea316..466563536a92 100644
--- a/drivers/cxl/cxlpci.h
+++ b/drivers/cxl/cxlpci.h
@@ -85,6 +85,19 @@ struct cdat_entry_header {
__le16 length;
} __packed;
+/*
+ * CXL v3.0 6.2.3 Table 6-4
+ * The table indicates that if PCIe Flit Mode is set, then CXL is in 256B flits
+ * mode, otherwise it's 68B flits mode.
+ */
+static inline bool cxl_pci_flit_256(struct pci_dev *pdev)
+{
+ u16 lnksta2;
+
+ pcie_capability_read_word(pdev, PCI_EXP_LNKSTA2, &lnksta2);
+ return lnksta2 & PCI_EXP_LNKSTA2_FLIT;
+}
+
int devm_cxl_port_enumerate_dports(struct cxl_port *port);
struct cxl_dev_state;
int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm,
@@ -93,4 +106,6 @@ void read_cdat_data(struct cxl_port *port);
void cxl_cor_error_detected(struct pci_dev *pdev);
pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
pci_channel_state_t state);
+long cxl_pci_get_latency(struct pci_dev *pdev);
+
#endif /* __CXL_PCI_H__ */
next prev parent reply other threads:[~2023-10-12 18:55 UTC|newest]
Thread overview: 35+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-10-12 18:53 [PATCH v11 00/22] cxl: Add support for QTG ID retrieval for CXL subsystem Dave Jiang
2023-10-12 18:53 ` [PATCH v11 01/22] cxl: Export QTG ids from CFMWS to sysfs as qos_class attribute Dave Jiang
2023-10-12 18:53 ` [PATCH v11 02/22] cxl: Add checksum verification to CDAT from CXL Dave Jiang
2023-10-12 18:53 ` [PATCH v11 03/22] cxl: Add support for reading CXL switch CDAT table Dave Jiang
2023-10-12 18:53 ` [PATCH v11 04/22] acpi: Move common tables helper functions to common lib Dave Jiang
2023-10-12 18:54 ` [PATCH v11 05/22] lib/firmware_table: tables: Add CDAT table parsing support Dave Jiang
2023-10-12 18:54 ` [PATCH v11 06/22] base/node / acpi: Change 'node_hmem_attrs' to 'access_coordinates' Dave Jiang
2023-10-28 4:51 ` Dan Williams
2023-10-12 18:54 ` [PATCH v11 07/22] acpi: numa: Create enum for memory_target access coordinates indexing Dave Jiang
2023-10-12 18:54 ` [PATCH v11 08/22] acpi: numa: Add genport target allocation to the HMAT parsing Dave Jiang
2023-10-12 18:54 ` [PATCH v11 09/22] acpi: Break out nesting for hmat_parse_locality() Dave Jiang
2023-10-12 18:54 ` [PATCH v11 10/22] acpi: numa: Add setting of generic port system locality attributes Dave Jiang
2023-10-12 18:54 ` [PATCH v11 11/22] acpi: numa: Add helper function to retrieve the performance attributes Dave Jiang
2023-10-12 18:54 ` [PATCH v11 12/22] cxl: Add callback to parse the DSMAS subtables from CDAT Dave Jiang
2023-10-28 4:08 ` Dan Williams
2023-10-12 18:54 ` [PATCH v11 13/22] cxl: Add callback to parse the DSLBIS subtable " Dave Jiang
2023-10-12 18:54 ` [PATCH v11 14/22] cxl: Add callback to parse the SSLBIS " Dave Jiang
2023-10-12 18:55 ` [PATCH v11 15/22] cxl: Add support for _DSM Function for retrieving QTG ID Dave Jiang
2023-10-16 10:56 ` Jonathan Cameron
2023-10-12 18:55 ` Dave Jiang [this message]
2023-10-12 18:55 ` [PATCH v11 17/22] cxl: Store the access coordinates for the generic ports Dave Jiang
2023-10-12 18:55 ` [PATCH v11 18/22] cxl: Add helper function that calculate performance data for downstream ports Dave Jiang
2023-10-12 18:55 ` [PATCH v11 19/22] cxl: Compute the entire CXL path latency and bandwidth data Dave Jiang
2023-10-12 18:55 ` [PATCH v11 20/22] cxl: Store QTG IDs and related info to the CXL memory device context Dave Jiang
2023-10-16 10:58 ` Jonathan Cameron
2023-10-27 4:48 ` Gregory Price
2023-10-27 17:17 ` Gregory Price
2023-10-28 4:23 ` Dan Williams
2023-10-12 18:55 ` [PATCH v11 21/22] cxl: Export sysfs attributes for memory device QoS class Dave Jiang
2023-10-16 10:59 ` Jonathan Cameron
2023-10-12 18:55 ` [PATCH v11 22/22] cxl: Check qos_class validity on memdev probe Dave Jiang
2023-10-16 11:04 ` Jonathan Cameron
2023-10-28 4:29 ` Dan Williams
2023-10-26 22:54 ` [PATCH v11 00/22] cxl: Add support for QTG ID retrieval for CXL subsystem Gregory Price
2023-10-30 16:18 ` Dave Jiang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=169713690650.2205276.6608428147494650680.stgit@djiang5-mobl3 \
--to=dave.jiang@intel.com \
--cc=Jonathan.Cameron@huawei.com \
--cc=alison.schofield@intel.com \
--cc=dan.j.williams@intel.com \
--cc=dave@stgolabs.net \
--cc=ira.weiny@intel.com \
--cc=linux-cxl@vger.kernel.org \
--cc=vishal.l.verma@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.