[PATCH 0/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[PATCH 0/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[Heiner Kallweit]

Background of this extension is a problem with the r8169 network driver.
Several combinations of board chipsets and network chip versions have
problems if ASPM is enabled, therefore we have to disable ASPM per
default. However especially on notebooks ASPM can provide significant
power-saving, therefore we want to give users the option to enable
ASPM. With the new sysfs attribute users can control which ASPM
link-states are disabled.

After few RFC's this is the submission-ready version. Added to the RFC
version has been documentation of the new sysfs attribute.

Heiner Kallweit (3):
  PCI/ASPM: add L1 sub-state support to pci_disable_link_state
  PCI/ASPM: allow to re-enable Clock PM
  PCI/ASPM: add sysfs attribute for controlling ASPM

 Documentation/ABI/testing/sysfs-bus-pci |  13 ++
 drivers/pci/pci.h                       |   8 +-
 drivers/pci/pcie/aspm.c                 | 211 ++++++++++++++++++++++--
 include/linux/pci-aspm.h                |   8 +-
 4 files changed, 219 insertions(+), 21 deletions(-)

-- 
2.21.0

[PATCH 1/3] PCI/ASPM: add L1 sub-state support to pci_disable_link_state

[Heiner Kallweit]

Add support for disabling states L1.1 and L1.2 to pci_disable_link_state.

Signed-off-by: Heiner Kallweit <hkallweit1@gmail.com>
---
 drivers/pci/pcie/aspm.c  | 13 ++++++++++---
 include/linux/pci-aspm.h |  8 +++++---
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index fd4cb7508..511f3e018 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -35,9 +35,9 @@
 #define ASPM_STATE_L1_1_PCIPM	(0x20)	/* PCI PM L1.1 state */
 #define ASPM_STATE_L1_2_PCIPM	(0x40)	/* PCI PM L1.2 state */
 #define ASPM_STATE_L1_SS_PCIPM	(ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM)
+#define ASPM_STATE_L1_1_MASK	(ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM)
 #define ASPM_STATE_L1_2_MASK	(ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM)
-#define ASPM_STATE_L1SS		(ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\
-				 ASPM_STATE_L1_2_MASK)
+#define ASPM_STATE_L1SS		(ASPM_STATE_L1_1_MASK | ASPM_STATE_L1_2_MASK)
 #define ASPM_STATE_L0S		(ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
 #define ASPM_STATE_ALL		(ASPM_STATE_L0S | ASPM_STATE_L1 |	\
 				 ASPM_STATE_L1SS)
@@ -1094,8 +1094,15 @@ static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
 	link = parent->link_state;
 	if (state & PCIE_LINK_STATE_L0S)
 		link->aspm_disable |= ASPM_STATE_L0S;
-	if (state & PCIE_LINK_STATE_L1)
+	if (state & PCIE_LINK_STATE_L1) {
 		link->aspm_disable |= ASPM_STATE_L1;
+		/* sub-states require L1 */
+		link->aspm_disable |= ASPM_STATE_L1SS;
+	}
+	if (state & PCIE_LINK_STATE_L1_1)
+		link->aspm_disable |= ASPM_STATE_L1_1_MASK;
+	if (state & PCIE_LINK_STATE_L1_2)
+		link->aspm_disable |= ASPM_STATE_L1_2_MASK;
 	pcie_config_aspm_link(link, policy_to_aspm_state(link));
 
 	if (state & PCIE_LINK_STATE_CLKPM) {
diff --git a/include/linux/pci-aspm.h b/include/linux/pci-aspm.h
index df28af5ce..e66c3e3d8 100644
--- a/include/linux/pci-aspm.h
+++ b/include/linux/pci-aspm.h
@@ -19,9 +19,11 @@
 
 #include <linux/pci.h>
 
-#define PCIE_LINK_STATE_L0S	1
-#define PCIE_LINK_STATE_L1	2
-#define PCIE_LINK_STATE_CLKPM	4
+#define PCIE_LINK_STATE_L0S	BIT(0)
+#define PCIE_LINK_STATE_L1	BIT(1)
+#define PCIE_LINK_STATE_CLKPM	BIT(2)
+#define PCIE_LINK_STATE_L1_1	BIT(3)
+#define PCIE_LINK_STATE_L1_2	BIT(4)
 
 #ifdef CONFIG_PCIEASPM
 void pci_disable_link_state(struct pci_dev *pdev, int state);
-- 
2.21.0

[PATCH 2/3] PCI/ASPM: allow to re-enable Clock PM

[Heiner Kallweit]

So far Clock PM can't be re-enabled once it has been disabled with a
call to pci_disable_link_state(). Reason is that clkpm_capable is
reset. Change this by adding a clkpm_disable field similar to
aspm_disable.

Signed-off-by: Heiner Kallweit <hkallweit1@gmail.com>
---
 drivers/pci/pcie/aspm.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index 511f3e018..7847be38e 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -65,6 +65,7 @@ struct pcie_link_state {
 	u32 clkpm_capable:1;		/* Clock PM capable? */
 	u32 clkpm_enabled:1;		/* Current Clock PM state */
 	u32 clkpm_default:1;		/* Default Clock PM state by BIOS */
+	u32 clkpm_disable:1;		/* Clock PM disabled */
 
 	/* Exit latencies */
 	struct aspm_latency latency_up;	/* Upstream direction exit latency */
@@ -162,8 +163,11 @@ static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
 
 static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
 {
-	/* Don't enable Clock PM if the link is not Clock PM capable */
-	if (!link->clkpm_capable)
+	/*
+	 * Don't enable Clock PM if the link is not Clock PM capable
+	 * or Clock PM is disabled
+	 */
+	if (!link->clkpm_capable || link->clkpm_disable)
 		enable = 0;
 	/* Need nothing if the specified equals to current state */
 	if (link->clkpm_enabled == enable)
@@ -193,7 +197,8 @@ static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
 	}
 	link->clkpm_enabled = enabled;
 	link->clkpm_default = enabled;
-	link->clkpm_capable = (blacklist) ? 0 : capable;
+	link->clkpm_capable = capable;
+	link->clkpm_disable = blacklist ? 1 : 0;
 }
 
 static bool pcie_retrain_link(struct pcie_link_state *link)
@@ -1105,10 +1110,9 @@ static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
 		link->aspm_disable |= ASPM_STATE_L1_2_MASK;
 	pcie_config_aspm_link(link, policy_to_aspm_state(link));
 
-	if (state & PCIE_LINK_STATE_CLKPM) {
-		link->clkpm_capable = 0;
-		pcie_set_clkpm(link, 0);
-	}
+	if (state & PCIE_LINK_STATE_CLKPM)
+		link->clkpm_disable = 1;
+	pcie_set_clkpm(link, policy_to_clkpm_state(link));
 	mutex_unlock(&aspm_lock);
 	if (sem)
 		up_read(&pci_bus_sem);
-- 
2.21.0

[PATCH 3/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[Heiner Kallweit]

Background of this extension is a problem with the r8169 network driver.
Several combinations of board chipsets and network chip versions have
problems if ASPM is enabled, therefore we have to disable ASPM per default.
However especially on notebooks ASPM can provide significant power-saving,
therefore we want to give users the option to enable ASPM. With the new sysfs
attribute users can control which ASPM link-states are enabled/disabled.

Signed-off-by: Heiner Kallweit <hkallweit1@gmail.com>
---
 Documentation/ABI/testing/sysfs-bus-pci |  13 ++
 drivers/pci/pci.h                       |   8 +-
 drivers/pci/pcie/aspm.c                 | 180 +++++++++++++++++++++++-
 3 files changed, 193 insertions(+), 8 deletions(-)

diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci
index 8bfee557e..38fe358de 100644
--- a/Documentation/ABI/testing/sysfs-bus-pci
+++ b/Documentation/ABI/testing/sysfs-bus-pci
@@ -347,3 +347,16 @@ Description:
 		If the device has any Peer-to-Peer memory registered, this
 	        file contains a '1' if the memory has been published for
 		use outside the driver that owns the device.
+
+What:		/sys/bus/pci/devices/.../power/aspm_link_states
+Date:		May 2019
+Contact:	Heiner Kallweit <hkallweit1@gmail.com>
+Description:
+		If ASPM is supported for an endpoint, then this file can be
+		used to enable / disable link states. A link state
+		displayed in brackets is enabled, otherwise it's disabled.
+		To control link states (case insensitive):
+		+state : enables a supported state
+		-state : disables a state
+		none : disables all link states
+		all : enables all supported link states
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 9cb99380c..06642b7de 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -499,17 +499,13 @@ void pcie_aspm_init_link_state(struct pci_dev *pdev);
 void pcie_aspm_exit_link_state(struct pci_dev *pdev);
 void pcie_aspm_pm_state_change(struct pci_dev *pdev);
 void pcie_aspm_powersave_config_link(struct pci_dev *pdev);
+void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev);
+void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev);
 #else
 static inline void pcie_aspm_init_link_state(struct pci_dev *pdev) { }
 static inline void pcie_aspm_exit_link_state(struct pci_dev *pdev) { }
 static inline void pcie_aspm_pm_state_change(struct pci_dev *pdev) { }
 static inline void pcie_aspm_powersave_config_link(struct pci_dev *pdev) { }
-#endif
-
-#ifdef CONFIG_PCIEASPM_DEBUG
-void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev);
-void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev);
-#else
 static inline void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) { }
 static inline void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) { }
 #endif
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index 7847be38e..f3822accb 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -42,6 +42,8 @@
 #define ASPM_STATE_ALL		(ASPM_STATE_L0S | ASPM_STATE_L1 |	\
 				 ASPM_STATE_L1SS)
 
+static const char power_group[] = "power";
+
 struct aspm_latency {
 	u32 l0s;			/* L0s latency (nsec) */
 	u32 l1;				/* L1 latency (nsec) */
@@ -1251,38 +1253,212 @@ static ssize_t clk_ctl_store(struct device *dev,
 
 static DEVICE_ATTR_RW(link_state);
 static DEVICE_ATTR_RW(clk_ctl);
+#endif
+
+struct aspm_sysfs_state {
+	const char *name;
+	int mask;
+};
+
+static const struct aspm_sysfs_state aspm_sysfs_states[] = {
+	{ "L0S",	ASPM_STATE_L0S		},
+	{ "L1",		ASPM_STATE_L1		},
+	{ "L1.1",	ASPM_STATE_L1_1_MASK	},
+	{ "L1.2",	ASPM_STATE_L1_2_MASK	},
+};
+
+static struct pcie_link_state *aspm_get_parent_link(struct pci_dev *pdev)
+{
+	struct pci_dev *parent = pdev->bus->self;
+
+	if (pdev->has_secondary_link)
+		parent = pdev;
+
+	return parent ? parent->link_state : NULL;
+}
+
+static bool pcie_check_aspm_endpoint(struct pci_dev *pdev)
+{
+	struct pcie_link_state *link;
+
+	if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ENDPOINT)
+		return false;
+
+	link = aspm_get_parent_link(pdev);
+
+	return link && link->aspm_support;
+}
+
+static ssize_t aspm_link_states_show(struct device *dev,
+				     struct device_attribute *attr, char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pcie_link_state *link;
+	int len = 0, i;
+
+	link = aspm_get_parent_link(pdev);
+	if (!link)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&aspm_lock);
+
+	for (i = 0; i < ARRAY_SIZE(aspm_sysfs_states); i++) {
+		const struct aspm_sysfs_state *st = aspm_sysfs_states + i;
+
+		if (link->aspm_enabled & st->mask)
+			len += scnprintf(buf + len, PAGE_SIZE - len, "[%s] ",
+					 st->name);
+		else
+			len += scnprintf(buf + len, PAGE_SIZE - len, "%s ",
+					 st->name);
+	}
+
+	if (link->clkpm_enabled)
+		len += scnprintf(buf + len, PAGE_SIZE - len, "[CLKPM] ");
+	else
+		len += scnprintf(buf + len, PAGE_SIZE - len, "CLKPM ");
+
+	mutex_unlock(&aspm_lock);
+
+	len += scnprintf(buf + len, PAGE_SIZE - len, "\n");
+
+	return len;
+}
+
+static ssize_t aspm_link_states_store(struct device *dev,
+				      struct device_attribute *attr,
+				      const char *buf, size_t len)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct pcie_link_state *link;
+	char *buftmp = (char *)buf, *tok;
+	unsigned int disable_aspm, disable_clkpm;
+	bool first = true, add;
+	int err = 0, i;
+
+	if (aspm_disabled)
+		return -EPERM;
+
+	link = aspm_get_parent_link(pdev);
+	if (!link)
+		return -EOPNOTSUPP;
+
+	down_read(&pci_bus_sem);
+	mutex_lock(&aspm_lock);
+
+	disable_aspm = link->aspm_disable;
+	disable_clkpm = link->clkpm_disable;
+
+	while ((tok = strsep(&buftmp, " \n")) != NULL) {
+		bool found = false;
+
+		if (!*tok)
+			continue;
+
+		if (first) {
+			if (!strcasecmp(tok, "none")) {
+				disable_aspm = ASPM_STATE_ALL;
+				disable_clkpm = 1;
+				break;
+			}
+			if (!strcasecmp(tok, "all")) {
+				disable_aspm = 0;
+				disable_clkpm = 0;
+				break;
+			}
+			first = false;
+		}
+
+		if (*tok != '+' && *tok != '-') {
+			err = -EINVAL;
+			goto out;
+		}
+
+		add = *tok++ == '+';
+
+		for (i = 0; i < ARRAY_SIZE(aspm_sysfs_states); i++) {
+			const struct aspm_sysfs_state *st =
+						aspm_sysfs_states + i;
+
+			if (!strcasecmp(tok, st->name)) {
+				if (add)
+					disable_aspm &= ~st->mask;
+				else
+					disable_aspm |= st->mask;
+				found = true;
+				break;
+			}
+		}
+
+		if (!found && !strcasecmp(tok, "clkpm")) {
+			disable_clkpm = add ? 0 : 1;
+			found = true;
+		}
+
+		if (!found) {
+			err = -EINVAL;
+			goto out;
+		}
+	}
+
+	if (disable_aspm & ASPM_STATE_L1)
+		disable_aspm |= ASPM_STATE_L1SS;
+
+	link->aspm_disable = disable_aspm;
+	link->clkpm_disable = disable_clkpm;
+
+	pcie_config_aspm_link(link, policy_to_aspm_state(link));
+	pcie_set_clkpm(link, policy_to_clkpm_state(link));
+out:
+	mutex_unlock(&aspm_lock);
+	up_read(&pci_bus_sem);
+
+	return err ?: len;
+}
+
+static DEVICE_ATTR_RW(aspm_link_states);
 
-static char power_group[] = "power";
 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
 {
 	struct pcie_link_state *link_state = pdev->link_state;
 
+	if (pcie_check_aspm_endpoint(pdev))
+		sysfs_add_file_to_group(&pdev->dev.kobj,
+			&dev_attr_aspm_link_states.attr, power_group);
+
 	if (!link_state)
 		return;
 
+#ifdef CONFIG_PCIEASPM_DEBUG
 	if (link_state->aspm_support)
 		sysfs_add_file_to_group(&pdev->dev.kobj,
 			&dev_attr_link_state.attr, power_group);
 	if (link_state->clkpm_capable)
 		sysfs_add_file_to_group(&pdev->dev.kobj,
 			&dev_attr_clk_ctl.attr, power_group);
+#endif
 }
 
 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
 {
 	struct pcie_link_state *link_state = pdev->link_state;
 
+	if (pcie_check_aspm_endpoint(pdev))
+		sysfs_remove_file_from_group(&pdev->dev.kobj,
+			&dev_attr_aspm_link_states.attr, power_group);
+
 	if (!link_state)
 		return;
 
+#ifdef CONFIG_PCIEASPM_DEBUG
 	if (link_state->aspm_support)
 		sysfs_remove_file_from_group(&pdev->dev.kobj,
 			&dev_attr_link_state.attr, power_group);
 	if (link_state->clkpm_capable)
 		sysfs_remove_file_from_group(&pdev->dev.kobj,
 			&dev_attr_clk_ctl.attr, power_group);
-}
 #endif
+}
 
 static int __init pcie_aspm_disable(char *str)
 {
-- 
2.21.0

Re: [PATCH 3/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[Bjorn Helgaas]

[+cc Greg, Rajat]

On Thu, May 23, 2019 at 10:05:35PM +0200, Heiner Kallweit wrote:
> Background of this extension is a problem with the r8169 network driver.
> Several combinations of board chipsets and network chip versions have
> problems if ASPM is enabled, therefore we have to disable ASPM per default.
> However especially on notebooks ASPM can provide significant power-saving,
> therefore we want to give users the option to enable ASPM. With the new sysfs
> attribute users can control which ASPM link-states are enabled/disabled.
> 
> Signed-off-by: Heiner Kallweit <hkallweit1@gmail.com>
> ---
>  Documentation/ABI/testing/sysfs-bus-pci |  13 ++
>  drivers/pci/pci.h                       |   8 +-
>  drivers/pci/pcie/aspm.c                 | 180 +++++++++++++++++++++++-
>  3 files changed, 193 insertions(+), 8 deletions(-)
> 
> diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci
> index 8bfee557e..38fe358de 100644
> --- a/Documentation/ABI/testing/sysfs-bus-pci
> +++ b/Documentation/ABI/testing/sysfs-bus-pci
> @@ -347,3 +347,16 @@ Description:
>  		If the device has any Peer-to-Peer memory registered, this
>  	        file contains a '1' if the memory has been published for
>  		use outside the driver that owns the device.
> +
> +What:		/sys/bus/pci/devices/.../power/aspm_link_states
> +Date:		May 2019
> +Contact:	Heiner Kallweit <hkallweit1@gmail.com>
> +Description:
> +		If ASPM is supported for an endpoint, then this file can be
> +		used to enable / disable link states. A link state
> +		displayed in brackets is enabled, otherwise it's disabled.
> +		To control link states (case insensitive):
> +		+state : enables a supported state
> +		-state : disables a state
> +		none : disables all link states
> +		all : enables all supported link states

IIUC this "aspm_link_states" file will contain things like this:

  L0S L1 L1.1 L1.2                 # All states supported, all disabled
  [L0S] L1                         # L0s enabled, L1 supported but disabled
  [L0S] [L1]                       # L0s and L1 enabled
  ...

and the control is by writing things like this to it:

  +L1                              # enables L1
  +L1.1                            # enables L1.1
  -L0S                             # disables L0s

I know this file structure is similar to protocol handling in
drivers/media/rc/rc-main.c, but Documentation/filesystems/sysfs.txt
suggests single values in a file, and Greg recently pointed out that
we screwed up some PCI AER stats [1].

So I'm thinking maybe we should split this into several files, e.g.,

  /sys/devices/pci*/.../power/aspm_l0s
  /sys/devices/pci*/.../power/aspm_l1
  /sys/devices/pci*/.../power/aspm_l1.1
  /sys/devices/pci*/.../power/aspm_l1.2

which would contain just 1/0 values, and we'd write 1/0 to
enable/disable things.

Since the L1 PM Substates control register has separate enable bits
for PCI-PM L1.1 and L1.2, we might also want a way to manage those.

Bjorn

[1] https://lore.kernel.org/r/20190621072911.GA21600@kroah.com

Re: [PATCH 3/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[Greg KH]

On Tue, Aug 20, 2019 at 05:34:00AM -0500, Bjorn Helgaas wrote:
> [+cc Greg, Rajat]
> 
> On Thu, May 23, 2019 at 10:05:35PM +0200, Heiner Kallweit wrote:
> > Background of this extension is a problem with the r8169 network driver.
> > Several combinations of board chipsets and network chip versions have
> > problems if ASPM is enabled, therefore we have to disable ASPM per default.
> > However especially on notebooks ASPM can provide significant power-saving,
> > therefore we want to give users the option to enable ASPM. With the new sysfs
> > attribute users can control which ASPM link-states are enabled/disabled.
> > 
> > Signed-off-by: Heiner Kallweit <hkallweit1@gmail.com>
> > ---
> >  Documentation/ABI/testing/sysfs-bus-pci |  13 ++
> >  drivers/pci/pci.h                       |   8 +-
> >  drivers/pci/pcie/aspm.c                 | 180 +++++++++++++++++++++++-
> >  3 files changed, 193 insertions(+), 8 deletions(-)
> > 
> > diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci
> > index 8bfee557e..38fe358de 100644
> > --- a/Documentation/ABI/testing/sysfs-bus-pci
> > +++ b/Documentation/ABI/testing/sysfs-bus-pci
> > @@ -347,3 +347,16 @@ Description:
> >  		If the device has any Peer-to-Peer memory registered, this
> >  	        file contains a '1' if the memory has been published for
> >  		use outside the driver that owns the device.
> > +
> > +What:		/sys/bus/pci/devices/.../power/aspm_link_states
> > +Date:		May 2019
> > +Contact:	Heiner Kallweit <hkallweit1@gmail.com>
> > +Description:
> > +		If ASPM is supported for an endpoint, then this file can be
> > +		used to enable / disable link states. A link state
> > +		displayed in brackets is enabled, otherwise it's disabled.
> > +		To control link states (case insensitive):
> > +		+state : enables a supported state
> > +		-state : disables a state
> > +		none : disables all link states
> > +		all : enables all supported link states
> 
> IIUC this "aspm_link_states" file will contain things like this:
> 
>   L0S L1 L1.1 L1.2                 # All states supported, all disabled
>   [L0S] L1                         # L0s enabled, L1 supported but disabled
>   [L0S] [L1]                       # L0s and L1 enabled
>   ...
> 
> and the control is by writing things like this to it:
> 
>   +L1                              # enables L1
>   +L1.1                            # enables L1.1
>   -L0S                             # disables L0s
> 
> I know this file structure is similar to protocol handling in
> drivers/media/rc/rc-main.c, but Documentation/filesystems/sysfs.txt
> suggests single values in a file, and Greg recently pointed out that
> we screwed up some PCI AER stats [1].
> 
> So I'm thinking maybe we should split this into several files, e.g.,
> 
>   /sys/devices/pci*/.../power/aspm_l0s
>   /sys/devices/pci*/.../power/aspm_l1
>   /sys/devices/pci*/.../power/aspm_l1.1
>   /sys/devices/pci*/.../power/aspm_l1.2
> 
> which would contain just 1/0 values, and we'd write 1/0 to
> enable/disable things.

Yes, that is much simpler for both userspace to handle as well as the
kernel, as no need for parsing anything "special" at all here.

If the file is present, the state is supported, and the kernel code
should be _much_ easier to write and maintain over time.

> Since the L1 PM Substates control register has separate enable bits
> for PCI-PM L1.1 and L1.2, we might also want a way to manage those.

That sounds good as well.

thanks,

greg k-h

Re: [PATCH 3/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[Rajat Jain]

On Tue, Aug 20, 2019 at 3:34 AM Bjorn Helgaas <helgaas@kernel.org> wrote:
>
> [+cc Greg, Rajat]
>
> On Thu, May 23, 2019 at 10:05:35PM +0200, Heiner Kallweit wrote:
> > Background of this extension is a problem with the r8169 network driver.
> > Several combinations of board chipsets and network chip versions have
> > problems if ASPM is enabled, therefore we have to disable ASPM per default.
> > However especially on notebooks ASPM can provide significant power-saving,
> > therefore we want to give users the option to enable ASPM. With the new sysfs
> > attribute users can control which ASPM link-states are enabled/disabled.
> >
> > Signed-off-by: Heiner Kallweit <hkallweit1@gmail.com>
> > ---
> >  Documentation/ABI/testing/sysfs-bus-pci |  13 ++
> >  drivers/pci/pci.h                       |   8 +-
> >  drivers/pci/pcie/aspm.c                 | 180 +++++++++++++++++++++++-
> >  3 files changed, 193 insertions(+), 8 deletions(-)
> >
> > diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci
> > index 8bfee557e..38fe358de 100644
> > --- a/Documentation/ABI/testing/sysfs-bus-pci
> > +++ b/Documentation/ABI/testing/sysfs-bus-pci
> > @@ -347,3 +347,16 @@ Description:
> >               If the device has any Peer-to-Peer memory registered, this
> >               file contains a '1' if the memory has been published for
> >               use outside the driver that owns the device.
> > +
> > +What:                /sys/bus/pci/devices/.../power/aspm_link_states
> > +Date:                May 2019
> > +Contact:     Heiner Kallweit <hkallweit1@gmail.com>
> > +Description:
> > +             If ASPM is supported for an endpoint, then this file can be
> > +             used to enable / disable link states. A link state
> > +             displayed in brackets is enabled, otherwise it's disabled.
> > +             To control link states (case insensitive):
> > +             +state : enables a supported state
> > +             -state : disables a state
> > +             none : disables all link states
> > +             all : enables all supported link states
>
> IIUC this "aspm_link_states" file will contain things like this:
>
>   L0S L1 L1.1 L1.2                 # All states supported, all disabled
>   [L0S] L1                         # L0s enabled, L1 supported but disabled
>   [L0S] [L1]                       # L0s and L1 enabled
>   ...
>
> and the control is by writing things like this to it:
>
>   +L1                              # enables L1
>   +L1.1                            # enables L1.1
>   -L0S                             # disables L0s
>
> I know this file structure is similar to protocol handling in
> drivers/media/rc/rc-main.c, but Documentation/filesystems/sysfs.txt
> suggests single values in a file, and Greg recently pointed out that
> we screwed up some PCI AER stats [1].
>
> So I'm thinking maybe we should split this into several files, e.g.,
>
>   /sys/devices/pci*/.../power/aspm_l0s
>   /sys/devices/pci*/.../power/aspm_l1
>   /sys/devices/pci*/.../power/aspm_l1.1
>   /sys/devices/pci*/.../power/aspm_l1.2
>
> which would contain just 1/0 values, and we'd write 1/0 to
> enable/disable things.
>
> Since the L1 PM Substates control register has separate enable bits
> for PCI-PM L1.1 and L1.2, we might also want a way to manage those.
>
> Bjorn
>
> [1] https://lore.kernel.org/r/20190621072911.GA21600@kroah.com


Sorry, this has been sitting on my plate for some time now. I'll work
on it and send out a patch later in this week.

Thanks,

Rajat

Re: [PATCH 3/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[Bjorn Helgaas]

On Tue, Aug 20, 2019 at 12:05:01PM -0700, Rajat Jain wrote:
> On Tue, Aug 20, 2019 at 3:34 AM Bjorn Helgaas <helgaas@kernel.org> wrote:
> > On Thu, May 23, 2019 at 10:05:35PM +0200, Heiner Kallweit wrote:
> > > Background of this extension is a problem with the r8169 network driver.
> > > Several combinations of board chipsets and network chip versions have
> > > problems if ASPM is enabled, therefore we have to disable ASPM per default.
> > > However especially on notebooks ASPM can provide significant power-saving,
> > > therefore we want to give users the option to enable ASPM. With the new sysfs
> > > attribute users can control which ASPM link-states are enabled/disabled.
> > >
> > > Signed-off-by: Heiner Kallweit <hkallweit1@gmail.com>
> > > ---
> > >  Documentation/ABI/testing/sysfs-bus-pci |  13 ++
> > >  drivers/pci/pci.h                       |   8 +-
> > >  drivers/pci/pcie/aspm.c                 | 180 +++++++++++++++++++++++-
> > >  3 files changed, 193 insertions(+), 8 deletions(-)
> > >
> > > diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci
> > > index 8bfee557e..38fe358de 100644
> > > --- a/Documentation/ABI/testing/sysfs-bus-pci
> > > +++ b/Documentation/ABI/testing/sysfs-bus-pci
> > > @@ -347,3 +347,16 @@ Description:
> > >               If the device has any Peer-to-Peer memory registered, this
> > >               file contains a '1' if the memory has been published for
> > >               use outside the driver that owns the device.
> > > +
> > > +What:                /sys/bus/pci/devices/.../power/aspm_link_states
> > > +Date:                May 2019
> > > +Contact:     Heiner Kallweit <hkallweit1@gmail.com>
> > > +Description:
> > > +             If ASPM is supported for an endpoint, then this file can be
> > > +             used to enable / disable link states. A link state
> > > +             displayed in brackets is enabled, otherwise it's disabled.
> > > +             To control link states (case insensitive):
> > > +             +state : enables a supported state
> > > +             -state : disables a state
> > > +             none : disables all link states
> > > +             all : enables all supported link states
> >
> > IIUC this "aspm_link_states" file will contain things like this:
> >
> >   L0S L1 L1.1 L1.2                 # All states supported, all disabled
> >   [L0S] L1                         # L0s enabled, L1 supported but disabled
> >   [L0S] [L1]                       # L0s and L1 enabled
> >   ...
> >
> > and the control is by writing things like this to it:
> >
> >   +L1                              # enables L1
> >   +L1.1                            # enables L1.1
> >   -L0S                             # disables L0s
> >
> > I know this file structure is similar to protocol handling in
> > drivers/media/rc/rc-main.c, but Documentation/filesystems/sysfs.txt
> > suggests single values in a file, and Greg recently pointed out that
> > we screwed up some PCI AER stats [1].
> >
> > So I'm thinking maybe we should split this into several files, e.g.,
> >
> >   /sys/devices/pci*/.../power/aspm_l0s
> >   /sys/devices/pci*/.../power/aspm_l1
> >   /sys/devices/pci*/.../power/aspm_l1.1
> >   /sys/devices/pci*/.../power/aspm_l1.2
> >
> > which would contain just 1/0 values, and we'd write 1/0 to
> > enable/disable things.
> >
> > Since the L1 PM Substates control register has separate enable bits
> > for PCI-PM L1.1 and L1.2, we might also want a way to manage those.
> >
> > Bjorn
> >
> > [1] https://lore.kernel.org/r/20190621072911.GA21600@kroah.com
> 
> Sorry, this has been sitting on my plate for some time now. I'll work
> on it and send out a patch later in this week.

Don't worry, I cc'd you mostly as FYI because you're a recent
contributor to aspm.c in this area, though I wouldn't complain about
AER stats sysfs patches either :)

Bjorn

Re: [PATCH 3/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[Rajat Jain]

On Tue, Aug 20, 2019 at 12:32 PM Bjorn Helgaas <helgaas@kernel.org> wrote:
>
> On Tue, Aug 20, 2019 at 12:05:01PM -0700, Rajat Jain wrote:
> > On Tue, Aug 20, 2019 at 3:34 AM Bjorn Helgaas <helgaas@kernel.org> wrote:
> > > On Thu, May 23, 2019 at 10:05:35PM +0200, Heiner Kallweit wrote:
> > > > Background of this extension is a problem with the r8169 network driver.
> > > > Several combinations of board chipsets and network chip versions have
> > > > problems if ASPM is enabled, therefore we have to disable ASPM per default.
> > > > However especially on notebooks ASPM can provide significant power-saving,
> > > > therefore we want to give users the option to enable ASPM. With the new sysfs
> > > > attribute users can control which ASPM link-states are enabled/disabled.
> > > >
> > > > Signed-off-by: Heiner Kallweit <hkallweit1@gmail.com>
> > > > ---
> > > >  Documentation/ABI/testing/sysfs-bus-pci |  13 ++
> > > >  drivers/pci/pci.h                       |   8 +-
> > > >  drivers/pci/pcie/aspm.c                 | 180 +++++++++++++++++++++++-
> > > >  3 files changed, 193 insertions(+), 8 deletions(-)
> > > >
> > > > diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci
> > > > index 8bfee557e..38fe358de 100644
> > > > --- a/Documentation/ABI/testing/sysfs-bus-pci
> > > > +++ b/Documentation/ABI/testing/sysfs-bus-pci
> > > > @@ -347,3 +347,16 @@ Description:
> > > >               If the device has any Peer-to-Peer memory registered, this
> > > >               file contains a '1' if the memory has been published for
> > > >               use outside the driver that owns the device.
> > > > +
> > > > +What:                /sys/bus/pci/devices/.../power/aspm_link_states
> > > > +Date:                May 2019
> > > > +Contact:     Heiner Kallweit <hkallweit1@gmail.com>
> > > > +Description:
> > > > +             If ASPM is supported for an endpoint, then this file can be
> > > > +             used to enable / disable link states. A link state
> > > > +             displayed in brackets is enabled, otherwise it's disabled.
> > > > +             To control link states (case insensitive):
> > > > +             +state : enables a supported state
> > > > +             -state : disables a state
> > > > +             none : disables all link states
> > > > +             all : enables all supported link states
> > >
> > > IIUC this "aspm_link_states" file will contain things like this:
> > >
> > >   L0S L1 L1.1 L1.2                 # All states supported, all disabled
> > >   [L0S] L1                         # L0s enabled, L1 supported but disabled
> > >   [L0S] [L1]                       # L0s and L1 enabled
> > >   ...
> > >
> > > and the control is by writing things like this to it:
> > >
> > >   +L1                              # enables L1
> > >   +L1.1                            # enables L1.1
> > >   -L0S                             # disables L0s
> > >
> > > I know this file structure is similar to protocol handling in
> > > drivers/media/rc/rc-main.c, but Documentation/filesystems/sysfs.txt
> > > suggests single values in a file, and Greg recently pointed out that
> > > we screwed up some PCI AER stats [1].
> > >
> > > So I'm thinking maybe we should split this into several files, e.g.,
> > >
> > >   /sys/devices/pci*/.../power/aspm_l0s
> > >   /sys/devices/pci*/.../power/aspm_l1
> > >   /sys/devices/pci*/.../power/aspm_l1.1
> > >   /sys/devices/pci*/.../power/aspm_l1.2
> > >
> > > which would contain just 1/0 values, and we'd write 1/0 to
> > > enable/disable things.
> > >
> > > Since the L1 PM Substates control register has separate enable bits
> > > for PCI-PM L1.1 and L1.2, we might also want a way to manage those.
> > >
> > > Bjorn
> > >
> > > [1] https://lore.kernel.org/r/20190621072911.GA21600@kroah.com
> >
> > Sorry, this has been sitting on my plate for some time now. I'll work
> > on it and send out a patch later in this week.
>
> Don't worry, I cc'd you mostly as FYI because you're a recent
> contributor to aspm.c in this area, though I wouldn't complain about
> AER stats sysfs patches either :)
>

May be we're digressing now, but I'd like to point out that there is
atleast one more file in ASPM that potentially violates the "1 value
per file" rule:

rajatja@rajat2:/sys/module/pcie_aspm/parameters$ cat policy
[default] performance powersave powersupersave
rajatja@rajat2:/sys/module/pcie_aspm/parameters$

... although I would argue in this case that it makes it much clear
what are the allowable values to write, and which is the current
selected one.

Thanks,

Rajat


> Bjorn

Re: [PATCH 3/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[Bjorn Helgaas]

On Tue, Aug 20, 2019 at 12:51:09PM -0700, Rajat Jain wrote:
> 
> May be we're digressing now, but I'd like to point out that there is
> atleast one more file in ASPM that potentially violates the "1 value
> per file" rule:
> 
> rajatja@rajat2:/sys/module/pcie_aspm/parameters$ cat policy
> [default] performance powersave powersupersave
> rajatja@rajat2:/sys/module/pcie_aspm/parameters$
> 
> ... although I would argue in this case that it makes it much clear
> what are the allowable values to write, and which is the current
> selected one.

Huh, that's a good point.  That "policy" file is a little problematic
for several reasons, one being the config options
(CONFIG_PCIEASPM_PERFORMANCE, CONFIG_PCIEASPM_POWERSAVE, etc) that
lock a distro into some default choice.

Maybe there's something we can do there, although there's legacy use
to consider (there are a zillion web pages that document
pcie_aspm/parameters/policy as a way to fix things), and it's
certainly beyond the scope of *this* series.

Bjorn

Re: [PATCH 3/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[Rajat Jain]

On Tue, Aug 20, 2019 at 1:48 PM Bjorn Helgaas <helgaas@kernel.org> wrote:
>
> On Tue, Aug 20, 2019 at 12:51:09PM -0700, Rajat Jain wrote:
> >
> > May be we're digressing now, but I'd like to point out that there is
> > atleast one more file in ASPM that potentially violates the "1 value
> > per file" rule:
> >
> > rajatja@rajat2:/sys/module/pcie_aspm/parameters$ cat policy
> > [default] performance powersave powersupersave
> > rajatja@rajat2:/sys/module/pcie_aspm/parameters$
> >
> > ... although I would argue in this case that it makes it much clear
> > what are the allowable values to write, and which is the current
> > selected one.
>
> Huh, that's a good point.  That "policy" file is a little problematic
> for several reasons, one being the config options
> (CONFIG_PCIEASPM_PERFORMANCE, CONFIG_PCIEASPM_POWERSAVE, etc) that
> lock a distro into some default choice.
>
> Maybe there's something we can do there, although there's legacy use
> to consider (there are a zillion web pages that document
> pcie_aspm/parameters/policy as a way to fix things), and it's
> certainly beyond the scope of *this* series.

Agreed!

>
> Bjorn

Re: [PATCH 0/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[Heiner Kallweit]

On 23.05.2019 22:03, Heiner Kallweit wrote:
> Background of this extension is a problem with the r8169 network driver.
> Several combinations of board chipsets and network chip versions have
> problems if ASPM is enabled, therefore we have to disable ASPM per
> default. However especially on notebooks ASPM can provide significant
> power-saving, therefore we want to give users the option to enable
> ASPM. With the new sysfs attribute users can control which ASPM
> link-states are disabled.
> 
> After few RFC's this is the submission-ready version. Added to the RFC
> version has been documentation of the new sysfs attribute.
> 
> Heiner Kallweit (3):
>   PCI/ASPM: add L1 sub-state support to pci_disable_link_state
>   PCI/ASPM: allow to re-enable Clock PM
>   PCI/ASPM: add sysfs attribute for controlling ASPM
> 
>  Documentation/ABI/testing/sysfs-bus-pci |  13 ++
>  drivers/pci/pci.h                       |   8 +-
>  drivers/pci/pcie/aspm.c                 | 211 ++++++++++++++++++++++--
>  include/linux/pci-aspm.h                |   8 +-
>  4 files changed, 219 insertions(+), 21 deletions(-)
> 
Can we get this into 5.3? The series was reviewed before in RFC state and
there have been no change requests.

Thanks, Heiner

Re: [PATCH 0/3] PCI/ASPM: add sysfs attribute for controlling ASPM

[AceLan Kao]

These patches work as expected and provide an interface to toggle PCI ASPM
link state, that's really helpful for PCI devices to check its ASPM
functionality by enabling/disabling it runtime.

Tested-by: AceLan Kao <acelan.kao@canonical.com>