[RESEND PATCH V3 0/6] PCI: Enable 10-Bit tag support for PCIe devices

[RESEND PATCH V3 0/6] PCI: Enable 10-Bit tag support for PCIe devices

[Dongdong Liu]

10-Bit Tag capability, introduced in PCIe-4.0 increases the total Tag
field size from 8 bits to 10 bits.

This patchset is to enable 10-Bit tag for PCIe EP devices (include VF) and
RP devices.

V2->V3:
- Use cached Device Capabilities Register suggested by Christoph.
- Fix code style to avoid > 80 char lines.
- Renamve devcap2 to pcie_devcap2.

V1->V2: Fix some comments by Christoph.
- Store the devcap2 value in the pci_dev instead of reading it multiple
  times.
  - Change pci_info to pci_dbg to avoid the noisy log.
  - Rename ext_10bit_tag_comp_path to ext_10bit_tag.
  - Fix the compile error.
  - Rebased on v5.13-rc1.

Dongdong Liu (6):
  PCI: Use cached Device Capabilities Register
  PCI: Use cached Device Capabilities 2 Register
  PCI: Add 10-Bit Tag register definitions
  PCI: Enable 10-Bit tag support for PCIe Endpoint devices
  PCI/IOV: Enable 10-Bit tag support for PCIe VF devices
  PCI: Enable 10-Bit tag support for PCIe RP devices

 drivers/media/pci/cobalt/cobalt-driver.c        |  4 +-
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c |  4 +-
 drivers/pci/iov.c                               |  8 +++
 drivers/pci/pci.c                               | 13 ++---
 drivers/pci/pcie/aspm.c                         | 11 ++--
 drivers/pci/pcie/portdrv_pci.c                  | 75 +++++++++++++++++++++++++
 drivers/pci/probe.c                             | 65 ++++++++++++++++-----
 drivers/pci/quirks.c                            |  3 +-
 include/linux/pci.h                             |  5 ++
 include/uapi/linux/pci_regs.h                   |  5 ++
 10 files changed, 156 insertions(+), 37 deletions(-)

--
2.7.4

[RESEND PATCH V3 1/6] PCI: Use cached Device Capabilities Register

[Dongdong Liu]

It will make sense to store the pcie_devcap value in the pci_dev
structure instead of reading Device Capabilities Register multiple
times. The fisrt place to use pcie_devcap is in set_pcie_port_type(),
get the pcie_devcap value here, then use cached pcie_devcap in the
needed place.

Acked-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
---
 drivers/media/pci/cobalt/cobalt-driver.c |  4 ++--
 drivers/pci/pci.c                        |  5 +----
 drivers/pci/pcie/aspm.c                  | 11 ++++-------
 drivers/pci/probe.c                      | 11 +++--------
 drivers/pci/quirks.c                     |  3 +--
 include/linux/pci.h                      |  1 +
 6 files changed, 12 insertions(+), 23 deletions(-)

diff --git a/drivers/media/pci/cobalt/cobalt-driver.c b/drivers/media/pci/cobalt/cobalt-driver.c
index 839503e..04e735f 100644
--- a/drivers/media/pci/cobalt/cobalt-driver.c
+++ b/drivers/media/pci/cobalt/cobalt-driver.c
@@ -193,11 +193,11 @@ void cobalt_pcie_status_show(struct cobalt *cobalt)
 		return;
 
 	/* Device */
-	pcie_capability_read_dword(pci_dev, PCI_EXP_DEVCAP, &capa);
 	pcie_capability_read_word(pci_dev, PCI_EXP_DEVCTL, &ctrl);
 	pcie_capability_read_word(pci_dev, PCI_EXP_DEVSTA, &stat);
 	cobalt_info("PCIe device capability 0x%08x: Max payload %d\n",
-		    capa, get_payload_size(capa & PCI_EXP_DEVCAP_PAYLOAD));
+		    capa,
+		    get_payload_size(pci_dev->pcie_devcap & PCI_EXP_DEVCAP_PAYLOAD));
 	cobalt_info("PCIe device control 0x%04x: Max payload %d. Max read request %d\n",
 		    ctrl,
 		    get_payload_size((ctrl & PCI_EXP_DEVCTL_PAYLOAD) >> 5),
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index b717680..68ccd77 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -4620,13 +4620,10 @@ EXPORT_SYMBOL(pci_wait_for_pending_transaction);
  */
 bool pcie_has_flr(struct pci_dev *dev)
 {
-	u32 cap;
-
 	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
 		return false;
 
-	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
-	return cap & PCI_EXP_DEVCAP_FLR;
+	return dev->pcie_devcap & PCI_EXP_DEVCAP_FLR;
 }
 EXPORT_SYMBOL_GPL(pcie_has_flr);
 
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index ac0557a..d637564 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -660,7 +660,7 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
 
 	/* Get and check endpoint acceptable latencies */
 	list_for_each_entry(child, &linkbus->devices, bus_list) {
-		u32 reg32, encoding;
+		u32 encoding;
 		struct aspm_latency *acceptable =
 			&link->acceptable[PCI_FUNC(child->devfn)];
 
@@ -668,12 +668,11 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
 		    pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)
 			continue;
 
-		pcie_capability_read_dword(child, PCI_EXP_DEVCAP, &reg32);
 		/* Calculate endpoint L0s acceptable latency */
-		encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
+		encoding = (child->pcie_devcap & PCI_EXP_DEVCAP_L0S) >> 6;
 		acceptable->l0s = calc_l0s_acceptable(encoding);
 		/* Calculate endpoint L1 acceptable latency */
-		encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
+		encoding = (child->pcie_devcap & PCI_EXP_DEVCAP_L1) >> 9;
 		acceptable->l1 = calc_l1_acceptable(encoding);
 
 		pcie_aspm_check_latency(child);
@@ -808,7 +807,6 @@ static void free_link_state(struct pcie_link_state *link)
 static int pcie_aspm_sanity_check(struct pci_dev *pdev)
 {
 	struct pci_dev *child;
-	u32 reg32;
 
 	/*
 	 * Some functions in a slot might not all be PCIe functions,
@@ -831,8 +829,7 @@ static int pcie_aspm_sanity_check(struct pci_dev *pdev)
 		 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
 		 * RBER bit to determine if a function is 1.1 version device
 		 */
-		pcie_capability_read_dword(child, PCI_EXP_DEVCAP, &reg32);
-		if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
+		if (!(child->pcie_devcap & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
 			pci_info(child, "disabling ASPM on pre-1.1 PCIe device.  You can enable it with 'pcie_aspm=force'\n");
 			return -EINVAL;
 		}
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 3a62d09..7963ab2 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -1497,8 +1497,8 @@ void set_pcie_port_type(struct pci_dev *pdev)
 	pdev->pcie_cap = pos;
 	pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, &reg16);
 	pdev->pcie_flags_reg = reg16;
-	pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, &reg16);
-	pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD;
+	pci_read_config_dword(pdev, pos + PCI_EXP_DEVCAP, &pdev->pcie_devcap);
+	pdev->pcie_mpss = pdev->pcie_devcap & PCI_EXP_DEVCAP_PAYLOAD;
 
 	parent = pci_upstream_bridge(pdev);
 	if (!parent)
@@ -2008,18 +2008,13 @@ static void pci_configure_mps(struct pci_dev *dev)
 int pci_configure_extended_tags(struct pci_dev *dev, void *ign)
 {
 	struct pci_host_bridge *host;
-	u32 cap;
 	u16 ctl;
 	int ret;
 
 	if (!pci_is_pcie(dev))
 		return 0;
 
-	ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
-	if (ret)
-		return 0;
-
-	if (!(cap & PCI_EXP_DEVCAP_EXT_TAG))
+	if (!(dev->pcie_devcap & PCI_EXP_DEVCAP_EXT_TAG))
 		return 0;
 
 	ret = pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index dcb229d..b89b438 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -5073,8 +5073,7 @@ static void quirk_intel_qat_vf_cap(struct pci_dev *pdev)
 		pdev->pcie_cap = pos;
 		pci_read_config_word(pdev, pos + PCI_EXP_FLAGS, &reg16);
 		pdev->pcie_flags_reg = reg16;
-		pci_read_config_word(pdev, pos + PCI_EXP_DEVCAP, &reg16);
-		pdev->pcie_mpss = reg16 & PCI_EXP_DEVCAP_PAYLOAD;
+		pdev->pcie_mpss = pdev->pcie_devcap & PCI_EXP_DEVCAP_PAYLOAD;
 
 		pdev->cfg_size = PCI_CFG_SPACE_EXP_SIZE;
 		if (pci_read_config_dword(pdev, PCI_CFG_SPACE_SIZE, &status) !=
diff --git a/include/linux/pci.h b/include/linux/pci.h
index c20211e..555a3ac 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -340,6 +340,7 @@ struct pci_dev {
 	u8		rom_base_reg;	/* Config register controlling ROM */
 	u8		pin;		/* Interrupt pin this device uses */
 	u16		pcie_flags_reg;	/* Cached PCIe Capabilities Register */
+	u32		pcie_devcap;	/* Cached Device Capabilities Register */
 	unsigned long	*dma_alias_mask;/* Mask of enabled devfn aliases */
 
 	struct pci_driver *driver;	/* Driver bound to this device */
-- 
2.7.4

Re: [RESEND PATCH V3 1/6] PCI: Use cached Device Capabilities Register

[Christoph Hellwig]

On Sun, Jun 13, 2021 at 05:29:10PM +0800, Dongdong Liu wrote:
> It will make sense to store the pcie_devcap value in the pci_dev
> structure instead of reading Device Capabilities Register multiple
> times. The fisrt place to use pcie_devcap is in set_pcie_port_type(),
> get the pcie_devcap value here, then use cached pcie_devcap in the
> needed place.
> 
> Acked-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
> Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
> ---
>  drivers/media/pci/cobalt/cobalt-driver.c |  4 ++--
>  drivers/pci/pci.c                        |  5 +----
>  drivers/pci/pcie/aspm.c                  | 11 ++++-------
>  drivers/pci/probe.c                      | 11 +++--------
>  drivers/pci/quirks.c                     |  3 +--
>  include/linux/pci.h                      |  1 +
>  6 files changed, 12 insertions(+), 23 deletions(-)
> 
> diff --git a/drivers/media/pci/cobalt/cobalt-driver.c b/drivers/media/pci/cobalt/cobalt-driver.c
> index 839503e..04e735f 100644
> --- a/drivers/media/pci/cobalt/cobalt-driver.c
> +++ b/drivers/media/pci/cobalt/cobalt-driver.c
> @@ -193,11 +193,11 @@ void cobalt_pcie_status_show(struct cobalt *cobalt)
>  		return;
>  
>  	/* Device */
> -	pcie_capability_read_dword(pci_dev, PCI_EXP_DEVCAP, &capa);
>  	pcie_capability_read_word(pci_dev, PCI_EXP_DEVCTL, &ctrl);
>  	pcie_capability_read_word(pci_dev, PCI_EXP_DEVSTA, &stat);
>  	cobalt_info("PCIe device capability 0x%08x: Max payload %d\n",
> -		    capa, get_payload_size(capa & PCI_EXP_DEVCAP_PAYLOAD));
> +		    capa,
> +		    get_payload_size(pci_dev->pcie_devcap & PCI_EXP_DEVCAP_PAYLOAD));

Overly long line.

> +		if (!(child->pcie_devcap & PCI_EXP_DEVCAP_RBER) && !aspm_force) {

Another one.

Otherwise looks good:

Reviewed-by: Christoph Hellwig <hch@lst.de>

Re: [RESEND PATCH V3 1/6] PCI: Use cached Device Capabilities Register

[Dongdong Liu]

On 2021/6/14 13:42, Christoph Hellwig wrote:
> On Sun, Jun 13, 2021 at 05:29:10PM +0800, Dongdong Liu wrote:
>> It will make sense to store the pcie_devcap value in the pci_dev
>> structure instead of reading Device Capabilities Register multiple
>> times. The fisrt place to use pcie_devcap is in set_pcie_port_type(),
>> get the pcie_devcap value here, then use cached pcie_devcap in the
>> needed place.
>>
>> Acked-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
>> Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
>> ---
>>  drivers/media/pci/cobalt/cobalt-driver.c |  4 ++--
>>  drivers/pci/pci.c                        |  5 +----
>>  drivers/pci/pcie/aspm.c                  | 11 ++++-------
>>  drivers/pci/probe.c                      | 11 +++--------
>>  drivers/pci/quirks.c                     |  3 +--
>>  include/linux/pci.h                      |  1 +
>>  6 files changed, 12 insertions(+), 23 deletions(-)
>>
>> diff --git a/drivers/media/pci/cobalt/cobalt-driver.c b/drivers/media/pci/cobalt/cobalt-driver.c
>> index 839503e..04e735f 100644
>> --- a/drivers/media/pci/cobalt/cobalt-driver.c
>> +++ b/drivers/media/pci/cobalt/cobalt-driver.c
>> @@ -193,11 +193,11 @@ void cobalt_pcie_status_show(struct cobalt *cobalt)
>>  		return;
>>
>>  	/* Device */
>> -	pcie_capability_read_dword(pci_dev, PCI_EXP_DEVCAP, &capa);
>>  	pcie_capability_read_word(pci_dev, PCI_EXP_DEVCTL, &ctrl);
>>  	pcie_capability_read_word(pci_dev, PCI_EXP_DEVSTA, &stat);
>>  	cobalt_info("PCIe device capability 0x%08x: Max payload %d\n",
>> -		    capa, get_payload_size(capa & PCI_EXP_DEVCAP_PAYLOAD));
>> +		    capa,
>> +		    get_payload_size(pci_dev->pcie_devcap & PCI_EXP_DEVCAP_PAYLOAD));
>
> Overly long line.
Will fix
>
>> +		if (!(child->pcie_devcap & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
>
> Another one.
Will fix.

Thanks,
Dongdong
>
> Otherwise looks good:
>
> Reviewed-by: Christoph Hellwig <hch@lst.de>
> .
>

Re: [RESEND PATCH V3 1/6] PCI: Use cached Device Capabilities Register

[kernel test robot]

[-- Attachment #1: Type: text/plain, Size: 9832 bytes --]

Hi Dongdong,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on pci/next]
[also build test WARNING on linuxtv-media/master linus/master v5.13-rc6 next-20210618]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Dongdong-Liu/PCI-Enable-10-Bit-tag-support-for-PCIe-devices/20210617-041115
base:   https://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci.git next
config: s390-randconfig-r032-20210618 (attached as .config)
compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project 64720f57bea6a6bf033feef4a5751ab9c0c3b401)
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # install s390 cross compiling tool for clang build
        # apt-get install binutils-s390x-linux-gnu
        # https://github.com/0day-ci/linux/commit/caefa7e6d0209bc08eb1934b58dae3aaa0b9dbba
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Dongdong-Liu/PCI-Enable-10-Bit-tag-support-for-PCIe-devices/20210617-041115
        git checkout caefa7e6d0209bc08eb1934b58dae3aaa0b9dbba
        # save the attached .config to linux build tree
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=s390 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>

All warnings (new ones prefixed by >>):

   In file included from drivers/media/pci/cobalt/cobalt-driver.c:18:
   In file included from drivers/media/pci/cobalt/cobalt-driver.h:16:
   In file included from include/linux/pci.h:39:
   In file included from include/linux/io.h:13:
   In file included from arch/s390/include/asm/io.h:75:
   include/asm-generic/io.h:464:31: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           val = __raw_readb(PCI_IOBASE + addr);
                             ~~~~~~~~~~ ^
   include/asm-generic/io.h:477:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           val = __le16_to_cpu((__le16 __force)__raw_readw(PCI_IOBASE + addr));
                                                           ~~~~~~~~~~ ^
   include/uapi/linux/byteorder/big_endian.h:36:59: note: expanded from macro '__le16_to_cpu'
   #define __le16_to_cpu(x) __swab16((__force __u16)(__le16)(x))
                                                             ^
   include/uapi/linux/swab.h:102:54: note: expanded from macro '__swab16'
   #define __swab16(x) (__u16)__builtin_bswap16((__u16)(x))
                                                        ^
   In file included from drivers/media/pci/cobalt/cobalt-driver.c:18:
   In file included from drivers/media/pci/cobalt/cobalt-driver.h:16:
   In file included from include/linux/pci.h:39:
   In file included from include/linux/io.h:13:
   In file included from arch/s390/include/asm/io.h:75:
   include/asm-generic/io.h:490:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           val = __le32_to_cpu((__le32 __force)__raw_readl(PCI_IOBASE + addr));
                                                           ~~~~~~~~~~ ^
   include/uapi/linux/byteorder/big_endian.h:34:59: note: expanded from macro '__le32_to_cpu'
   #define __le32_to_cpu(x) __swab32((__force __u32)(__le32)(x))
                                                             ^
   include/uapi/linux/swab.h:115:54: note: expanded from macro '__swab32'
   #define __swab32(x) (__u32)__builtin_bswap32((__u32)(x))
                                                        ^
   In file included from drivers/media/pci/cobalt/cobalt-driver.c:18:
   In file included from drivers/media/pci/cobalt/cobalt-driver.h:16:
   In file included from include/linux/pci.h:39:
   In file included from include/linux/io.h:13:
   In file included from arch/s390/include/asm/io.h:75:
   include/asm-generic/io.h:501:33: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           __raw_writeb(value, PCI_IOBASE + addr);
                               ~~~~~~~~~~ ^
   include/asm-generic/io.h:511:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           __raw_writew((u16 __force)cpu_to_le16(value), PCI_IOBASE + addr);
                                                         ~~~~~~~~~~ ^
   include/asm-generic/io.h:521:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           __raw_writel((u32 __force)cpu_to_le32(value), PCI_IOBASE + addr);
                                                         ~~~~~~~~~~ ^
   include/asm-generic/io.h:609:20: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           readsb(PCI_IOBASE + addr, buffer, count);
                  ~~~~~~~~~~ ^
   include/asm-generic/io.h:617:20: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           readsw(PCI_IOBASE + addr, buffer, count);
                  ~~~~~~~~~~ ^
   include/asm-generic/io.h:625:20: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           readsl(PCI_IOBASE + addr, buffer, count);
                  ~~~~~~~~~~ ^
   include/asm-generic/io.h:634:21: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           writesb(PCI_IOBASE + addr, buffer, count);
                   ~~~~~~~~~~ ^
   include/asm-generic/io.h:643:21: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           writesw(PCI_IOBASE + addr, buffer, count);
                   ~~~~~~~~~~ ^
   include/asm-generic/io.h:652:21: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
           writesl(PCI_IOBASE + addr, buffer, count);
                   ~~~~~~~~~~ ^
>> drivers/media/pci/cobalt/cobalt-driver.c:199:7: warning: variable 'capa' is uninitialized when used here [-Wuninitialized]
                       capa,
                       ^~~~
   drivers/media/pci/cobalt/cobalt-driver.h:160:71: note: expanded from macro 'cobalt_info'
   #define cobalt_info(fmt, arg...) v4l2_info(&cobalt->v4l2_dev, fmt, ## arg)
                                                                         ^~~
   include/media/v4l2-common.h:67:39: note: expanded from macro 'v4l2_info'
           v4l2_printk(KERN_INFO, dev, fmt , ## arg)
                                                ^~~
   include/media/v4l2-common.h:58:44: note: expanded from macro 'v4l2_printk'
           printk(level "%s: " fmt, (dev)->name , ## arg)
                                                     ^~~
   drivers/media/pci/cobalt/cobalt-driver.c:189:10: note: initialize the variable 'capa' to silence this warning
           u32 capa;
                   ^
                    = 0
   13 warnings generated.


vim +/capa +199 drivers/media/pci/cobalt/cobalt-driver.c

   184	
   185	void cobalt_pcie_status_show(struct cobalt *cobalt)
   186	{
   187		struct pci_dev *pci_dev = cobalt->pci_dev;
   188		struct pci_dev *pci_bus_dev = cobalt->pci_dev->bus->self;
   189		u32 capa;
   190		u16 stat, ctrl;
   191	
   192		if (!pci_is_pcie(pci_dev) || !pci_is_pcie(pci_bus_dev))
   193			return;
   194	
   195		/* Device */
   196		pcie_capability_read_word(pci_dev, PCI_EXP_DEVCTL, &ctrl);
   197		pcie_capability_read_word(pci_dev, PCI_EXP_DEVSTA, &stat);
   198		cobalt_info("PCIe device capability 0x%08x: Max payload %d\n",
 > 199			    capa,
   200			    get_payload_size(pci_dev->pcie_devcap & PCI_EXP_DEVCAP_PAYLOAD));
   201		cobalt_info("PCIe device control 0x%04x: Max payload %d. Max read request %d\n",
   202			    ctrl,
   203			    get_payload_size((ctrl & PCI_EXP_DEVCTL_PAYLOAD) >> 5),
   204			    get_payload_size((ctrl & PCI_EXP_DEVCTL_READRQ) >> 12));
   205		cobalt_info("PCIe device status 0x%04x\n", stat);
   206	
   207		/* Link */
   208		pcie_capability_read_dword(pci_dev, PCI_EXP_LNKCAP, &capa);
   209		pcie_capability_read_word(pci_dev, PCI_EXP_LNKCTL, &ctrl);
   210		pcie_capability_read_word(pci_dev, PCI_EXP_LNKSTA, &stat);
   211		cobalt_info("PCIe link capability 0x%08x: %s per lane and %u lanes\n",
   212				capa, get_link_speed(capa),
   213				(capa & PCI_EXP_LNKCAP_MLW) >> 4);
   214		cobalt_info("PCIe link control 0x%04x\n", ctrl);
   215		cobalt_info("PCIe link status 0x%04x: %s per lane and %u lanes\n",
   216			    stat, get_link_speed(stat),
   217			    (stat & PCI_EXP_LNKSTA_NLW) >> 4);
   218	
   219		/* Bus */
   220		pcie_capability_read_dword(pci_bus_dev, PCI_EXP_LNKCAP, &capa);
   221		cobalt_info("PCIe bus link capability 0x%08x: %s per lane and %u lanes\n",
   222				capa, get_link_speed(capa),
   223				(capa & PCI_EXP_LNKCAP_MLW) >> 4);
   224	
   225		/* Slot */
   226		pcie_capability_read_dword(pci_dev, PCI_EXP_SLTCAP, &capa);
   227		pcie_capability_read_word(pci_dev, PCI_EXP_SLTCTL, &ctrl);
   228		pcie_capability_read_word(pci_dev, PCI_EXP_SLTSTA, &stat);
   229		cobalt_info("PCIe slot capability 0x%08x\n", capa);
   230		cobalt_info("PCIe slot control 0x%04x\n", ctrl);
   231		cobalt_info("PCIe slot status 0x%04x\n", stat);
   232	}
   233	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 36991 bytes --]

Re: [RESEND PATCH V3 1/6] PCI: Use cached Device Capabilities Register

[Dongdong Liu]

On 2021/6/18 22:51, kernel test robot wrote:
> Hi Dongdong,
>
> Thank you for the patch! Perhaps something to improve:
>
> [auto build test WARNING on pci/next]
> [also build test WARNING on linuxtv-media/master linus/master v5.13-rc6 next-20210618]
> [If your patch is applied to the wrong git tree, kindly drop us a note.
> And when submitting patch, we suggest to use '--base' as documented in
> https://git-scm.com/docs/git-format-patch]
>
> url:    https://github.com/0day-ci/linux/commits/Dongdong-Liu/PCI-Enable-10-Bit-tag-support-for-PCIe-devices/20210617-041115
> base:   https://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci.git next
> config: s390-randconfig-r032-20210618 (attached as .config)
> compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project 64720f57bea6a6bf033feef4a5751ab9c0c3b401)
> reproduce (this is a W=1 build):
>         wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
>         chmod +x ~/bin/make.cross
>         # install s390 cross compiling tool for clang build
>         # apt-get install binutils-s390x-linux-gnu
>         # https://github.com/0day-ci/linux/commit/caefa7e6d0209bc08eb1934b58dae3aaa0b9dbba
>         git remote add linux-review https://github.com/0day-ci/linux
>         git fetch --no-tags linux-review Dongdong-Liu/PCI-Enable-10-Bit-tag-support-for-PCIe-devices/20210617-041115
>         git checkout caefa7e6d0209bc08eb1934b58dae3aaa0b9dbba
>         # save the attached .config to linux build tree
>         COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=s390
>
> If you fix the issue, kindly add following tag as appropriate
> Reported-by: kernel test robot <lkp@intel.com>
>
> All warnings (new ones prefixed by >>):
>
>    In file included from drivers/media/pci/cobalt/cobalt-driver.c:18:
>    In file included from drivers/media/pci/cobalt/cobalt-driver.h:16:
>    In file included from include/linux/pci.h:39:
>    In file included from include/linux/io.h:13:
>    In file included from arch/s390/include/asm/io.h:75:
>    include/asm-generic/io.h:464:31: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            val = __raw_readb(PCI_IOBASE + addr);
>                              ~~~~~~~~~~ ^
>    include/asm-generic/io.h:477:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            val = __le16_to_cpu((__le16 __force)__raw_readw(PCI_IOBASE + addr));
>                                                            ~~~~~~~~~~ ^
>    include/uapi/linux/byteorder/big_endian.h:36:59: note: expanded from macro '__le16_to_cpu'
>    #define __le16_to_cpu(x) __swab16((__force __u16)(__le16)(x))
>                                                              ^
>    include/uapi/linux/swab.h:102:54: note: expanded from macro '__swab16'
>    #define __swab16(x) (__u16)__builtin_bswap16((__u16)(x))
>                                                         ^
>    In file included from drivers/media/pci/cobalt/cobalt-driver.c:18:
>    In file included from drivers/media/pci/cobalt/cobalt-driver.h:16:
>    In file included from include/linux/pci.h:39:
>    In file included from include/linux/io.h:13:
>    In file included from arch/s390/include/asm/io.h:75:
>    include/asm-generic/io.h:490:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            val = __le32_to_cpu((__le32 __force)__raw_readl(PCI_IOBASE + addr));
>                                                            ~~~~~~~~~~ ^
>    include/uapi/linux/byteorder/big_endian.h:34:59: note: expanded from macro '__le32_to_cpu'
>    #define __le32_to_cpu(x) __swab32((__force __u32)(__le32)(x))
>                                                              ^
>    include/uapi/linux/swab.h:115:54: note: expanded from macro '__swab32'
>    #define __swab32(x) (__u32)__builtin_bswap32((__u32)(x))
>                                                         ^
>    In file included from drivers/media/pci/cobalt/cobalt-driver.c:18:
>    In file included from drivers/media/pci/cobalt/cobalt-driver.h:16:
>    In file included from include/linux/pci.h:39:
>    In file included from include/linux/io.h:13:
>    In file included from arch/s390/include/asm/io.h:75:
>    include/asm-generic/io.h:501:33: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            __raw_writeb(value, PCI_IOBASE + addr);
>                                ~~~~~~~~~~ ^
>    include/asm-generic/io.h:511:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            __raw_writew((u16 __force)cpu_to_le16(value), PCI_IOBASE + addr);
>                                                          ~~~~~~~~~~ ^
>    include/asm-generic/io.h:521:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            __raw_writel((u32 __force)cpu_to_le32(value), PCI_IOBASE + addr);
>                                                          ~~~~~~~~~~ ^
>    include/asm-generic/io.h:609:20: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            readsb(PCI_IOBASE + addr, buffer, count);
>                   ~~~~~~~~~~ ^
>    include/asm-generic/io.h:617:20: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            readsw(PCI_IOBASE + addr, buffer, count);
>                   ~~~~~~~~~~ ^
>    include/asm-generic/io.h:625:20: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            readsl(PCI_IOBASE + addr, buffer, count);
>                   ~~~~~~~~~~ ^
>    include/asm-generic/io.h:634:21: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            writesb(PCI_IOBASE + addr, buffer, count);
>                    ~~~~~~~~~~ ^
>    include/asm-generic/io.h:643:21: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            writesw(PCI_IOBASE + addr, buffer, count);
>                    ~~~~~~~~~~ ^
>    include/asm-generic/io.h:652:21: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
>            writesl(PCI_IOBASE + addr, buffer, count);
>                    ~~~~~~~~~~ ^
>>> drivers/media/pci/cobalt/cobalt-driver.c:199:7: warning: variable 'capa' is uninitialized when used here [-Wuninitialized]
>                        capa,
>                        ^~~~
>    drivers/media/pci/cobalt/cobalt-driver.h:160:71: note: expanded from macro 'cobalt_info'
>    #define cobalt_info(fmt, arg...) v4l2_info(&cobalt->v4l2_dev, fmt, ## arg)
>                                                                          ^~~
>    include/media/v4l2-common.h:67:39: note: expanded from macro 'v4l2_info'
>            v4l2_printk(KERN_INFO, dev, fmt , ## arg)
>                                                 ^~~
>    include/media/v4l2-common.h:58:44: note: expanded from macro 'v4l2_printk'
>            printk(level "%s: " fmt, (dev)->name , ## arg)
>                                                      ^~~
>    drivers/media/pci/cobalt/cobalt-driver.c:189:10: note: initialize the variable 'capa' to silence this warning
>            u32 capa;
>                    ^
>                     = 0
>    13 warnings generated.
>
>
> vim +/capa +199 drivers/media/pci/cobalt/cobalt-driver.c
>
>    184	
>    185	void cobalt_pcie_status_show(struct cobalt *cobalt)
>    186	{
>    187		struct pci_dev *pci_dev = cobalt->pci_dev;
>    188		struct pci_dev *pci_bus_dev = cobalt->pci_dev->bus->self;
>    189		u32 capa;
>    190		u16 stat, ctrl;
>    191	
>    192		if (!pci_is_pcie(pci_dev) || !pci_is_pcie(pci_bus_dev))
>    193			return;
>    194	
>    195		/* Device */
>    196		pcie_capability_read_word(pci_dev, PCI_EXP_DEVCTL, &ctrl);
>    197		pcie_capability_read_word(pci_dev, PCI_EXP_DEVSTA, &stat);
>    198		cobalt_info("PCIe device capability 0x%08x: Max payload %d\n",
>  > 199			    capa,
Will fix with pci_dev->pcie_devcap.

Thanks,
Dongdong
>    200			    get_payload_size(pci_dev->pcie_devcap & PCI_EXP_DEVCAP_PAYLOAD));
>    201		cobalt_info("PCIe device control 0x%04x: Max payload %d. Max read request %d\n",
>    202			    ctrl,
>    203			    get_payload_size((ctrl & PCI_EXP_DEVCTL_PAYLOAD) >> 5),
>    204			    get_payload_size((ctrl & PCI_EXP_DEVCTL_READRQ) >> 12));
>    205		cobalt_info("PCIe device status 0x%04x\n", stat);
>    206	
>    207		/* Link */
>    208		pcie_capability_read_dword(pci_dev, PCI_EXP_LNKCAP, &capa);
>    209		pcie_capability_read_word(pci_dev, PCI_EXP_LNKCTL, &ctrl);
>    210		pcie_capability_read_word(pci_dev, PCI_EXP_LNKSTA, &stat);
>    211		cobalt_info("PCIe link capability 0x%08x: %s per lane and %u lanes\n",
>    212				capa, get_link_speed(capa),
>    213				(capa & PCI_EXP_LNKCAP_MLW) >> 4);
>    214		cobalt_info("PCIe link control 0x%04x\n", ctrl);
>    215		cobalt_info("PCIe link status 0x%04x: %s per lane and %u lanes\n",
>    216			    stat, get_link_speed(stat),
>    217			    (stat & PCI_EXP_LNKSTA_NLW) >> 4);
>    218	
>    219		/* Bus */
>    220		pcie_capability_read_dword(pci_bus_dev, PCI_EXP_LNKCAP, &capa);
>    221		cobalt_info("PCIe bus link capability 0x%08x: %s per lane and %u lanes\n",
>    222				capa, get_link_speed(capa),
>    223				(capa & PCI_EXP_LNKCAP_MLW) >> 4);
>    224	
>    225		/* Slot */
>    226		pcie_capability_read_dword(pci_dev, PCI_EXP_SLTCAP, &capa);
>    227		pcie_capability_read_word(pci_dev, PCI_EXP_SLTCTL, &ctrl);
>    228		pcie_capability_read_word(pci_dev, PCI_EXP_SLTSTA, &stat);
>    229		cobalt_info("PCIe slot capability 0x%08x\n", capa);
>    230		cobalt_info("PCIe slot control 0x%04x\n", ctrl);
>    231		cobalt_info("PCIe slot status 0x%04x\n", stat);
>    232	}
>    233	
>
> ---
> 0-DAY CI Kernel Test Service, Intel Corporation
> https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
>

[RESEND PATCH V3 2/6] PCI: Use cached Device Capabilities 2 Register

[Dongdong Liu]

It will make sense to store the pcie_devcap2 value in the pci_dev
structure instead of reading Device Capabilities 2 Register multiple
times. Add pci_init_devcap2() to get the pcie_devcap2 value, then
use cached pcie_devcap2 in the needed place.

Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
---
 drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c |  4 +---
 drivers/pci/pci.c                               |  8 +++-----
 drivers/pci/probe.c                             | 18 ++++++++++++------
 include/linux/pci.h                             |  2 ++
 4 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 6264bc6..7896c96 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -6303,7 +6303,6 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
 		struct pci_dev *pbridge;
 		struct port_info *pi;
 		char name[IFNAMSIZ];
-		u32 devcap2;
 		u16 flags;
 
 		/* If we want to instantiate Virtual Functions, then our
@@ -6313,10 +6312,9 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs)
 		 */
 		pbridge = pdev->bus->self;
 		pcie_capability_read_word(pbridge, PCI_EXP_FLAGS, &flags);
-		pcie_capability_read_dword(pbridge, PCI_EXP_DEVCAP2, &devcap2);
 
 		if ((flags & PCI_EXP_FLAGS_VERS) < 2 ||
-		    !(devcap2 & PCI_EXP_DEVCAP2_ARI)) {
+		    !(pbridge->pcie_devcap2 & PCI_EXP_DEVCAP2_ARI)) {
 			/* Our parent bridge does not support ARI so issue a
 			 * warning and skip instantiating the VFs.  They
 			 * won't be reachable.
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index 68ccd77..11103bb 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -3690,7 +3690,7 @@ int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
 {
 	struct pci_bus *bus = dev->bus;
 	struct pci_dev *bridge;
-	u32 cap, ctl2;
+	u32 ctl2;
 
 	if (!pci_is_pcie(dev))
 		return -EINVAL;
@@ -3714,19 +3714,17 @@ int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
 	while (bus->parent) {
 		bridge = bus->self;
 
-		pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
-
 		switch (pci_pcie_type(bridge)) {
 		/* Ensure switch ports support AtomicOp routing */
 		case PCI_EXP_TYPE_UPSTREAM:
 		case PCI_EXP_TYPE_DOWNSTREAM:
-			if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
+			if (!(bridge->pcie_devcap2 & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
 				return -EINVAL;
 			break;
 
 		/* Ensure root port supports all the sizes we care about */
 		case PCI_EXP_TYPE_ROOT_PORT:
-			if ((cap & cap_mask) != cap_mask)
+			if ((bridge->pcie_devcap2 & cap_mask) != cap_mask)
 				return -EINVAL;
 			break;
 		}
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index 7963ab2..b9942cc 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -2093,7 +2093,7 @@ static void pci_configure_ltr(struct pci_dev *dev)
 #ifdef CONFIG_PCIEASPM
 	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
 	struct pci_dev *bridge;
-	u32 cap, ctl;
+	u32 ctl;
 
 	if (!pci_is_pcie(dev))
 		return;
@@ -2101,8 +2101,7 @@ static void pci_configure_ltr(struct pci_dev *dev)
 	/* Read L1 PM substate capabilities */
 	dev->l1ss = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_L1SS);
 
-	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &cap);
-	if (!(cap & PCI_EXP_DEVCAP2_LTR))
+	if (!(dev->pcie_devcap2 & PCI_EXP_DEVCAP2_LTR))
 		return;
 
 	pcie_capability_read_dword(dev, PCI_EXP_DEVCTL2, &ctl);
@@ -2142,13 +2141,11 @@ static void pci_configure_eetlp_prefix(struct pci_dev *dev)
 #ifdef CONFIG_PCI_PASID
 	struct pci_dev *bridge;
 	int pcie_type;
-	u32 cap;
 
 	if (!pci_is_pcie(dev))
 		return;
 
-	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &cap);
-	if (!(cap & PCI_EXP_DEVCAP2_EE_PREFIX))
+	if (!(dev->pcie_devcap2 & PCI_EXP_DEVCAP2_EE_PREFIX))
 		return;
 
 	pcie_type = pci_pcie_type(dev);
@@ -2376,6 +2373,14 @@ void pcie_report_downtraining(struct pci_dev *dev)
 	__pcie_print_link_status(dev, false);
 }
 
+static void pci_init_devcap2(struct pci_dev *dev)
+{
+	if (!pci_is_pcie(dev))
+		return;
+
+	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &dev->pcie_devcap2);
+}
+
 static void pci_init_capabilities(struct pci_dev *dev)
 {
 	pci_ea_init(dev);		/* Enhanced Allocation */
@@ -2452,6 +2457,7 @@ void pci_device_add(struct pci_dev *dev, struct pci_bus *bus)
 {
 	int ret;
 
+	pci_init_devcap2(dev);
 	pci_configure_device(dev);
 
 	device_initialize(&dev->dev);
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 555a3ac..2965620 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -341,6 +341,8 @@ struct pci_dev {
 	u8		pin;		/* Interrupt pin this device uses */
 	u16		pcie_flags_reg;	/* Cached PCIe Capabilities Register */
 	u32		pcie_devcap;	/* Cached Device Capabilities Register */
+	u32		pcie_devcap2;	/* Cached Device Capabilities 2
+					   Register */
 	unsigned long	*dma_alias_mask;/* Mask of enabled devfn aliases */
 
 	struct pci_driver *driver;	/* Driver bound to this device */
-- 
2.7.4

Re: [RESEND PATCH V3 2/6] PCI: Use cached Device Capabilities 2 Register

[Christoph Hellwig]

> +			if (!(bridge->pcie_devcap2 & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))

Overly long line.

> +static void pci_init_devcap2(struct pci_dev *dev)
> +{
> +	if (!pci_is_pcie(dev))
> +		return;
> +
> +	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &dev->pcie_devcap2);
> +}

Wouldn't it make sene to merge this into set_pcie_port_type?

Re: [RESEND PATCH V3 2/6] PCI: Use cached Device Capabilities 2 Register

[Dongdong Liu]

On 2021/6/14 13:49, Christoph Hellwig wrote:
>> +			if (!(bridge->pcie_devcap2 & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
>
> Overly long line.
>
Will fix.
>> +static void pci_init_devcap2(struct pci_dev *dev)
>> +{
>> +	if (!pci_is_pcie(dev))
>> +		return;
>> +
>> +	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, &dev->pcie_devcap2);
>> +}
>
> Wouldn't it make sene to merge this into set_pcie_port_type?
Good suggestion, will do.

Thanks,
Dongdong
> .
>

[RESEND PATCH V3 3/6] PCI: Add 10-Bit Tag register definitions

[Dongdong Liu]

Add 10-Bit Tag register definitions for use in subsequen patches.
See the PCIe 5.0 spec section 7.5.3.15 and 9.3.3.2.

Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
---
 include/uapi/linux/pci_regs.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h
index e709ae8..cf1ddb8 100644
--- a/include/uapi/linux/pci_regs.h
+++ b/include/uapi/linux/pci_regs.h
@@ -648,6 +648,8 @@
 #define  PCI_EXP_DEVCAP2_ATOMIC_COMP64	0x00000100 /* 64b AtomicOp completion */
 #define  PCI_EXP_DEVCAP2_ATOMIC_COMP128	0x00000200 /* 128b AtomicOp completion */
 #define  PCI_EXP_DEVCAP2_LTR		0x00000800 /* Latency tolerance reporting */
+#define  PCI_EXP_DEVCAP2_10BIT_TAG_COMP 0x00010000 /* 10-Bit Tag Completer Supported */
+#define  PCI_EXP_DEVCAP2_10BIT_TAG_REQ  0x00020000 /* 10-Bit Tag Requester Supported */
 #define  PCI_EXP_DEVCAP2_OBFF_MASK	0x000c0000 /* OBFF support mechanism */
 #define  PCI_EXP_DEVCAP2_OBFF_MSG	0x00040000 /* New message signaling */
 #define  PCI_EXP_DEVCAP2_OBFF_WAKE	0x00080000 /* Re-use WAKE# for OBFF */
@@ -661,6 +663,7 @@
 #define  PCI_EXP_DEVCTL2_IDO_REQ_EN	0x0100	/* Allow IDO for requests */
 #define  PCI_EXP_DEVCTL2_IDO_CMP_EN	0x0200	/* Allow IDO for completions */
 #define  PCI_EXP_DEVCTL2_LTR_EN		0x0400	/* Enable LTR mechanism */
+#define  PCI_EXP_DEVCTL2_10BIT_TAG_REQ_EN 0x1000 /* 10-Bit Tag Requester Enable */
 #define  PCI_EXP_DEVCTL2_OBFF_MSGA_EN	0x2000	/* Enable OBFF Message type A */
 #define  PCI_EXP_DEVCTL2_OBFF_MSGB_EN	0x4000	/* Enable OBFF Message type B */
 #define  PCI_EXP_DEVCTL2_OBFF_WAKE_EN	0x6000	/* OBFF using WAKE# signaling */
@@ -931,6 +934,7 @@
 /* Single Root I/O Virtualization */
 #define PCI_SRIOV_CAP		0x04	/* SR-IOV Capabilities */
 #define  PCI_SRIOV_CAP_VFM	0x00000001  /* VF Migration Capable */
+#define  PCI_SRIOV_CAP_VF_10BIT_TAG_REQ	0x00000004 /* VF 10-Bit Tag Requester Supported */
 #define  PCI_SRIOV_CAP_INTR(x)	((x) >> 21) /* Interrupt Message Number */
 #define PCI_SRIOV_CTRL		0x08	/* SR-IOV Control */
 #define  PCI_SRIOV_CTRL_VFE	0x0001	/* VF Enable */
@@ -938,6 +942,7 @@
 #define  PCI_SRIOV_CTRL_INTR	0x0004	/* VF Migration Interrupt Enable */
 #define  PCI_SRIOV_CTRL_MSE	0x0008	/* VF Memory Space Enable */
 #define  PCI_SRIOV_CTRL_ARI	0x0010	/* ARI Capable Hierarchy */
+#define  PCI_SRIOV_CTRL_VF_10BIT_TAG_REQ_EN 0x0020 /* VF 10-Bit Tag Requester Enable */
 #define PCI_SRIOV_STATUS	0x0a	/* SR-IOV Status */
 #define  PCI_SRIOV_STATUS_VFM	0x0001	/* VF Migration Status */
 #define PCI_SRIOV_INITIAL_VF	0x0c	/* Initial VFs */
-- 
2.7.4

Re: [RESEND PATCH V3 3/6] PCI: Add 10-Bit Tag register definitions

[Christoph Hellwig]

Looks good,

Reviewed-by: Christoph Hellwig <hch@lst.de>

[RESEND PATCH V3 4/6] PCI: Enable 10-Bit tag support for PCIe Endpoint devices

[Dongdong Liu]

10-Bit Tag capability, introduced in PCIe-4.0 increases the total Tag
field size from 8 bits to 10 bits.

For platforms where the RC supports 10-Bit Tag Completer capability,
it is highly recommended for platform firmware or operating software
that configures PCIe hierarchies to Set the 10-Bit Tag Requester Enable
bit automatically in Endpoints with 10-Bit Tag Requester capability. This
enables the important class of 10-Bit Tag capable adapters that send
Memory Read Requests only to host memory.

Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
---
 drivers/pci/probe.c | 36 ++++++++++++++++++++++++++++++++++++
 include/linux/pci.h |  2 ++
 2 files changed, 38 insertions(+)

diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index b9942cc..dfcd3d2 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -2046,6 +2046,41 @@ int pci_configure_extended_tags(struct pci_dev *dev, void *ign)
 	return 0;
 }
 
+static void pci_configure_10bit_tags(struct pci_dev *dev)
+{
+	struct pci_dev *bridge;
+
+	if (!pci_is_pcie(dev))
+		return;
+
+	if (!(dev->pcie_devcap2 & PCI_EXP_DEVCAP2_10BIT_TAG_COMP))
+		return;
+
+	if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) {
+		dev->ext_10bit_tag = 1;
+		return;
+	}
+
+	bridge = pci_upstream_bridge(dev);
+	if (bridge && bridge->ext_10bit_tag)
+		dev->ext_10bit_tag = 1;
+
+	/*
+	 * 10-Bit Tag Requester Enable in Device Control 2 Register is RsvdP
+	 * for VF.
+	 */
+	if (dev->is_virtfn)
+		return;
+
+	if (pci_pcie_type(dev) == PCI_EXP_TYPE_ENDPOINT &&
+	    dev->ext_10bit_tag == 1 &&
+	    (dev->pcie_devcap2 & PCI_EXP_DEVCAP2_10BIT_TAG_REQ)) {
+		pci_dbg(dev, "enabling 10-Bit Tag Requester\n");
+		pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
+					PCI_EXP_DEVCTL2_10BIT_TAG_REQ_EN);
+	}
+}
+
 /**
  * pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable
  * @dev: PCI device to query
@@ -2182,6 +2217,7 @@ static void pci_configure_device(struct pci_dev *dev)
 {
 	pci_configure_mps(dev);
 	pci_configure_extended_tags(dev, NULL);
+	pci_configure_10bit_tags(dev);
 	pci_configure_relaxed_ordering(dev);
 	pci_configure_ltr(dev);
 	pci_configure_eetlp_prefix(dev);
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 2965620..f2b2b5b 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -393,6 +393,8 @@ struct pci_dev {
 #endif
 	unsigned int	eetlp_prefix_path:1;	/* End-to-End TLP Prefix */
 
+	unsigned int	ext_10bit_tag:1; /* 10-Bit Tag Completer Supported
+					    from root to here */
 	pci_channel_state_t error_state;	/* Current connectivity state */
 	struct device	dev;			/* Generic device interface */
 
-- 
2.7.4

Re: [RESEND PATCH V3 4/6] PCI: Enable 10-Bit tag support for PCIe Endpoint devices

[Christoph Hellwig]

On Sun, Jun 13, 2021 at 05:29:13PM +0800, Dongdong Liu wrote:
> +static void pci_configure_10bit_tags(struct pci_dev *dev)
> +{
> +	struct pci_dev *bridge;
> +
> +	if (!pci_is_pcie(dev))
> +		return;
> +
> +	if (!(dev->pcie_devcap2 & PCI_EXP_DEVCAP2_10BIT_TAG_COMP))
> +		return;

Doesn't the second check imply the first one?

Otherwise looks good:

Reviewed-by: Christoph Hellwig <hch@lst.de>

Re: [RESEND PATCH V3 4/6] PCI: Enable 10-Bit tag support for PCIe Endpoint devices

[Dongdong Liu]

On 2021/6/14 13:54, Christoph Hellwig wrote:
> On Sun, Jun 13, 2021 at 05:29:13PM +0800, Dongdong Liu wrote:
>> +static void pci_configure_10bit_tags(struct pci_dev *dev)
>> +{
>> +	struct pci_dev *bridge;
>> +
>> +	if (!pci_is_pcie(dev))
>> +		return;
>> +
>> +	if (!(dev->pcie_devcap2 & PCI_EXP_DEVCAP2_10BIT_TAG_COMP))
>> +		return;
>
> Doesn't the second check imply the first one?
Yes, no need the first one, will delete.

Thanks,
Dongdong
>
> Otherwise looks good:
>
> Reviewed-by: Christoph Hellwig <hch@lst.de>
> .
>

[RESEND PATCH V3 5/6] PCI/IOV: Enable 10-Bit tag support for PCIe VF devices

[Dongdong Liu]

Enable VF 10-Bit Tag Requester when it's upstream component support
10-bit Tag Completer.

Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
---
 drivers/pci/iov.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index afc06e6..3eb4348 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -627,6 +627,10 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 
 	pci_iov_set_numvfs(dev, nr_virtfn);
 	iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE;
+	if ((iov->cap & PCI_SRIOV_CAP_VF_10BIT_TAG_REQ) &&
+	    dev->ext_10bit_tag)
+		iov->ctrl |= PCI_SRIOV_CTRL_VF_10BIT_TAG_REQ_EN;
+
 	pci_cfg_access_lock(dev);
 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
 	msleep(100);
@@ -643,6 +647,8 @@ static int sriov_enable(struct pci_dev *dev, int nr_virtfn)
 
 err_pcibios:
 	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
+	if (iov->ctrl & PCI_SRIOV_CTRL_VF_10BIT_TAG_REQ_EN)
+		iov->ctrl &= ~PCI_SRIOV_CTRL_VF_10BIT_TAG_REQ_EN;
 	pci_cfg_access_lock(dev);
 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
 	ssleep(1);
@@ -675,6 +681,8 @@ static void sriov_disable(struct pci_dev *dev)
 
 	sriov_del_vfs(dev);
 	iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE);
+	if (iov->ctrl & PCI_SRIOV_CTRL_VF_10BIT_TAG_REQ_EN)
+		iov->ctrl &= ~PCI_SRIOV_CTRL_VF_10BIT_TAG_REQ_EN;
 	pci_cfg_access_lock(dev);
 	pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl);
 	ssleep(1);
-- 
2.7.4

Re: [RESEND PATCH V3 5/6] PCI/IOV: Enable 10-Bit tag support for PCIe VF devices

[Christoph Hellwig]

On Sun, Jun 13, 2021 at 05:29:14PM +0800, Dongdong Liu wrote:
> Enable VF 10-Bit Tag Requester when it's upstream component support
> 10-bit Tag Completer.
> 
> Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>

Looks good,

Reviewed-by: Christoph Hellwig <hch@lst.de>

[RESEND PATCH V3 6/6] PCI: Enable 10-Bit tag support for PCIe RP devices

[Dongdong Liu]

PCIe spec 5.0r1.0 section 2.2.6.2 implementation note, In configurations
where a Requester with 10-Bit Tag Requester capability needs to target
multiple Completers, one needs to ensure that the Requester sends 10-Bit
Tag Requests only to Completers that have 10-Bit Tag Completer capability.
So we enable 10-Bit Tag Requester for root port only when the devices
under the root port support 10-Bit Tag Completer.

Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
---
 drivers/pci/pcie/portdrv_pci.c | 75 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c
index c7ff1ee..baf413f 100644
--- a/drivers/pci/pcie/portdrv_pci.c
+++ b/drivers/pci/pcie/portdrv_pci.c
@@ -90,6 +90,78 @@ static const struct dev_pm_ops pcie_portdrv_pm_ops = {
 #define PCIE_PORTDRV_PM_OPS	NULL
 #endif /* !PM */
 
+static int pci_10bit_tag_comp_support(struct pci_dev *dev, void *data)
+{
+	u8 *support = data;
+
+	if (*support == 0)
+		return 0;
+
+	if (!pci_is_pcie(dev)) {
+		*support = 0;
+		return 0;
+	}
+
+	/*
+	 * PCIe spec 5.0r1.0 section 2.2.6.2 implementation note.
+	 * For configurations where a Requester with 10-Bit Tag Requester
+	 * capability targets Completers where some do and some do not have
+	 * 10-Bit Tag Completer capability, how the Requester determines which
+	 * NPRs include 10-Bit Tags is outside the scope of this specification.
+	 * So we do not consider hotplug scenario.
+	 */
+	if (dev->is_hotplug_bridge) {
+		*support = 0;
+		return 0;
+	}
+
+	if (!(dev->pcie_devcap2 & PCI_EXP_DEVCAP2_10BIT_TAG_COMP)) {
+		*support = 0;
+		return 0;
+	}
+
+	return 0;
+}
+
+static void pci_configure_rp_10bit_tag(struct pci_dev *dev)
+{
+	u8 support = 1;
+	struct pci_dev *pchild;
+
+	if (dev->subordinate == NULL)
+		return;
+
+	/* If no devices under the root port, no need to enable 10-Bit Tag. */
+	pchild = list_first_entry_or_null(&dev->subordinate->devices,
+					  struct pci_dev, bus_list);
+	if (pchild == NULL)
+		return;
+
+	pci_10bit_tag_comp_support(dev, &support);
+	if (!support)
+		return;
+
+	/*
+	 * PCIe spec 5.0r1.0 section 2.2.6.2 implementation note.
+	 * In configurations where a Requester with 10-Bit Tag Requester
+	 * capability needs to target multiple Completers, one needs to ensure
+	 * that the Requester sends 10-Bit Tag Requests only to Completers
+	 * that have 10-Bit Tag Completer capability. So we enable 10-Bit Tag
+	 * Requester for root port only when the devices under the root port
+	 * support 10-Bit Tag Completer.
+	 */
+	pci_walk_bus(dev->subordinate, pci_10bit_tag_comp_support, &support);
+	if (!support)
+		return;
+
+	if (!(dev->pcie_devcap2 & PCI_EXP_DEVCAP2_10BIT_TAG_REQ))
+		return;
+
+	pci_dbg(dev, "enabling 10-Bit Tag Requester\n");
+	pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
+				 PCI_EXP_DEVCTL2_10BIT_TAG_REQ_EN);
+}
+
 /*
  * pcie_portdrv_probe - Probe PCI-Express port devices
  * @dev: PCI-Express port device being probed
@@ -111,6 +183,9 @@ static int pcie_portdrv_probe(struct pci_dev *dev,
 	     (type != PCI_EXP_TYPE_RC_EC)))
 		return -ENODEV;
 
+	if (type == PCI_EXP_TYPE_ROOT_PORT)
+		pci_configure_rp_10bit_tag(dev);
+
 	if (type == PCI_EXP_TYPE_RC_EC)
 		pcie_link_rcec(dev);
 
-- 
2.7.4

Re: [RESEND PATCH V3 6/6] PCI: Enable 10-Bit tag support for PCIe RP devices

[Christoph Hellwig]

On Sun, Jun 13, 2021 at 05:29:15PM +0800, Dongdong Liu wrote:
> PCIe spec 5.0r1.0 section 2.2.6.2 implementation note, In configurations
> where a Requester with 10-Bit Tag Requester capability needs to target
> multiple Completers, one needs to ensure that the Requester sends 10-Bit
> Tag Requests only to Completers that have 10-Bit Tag Completer capability.
> So we enable 10-Bit Tag Requester for root port only when the devices
> under the root port support 10-Bit Tag Completer.
> 
> Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
> ---
>  drivers/pci/pcie/portdrv_pci.c | 75 ++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 75 insertions(+)
> 
> diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c
> index c7ff1ee..baf413f 100644
> --- a/drivers/pci/pcie/portdrv_pci.c
> +++ b/drivers/pci/pcie/portdrv_pci.c
> @@ -90,6 +90,78 @@ static const struct dev_pm_ops pcie_portdrv_pm_ops = {
>  #define PCIE_PORTDRV_PM_OPS	NULL
>  #endif /* !PM */
>  
> +static int pci_10bit_tag_comp_support(struct pci_dev *dev, void *data)
> +{
> +	u8 *support = data;
> +
> +	if (*support == 0)
> +		return 0;
> +
> +	if (!pci_is_pcie(dev)) {
> +		*support = 0;
> +		return 0;
> +	}
> +
> +	/*
> +	 * PCIe spec 5.0r1.0 section 2.2.6.2 implementation note.
> +	 * For configurations where a Requester with 10-Bit Tag Requester
> +	 * capability targets Completers where some do and some do not have
> +	 * 10-Bit Tag Completer capability, how the Requester determines which
> +	 * NPRs include 10-Bit Tags is outside the scope of this specification.
> +	 * So we do not consider hotplug scenario.
> +	 */
> +	if (dev->is_hotplug_bridge) {
> +		*support = 0;
> +		return 0;
> +	}
> +
> +	if (!(dev->pcie_devcap2 & PCI_EXP_DEVCAP2_10BIT_TAG_COMP)) {
> +		*support = 0;
> +		return 0;
> +	}
> +
> +	return 0;
> +}
> +
> +static void pci_configure_rp_10bit_tag(struct pci_dev *dev)
> +{
> +	u8 support = 1;
> +	struct pci_dev *pchild;
> +
> +	if (dev->subordinate == NULL)
> +		return;
> +
> +	/* If no devices under the root port, no need to enable 10-Bit Tag. */
> +	pchild = list_first_entry_or_null(&dev->subordinate->devices,
> +					  struct pci_dev, bus_list);
> +	if (pchild == NULL)
> +		return;

pchild is never used after this check, so this could be simplified to
a list_empty(&dev->subordinate->devices).

Re: [RESEND PATCH V3 6/6] PCI: Enable 10-Bit tag support for PCIe RP devices

[Dongdong Liu]

On 2021/6/14 13:57, Christoph Hellwig wrote:
> On Sun, Jun 13, 2021 at 05:29:15PM +0800, Dongdong Liu wrote:
>> PCIe spec 5.0r1.0 section 2.2.6.2 implementation note, In configurations
>> where a Requester with 10-Bit Tag Requester capability needs to target
>> multiple Completers, one needs to ensure that the Requester sends 10-Bit
>> Tag Requests only to Completers that have 10-Bit Tag Completer capability.
>> So we enable 10-Bit Tag Requester for root port only when the devices
>> under the root port support 10-Bit Tag Completer.
>>
>> Signed-off-by: Dongdong Liu <liudongdong3@huawei.com>
>> ---
>>  drivers/pci/pcie/portdrv_pci.c | 75 ++++++++++++++++++++++++++++++++++++++++++
>>  1 file changed, 75 insertions(+)
>>
>> diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c
>> index c7ff1ee..baf413f 100644
>> --- a/drivers/pci/pcie/portdrv_pci.c
>> +++ b/drivers/pci/pcie/portdrv_pci.c
>> @@ -90,6 +90,78 @@ static const struct dev_pm_ops pcie_portdrv_pm_ops = {
>>  #define PCIE_PORTDRV_PM_OPS	NULL
>>  #endif /* !PM */
>>
>> +static int pci_10bit_tag_comp_support(struct pci_dev *dev, void *data)
>> +{
>> +	u8 *support = data;
>> +
>> +	if (*support == 0)
>> +		return 0;
>> +
>> +	if (!pci_is_pcie(dev)) {
>> +		*support = 0;
>> +		return 0;
>> +	}
>> +
>> +	/*
>> +	 * PCIe spec 5.0r1.0 section 2.2.6.2 implementation note.
>> +	 * For configurations where a Requester with 10-Bit Tag Requester
>> +	 * capability targets Completers where some do and some do not have
>> +	 * 10-Bit Tag Completer capability, how the Requester determines which
>> +	 * NPRs include 10-Bit Tags is outside the scope of this specification.
>> +	 * So we do not consider hotplug scenario.
>> +	 */
>> +	if (dev->is_hotplug_bridge) {
>> +		*support = 0;
>> +		return 0;
>> +	}
>> +
>> +	if (!(dev->pcie_devcap2 & PCI_EXP_DEVCAP2_10BIT_TAG_COMP)) {
>> +		*support = 0;
>> +		return 0;
>> +	}
>> +
>> +	return 0;
>> +}
>> +
>> +static void pci_configure_rp_10bit_tag(struct pci_dev *dev)
>> +{
>> +	u8 support = 1;
>> +	struct pci_dev *pchild;
>> +
>> +	if (dev->subordinate == NULL)
>> +		return;
>> +
>> +	/* If no devices under the root port, no need to enable 10-Bit Tag. */
>> +	pchild = list_first_entry_or_null(&dev->subordinate->devices,
>> +					  struct pci_dev, bus_list);
>> +	if (pchild == NULL)
>> +		return;
>
> pchild is never used after this check, so this could be simplified to
> a list_empty(&dev->subordinate->devices).
Good point, will do.

Thanks,
Dongdong
> .
>