Add multiple PFs support for DWC, different PF have different config space we use pf-offset property which get from the DTS to access the different pF config space. Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> --- v2: - Remove duplicate redundant code. - Reimplement the PF config space access way. drivers/pci/controller/dwc/pcie-designware-ep.c | 122 ++++++++++++++++-------- drivers/pci/controller/dwc/pcie-designware.c | 59 ++++++++---- drivers/pci/controller/dwc/pcie-designware.h | 11 ++- 3 files changed, 134 insertions(+), 58 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 2bf5a35..3e2b740 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -19,12 +19,17 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) pci_epc_linkup(epc); } -static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar, - int flags) +static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, + enum pci_barno bar, int flags) { u32 reg; + unsigned int func_offset = 0; + struct dw_pcie_ep *ep = &pci->ep; - reg = PCI_BASE_ADDRESS_0 + (4 * bar); + if (ep->ops->func_conf_select) + func_offset = ep->ops->func_conf_select(ep, func_no); + + reg = func_offset + PCI_BASE_ADDRESS_0 + (4 * bar); dw_pcie_dbi_ro_wr_en(pci); dw_pcie_writel_dbi2(pci, reg, 0x0); dw_pcie_writel_dbi(pci, reg, 0x0); @@ -37,7 +42,12 @@ static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar, void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) { - __dw_pcie_ep_reset_bar(pci, bar, 0); + u8 func_no, funcs; + + funcs = pci->ep.epc->max_functions; + + for (func_no = 0; func_no < funcs; func_no++) + __dw_pcie_ep_reset_bar(pci, func_no, bar, 0); } static u8 __dw_pcie_ep_find_next_cap(struct dw_pcie *pci, u8 cap_ptr, @@ -78,28 +88,32 @@ static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, { struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + unsigned int func_offset = 0; + + if (ep->ops->func_conf_select) + func_offset = ep->ops->func_conf_select(ep, func_no); dw_pcie_dbi_ro_wr_en(pci); - dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, hdr->vendorid); - dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, hdr->deviceid); - dw_pcie_writeb_dbi(pci, PCI_REVISION_ID, hdr->revid); - dw_pcie_writeb_dbi(pci, PCI_CLASS_PROG, hdr->progif_code); - dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, + dw_pcie_writew_dbi(pci, func_offset + PCI_VENDOR_ID, hdr->vendorid); + dw_pcie_writew_dbi(pci, func_offset + PCI_DEVICE_ID, hdr->deviceid); + dw_pcie_writeb_dbi(pci, func_offset + PCI_REVISION_ID, hdr->revid); + dw_pcie_writeb_dbi(pci, func_offset + PCI_CLASS_PROG, hdr->progif_code); + dw_pcie_writew_dbi(pci, func_offset + PCI_CLASS_DEVICE, hdr->subclass_code | hdr->baseclass_code << 8); - dw_pcie_writeb_dbi(pci, PCI_CACHE_LINE_SIZE, + dw_pcie_writeb_dbi(pci, func_offset + PCI_CACHE_LINE_SIZE, hdr->cache_line_size); - dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_VENDOR_ID, + dw_pcie_writew_dbi(pci, func_offset + PCI_SUBSYSTEM_VENDOR_ID, hdr->subsys_vendor_id); - dw_pcie_writew_dbi(pci, PCI_SUBSYSTEM_ID, hdr->subsys_id); - dw_pcie_writeb_dbi(pci, PCI_INTERRUPT_PIN, + dw_pcie_writew_dbi(pci, func_offset + PCI_SUBSYSTEM_ID, hdr->subsys_id); + dw_pcie_writeb_dbi(pci, func_offset + PCI_INTERRUPT_PIN, hdr->interrupt_pin); dw_pcie_dbi_ro_wr_dis(pci); return 0; } -static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar, - dma_addr_t cpu_addr, +static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, u8 func_no, + enum pci_barno bar, dma_addr_t cpu_addr, enum dw_pcie_as_type as_type) { int ret; @@ -112,7 +126,7 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar, return -EINVAL; } - ret = dw_pcie_prog_inbound_atu(pci, free_win, bar, cpu_addr, + ret = dw_pcie_prog_inbound_atu(pci, func_no, free_win, bar, cpu_addr, as_type); if (ret < 0) { dev_err(pci->dev, "Failed to program IB window\n"); @@ -125,7 +139,8 @@ static int dw_pcie_ep_inbound_atu(struct dw_pcie_ep *ep, enum pci_barno bar, return 0; } -static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr, +static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, u8 func_no, + phys_addr_t phys_addr, u64 pci_addr, size_t size) { u32 free_win; @@ -137,8 +152,8 @@ static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, phys_addr_t phys_addr, return -EINVAL; } - dw_pcie_prog_outbound_atu(pci, free_win, PCIE_ATU_TYPE_MEM, - phys_addr, pci_addr, size); + dw_pcie_prog_ep_outbound_atu(pci, func_no, free_win, PCIE_ATU_TYPE_MEM, + phys_addr, pci_addr, size); set_bit(free_win, ep->ob_window_map); ep->outbound_addr[free_win] = phys_addr; @@ -154,7 +169,7 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, enum pci_barno bar = epf_bar->barno; u32 atu_index = ep->bar_to_atu[bar]; - __dw_pcie_ep_reset_bar(pci, bar, epf_bar->flags); + __dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags); dw_pcie_disable_atu(pci, atu_index, DW_PCIE_REGION_INBOUND); clear_bit(atu_index, ep->ib_window_map); @@ -170,14 +185,21 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, size_t size = epf_bar->size; int flags = epf_bar->flags; enum dw_pcie_as_type as_type; - u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); + u32 reg; + unsigned int func_offset = 0; + + if (ep->ops->func_conf_select) + func_offset = ep->ops->func_conf_select(ep, func_no); + + reg = PCI_BASE_ADDRESS_0 + (4 * bar) + func_offset; if (!(flags & PCI_BASE_ADDRESS_SPACE)) as_type = DW_PCIE_AS_MEM; else as_type = DW_PCIE_AS_IO; - ret = dw_pcie_ep_inbound_atu(ep, bar, epf_bar->phys_addr, as_type); + ret = dw_pcie_ep_inbound_atu(ep, func_no, bar, + epf_bar->phys_addr, as_type); if (ret) return ret; @@ -235,7 +257,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); - ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size); + ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr, size); if (ret) { dev_err(pci->dev, "Failed to enable address\n"); return ret; @@ -249,11 +271,15 @@ static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); u32 val, reg; + unsigned int func_offset = 0; + + if (ep->ops->func_conf_select) + func_offset = ep->ops->func_conf_select(ep, func_no); if (!ep->msi_cap) return -EINVAL; - reg = ep->msi_cap + PCI_MSI_FLAGS; + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; val = dw_pcie_readw_dbi(pci, reg); if (!(val & PCI_MSI_FLAGS_ENABLE)) return -EINVAL; @@ -268,11 +294,15 @@ static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); u32 val, reg; + unsigned int func_offset = 0; + + if (ep->ops->func_conf_select) + func_offset = ep->ops->func_conf_select(ep, func_no); if (!ep->msi_cap) return -EINVAL; - reg = ep->msi_cap + PCI_MSI_FLAGS; + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; val = dw_pcie_readw_dbi(pci, reg); val &= ~PCI_MSI_FLAGS_QMASK; val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; @@ -288,11 +318,15 @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); u32 val, reg; + unsigned int func_offset = 0; + + if (ep->ops->func_conf_select) + func_offset = ep->ops->func_conf_select(ep, func_no); if (!ep->msix_cap) return -EINVAL; - reg = ep->msix_cap + PCI_MSIX_FLAGS; + reg = ep->msix_cap + func_offset + PCI_MSIX_FLAGS; val = dw_pcie_readw_dbi(pci, reg); if (!(val & PCI_MSIX_FLAGS_ENABLE)) return -EINVAL; @@ -307,11 +341,15 @@ static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts) struct dw_pcie_ep *ep = epc_get_drvdata(epc); struct dw_pcie *pci = to_dw_pcie_from_ep(ep); u32 val, reg; + unsigned int func_offset = 0; + + if (ep->ops->func_conf_select) + func_offset = ep->ops->func_conf_select(ep, func_no); if (!ep->msix_cap) return -EINVAL; - reg = ep->msix_cap + PCI_MSIX_FLAGS; + reg = ep->msix_cap + func_offset + PCI_MSIX_FLAGS; val = dw_pcie_readw_dbi(pci, reg); val &= ~PCI_MSIX_FLAGS_QSIZE; val |= interrupts; @@ -398,29 +436,33 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, struct dw_pcie *pci = to_dw_pcie_from_ep(ep); struct pci_epc *epc = ep->epc; unsigned int aligned_offset; + unsigned int func_offset = 0; u16 msg_ctrl, msg_data; u32 msg_addr_lower, msg_addr_upper, reg; u64 msg_addr; bool has_upper; int ret; + if (ep->ops->func_conf_select) + func_offset = ep->ops->func_conf_select(ep, func_no); + if (!ep->msi_cap) return -EINVAL; /* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */ - reg = ep->msi_cap + PCI_MSI_FLAGS; + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; msg_ctrl = dw_pcie_readw_dbi(pci, reg); has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT); - reg = ep->msi_cap + PCI_MSI_ADDRESS_LO; + reg = ep->msi_cap + func_offset + PCI_MSI_ADDRESS_LO; msg_addr_lower = dw_pcie_readl_dbi(pci, reg); if (has_upper) { - reg = ep->msi_cap + PCI_MSI_ADDRESS_HI; + reg = ep->msi_cap + func_offset + PCI_MSI_ADDRESS_HI; msg_addr_upper = dw_pcie_readl_dbi(pci, reg); - reg = ep->msi_cap + PCI_MSI_DATA_64; + reg = ep->msi_cap + func_offset + PCI_MSI_DATA_64; msg_data = dw_pcie_readw_dbi(pci, reg); } else { msg_addr_upper = 0; - reg = ep->msi_cap + PCI_MSI_DATA_32; + reg = ep->msi_cap + func_offset + PCI_MSI_DATA_32; msg_data = dw_pcie_readw_dbi(pci, reg); } aligned_offset = msg_addr_lower & (epc->mem->page_size - 1); @@ -439,11 +481,12 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, } int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, - u16 interrupt_num) + u16 interrupt_num) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); struct pci_epc *epc = ep->epc; u16 tbl_offset, bir; + unsigned int func_offset = 0; u32 bar_addr_upper, bar_addr_lower; u32 msg_addr_upper, msg_addr_lower; u32 reg, msg_data, vec_ctrl; @@ -451,12 +494,15 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, void __iomem *msix_tbl; int ret; - reg = ep->msix_cap + PCI_MSIX_TABLE; + if (ep->ops->func_conf_select) + func_offset = ep->ops->func_conf_select(ep, func_no); + + reg = ep->msix_cap + func_offset + PCI_MSIX_TABLE; tbl_offset = dw_pcie_readl_dbi(pci, reg); bir = (tbl_offset & PCI_MSIX_TABLE_BIR); tbl_offset &= PCI_MSIX_TABLE_OFFSET; - reg = PCI_BASE_ADDRESS_0 + (4 * bir); + reg = PCI_BASE_ADDRESS_0 + func_offset + (4 * bir); bar_addr_upper = 0; bar_addr_lower = dw_pcie_readl_dbi(pci, reg); reg_u64 = (bar_addr_lower & PCI_BASE_ADDRESS_MEM_TYPE_MASK); @@ -592,13 +638,13 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) ep->epc = epc; epc_set_drvdata(epc, ep); - if (ep->ops->ep_init) - ep->ops->ep_init(ep); - ret = of_property_read_u8(np, "max-functions", &epc->max_functions); if (ret < 0) epc->max_functions = 1; + if (ep->ops->ep_init) + ep->ops->ep_init(ep); + ret = __pci_epc_mem_init(epc, ep->phys_base, ep->addr_size, ep->page_size); if (ret < 0) { diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c index 7d25102..305e73d 100644 --- a/drivers/pci/controller/dwc/pcie-designware.c +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -158,9 +158,10 @@ static void dw_pcie_writel_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg, dw_pcie_writel_atu(pci, offset + reg, val); } -static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, - int type, u64 cpu_addr, - u64 pci_addr, u32 size) +static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, u8 func_no, + int index, int type, + u64 cpu_addr, u64 pci_addr, + u32 size) { u32 retries, val; @@ -175,7 +176,7 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET, upper_32_bits(pci_addr)); dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, - type); + type | PCIE_ATU_FUNC_NUM(func_no)); dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2, PCIE_ATU_ENABLE); @@ -194,8 +195,9 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, dev_err(pci->dev, "Outbound iATU is not being enabled\n"); } -void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, - u64 cpu_addr, u64 pci_addr, u32 size) +static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no, + int index, int type, u64 cpu_addr, + u64 pci_addr, u32 size) { u32 retries, val; @@ -203,8 +205,8 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr); if (pci->iatu_unroll_enabled) { - dw_pcie_prog_outbound_atu_unroll(pci, index, type, cpu_addr, - pci_addr, size); + dw_pcie_prog_outbound_atu_unroll(pci, func_no, index, type, + cpu_addr, pci_addr, size); return; } @@ -220,7 +222,8 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, lower_32_bits(pci_addr)); dw_pcie_writel_dbi(pci, PCIE_ATU_UPPER_TARGET, upper_32_bits(pci_addr)); - dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type); + dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type | + PCIE_ATU_FUNC_NUM(func_no)); dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE); /* @@ -237,6 +240,21 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, dev_err(pci->dev, "Outbound iATU is not being enabled\n"); } +void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, + u64 cpu_addr, u64 pci_addr, u32 size) +{ + __dw_pcie_prog_outbound_atu(pci, 0, index, type, + cpu_addr, pci_addr, size); +} + +void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index, + int type, u64 cpu_addr, u64 pci_addr, + u32 size) +{ + __dw_pcie_prog_outbound_atu(pci, func_no, index, type, + cpu_addr, pci_addr, size); +} + static u32 dw_pcie_readl_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg) { u32 offset = PCIE_GET_ATU_INB_UNR_REG_OFFSET(index); @@ -252,8 +270,8 @@ static void dw_pcie_writel_ib_unroll(struct dw_pcie *pci, u32 index, u32 reg, dw_pcie_writel_atu(pci, offset + reg, val); } -static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index, - int bar, u64 cpu_addr, +static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, u8 func_no, + int index, int bar, u64 cpu_addr, enum dw_pcie_as_type as_type) { int type; @@ -275,8 +293,10 @@ static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index, return -EINVAL; } - dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, type); + dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, type | + PCIE_ATU_FUNC_NUM(func_no)); dw_pcie_writel_ib_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2, + PCIE_ATU_FUNC_NUM_MATCH_EN | PCIE_ATU_ENABLE | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8)); @@ -297,14 +317,15 @@ static int dw_pcie_prog_inbound_atu_unroll(struct dw_pcie *pci, int index, return -EBUSY; } -int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar, - u64 cpu_addr, enum dw_pcie_as_type as_type) +int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, + int bar, u64 cpu_addr, + enum dw_pcie_as_type as_type) { int type; u32 retries, val; if (pci->iatu_unroll_enabled) - return dw_pcie_prog_inbound_atu_unroll(pci, index, bar, + return dw_pcie_prog_inbound_atu_unroll(pci, func_no, index, bar, cpu_addr, as_type); dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, PCIE_ATU_REGION_INBOUND | @@ -323,9 +344,11 @@ int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar, return -EINVAL; } - dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type); - dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE - | PCIE_ATU_BAR_MODE_ENABLE | (bar << 8)); + dw_pcie_writel_dbi(pci, PCIE_ATU_CR1, type | + PCIE_ATU_FUNC_NUM(func_no)); + dw_pcie_writel_dbi(pci, PCIE_ATU_CR2, PCIE_ATU_ENABLE | + PCIE_ATU_FUNC_NUM_MATCH_EN | + PCIE_ATU_BAR_MODE_ENABLE | (bar << 8)); /* * Make sure ATU enable takes effect before any subsequent config diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index ffed084..a0fdbf7 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -71,9 +71,11 @@ #define PCIE_ATU_TYPE_IO 0x2 #define PCIE_ATU_TYPE_CFG0 0x4 #define PCIE_ATU_TYPE_CFG1 0x5 +#define PCIE_ATU_FUNC_NUM(pf) (pf << 20) #define PCIE_ATU_CR2 0x908 #define PCIE_ATU_ENABLE BIT(31) #define PCIE_ATU_BAR_MODE_ENABLE BIT(30) +#define PCIE_ATU_FUNC_NUM_MATCH_EN BIT(19) #define PCIE_ATU_LOWER_BASE 0x90C #define PCIE_ATU_UPPER_BASE 0x910 #define PCIE_ATU_LIMIT 0x914 @@ -197,6 +199,7 @@ struct dw_pcie_ep_ops { int (*raise_irq)(struct dw_pcie_ep *ep, u8 func_no, enum pci_epc_irq_type type, u16 interrupt_num); const struct pci_epc_features* (*get_features)(struct dw_pcie_ep *ep); + unsigned int (*func_conf_select)(struct dw_pcie_ep *ep, u8 func_no); }; struct dw_pcie_ep { @@ -265,8 +268,12 @@ int dw_pcie_wait_for_link(struct dw_pcie *pci); void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, u64 cpu_addr, u64 pci_addr, u32 size); -int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar, - u64 cpu_addr, enum dw_pcie_as_type as_type); +void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index, + int type, u64 cpu_addr, u64 pci_addr, + u32 size); +int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, + int bar, u64 cpu_addr, + enum dw_pcie_as_type as_type); void dw_pcie_disable_atu(struct dw_pcie *pci, int index, enum dw_pcie_region_type type); void dw_pcie_setup(struct dw_pcie *pci); -- 2.9.5
Add the doorbell mode of MSI-X in EP mode. Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> --- v2: - Remove the macro of no used. drivers/pci/controller/dwc/pcie-designware-ep.c | 14 ++++++++++++++ drivers/pci/controller/dwc/pcie-designware.h | 12 ++++++++++++ 2 files changed, 26 insertions(+) diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index 3e2b740..b8388f8 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -480,6 +480,20 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, return 0; } +int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, + u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + u32 msg_data; + + msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) | + (interrupt_num - 1); + + dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data); + + return 0; +} + int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num) { diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h index a0fdbf7..895a9ef 100644 --- a/drivers/pci/controller/dwc/pcie-designware.h +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -88,6 +88,9 @@ #define PCIE_MISC_CONTROL_1_OFF 0x8BC #define PCIE_DBI_RO_WR_EN BIT(0) +#define PCIE_MSIX_DOORBELL 0x948 +#define PCIE_MSIX_DOORBELL_PF_SHIFT 24 + /* * iATU Unroll-specific register definitions * From 4.80 core version the address translation will be made by unroll @@ -400,6 +403,8 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, u8 interrupt_num); int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num); +int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, + u16 interrupt_num); void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar); #else static inline void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) @@ -432,6 +437,13 @@ static inline int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, return 0; } +static inline int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, + u8 func_no, + u16 interrupt_num) +{ + return 0; +} + static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) { } -- 2.9.5
Move the function of getting MSI capability to the front of init function, because the init function of the EP platform driver will use the return value by the function of getting MSI capability. Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> --- v2: - No change. drivers/pci/controller/dwc/pcie-designware-ep.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c index b8388f8..0a6c199 100644 --- a/drivers/pci/controller/dwc/pcie-designware-ep.c +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -656,6 +656,10 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) if (ret < 0) epc->max_functions = 1; + ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI); + + ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX); + if (ep->ops->ep_init) ep->ops->ep_init(ep); @@ -672,9 +676,6 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) dev_err(dev, "Failed to reserve memory for MSI/MSI-X\n"); return -ENOMEM; } - ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI); - - ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX); offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); if (offset) { -- 2.9.5
Add compatible strings for ls1088a and ls2088a. Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> --- v2: - No change. Documentation/devicetree/bindings/pci/layerscape-pci.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/pci/layerscape-pci.txt b/Documentation/devicetree/bindings/pci/layerscape-pci.txt index e20ceaa..16f592e 100644 --- a/Documentation/devicetree/bindings/pci/layerscape-pci.txt +++ b/Documentation/devicetree/bindings/pci/layerscape-pci.txt @@ -22,7 +22,10 @@ Required properties: "fsl,ls1043a-pcie" "fsl,ls1012a-pcie" EP mode: - "fsl,ls1046a-pcie-ep", "fsl,ls-pcie-ep" + "fsl,ls-pcie-ep" + "fsl,ls1046a-pcie-ep" + "fsl,ls1088a-pcie-ep" + "fsl,ls2088a-pcie-ep" - reg: base addresses and lengths of the PCIe controller register blocks. - interrupts: A list of interrupt outputs of the controller. Must contain an entry for each entry in the interrupt-names property. -- 2.9.5
Fix some format issue of the code in EP driver. Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> --- v2: - No change. drivers/pci/controller/dwc/pci-layerscape-ep.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index be61d96..4e92a95 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -62,7 +62,7 @@ static void ls_pcie_ep_init(struct dw_pcie_ep *ep) } static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, - enum pci_epc_irq_type type, u16 interrupt_num) + enum pci_epc_irq_type type, u16 interrupt_num) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); @@ -86,7 +86,7 @@ static const struct dw_pcie_ep_ops pcie_ep_ops = { }; static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, - struct platform_device *pdev) + struct platform_device *pdev) { struct dw_pcie *pci = pcie->pci; struct device *dev = pci->dev; -- 2.9.5
The different PCIe controller in one board may be have different capability of MSI or MSIX, so change the way of getting the MSI capability, make it more flexible. Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> --- v2: - Remove the repeated assignment code. drivers/pci/controller/dwc/pci-layerscape-ep.c | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index 4e92a95..8461f62 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -22,6 +22,7 @@ struct ls_pcie_ep { struct dw_pcie *pci; + struct pci_epc_features *ls_epc; }; #define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) @@ -40,25 +41,26 @@ static const struct of_device_id ls_pcie_ep_of_match[] = { { }, }; -static const struct pci_epc_features ls_pcie_epc_features = { - .linkup_notifier = false, - .msi_capable = true, - .msix_capable = false, -}; - static const struct pci_epc_features* ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { - return &ls_pcie_epc_features; + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); + + return pcie->ls_epc; } static void ls_pcie_ep_init(struct dw_pcie_ep *ep) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); enum pci_barno bar; for (bar = BAR_0; bar <= BAR_5; bar++) dw_pcie_ep_reset_bar(pci, bar); + + pcie->ls_epc->msi_capable = ep->msi_cap ? true : false; + pcie->ls_epc->msix_capable = ep->msix_cap ? true : false; } static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, @@ -118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct dw_pcie *pci; struct ls_pcie_ep *pcie; + struct pci_epc_features *ls_epc; struct resource *dbi_base; int ret; @@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) if (!pci) return -ENOMEM; + ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL); + if (!ls_epc) + return -ENOMEM; + dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); if (IS_ERR(pci->dbi_base)) @@ -139,6 +146,11 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) pci->ops = &ls_pcie_ep_ops; pcie->pci = pci; + ls_epc->linkup_notifier = false, + ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4), + + pcie->ls_epc = ls_epc; + platform_set_drvdata(pdev, pcie); ret = ls_add_pcie_ep(pcie, pdev); -- 2.9.5
The layerscape platform use the doorbell way to trigger MSIX interrupt in EP mode. Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> --- v2: - No change. drivers/pci/controller/dwc/pci-layerscape-ep.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index 8461f62..7ca5fe8 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -74,7 +74,8 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, case PCI_EPC_IRQ_MSI: return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); case PCI_EPC_IRQ_MSIX: - return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); + return dw_pcie_ep_raise_msix_irq_doorbell(ep, func_no, + interrupt_num); default: dev_err(pci->dev, "UNKNOWN IRQ type\n"); return -EINVAL; -- 2.9.5
Add PCIe EP mode support for ls1088a and ls2088a, there are some difference between LS1 and LS2 platform, so refactor the code of the EP driver. Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> --- v2: - New mechanism for layerscape EP driver. drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 ++++++++++++++++++++------ 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index 7ca5fe8..2a66f07 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -20,27 +20,29 @@ #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ -struct ls_pcie_ep { - struct dw_pcie *pci; - struct pci_epc_features *ls_epc; +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) + +struct ls_pcie_ep_drvdata { + u32 func_offset; + const struct dw_pcie_ep_ops *ops; + const struct dw_pcie_ops *dw_pcie_ops; }; -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) +struct ls_pcie_ep { + struct dw_pcie *pci; + struct pci_epc_features *ls_epc; + const struct ls_pcie_ep_drvdata *drvdata; +}; static int ls_pcie_establish_link(struct dw_pcie *pci) { return 0; } -static const struct dw_pcie_ops ls_pcie_ep_ops = { +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { .start_link = ls_pcie_establish_link, }; -static const struct of_device_id ls_pcie_ep_of_match[] = { - { .compatible = "fsl,ls-pcie-ep",}, - { }, -}; - static const struct pci_epc_features* ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { @@ -82,10 +84,44 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, } } -static const struct dw_pcie_ep_ops pcie_ep_ops = { +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, + u8 func_no) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); + u8 header_type; + + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); + + if (header_type & (1 << 7)) + return pcie->drvdata->func_offset * func_no; + else + return 0; +} + +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { .ep_init = ls_pcie_ep_init, .raise_irq = ls_pcie_ep_raise_irq, .get_features = ls_pcie_ep_get_features, + .func_conf_select = ls_pcie_ep_func_conf_select, +}; + +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { + .ops = &ls_pcie_ep_ops, + .dw_pcie_ops = &dw_ls_pcie_ep_ops, +}; + +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { + .func_offset = 0x20000, + .ops = &ls_pcie_ep_ops, + .dw_pcie_ops = &dw_ls_pcie_ep_ops, +}; + +static const struct of_device_id ls_pcie_ep_of_match[] = { + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, + { }, }; static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ -98,7 +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, int ret; ep = &pci->ep; - ep->ops = &pcie_ep_ops; + ep->ops = pcie->drvdata->ops; res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); if (!res) @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) if (!ls_epc) return -ENOMEM; - dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); - if (IS_ERR(pci->dbi_base)) - return PTR_ERR(pci->dbi_base); + pcie->drvdata = of_device_get_match_data(dev); - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; pci->dev = dev; - pci->ops = &ls_pcie_ep_ops; + pci->ops = pcie->drvdata->dw_pcie_ops; + pcie->pci = pci; ls_epc->linkup_notifier = false, @@ -152,6 +185,13 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) pcie->ls_epc = ls_epc; + dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; + platform_set_drvdata(pdev, pcie); ret = ls_add_pcie_ep(pcie, pdev); -- 2.9.5
Add PCIe EP node for ls1088a to support EP mode. Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> --- v2: - Remove the pf-offset proparty. arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi index dfbead4..79109ad 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi @@ -471,6 +471,17 @@ status = "disabled"; }; + pcie_ep@3400000 { + compatible = "fsl,ls1088a-pcie-ep","fsl,ls-pcie-ep"; + reg = <0x00 0x03400000 0x0 0x00100000 + 0x20 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + num-ib-windows = <24>; + num-ob-windows = <128>; + max-functions = /bits/ 8 <2>; + status = "disabled"; + }; + pcie@3500000 { compatible = "fsl,ls1088a-pcie"; reg = <0x00 0x03500000 0x0 0x00100000 /* controller registers */ @@ -497,6 +508,16 @@ status = "disabled"; }; + pcie_ep@3500000 { + compatible = "fsl,ls1088a-pcie-ep","fsl,ls-pcie-ep"; + reg = <0x00 0x03500000 0x0 0x00100000 + 0x28 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + num-ib-windows = <6>; + num-ob-windows = <8>; + status = "disabled"; + }; + pcie@3600000 { compatible = "fsl,ls1088a-pcie"; reg = <0x00 0x03600000 0x0 0x00100000 /* controller registers */ @@ -523,6 +544,16 @@ status = "disabled"; }; + pcie_ep@3600000 { + compatible = "fsl,ls1088a-pcie-ep","fsl,ls-pcie-ep"; + reg = <0x00 0x03600000 0x0 0x00100000 + 0x30 0x00000000 0x8 0x00000000>; + reg-names = "regs", "addr_space"; + num-ib-windows = <6>; + num-ob-windows = <8>; + status = "disabled"; + }; + smmu: iommu@5000000 { compatible = "arm,mmu-500"; reg = <0 0x5000000 0 0x800000>; -- 2.9.5
Add LS1088a in pci_device_id table so that pci-epf-test can be used for testing PCIe EP in LS1088a. Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> --- v2: - No change. drivers/misc/pci_endpoint_test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c index 6e208a0..d531951 100644 --- a/drivers/misc/pci_endpoint_test.c +++ b/drivers/misc/pci_endpoint_test.c @@ -793,6 +793,7 @@ static const struct pci_device_id pci_endpoint_test_tbl[] = { { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x) }, { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x) }, { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0) }, + { PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x80c0) }, { PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) }, { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654), .driver_data = (kernel_ulong_t)&am654_data -- 2.9.5
Hi, On 22/08/19 4:52 PM, Xiaowei Bao wrote: > The different PCIe controller in one board may be have different > capability of MSI or MSIX, so change the way of getting the MSI > capability, make it more flexible. please use different pci_epc_features table for different boards. Thanks Kishon > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > --- > v2: > - Remove the repeated assignment code. > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 26 +++++++++++++++++++------- > 1 file changed, 19 insertions(+), 7 deletions(-) > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c > index 4e92a95..8461f62 100644 > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > @@ -22,6 +22,7 @@ > > struct ls_pcie_ep { > struct dw_pcie *pci; > + struct pci_epc_features *ls_epc; > }; > > #define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > @@ -40,25 +41,26 @@ static const struct of_device_id ls_pcie_ep_of_match[] = { > { }, > }; > > -static const struct pci_epc_features ls_pcie_epc_features = { > - .linkup_notifier = false, > - .msi_capable = true, > - .msix_capable = false, > -}; > - > static const struct pci_epc_features* > ls_pcie_ep_get_features(struct dw_pcie_ep *ep) > { > - return &ls_pcie_epc_features; > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > + > + return pcie->ls_epc; > } > > static void ls_pcie_ep_init(struct dw_pcie_ep *ep) > { > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > enum pci_barno bar; > > for (bar = BAR_0; bar <= BAR_5; bar++) > dw_pcie_ep_reset_bar(pci, bar); > + > + pcie->ls_epc->msi_capable = ep->msi_cap ? true : false; > + pcie->ls_epc->msix_capable = ep->msix_cap ? true : false; > } > > static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > @@ -118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) > struct device *dev = &pdev->dev; > struct dw_pcie *pci; > struct ls_pcie_ep *pcie; > + struct pci_epc_features *ls_epc; > struct resource *dbi_base; > int ret; > > @@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) > if (!pci) > return -ENOMEM; > > + ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL); > + if (!ls_epc) > + return -ENOMEM; > + > dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); > pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > if (IS_ERR(pci->dbi_base)) > @@ -139,6 +146,11 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) > pci->ops = &ls_pcie_ep_ops; > pcie->pci = pci; > > + ls_epc->linkup_notifier = false, > + ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4), > + > + pcie->ls_epc = ls_epc; > + > platform_set_drvdata(pdev, pcie); > > ret = ls_add_pcie_ep(pcie, pdev); >
> -----Original Message----- > From: Kishon Vijay Abraham I <kishon@ti.com> > Sent: 2019年8月22日 19:44 > To: Xiaowei Bao <xiaowei.bao@nxp.com>; bhelgaas@google.com; > robh+dt@kernel.org; mark.rutland@arm.com; shawnguo@kernel.org; Leo Li > <leoyang.li@nxp.com>; lorenzo.pieralisi@arm.co; arnd@arndb.de; > gregkh@linuxfoundation.org; M.h. Lian <minghuan.lian@nxp.com>; Mingkai > Hu <mingkai.hu@nxp.com>; Roy Zang <roy.zang@nxp.com>; > jingoohan1@gmail.com; gustavo.pimentel@synopsys.com; > linux-pci@vger.kernel.org; devicetree@vger.kernel.org; > linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org; > linuxppc-dev@lists.ozlabs.org; andrew.murray@arm.com > Subject: Re: [PATCH v2 06/10] PCI: layerscape: Modify the way of getting > capability with different PEX > > Hi, > > On 22/08/19 4:52 PM, Xiaowei Bao wrote: > > The different PCIe controller in one board may be have different > > capability of MSI or MSIX, so change the way of getting the MSI > > capability, make it more flexible. > > please use different pci_epc_features table for different boards. Thanks, I think that it will be more flexible to dynamically get MSI or MSIX capability, Thus, we will not need to define the pci_epc_feature for different boards. > > Thanks > Kishon > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > --- > > v2: > > - Remove the repeated assignment code. > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 26 > > +++++++++++++++++++------- > > 1 file changed, 19 insertions(+), 7 deletions(-) > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > index 4e92a95..8461f62 100644 > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > @@ -22,6 +22,7 @@ > > > > struct ls_pcie_ep { > > struct dw_pcie *pci; > > + struct pci_epc_features *ls_epc; > > }; > > > > #define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > @@ -40,25 +41,26 @@ static const struct of_device_id > ls_pcie_ep_of_match[] = { > > { }, > > }; > > > > -static const struct pci_epc_features ls_pcie_epc_features = { > > - .linkup_notifier = false, > > - .msi_capable = true, > > - .msix_capable = false, > > -}; > > - > > static const struct pci_epc_features* ls_pcie_ep_get_features(struct > > dw_pcie_ep *ep) { > > - return &ls_pcie_epc_features; > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > + > > + return pcie->ls_epc; > > } > > > > static void ls_pcie_ep_init(struct dw_pcie_ep *ep) { > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > enum pci_barno bar; > > > > for (bar = BAR_0; bar <= BAR_5; bar++) > > dw_pcie_ep_reset_bar(pci, bar); > > + > > + pcie->ls_epc->msi_capable = ep->msi_cap ? true : false; > > + pcie->ls_epc->msix_capable = ep->msix_cap ? true : false; > > } > > > > static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, @@ > > -118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct platform_device > *pdev) > > struct device *dev = &pdev->dev; > > struct dw_pcie *pci; > > struct ls_pcie_ep *pcie; > > + struct pci_epc_features *ls_epc; > > struct resource *dbi_base; > > int ret; > > > > @@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct > platform_device *pdev) > > if (!pci) > > return -ENOMEM; > > > > + ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL); > > + if (!ls_epc) > > + return -ENOMEM; > > + > > dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, > "regs"); > > pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > if (IS_ERR(pci->dbi_base)) > > @@ -139,6 +146,11 @@ static int __init ls_pcie_ep_probe(struct > platform_device *pdev) > > pci->ops = &ls_pcie_ep_ops; > > pcie->pci = pci; > > > > + ls_epc->linkup_notifier = false, > > + ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4), > > + > > + pcie->ls_epc = ls_epc; > > + > > platform_set_drvdata(pdev, pcie); > > > > ret = ls_add_pcie_ep(pcie, pdev); > >
Hi, (Fixed Lorenzo's email address. All the patches in the series have wrong email id) On 23/08/19 8:09 AM, Xiaowei Bao wrote: > > >> -----Original Message----- >> From: Kishon Vijay Abraham I <kishon@ti.com> >> Sent: 2019年8月22日 19:44 >> To: Xiaowei Bao <xiaowei.bao@nxp.com>; bhelgaas@google.com; >> robh+dt@kernel.org; mark.rutland@arm.com; shawnguo@kernel.org; Leo Li >> <leoyang.li@nxp.com>; lorenzo.pieralisi@arm.co; arnd@arndb.de; >> gregkh@linuxfoundation.org; M.h. Lian <minghuan.lian@nxp.com>; Mingkai >> Hu <mingkai.hu@nxp.com>; Roy Zang <roy.zang@nxp.com>; >> jingoohan1@gmail.com; gustavo.pimentel@synopsys.com; >> linux-pci@vger.kernel.org; devicetree@vger.kernel.org; >> linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org; >> linuxppc-dev@lists.ozlabs.org; andrew.murray@arm.com >> Subject: Re: [PATCH v2 06/10] PCI: layerscape: Modify the way of getting >> capability with different PEX >> >> Hi, >> >> On 22/08/19 4:52 PM, Xiaowei Bao wrote: >>> The different PCIe controller in one board may be have different >>> capability of MSI or MSIX, so change the way of getting the MSI >>> capability, make it more flexible. >> >> please use different pci_epc_features table for different boards. > Thanks, I think that it will be more flexible to dynamically get MSI or MSIX capability, > Thus, we will not need to define the pci_epc_feature for different boards. Is the restriction because you cannot have different compatible for different boards? Thanks Kishon >> >> Thanks >> Kishon >>> >>> Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> >>> --- >>> v2: >>> - Remove the repeated assignment code. >>> >>> drivers/pci/controller/dwc/pci-layerscape-ep.c | 26 >>> +++++++++++++++++++------- >>> 1 file changed, 19 insertions(+), 7 deletions(-) >>> >>> diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c >>> b/drivers/pci/controller/dwc/pci-layerscape-ep.c >>> index 4e92a95..8461f62 100644 >>> --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c >>> +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c >>> @@ -22,6 +22,7 @@ >>> >>> struct ls_pcie_ep { >>> struct dw_pcie *pci; >>> + struct pci_epc_features *ls_epc; >>> }; >>> >>> #define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) >>> @@ -40,25 +41,26 @@ static const struct of_device_id >> ls_pcie_ep_of_match[] = { >>> { }, >>> }; >>> >>> -static const struct pci_epc_features ls_pcie_epc_features = { >>> - .linkup_notifier = false, >>> - .msi_capable = true, >>> - .msix_capable = false, >>> -}; >>> - >>> static const struct pci_epc_features* ls_pcie_ep_get_features(struct >>> dw_pcie_ep *ep) { >>> - return &ls_pcie_epc_features; >>> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); >>> + >>> + return pcie->ls_epc; >>> } >>> >>> static void ls_pcie_ep_init(struct dw_pcie_ep *ep) { >>> struct dw_pcie *pci = to_dw_pcie_from_ep(ep); >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); >>> enum pci_barno bar; >>> >>> for (bar = BAR_0; bar <= BAR_5; bar++) >>> dw_pcie_ep_reset_bar(pci, bar); >>> + >>> + pcie->ls_epc->msi_capable = ep->msi_cap ? true : false; >>> + pcie->ls_epc->msix_capable = ep->msix_cap ? true : false; >>> } >>> >>> static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, @@ >>> -118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct platform_device >> *pdev) >>> struct device *dev = &pdev->dev; >>> struct dw_pcie *pci; >>> struct ls_pcie_ep *pcie; >>> + struct pci_epc_features *ls_epc; >>> struct resource *dbi_base; >>> int ret; >>> >>> @@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct >> platform_device *pdev) >>> if (!pci) >>> return -ENOMEM; >>> >>> + ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL); >>> + if (!ls_epc) >>> + return -ENOMEM; >>> + >>> dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, >> "regs"); >>> pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); >>> if (IS_ERR(pci->dbi_base)) >>> @@ -139,6 +146,11 @@ static int __init ls_pcie_ep_probe(struct >> platform_device *pdev) >>> pci->ops = &ls_pcie_ep_ops; >>> pcie->pci = pci; >>> >>> + ls_epc->linkup_notifier = false, >>> + ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4), >>> + >>> + pcie->ls_epc = ls_epc; >>> + >>> platform_set_drvdata(pdev, pcie); >>> >>> ret = ls_add_pcie_ep(pcie, pdev); >>>
> -----Original Message----- > From: Kishon Vijay Abraham I <kishon@ti.com> > Sent: 2019年8月23日 11:40 > To: Xiaowei Bao <xiaowei.bao@nxp.com>; bhelgaas@google.com; > robh+dt@kernel.org; mark.rutland@arm.com; shawnguo@kernel.org; Leo Li > <leoyang.li@nxp.com>; lorenzo.pieralisi@arm.co > <lorenzo.pieralisi@arm.com>; arnd@arndb.de; gregkh@linuxfoundation.org; > M.h. Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; > Roy Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org; > andrew.murray@arm.com > Subject: Re: [PATCH v2 06/10] PCI: layerscape: Modify the way of getting > capability with different PEX > > Hi, > > (Fixed Lorenzo's email address. All the patches in the series have wrong email > id) > > On 23/08/19 8:09 AM, Xiaowei Bao wrote: > > > > > >> -----Original Message----- > >> From: Kishon Vijay Abraham I <kishon@ti.com> > >> Sent: 2019年8月22日 19:44 > >> To: Xiaowei Bao <xiaowei.bao@nxp.com>; bhelgaas@google.com; > >> robh+dt@kernel.org; mark.rutland@arm.com; shawnguo@kernel.org; Leo > Li > >> <leoyang.li@nxp.com>; lorenzo.pieralisi@arm.co; arnd@arndb.de; > >> gregkh@linuxfoundation.org; M.h. Lian <minghuan.lian@nxp.com>; > >> Mingkai Hu <mingkai.hu@nxp.com>; Roy Zang <roy.zang@nxp.com>; > >> jingoohan1@gmail.com; gustavo.pimentel@synopsys.com; > >> linux-pci@vger.kernel.org; devicetree@vger.kernel.org; > >> linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org; > >> linuxppc-dev@lists.ozlabs.org; andrew.murray@arm.com > >> Subject: Re: [PATCH v2 06/10] PCI: layerscape: Modify the way of > >> getting capability with different PEX > >> > >> Hi, > >> > >> On 22/08/19 4:52 PM, Xiaowei Bao wrote: > >>> The different PCIe controller in one board may be have different > >>> capability of MSI or MSIX, so change the way of getting the MSI > >>> capability, make it more flexible. > >> > >> please use different pci_epc_features table for different boards. > > Thanks, I think that it will be more flexible to dynamically get MSI > > or MSIX capability, Thus, we will not need to define the pci_epc_feature for > different boards. > > Is the restriction because you cannot have different compatible for different > boards? Sorry, I am not very clear what your mean, I think even if I use the same compatible with different boards, each boards will enter the probe function, in there I will get the MSI or MSIX PCIe capability of the current controller in this board. Why do I need to define the pci_epc_feature for different boards? > > Thanks > Kishon > > >> > >> Thanks > >> Kishon > >>> > >>> Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > >>> --- > >>> v2: > >>> - Remove the repeated assignment code. > >>> > >>> drivers/pci/controller/dwc/pci-layerscape-ep.c | 26 > >>> +++++++++++++++++++------- > >>> 1 file changed, 19 insertions(+), 7 deletions(-) > >>> > >>> diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > >>> b/drivers/pci/controller/dwc/pci-layerscape-ep.c > >>> index 4e92a95..8461f62 100644 > >>> --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > >>> +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > >>> @@ -22,6 +22,7 @@ > >>> > >>> struct ls_pcie_ep { > >>> struct dw_pcie *pci; > >>> + struct pci_epc_features *ls_epc; > >>> }; > >>> > >>> #define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > >>> @@ -40,25 +41,26 @@ static const struct of_device_id > >> ls_pcie_ep_of_match[] = { > >>> { }, > >>> }; > >>> > >>> -static const struct pci_epc_features ls_pcie_epc_features = { > >>> - .linkup_notifier = false, > >>> - .msi_capable = true, > >>> - .msix_capable = false, > >>> -}; > >>> - > >>> static const struct pci_epc_features* > >>> ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { > >>> - return &ls_pcie_epc_features; > >>> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > >>> + > >>> + return pcie->ls_epc; > >>> } > >>> > >>> static void ls_pcie_ep_init(struct dw_pcie_ep *ep) { > >>> struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > >>> enum pci_barno bar; > >>> > >>> for (bar = BAR_0; bar <= BAR_5; bar++) > >>> dw_pcie_ep_reset_bar(pci, bar); > >>> + > >>> + pcie->ls_epc->msi_capable = ep->msi_cap ? true : false; > >>> + pcie->ls_epc->msix_capable = ep->msix_cap ? true : false; > >>> } > >>> > >>> static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > >>> @@ > >>> -118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct > >>> platform_device > >> *pdev) > >>> struct device *dev = &pdev->dev; > >>> struct dw_pcie *pci; > >>> struct ls_pcie_ep *pcie; > >>> + struct pci_epc_features *ls_epc; > >>> struct resource *dbi_base; > >>> int ret; > >>> > >>> @@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct > >> platform_device *pdev) > >>> if (!pci) > >>> return -ENOMEM; > >>> > >>> + ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL); > >>> + if (!ls_epc) > >>> + return -ENOMEM; > >>> + > >>> dbi_base = platform_get_resource_byname(pdev, > IORESOURCE_MEM, > >> "regs"); > >>> pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > >>> if (IS_ERR(pci->dbi_base)) > >>> @@ -139,6 +146,11 @@ static int __init ls_pcie_ep_probe(struct > >> platform_device *pdev) > >>> pci->ops = &ls_pcie_ep_ops; > >>> pcie->pci = pci; > >>> > >>> + ls_epc->linkup_notifier = false, > >>> + ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4), > >>> + > >>> + pcie->ls_epc = ls_epc; > >>> + > >>> platform_set_drvdata(pdev, pcie); > >>> > >>> ret = ls_add_pcie_ep(pcie, pdev); > >>>
On Thu, Aug 22, 2019 at 07:22:33PM +0800, Xiaowei Bao wrote: > Add multiple PFs support for DWC, different PF have different config space > we use pf-offset property which get from the DTS to access the different pF > config space. It looks like you're missing a --cover-letter again. > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > --- > v2: > - Remove duplicate redundant code. > - Reimplement the PF config space access way. > > drivers/pci/controller/dwc/pcie-designware-ep.c | 122 ++++++++++++++++-------- > drivers/pci/controller/dwc/pcie-designware.c | 59 ++++++++---- > drivers/pci/controller/dwc/pcie-designware.h | 11 ++- > 3 files changed, 134 insertions(+), 58 deletions(-) > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c > index 2bf5a35..3e2b740 100644 > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > @@ -19,12 +19,17 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) > pci_epc_linkup(epc); > } > > -static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar, > - int flags) > +static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, > + enum pci_barno bar, int flags) > { > u32 reg; > + unsigned int func_offset = 0; > + struct dw_pcie_ep *ep = &pci->ep; > > - reg = PCI_BASE_ADDRESS_0 + (4 * bar); > + if (ep->ops->func_conf_select) > + func_offset = ep->ops->func_conf_select(ep, func_no); > + > + reg = func_offset + PCI_BASE_ADDRESS_0 + (4 * bar); This pattern of checking if func_conf_select exists and using it to get an offset is repeated a lot throughout this file. You could move this functionality into a new function (similar to dw_pcie_read_dbi etc). Or perhaps a new variant of dw_pcie_writel_ should be created that writes takes a func_no argument. > dw_pcie_dbi_ro_wr_en(pci); > dw_pcie_writel_dbi2(pci, reg, 0x0); > dw_pcie_writel_dbi(pci, reg, 0x0); > @@ -235,7 +257,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > - ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size); > + ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr, size); > if (ret) { > dev_err(pci->dev, "Failed to enable address\n"); > return ret; > @@ -249,11 +271,15 @@ static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no) > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > u32 val, reg; > + unsigned int func_offset = 0; > + > + if (ep->ops->func_conf_select) > + func_offset = ep->ops->func_conf_select(ep, func_no); > > if (!ep->msi_cap) > return -EINVAL; > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; This makes me nervous. From a PCI viewpoint, each function has it's own capability structure and within each function there may exist a MSI capability. Yet what we're doing here is using dw_pcie_ep_find_capability to get the list of capabilities for function 0, and then applying offsets from that for subsequent functions. I.e. we're applying DW specific knowledge to find the correct capability, rather than following the general PCI approach. I think the above hunk shouldn't be required - but instead dw_pcie_ep_find_capability is updated to take a func_no parameter. Have I understood this correctly? > val = dw_pcie_readw_dbi(pci, reg); > if (!(val & PCI_MSI_FLAGS_ENABLE)) > return -EINVAL; > @@ -268,11 +294,15 @@ static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 interrupts) > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > u32 val, reg; > + unsigned int func_offset = 0; > + > + if (ep->ops->func_conf_select) > + func_offset = ep->ops->func_conf_select(ep, func_no); > > if (!ep->msi_cap) > return -EINVAL; > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > val = dw_pcie_readw_dbi(pci, reg); > val &= ~PCI_MSI_FLAGS_QMASK; > val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; > @@ -288,11 +318,15 @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > u32 val, reg; > + unsigned int func_offset = 0; > + > + if (ep->ops->func_conf_select) > + func_offset = ep->ops->func_conf_select(ep, func_no); > > if (!ep->msix_cap) > return -EINVAL; > > - reg = ep->msix_cap + PCI_MSIX_FLAGS; > + reg = ep->msix_cap + func_offset + PCI_MSIX_FLAGS; Same for MSIX. > val = dw_pcie_readw_dbi(pci, reg); > if (!(val & PCI_MSIX_FLAGS_ENABLE)) > return -EINVAL; > @@ -307,11 +341,15 @@ static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u16 interrupts) > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > u32 val, reg; > + unsigned int func_offset = 0; > + > + if (ep->ops->func_conf_select) > + func_offset = ep->ops->func_conf_select(ep, func_no); > > if (!ep->msix_cap) > return -EINVAL; > > - reg = ep->msix_cap + PCI_MSIX_FLAGS; > + reg = ep->msix_cap + func_offset + PCI_MSIX_FLAGS; > val = dw_pcie_readw_dbi(pci, reg); > val &= ~PCI_MSIX_FLAGS_QSIZE; > val |= interrupts; > @@ -398,29 +436,33 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > struct pci_epc *epc = ep->epc; > unsigned int aligned_offset; > + unsigned int func_offset = 0; > u16 msg_ctrl, msg_data; > u32 msg_addr_lower, msg_addr_upper, reg; > u64 msg_addr; > bool has_upper; > int ret; > > + if (ep->ops->func_conf_select) > + func_offset = ep->ops->func_conf_select(ep, func_no); > + You could probably move this hunk below the test for msi_cap to save some cycles. > if (!ep->msi_cap) > return -EINVAL; > > /* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */ > - reg = ep->msi_cap + PCI_MSI_FLAGS; > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > msg_ctrl = dw_pcie_readw_dbi(pci, reg); > has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT); > - reg = ep->msi_cap + PCI_MSI_ADDRESS_LO; > + reg = ep->msi_cap + func_offset + PCI_MSI_ADDRESS_LO; > msg_addr_lower = dw_pcie_readl_dbi(pci, reg); > if (has_upper) { > - reg = ep->msi_cap + PCI_MSI_ADDRESS_HI; > + reg = ep->msi_cap + func_offset + PCI_MSI_ADDRESS_HI; > msg_addr_upper = dw_pcie_readl_dbi(pci, reg); > - reg = ep->msi_cap + PCI_MSI_DATA_64; > + reg = ep->msi_cap + func_offset + PCI_MSI_DATA_64; > msg_data = dw_pcie_readw_dbi(pci, reg); > } else { > msg_addr_upper = 0; > - reg = ep->msi_cap + PCI_MSI_DATA_32; > + reg = ep->msi_cap + func_offset + PCI_MSI_DATA_32; > msg_data = dw_pcie_readw_dbi(pci, reg); > } > aligned_offset = msg_addr_lower & (epc->mem->page_size - 1); > diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c > index 7d25102..305e73d 100644 > --- a/drivers/pci/controller/dwc/pcie-designware.c > +++ b/drivers/pci/controller/dwc/pcie-designware.c > @@ -158,9 +158,10 @@ static void dw_pcie_writel_ob_unroll(struct dw_pcie *pci, u32 index, u32 reg, > dw_pcie_writel_atu(pci, offset + reg, val); > } > > -static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, > - int type, u64 cpu_addr, > - u64 pci_addr, u32 size) > +static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, u8 func_no, > + int index, int type, > + u64 cpu_addr, u64 pci_addr, > + u32 size) > { > u32 retries, val; > > @@ -175,7 +176,7 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, > dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET, > upper_32_bits(pci_addr)); > dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, > - type); > + type | PCIE_ATU_FUNC_NUM(func_no)); Much better :) > dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2, > PCIE_ATU_ENABLE); > > @@ -194,8 +195,9 @@ static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, > dev_err(pci->dev, "Outbound iATU is not being enabled\n"); > } > > -void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, > - u64 cpu_addr, u64 pci_addr, u32 size) > +static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 func_no, > + int index, int type, u64 cpu_addr, > + u64 pci_addr, u32 size) > { > u32 retries, val; > > @@ -203,8 +205,8 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, > cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr); > > if (pci->iatu_unroll_enabled) { > - dw_pcie_prog_outbound_atu_unroll(pci, index, type, cpu_addr, > - pci_addr, size); > + dw_pcie_prog_outbound_atu_unroll(pci, func_no, index, type, > + cpu_addr, pci_addr, size); > return; > } > > diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h > index ffed084..a0fdbf7 100644 > --- a/drivers/pci/controller/dwc/pcie-designware.h > +++ b/drivers/pci/controller/dwc/pcie-designware.h > @@ -71,9 +71,11 @@ > #define PCIE_ATU_TYPE_IO 0x2 > #define PCIE_ATU_TYPE_CFG0 0x4 > #define PCIE_ATU_TYPE_CFG1 0x5 > +#define PCIE_ATU_FUNC_NUM(pf) (pf << 20) "Macro argument 'pf' may be better as '(pf)' to avoid precedence issues" > #define PCIE_ATU_CR2 0x908 > #define PCIE_ATU_ENABLE BIT(31) > #define PCIE_ATU_BAR_MODE_ENABLE BIT(30) > +#define PCIE_ATU_FUNC_NUM_MATCH_EN BIT(19) > #define PCIE_ATU_LOWER_BASE 0x90C > #define PCIE_ATU_UPPER_BASE 0x910 > #define PCIE_ATU_LIMIT 0x914 > @@ -197,6 +199,7 @@ struct dw_pcie_ep_ops { > int (*raise_irq)(struct dw_pcie_ep *ep, u8 func_no, > enum pci_epc_irq_type type, u16 interrupt_num); > const struct pci_epc_features* (*get_features)(struct dw_pcie_ep *ep); > + unsigned int (*func_conf_select)(struct dw_pcie_ep *ep, u8 func_no); Given that this function will return an offset, I'm not sure the name you have is suitable. Something like get_pf_offset or similar is more descriptive. Thanks, Andrew Murray > }; > > struct dw_pcie_ep { > @@ -265,8 +268,12 @@ int dw_pcie_wait_for_link(struct dw_pcie *pci); > void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, > int type, u64 cpu_addr, u64 pci_addr, > u32 size); > -int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar, > - u64 cpu_addr, enum dw_pcie_as_type as_type); > +void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int index, > + int type, u64 cpu_addr, u64 pci_addr, > + u32 size); > +int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, > + int bar, u64 cpu_addr, > + enum dw_pcie_as_type as_type); > void dw_pcie_disable_atu(struct dw_pcie *pci, int index, > enum dw_pcie_region_type type); > void dw_pcie_setup(struct dw_pcie *pci); > -- > 2.9.5 >
On Thu, Aug 22, 2019 at 07:22:34PM +0800, Xiaowei Bao wrote: > Add the doorbell mode of MSI-X in EP mode. > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > --- > v2: > - Remove the macro of no used. > > drivers/pci/controller/dwc/pcie-designware-ep.c | 14 ++++++++++++++ > drivers/pci/controller/dwc/pcie-designware.h | 12 ++++++++++++ > 2 files changed, 26 insertions(+) > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c > index 3e2b740..b8388f8 100644 > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > @@ -480,6 +480,20 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, > return 0; > } > > +int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, > + u16 interrupt_num) > +{ > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > + u32 msg_data; > + > + msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) | > + (interrupt_num - 1); > + > + dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data); > + > + return 0; > +} > + > int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, > u16 interrupt_num) > { > diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h > index a0fdbf7..895a9ef 100644 > --- a/drivers/pci/controller/dwc/pcie-designware.h > +++ b/drivers/pci/controller/dwc/pcie-designware.h > @@ -88,6 +88,9 @@ > #define PCIE_MISC_CONTROL_1_OFF 0x8BC > #define PCIE_DBI_RO_WR_EN BIT(0) > > +#define PCIE_MSIX_DOORBELL 0x948 > +#define PCIE_MSIX_DOORBELL_PF_SHIFT 24 > + > /* > * iATU Unroll-specific register definitions > * From 4.80 core version the address translation will be made by unroll > @@ -400,6 +403,8 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, > u8 interrupt_num); > int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, > u16 interrupt_num); > +int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, > + u16 interrupt_num); > void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar); > #else > static inline void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) > @@ -432,6 +437,13 @@ static inline int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, > return 0; > } > > +static inline int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, > + u8 func_no, > + u16 interrupt_num) > +{ > + return 0; > +} > + Looks OK to me. Reviewed-by: Andrew Murray <andrew.murray@arm.com> > static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) > { > } > -- > 2.9.5 >
On Thu, Aug 22, 2019 at 07:22:35PM +0800, Xiaowei Bao wrote: > Move the function of getting MSI capability to the front of init > function, because the init function of the EP platform driver will use > the return value by the function of getting MSI capability. > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> Reviewed-by: Andrew Murray <andrew.murray@arm.com> > --- > v2: > - No change. > > drivers/pci/controller/dwc/pcie-designware-ep.c | 7 ++++--- > 1 file changed, 4 insertions(+), 3 deletions(-) > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c > index b8388f8..0a6c199 100644 > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > @@ -656,6 +656,10 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) > if (ret < 0) > epc->max_functions = 1; > > + ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI); > + > + ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX); > + > if (ep->ops->ep_init) > ep->ops->ep_init(ep); > > @@ -672,9 +676,6 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) > dev_err(dev, "Failed to reserve memory for MSI/MSI-X\n"); > return -ENOMEM; > } > - ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI); > - > - ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX); > > offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); > if (offset) { > -- > 2.9.5 >
On Thu, Aug 22, 2019 at 07:22:37PM +0800, Xiaowei Bao wrote: > Fix some format issue of the code in EP driver. > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> Reviewed-by: Andrew Murray <andrew.murray@arm.com> > --- > v2: > - No change. > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 4 ++-- > 1 file changed, 2 insertions(+), 2 deletions(-) > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c > index be61d96..4e92a95 100644 > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > @@ -62,7 +62,7 @@ static void ls_pcie_ep_init(struct dw_pcie_ep *ep) > } > > static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > - enum pci_epc_irq_type type, u16 interrupt_num) > + enum pci_epc_irq_type type, u16 interrupt_num) > { > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > @@ -86,7 +86,7 @@ static const struct dw_pcie_ep_ops pcie_ep_ops = { > }; > > static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, > - struct platform_device *pdev) > + struct platform_device *pdev) > { > struct dw_pcie *pci = pcie->pci; > struct device *dev = pci->dev; > -- > 2.9.5 >
On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote: > The layerscape platform use the doorbell way to trigger MSIX > interrupt in EP mode. > I have no problems with this patch, however... Are you able to add to this message a reason for why you are making this change? Did dw_pcie_ep_raise_msix_irq not work when func_no != 0? Or did it work yet dw_pcie_ep_raise_msix_irq_doorbell is more efficient? Thanks, Andrew Murray > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > --- > v2: > - No change. > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 3 ++- > 1 file changed, 2 insertions(+), 1 deletion(-) > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c > index 8461f62..7ca5fe8 100644 > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > @@ -74,7 +74,8 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > case PCI_EPC_IRQ_MSI: > return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); > case PCI_EPC_IRQ_MSIX: > - return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); > + return dw_pcie_ep_raise_msix_irq_doorbell(ep, func_no, > + interrupt_num); > default: > dev_err(pci->dev, "UNKNOWN IRQ type\n"); > return -EINVAL; > -- > 2.9.5 >
On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: > Add PCIe EP mode support for ls1088a and ls2088a, there are some > difference between LS1 and LS2 platform, so refactor the code of > the EP driver. > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > --- > v2: > - New mechanism for layerscape EP driver. Was there a v1 of this patch? > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 ++++++++++++++++++++------ > 1 file changed, 58 insertions(+), 18 deletions(-) > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c > index 7ca5fe8..2a66f07 100644 > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > @@ -20,27 +20,29 @@ > > #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ > > -struct ls_pcie_ep { > - struct dw_pcie *pci; > - struct pci_epc_features *ls_epc; > +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > + > +struct ls_pcie_ep_drvdata { > + u32 func_offset; > + const struct dw_pcie_ep_ops *ops; > + const struct dw_pcie_ops *dw_pcie_ops; > }; > > -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > +struct ls_pcie_ep { > + struct dw_pcie *pci; > + struct pci_epc_features *ls_epc; > + const struct ls_pcie_ep_drvdata *drvdata; > +}; > > static int ls_pcie_establish_link(struct dw_pcie *pci) > { > return 0; > } > > -static const struct dw_pcie_ops ls_pcie_ep_ops = { > +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { > .start_link = ls_pcie_establish_link, > }; > > -static const struct of_device_id ls_pcie_ep_of_match[] = { > - { .compatible = "fsl,ls-pcie-ep",}, > - { }, > -}; > - > static const struct pci_epc_features* > ls_pcie_ep_get_features(struct dw_pcie_ep *ep) > { > @@ -82,10 +84,44 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > } > } > > -static const struct dw_pcie_ep_ops pcie_ep_ops = { > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > + u8 func_no) > +{ > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > + u8 header_type; > + > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > + > + if (header_type & (1 << 7)) > + return pcie->drvdata->func_offset * func_no; > + else > + return 0; It looks like there isn't a PCI define for multi function, the nearest I could find was PCI_HEADER_TYPE_MULTIDEVICE in hotplug/ibmphp.h. A comment above the test might be helpful to explain the test. As the ls_pcie_ep_drvdata structures are static, the unset .func_offset will be initialised to 0, so you could just drop the test above. However something to the effect of the following may help spot misconfiguration: WARN_ON(func_no && !pcie->drvdata->func_offset); return pcie->drvdata->func_offset * func_no; The WARN is probably quite useful as if you are attempting to use non-zero functions and func_offset isn't set - then things may appear to work normally but actually will break horribly. Thanks, Andrew Murray > +} > + > +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { > .ep_init = ls_pcie_ep_init, > .raise_irq = ls_pcie_ep_raise_irq, > .get_features = ls_pcie_ep_get_features, > + .func_conf_select = ls_pcie_ep_func_conf_select, > +}; > + > +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { > + .ops = &ls_pcie_ep_ops, > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, > +}; > + > +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { > + .func_offset = 0x20000, > + .ops = &ls_pcie_ep_ops, > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, > +}; > + > +static const struct of_device_id ls_pcie_ep_of_match[] = { > + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, > + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, > + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, > + { }, > }; > > static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, > @@ -98,7 +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, > int ret; > > ep = &pci->ep; > - ep->ops = &pcie_ep_ops; > + ep->ops = pcie->drvdata->ops; > > res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); > if (!res) > @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) > if (!ls_epc) > return -ENOMEM; > > - dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); > - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > - if (IS_ERR(pci->dbi_base)) > - return PTR_ERR(pci->dbi_base); > + pcie->drvdata = of_device_get_match_data(dev); > > - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > pci->dev = dev; > - pci->ops = &ls_pcie_ep_ops; > + pci->ops = pcie->drvdata->dw_pcie_ops; > + > pcie->pci = pci; > > ls_epc->linkup_notifier = false, > @@ -152,6 +185,13 @@ static int __init ls_pcie_ep_probe(struct platform_device *pdev) > > pcie->ls_epc = ls_epc; > > + dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); > + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > + if (IS_ERR(pci->dbi_base)) > + return PTR_ERR(pci->dbi_base); > + > + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > + > platform_set_drvdata(pdev, pcie); > > ret = ls_add_pcie_ep(pcie, pdev); > -- > 2.9.5 >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月23日 21:25 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 01/10] PCI: designware-ep: Add multiple PFs support > for DWC > > On Thu, Aug 22, 2019 at 07:22:33PM +0800, Xiaowei Bao wrote: > > Add multiple PFs support for DWC, different PF have different config > > space we use pf-offset property which get from the DTS to access the > > different pF config space. > > It looks like you're missing a --cover-letter again. > > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > --- > > v2: > > - Remove duplicate redundant code. > > - Reimplement the PF config space access way. > > > > drivers/pci/controller/dwc/pcie-designware-ep.c | 122 > ++++++++++++++++-------- > > drivers/pci/controller/dwc/pcie-designware.c | 59 ++++++++---- > > drivers/pci/controller/dwc/pcie-designware.h | 11 ++- > > 3 files changed, 134 insertions(+), 58 deletions(-) > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c > > b/drivers/pci/controller/dwc/pcie-designware-ep.c > > index 2bf5a35..3e2b740 100644 > > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > > @@ -19,12 +19,17 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) > > pci_epc_linkup(epc); > > } > > > > -static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno > bar, > > - int flags) > > +static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, > > + enum pci_barno bar, int flags) > > { > > u32 reg; > > + unsigned int func_offset = 0; > > + struct dw_pcie_ep *ep = &pci->ep; > > > > - reg = PCI_BASE_ADDRESS_0 + (4 * bar); > > + if (ep->ops->func_conf_select) > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > + > > + reg = func_offset + PCI_BASE_ADDRESS_0 + (4 * bar); > > This pattern of checking if func_conf_select exists and using it to get an offset > is repeated a lot throughout this file. You could move this functionality into a > new function (similar to dw_pcie_read_dbi etc). Or perhaps a new variant of > dw_pcie_writel_ should be created that writes takes a func_no argument. Thanks for your comments, I thought about this method before, but there is a issue about the method of access the different func config space, due to our platform use this method that different func have different offset from dbi_base to access the different config space, but others platform maybe use the way that write a register to implement different func config space access, so I think reserve a callback function to different platform to implement the own method, my point is that, if use register method they can implement the code in this function and return offset is 0, if use offset method, they can return the offset value which can be use by dw_pcie_ep driver. > > > > dw_pcie_dbi_ro_wr_en(pci); > > dw_pcie_writel_dbi2(pci, reg, 0x0); > > dw_pcie_writel_dbi(pci, reg, 0x0); > > > > @@ -235,7 +257,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc > *epc, u8 func_no, > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > - ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size); > > + ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr, size); > > if (ret) { > > dev_err(pci->dev, "Failed to enable address\n"); > > return ret; > > @@ -249,11 +271,15 @@ static int dw_pcie_ep_get_msi(struct pci_epc > *epc, u8 func_no) > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > u32 val, reg; > > + unsigned int func_offset = 0; > > + > > + if (ep->ops->func_conf_select) > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > if (!ep->msi_cap) > > return -EINVAL; > > > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > > This makes me nervous. > > From a PCI viewpoint, each function has it's own capability structure and > within each function there may exist a MSI capability. Yet what we're doing > here is using dw_pcie_ep_find_capability to get the list of capabilities for > function 0, and then applying offsets from that for subsequent functions. I.e. > we're applying DW specific knowledge to find the correct capability, rather > than following the general PCI approach. > > I think the above hunk shouldn't be required - but instead > dw_pcie_ep_find_capability is updated to take a func_no parameter. > > Have I understood this correctly? Yes, this is a issue, I think the different func maybe have different capability, but the dw_pcie_ep_find_capability function is called by dw_pcie_ep_init function, we can't add func_no parameter to dw_pcie_ep_find_capability, I will try to fix it use a new patch, I think move this function to ep_init callback function If better, thanks. > > > val = dw_pcie_readw_dbi(pci, reg); > > if (!(val & PCI_MSI_FLAGS_ENABLE)) > > return -EINVAL; > > @@ -268,11 +294,15 @@ static int dw_pcie_ep_set_msi(struct pci_epc > *epc, u8 func_no, u8 interrupts) > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > u32 val, reg; > > + unsigned int func_offset = 0; > > + > > + if (ep->ops->func_conf_select) > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > if (!ep->msi_cap) > > return -EINVAL; > > > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > > val = dw_pcie_readw_dbi(pci, reg); > > val &= ~PCI_MSI_FLAGS_QMASK; > > val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; @@ -288,11 +318,15 > > @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > u32 val, reg; > > + unsigned int func_offset = 0; > > + > > + if (ep->ops->func_conf_select) > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > if (!ep->msix_cap) > > return -EINVAL; > > > > - reg = ep->msix_cap + PCI_MSIX_FLAGS; > > + reg = ep->msix_cap + func_offset + PCI_MSIX_FLAGS; > > Same for MSIX. Yes. > > > val = dw_pcie_readw_dbi(pci, reg); > > if (!(val & PCI_MSIX_FLAGS_ENABLE)) > > return -EINVAL; > > @@ -307,11 +341,15 @@ static int dw_pcie_ep_set_msix(struct pci_epc > *epc, u8 func_no, u16 interrupts) > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > u32 val, reg; > > + unsigned int func_offset = 0; > > + > > + if (ep->ops->func_conf_select) > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > if (!ep->msix_cap) > > return -EINVAL; > > > > - reg = ep->msix_cap + PCI_MSIX_FLAGS; > > + reg = ep->msix_cap + func_offset + PCI_MSIX_FLAGS; > > val = dw_pcie_readw_dbi(pci, reg); > > val &= ~PCI_MSIX_FLAGS_QSIZE; > > val |= interrupts; > > @@ -398,29 +436,33 @@ int dw_pcie_ep_raise_msi_irq(struct > dw_pcie_ep *ep, u8 func_no, > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > struct pci_epc *epc = ep->epc; > > unsigned int aligned_offset; > > + unsigned int func_offset = 0; > > u16 msg_ctrl, msg_data; > > u32 msg_addr_lower, msg_addr_upper, reg; > > u64 msg_addr; > > bool has_upper; > > int ret; > > > > + if (ep->ops->func_conf_select) > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > + > > You could probably move this hunk below the test for msi_cap to save some > cycles. Sorry, I didn't understand the means, please explain it detailly, thanks a lot, ^_^ > > > if (!ep->msi_cap) > > return -EINVAL; > > > > /* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */ > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > > msg_ctrl = dw_pcie_readw_dbi(pci, reg); > > has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT); > > - reg = ep->msi_cap + PCI_MSI_ADDRESS_LO; > > + reg = ep->msi_cap + func_offset + PCI_MSI_ADDRESS_LO; > > msg_addr_lower = dw_pcie_readl_dbi(pci, reg); > > if (has_upper) { > > - reg = ep->msi_cap + PCI_MSI_ADDRESS_HI; > > + reg = ep->msi_cap + func_offset + PCI_MSI_ADDRESS_HI; > > msg_addr_upper = dw_pcie_readl_dbi(pci, reg); > > - reg = ep->msi_cap + PCI_MSI_DATA_64; > > + reg = ep->msi_cap + func_offset + PCI_MSI_DATA_64; > > msg_data = dw_pcie_readw_dbi(pci, reg); > > } else { > > msg_addr_upper = 0; > > - reg = ep->msi_cap + PCI_MSI_DATA_32; > > + reg = ep->msi_cap + func_offset + PCI_MSI_DATA_32; > > msg_data = dw_pcie_readw_dbi(pci, reg); > > } > > aligned_offset = msg_addr_lower & (epc->mem->page_size - 1); > > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware.c > > b/drivers/pci/controller/dwc/pcie-designware.c > > index 7d25102..305e73d 100644 > > --- a/drivers/pci/controller/dwc/pcie-designware.c > > +++ b/drivers/pci/controller/dwc/pcie-designware.c > > @@ -158,9 +158,10 @@ static void dw_pcie_writel_ob_unroll(struct > dw_pcie *pci, u32 index, u32 reg, > > dw_pcie_writel_atu(pci, offset + reg, val); } > > > > -static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int > index, > > - int type, u64 cpu_addr, > > - u64 pci_addr, u32 size) > > +static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, u8 > func_no, > > + int index, int type, > > + u64 cpu_addr, u64 pci_addr, > > + u32 size) > > { > > u32 retries, val; > > > > @@ -175,7 +176,7 @@ static void > dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, > > dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET, > > upper_32_bits(pci_addr)); > > dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, > > - type); > > + type | PCIE_ATU_FUNC_NUM(func_no)); > > Much better :) Do you mean that use the expression "a? b:c" > > > dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2, > > PCIE_ATU_ENABLE); > > > > @@ -194,8 +195,9 @@ static void > dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, > > dev_err(pci->dev, "Outbound iATU is not being enabled\n"); } > > > > -void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, > > - u64 cpu_addr, u64 pci_addr, u32 size) > > +static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 > func_no, > > + int index, int type, u64 cpu_addr, > > + u64 pci_addr, u32 size) > > { > > u32 retries, val; > > > > @@ -203,8 +205,8 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie > *pci, int index, int type, > > cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr); > > > > if (pci->iatu_unroll_enabled) { > > - dw_pcie_prog_outbound_atu_unroll(pci, index, type, cpu_addr, > > - pci_addr, size); > > + dw_pcie_prog_outbound_atu_unroll(pci, func_no, index, type, > > + cpu_addr, pci_addr, size); > > return; > > } > > > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware.h > > b/drivers/pci/controller/dwc/pcie-designware.h > > index ffed084..a0fdbf7 100644 > > --- a/drivers/pci/controller/dwc/pcie-designware.h > > +++ b/drivers/pci/controller/dwc/pcie-designware.h > > @@ -71,9 +71,11 @@ > > #define PCIE_ATU_TYPE_IO 0x2 > > #define PCIE_ATU_TYPE_CFG0 0x4 > > #define PCIE_ATU_TYPE_CFG1 0x5 > > +#define PCIE_ATU_FUNC_NUM(pf) (pf << 20) > > "Macro argument 'pf' may be better as '(pf)' to avoid precedence issues" > > > #define PCIE_ATU_CR2 0x908 > > #define PCIE_ATU_ENABLE BIT(31) > > #define PCIE_ATU_BAR_MODE_ENABLE BIT(30) > > +#define PCIE_ATU_FUNC_NUM_MATCH_EN BIT(19) > > #define PCIE_ATU_LOWER_BASE 0x90C > > #define PCIE_ATU_UPPER_BASE 0x910 > > #define PCIE_ATU_LIMIT 0x914 > > @@ -197,6 +199,7 @@ struct dw_pcie_ep_ops { > > int (*raise_irq)(struct dw_pcie_ep *ep, u8 func_no, > > enum pci_epc_irq_type type, u16 interrupt_num); > > const struct pci_epc_features* (*get_features)(struct dw_pcie_ep > > *ep); > > + unsigned int (*func_conf_select)(struct dw_pcie_ep *ep, u8 func_no); > > Given that this function will return an offset, I'm not sure the name you have > is suitable. Something like get_pf_offset or similar is more descriptive. As above explain, my initial view is that this function can return 0 or offset depends on the platform implement mechanism, so I named it func_conf_select, I think add a comment for this function, like this: /* * provide a method to implement the method of different func config space access, * if use offset method, return the offset from dbi_base, if your register method, implement * the code in this callback function and return 0. */ How about it? > > Thanks, > > Andrew Murray > > > }; > > > > struct dw_pcie_ep { > > @@ -265,8 +268,12 @@ int dw_pcie_wait_for_link(struct dw_pcie *pci); > > void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, > > int type, u64 cpu_addr, u64 pci_addr, > > u32 size); > > -int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar, > > - u64 cpu_addr, enum dw_pcie_as_type as_type); > > +void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int > index, > > + int type, u64 cpu_addr, u64 pci_addr, > > + u32 size); > > +int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, > > + int bar, u64 cpu_addr, > > + enum dw_pcie_as_type as_type); > > void dw_pcie_disable_atu(struct dw_pcie *pci, int index, > > enum dw_pcie_region_type type); > > void dw_pcie_setup(struct dw_pcie *pci); > > -- > > 2.9.5 > >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月23日 21:36 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 02/10] PCI: designware-ep: Add the doorbell mode of > MSI-X in EP mode > > On Thu, Aug 22, 2019 at 07:22:34PM +0800, Xiaowei Bao wrote: > > Add the doorbell mode of MSI-X in EP mode. > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > --- > > v2: > > - Remove the macro of no used. > > > > drivers/pci/controller/dwc/pcie-designware-ep.c | 14 ++++++++++++++ > > drivers/pci/controller/dwc/pcie-designware.h | 12 ++++++++++++ > > 2 files changed, 26 insertions(+) > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c > > b/drivers/pci/controller/dwc/pcie-designware-ep.c > > index 3e2b740..b8388f8 100644 > > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > > @@ -480,6 +480,20 @@ int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep > *ep, u8 func_no, > > return 0; > > } > > > > +int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 > func_no, > > + u16 interrupt_num) > > +{ > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > + u32 msg_data; > > + > > + msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) | > > + (interrupt_num - 1); > > + > > + dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data); > > + > > + return 0; > > +} > > + > > int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, > > u16 interrupt_num) > > { > > diff --git a/drivers/pci/controller/dwc/pcie-designware.h > > b/drivers/pci/controller/dwc/pcie-designware.h > > index a0fdbf7..895a9ef 100644 > > --- a/drivers/pci/controller/dwc/pcie-designware.h > > +++ b/drivers/pci/controller/dwc/pcie-designware.h > > @@ -88,6 +88,9 @@ > > #define PCIE_MISC_CONTROL_1_OFF 0x8BC > > #define PCIE_DBI_RO_WR_EN BIT(0) > > > > +#define PCIE_MSIX_DOORBELL 0x948 > > +#define PCIE_MSIX_DOORBELL_PF_SHIFT 24 > > + > > /* > > * iATU Unroll-specific register definitions > > * From 4.80 core version the address translation will be made by > > unroll @@ -400,6 +403,8 @@ int dw_pcie_ep_raise_msi_irq(struct > dw_pcie_ep *ep, u8 func_no, > > u8 interrupt_num); > > int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, > > u16 interrupt_num); > > +int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 > func_no, > > + u16 interrupt_num); > > void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar); > > #else static inline void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) @@ > > -432,6 +437,13 @@ static inline int dw_pcie_ep_raise_msix_irq(struct > dw_pcie_ep *ep, u8 func_no, > > return 0; > > } > > > > +static inline int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep > *ep, > > + u8 func_no, > > + u16 interrupt_num) > > +{ > > + return 0; > > +} > > + > > Looks OK to me. > > Reviewed-by: Andrew Murray <andrew.murray@arm.com> Thanks a lot. > > > static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum > > pci_barno bar) { } > > -- > > 2.9.5 > >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月23日 21:45 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 05/10] PCI: layerscape: Fix some format issue of the > code > > On Thu, Aug 22, 2019 at 07:22:37PM +0800, Xiaowei Bao wrote: > > Fix some format issue of the code in EP driver. > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > Reviewed-by: Andrew Murray <andrew.murray@arm.com> Thanks. > > > > --- > > v2: > > - No change. > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 4 ++-- > > 1 file changed, 2 insertions(+), 2 deletions(-) > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > index be61d96..4e92a95 100644 > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > @@ -62,7 +62,7 @@ static void ls_pcie_ep_init(struct dw_pcie_ep *ep) > > } > > > > static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > > - enum pci_epc_irq_type type, u16 interrupt_num) > > + enum pci_epc_irq_type type, u16 interrupt_num) > > { > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > @@ -86,7 +86,7 @@ static const struct dw_pcie_ep_ops pcie_ep_ops = { > > }; > > > > static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, > > - struct platform_device *pdev) > > + struct platform_device *pdev) > > { > > struct dw_pcie *pci = pcie->pci; > > struct device *dev = pci->dev; > > -- > > 2.9.5 > >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月23日 21:58 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to the > doorbell way > > On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote: > > The layerscape platform use the doorbell way to trigger MSIX interrupt > > in EP mode. > > > > I have no problems with this patch, however... > > Are you able to add to this message a reason for why you are making this > change? Did dw_pcie_ep_raise_msix_irq not work when func_no != 0? Or did > it work yet dw_pcie_ep_raise_msix_irq_doorbell is more efficient? The fact is that, this driver is verified in ls1046a platform of NXP before, and ls1046a don't support MSIX feature, so I set the msix_capable of pci_epc_features struct is false, but in other platform, e.g. ls1088a, it support the MSIX feature, I verified the MSIX feature in ls1088a, it is not OK, so I changed to another way. Thanks. > > Thanks, > > Andrew Murray > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > --- > > v2: > > - No change. > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 3 ++- > > 1 file changed, 2 insertions(+), 1 deletion(-) > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > index 8461f62..7ca5fe8 100644 > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > @@ -74,7 +74,8 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, > u8 func_no, > > case PCI_EPC_IRQ_MSI: > > return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); > > case PCI_EPC_IRQ_MSIX: > > - return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); > > + return dw_pcie_ep_raise_msix_irq_doorbell(ep, func_no, > > + interrupt_num); > > default: > > dev_err(pci->dev, "UNKNOWN IRQ type\n"); > > return -EINVAL; > > -- > > 2.9.5 > >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月23日 22:28 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support for > ls1088a and ls2088a > > On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: > > Add PCIe EP mode support for ls1088a and ls2088a, there are some > > difference between LS1 and LS2 platform, so refactor the code of the > > EP driver. > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > --- > > v2: > > - New mechanism for layerscape EP driver. > > Was there a v1 of this patch? Yes, but I don't know how to comments, ^_^ > > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 > > ++++++++++++++++++++------ > > 1 file changed, 58 insertions(+), 18 deletions(-) > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > index 7ca5fe8..2a66f07 100644 > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > @@ -20,27 +20,29 @@ > > > > #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ > > > > -struct ls_pcie_ep { > > - struct dw_pcie *pci; > > - struct pci_epc_features *ls_epc; > > +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > + > > +struct ls_pcie_ep_drvdata { > > + u32 func_offset; > > + const struct dw_pcie_ep_ops *ops; > > + const struct dw_pcie_ops *dw_pcie_ops; > > }; > > > > -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > +struct ls_pcie_ep { > > + struct dw_pcie *pci; > > + struct pci_epc_features *ls_epc; > > + const struct ls_pcie_ep_drvdata *drvdata; }; > > > > static int ls_pcie_establish_link(struct dw_pcie *pci) { > > return 0; > > } > > > > -static const struct dw_pcie_ops ls_pcie_ep_ops = { > > +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { > > .start_link = ls_pcie_establish_link, }; > > > > -static const struct of_device_id ls_pcie_ep_of_match[] = { > > - { .compatible = "fsl,ls-pcie-ep",}, > > - { }, > > -}; > > - > > static const struct pci_epc_features* ls_pcie_ep_get_features(struct > > dw_pcie_ep *ep) { @@ -82,10 +84,44 @@ static int > > ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > > } > > } > > > > -static const struct dw_pcie_ep_ops pcie_ep_ops = { > > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > > + u8 func_no) > > +{ > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > + u8 header_type; > > + > > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > + > > + if (header_type & (1 << 7)) > > + return pcie->drvdata->func_offset * func_no; > > + else > > + return 0; > > It looks like there isn't a PCI define for multi function, the nearest I could find > was PCI_HEADER_TYPE_MULTIDEVICE in hotplug/ibmphp.h. A comment > above the test might be helpful to explain the test. Yes, I have not find the PCI_HEADER_TYPE_MULTIDEVICE define. OK, I will add The comments in next version patch. > > As the ls_pcie_ep_drvdata structures are static, the unset .func_offset will be > initialised to 0, so you could just drop the test above. OK, thanks > > However something to the effect of the following may help spot > misconfiguration: > > WARN_ON(func_no && !pcie->drvdata->func_offset); return > pcie->drvdata->func_offset * func_no; Thanks a lot, this looks better. > > The WARN is probably quite useful as if you are attempting to use non-zero > functions and func_offset isn't set - then things may appear to work normally > but actually will break horribly. got it, thanks. > > Thanks, > > Andrew Murray > > > +} > > + > > +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { > > .ep_init = ls_pcie_ep_init, > > .raise_irq = ls_pcie_ep_raise_irq, > > .get_features = ls_pcie_ep_get_features, > > + .func_conf_select = ls_pcie_ep_func_conf_select, }; > > + > > +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { > > + .ops = &ls_pcie_ep_ops, > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, > > +}; > > + > > +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { > > + .func_offset = 0x20000, > > + .ops = &ls_pcie_ep_ops, > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, > > +}; > > + > > +static const struct of_device_id ls_pcie_ep_of_match[] = { > > + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, > > + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, > > + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, > > + { }, > > }; > > > > static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ -98,7 > > +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, > > int ret; > > > > ep = &pci->ep; > > - ep->ops = &pcie_ep_ops; > > + ep->ops = pcie->drvdata->ops; > > > > res = platform_get_resource_byname(pdev, IORESOURCE_MEM, > "addr_space"); > > if (!res) > > @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct > platform_device *pdev) > > if (!ls_epc) > > return -ENOMEM; > > > > - dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, > "regs"); > > - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > - if (IS_ERR(pci->dbi_base)) > > - return PTR_ERR(pci->dbi_base); > > + pcie->drvdata = of_device_get_match_data(dev); > > > > - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > pci->dev = dev; > > - pci->ops = &ls_pcie_ep_ops; > > + pci->ops = pcie->drvdata->dw_pcie_ops; > > + > > pcie->pci = pci; > > > > ls_epc->linkup_notifier = false, > > @@ -152,6 +185,13 @@ static int __init ls_pcie_ep_probe(struct > > platform_device *pdev) > > > > pcie->ls_epc = ls_epc; > > > > + dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, > "regs"); > > + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > + if (IS_ERR(pci->dbi_base)) > > + return PTR_ERR(pci->dbi_base); > > + > > + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > + > > platform_set_drvdata(pdev, pcie); > > > > ret = ls_add_pcie_ep(pcie, pdev); > > -- > > 2.9.5 > >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月23日 21:39 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 03/10] PCI: designware-ep: Move the function of > getting MSI capability forward > > On Thu, Aug 22, 2019 at 07:22:35PM +0800, Xiaowei Bao wrote: > > Move the function of getting MSI capability to the front of init > > function, because the init function of the EP platform driver will use > > the return value by the function of getting MSI capability. > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > Reviewed-by: Andrew Murray <andrew.murray@arm.com> Thanks a lot, I think move this to ep_init is better. > > > --- > > v2: > > - No change. > > > > drivers/pci/controller/dwc/pcie-designware-ep.c | 7 ++++--- > > 1 file changed, 4 insertions(+), 3 deletions(-) > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c > > b/drivers/pci/controller/dwc/pcie-designware-ep.c > > index b8388f8..0a6c199 100644 > > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > > @@ -656,6 +656,10 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) > > if (ret < 0) > > epc->max_functions = 1; > > > > + ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI); > > + > > + ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX); > > + > > if (ep->ops->ep_init) > > ep->ops->ep_init(ep); > > > > @@ -672,9 +676,6 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) > > dev_err(dev, "Failed to reserve memory for MSI/MSI-X\n"); > > return -ENOMEM; > > } > > - ep->msi_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSI); > > - > > - ep->msix_cap = dw_pcie_ep_find_capability(pci, PCI_CAP_ID_MSIX); > > > > offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR); > > if (offset) { > > -- > > 2.9.5 > >
Le 24/08/2019 à 02:18, Xiaowei Bao a écrit : > > >> -----Original Message----- >> From: Andrew Murray <andrew.murray@arm.com> >> Sent: 2019年8月23日 22:28 >> To: Xiaowei Bao <xiaowei.bao@nxp.com> >> Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; >> shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; >> lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. >> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy >> Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; >> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; >> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; >> linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org >> Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support for >> ls1088a and ls2088a >> >> On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: >>> Add PCIe EP mode support for ls1088a and ls2088a, there are some >>> difference between LS1 and LS2 platform, so refactor the code of the >>> EP driver. >>> >>> Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> >>> --- >>> v2: >>> - New mechanism for layerscape EP driver. >> >> Was there a v1 of this patch? > > Yes, but I don't know how to comments, ^_^ As far as I can see, in the previous version of the series (https://patchwork.ozlabs.org/project/linuxppc-dev/list/?series=125315&state=*), the 8/10 was something completely different, and I can't find any other patch in the series that could have been the v1 of this patch. Christophe > >> >>> >>> drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 >>> ++++++++++++++++++++------ >>> 1 file changed, 58 insertions(+), 18 deletions(-) >>> >>> diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c >>> b/drivers/pci/controller/dwc/pci-layerscape-ep.c >>> index 7ca5fe8..2a66f07 100644 >>> --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c >>> +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c >>> @@ -20,27 +20,29 @@ >>> >>> #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ >>> >>> -struct ls_pcie_ep { >>> - struct dw_pcie *pci; >>> - struct pci_epc_features *ls_epc; >>> +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) >>> + >>> +struct ls_pcie_ep_drvdata { >>> + u32 func_offset; >>> + const struct dw_pcie_ep_ops *ops; >>> + const struct dw_pcie_ops *dw_pcie_ops; >>> }; >>> >>> -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) >>> +struct ls_pcie_ep { >>> + struct dw_pcie *pci; >>> + struct pci_epc_features *ls_epc; >>> + const struct ls_pcie_ep_drvdata *drvdata; }; >>> >>> static int ls_pcie_establish_link(struct dw_pcie *pci) { >>> return 0; >>> } >>> >>> -static const struct dw_pcie_ops ls_pcie_ep_ops = { >>> +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { >>> .start_link = ls_pcie_establish_link, }; >>> >>> -static const struct of_device_id ls_pcie_ep_of_match[] = { >>> - { .compatible = "fsl,ls-pcie-ep",}, >>> - { }, >>> -}; >>> - >>> static const struct pci_epc_features* ls_pcie_ep_get_features(struct >>> dw_pcie_ep *ep) { @@ -82,10 +84,44 @@ static int >>> ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, >>> } >>> } >>> >>> -static const struct dw_pcie_ep_ops pcie_ep_ops = { >>> +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, >>> + u8 func_no) >>> +{ >>> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); >>> + u8 header_type; >>> + >>> + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); >>> + >>> + if (header_type & (1 << 7)) >>> + return pcie->drvdata->func_offset * func_no; >>> + else >>> + return 0; >> >> It looks like there isn't a PCI define for multi function, the nearest I could find >> was PCI_HEADER_TYPE_MULTIDEVICE in hotplug/ibmphp.h. A comment >> above the test might be helpful to explain the test. > > Yes, I have not find the PCI_HEADER_TYPE_MULTIDEVICE define. OK, I will add > The comments in next version patch. > >> >> As the ls_pcie_ep_drvdata structures are static, the unset .func_offset will be >> initialised to 0, so you could just drop the test above. > > OK, thanks > >> >> However something to the effect of the following may help spot >> misconfiguration: >> >> WARN_ON(func_no && !pcie->drvdata->func_offset); return >> pcie->drvdata->func_offset * func_no; > > Thanks a lot, this looks better. > >> >> The WARN is probably quite useful as if you are attempting to use non-zero >> functions and func_offset isn't set - then things may appear to work normally >> but actually will break horribly. > > got it, thanks. > >> >> Thanks, >> >> Andrew Murray >> >>> +} >>> + >>> +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { >>> .ep_init = ls_pcie_ep_init, >>> .raise_irq = ls_pcie_ep_raise_irq, >>> .get_features = ls_pcie_ep_get_features, >>> + .func_conf_select = ls_pcie_ep_func_conf_select, }; >>> + >>> +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { >>> + .ops = &ls_pcie_ep_ops, >>> + .dw_pcie_ops = &dw_ls_pcie_ep_ops, >>> +}; >>> + >>> +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { >>> + .func_offset = 0x20000, >>> + .ops = &ls_pcie_ep_ops, >>> + .dw_pcie_ops = &dw_ls_pcie_ep_ops, >>> +}; >>> + >>> +static const struct of_device_id ls_pcie_ep_of_match[] = { >>> + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, >>> + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, >>> + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, >>> + { }, >>> }; >>> >>> static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ -98,7 >>> +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, >>> int ret; >>> >>> ep = &pci->ep; >>> - ep->ops = &pcie_ep_ops; >>> + ep->ops = pcie->drvdata->ops; >>> >>> res = platform_get_resource_byname(pdev, IORESOURCE_MEM, >> "addr_space"); >>> if (!res) >>> @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct >> platform_device *pdev) >>> if (!ls_epc) >>> return -ENOMEM; >>> >>> - dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, >> "regs"); >>> - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); >>> - if (IS_ERR(pci->dbi_base)) >>> - return PTR_ERR(pci->dbi_base); >>> + pcie->drvdata = of_device_get_match_data(dev); >>> >>> - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; >>> pci->dev = dev; >>> - pci->ops = &ls_pcie_ep_ops; >>> + pci->ops = pcie->drvdata->dw_pcie_ops; >>> + >>> pcie->pci = pci; >>> >>> ls_epc->linkup_notifier = false, >>> @@ -152,6 +185,13 @@ static int __init ls_pcie_ep_probe(struct >>> platform_device *pdev) >>> >>> pcie->ls_epc = ls_epc; >>> >>> + dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, >> "regs"); >>> + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); >>> + if (IS_ERR(pci->dbi_base)) >>> + return PTR_ERR(pci->dbi_base); >>> + >>> + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; >>> + >>> platform_set_drvdata(pdev, pcie); >>> >>> ret = ls_add_pcie_ep(pcie, pdev); >>> -- >>> 2.9.5 >>> --- L'absence de virus dans ce courrier électronique a été vérifiée par le logiciel antivirus Avast. https://www.avast.com/antivirus
> -----Original Message----- > From: christophe leroy <christophe.leroy@c-s.fr> > Sent: 2019年8月24日 14:45 > To: Xiaowei Bao <xiaowei.bao@nxp.com>; Andrew Murray > <andrew.murray@arm.com> > Cc: mark.rutland@arm.com; Roy Zang <roy.zang@nxp.com>; > lorenzo.pieralisi@arm.co; arnd@arndb.de; devicetree@vger.kernel.org; > gregkh@linuxfoundation.org; linuxppc-dev@lists.ozlabs.org; > linux-pci@vger.kernel.org; linux-kernel@vger.kernel.org; kishon@ti.com; M.h. > Lian <minghuan.lian@nxp.com>; robh+dt@kernel.org; > gustavo.pimentel@synopsys.com; jingoohan1@gmail.com; > bhelgaas@google.com; Leo Li <leoyang.li@nxp.com>; shawnguo@kernel.org; > Mingkai Hu <mingkai.hu@nxp.com>; linux-arm-kernel@lists.infradead.org > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support for > ls1088a and ls2088a > > > > Le 24/08/2019 à 02:18, Xiaowei Bao a écrit : > > > > > >> -----Original Message----- > >> From: Andrew Murray <andrew.murray@arm.com> > >> Sent: 2019年8月23日 22:28 > >> To: Xiaowei Bao <xiaowei.bao@nxp.com> > >> Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > >> shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > >> lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; > M.h. > >> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > >> Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > >> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > >> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > >> linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > >> Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support > >> for ls1088a and ls2088a > >> > >> On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: > >>> Add PCIe EP mode support for ls1088a and ls2088a, there are some > >>> difference between LS1 and LS2 platform, so refactor the code of the > >>> EP driver. > >>> > >>> Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > >>> --- > >>> v2: > >>> - New mechanism for layerscape EP driver. > >> > >> Was there a v1 of this patch? > > > > Yes, but I don't know how to comments, ^_^ > > As far as I can see, in the previous version of the series > (https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpatch > work.ozlabs.org%2Fproject%2Flinuxppc-dev%2Flist%2F%3Fseries%3D125315 > %26state%3D*&data=02%7C01%7Cxiaowei.bao%40nxp.com%7C1befe9 > a67c8046f9535e08d7285eaab6%7C686ea1d3bc2b4c6fa92cd99c5c301635% > 7C0%7C0%7C637022259387139020&sdata=p4wbycd04Z7qRUfAoZtwc > UP7pR%2FuA3%2FjVcWMz6YyQVQ%3D&reserved=0), > the 8/10 was something completely different, and I can't find any other patch > in the series that could have been the v1 of this patch. Thanks, I will correct it to v1 in next version patch. > > Christophe > > > > >> > >>> > >>> drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 > >>> ++++++++++++++++++++------ > >>> 1 file changed, 58 insertions(+), 18 deletions(-) > >>> > >>> diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > >>> b/drivers/pci/controller/dwc/pci-layerscape-ep.c > >>> index 7ca5fe8..2a66f07 100644 > >>> --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > >>> +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > >>> @@ -20,27 +20,29 @@ > >>> > >>> #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ > >>> > >>> -struct ls_pcie_ep { > >>> - struct dw_pcie *pci; > >>> - struct pci_epc_features *ls_epc; > >>> +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > >>> + > >>> +struct ls_pcie_ep_drvdata { > >>> + u32 func_offset; > >>> + const struct dw_pcie_ep_ops *ops; > >>> + const struct dw_pcie_ops *dw_pcie_ops; > >>> }; > >>> > >>> -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > >>> +struct ls_pcie_ep { > >>> + struct dw_pcie *pci; > >>> + struct pci_epc_features *ls_epc; > >>> + const struct ls_pcie_ep_drvdata *drvdata; }; > >>> > >>> static int ls_pcie_establish_link(struct dw_pcie *pci) { > >>> return 0; > >>> } > >>> > >>> -static const struct dw_pcie_ops ls_pcie_ep_ops = { > >>> +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { > >>> .start_link = ls_pcie_establish_link, }; > >>> > >>> -static const struct of_device_id ls_pcie_ep_of_match[] = { > >>> - { .compatible = "fsl,ls-pcie-ep",}, > >>> - { }, > >>> -}; > >>> - > >>> static const struct pci_epc_features* > >>> ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { @@ -82,10 +84,44 > >>> @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > >>> } > >>> } > >>> > >>> -static const struct dw_pcie_ep_ops pcie_ep_ops = { > >>> +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > >>> + u8 func_no) > >>> +{ > >>> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > >>> + u8 header_type; > >>> + > >>> + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > >>> + > >>> + if (header_type & (1 << 7)) > >>> + return pcie->drvdata->func_offset * func_no; > >>> + else > >>> + return 0; > >> > >> It looks like there isn't a PCI define for multi function, the > >> nearest I could find was PCI_HEADER_TYPE_MULTIDEVICE in > >> hotplug/ibmphp.h. A comment above the test might be helpful to explain > the test. > > > > Yes, I have not find the PCI_HEADER_TYPE_MULTIDEVICE define. OK, I > > will add The comments in next version patch. > > > >> > >> As the ls_pcie_ep_drvdata structures are static, the unset > >> .func_offset will be initialised to 0, so you could just drop the test above. > > > > OK, thanks > > > >> > >> However something to the effect of the following may help spot > >> misconfiguration: > >> > >> WARN_ON(func_no && !pcie->drvdata->func_offset); return > >> pcie->drvdata->func_offset * func_no; > > > > Thanks a lot, this looks better. > > > >> > >> The WARN is probably quite useful as if you are attempting to use > >> non-zero functions and func_offset isn't set - then things may appear > >> to work normally but actually will break horribly. > > > > got it, thanks. > > > >> > >> Thanks, > >> > >> Andrew Murray > >> > >>> +} > >>> + > >>> +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { > >>> .ep_init = ls_pcie_ep_init, > >>> .raise_irq = ls_pcie_ep_raise_irq, > >>> .get_features = ls_pcie_ep_get_features, > >>> + .func_conf_select = ls_pcie_ep_func_conf_select, }; > >>> + > >>> +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { > >>> + .ops = &ls_pcie_ep_ops, > >>> + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > >>> + > >>> +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { > >>> + .func_offset = 0x20000, > >>> + .ops = &ls_pcie_ep_ops, > >>> + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > >>> + > >>> +static const struct of_device_id ls_pcie_ep_of_match[] = { > >>> + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, > >>> + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, > >>> + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, > >>> + { }, > >>> }; > >>> > >>> static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ -98,7 > >>> +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, > >>> int ret; > >>> > >>> ep = &pci->ep; > >>> - ep->ops = &pcie_ep_ops; > >>> + ep->ops = pcie->drvdata->ops; > >>> > >>> res = platform_get_resource_byname(pdev, IORESOURCE_MEM, > >> "addr_space"); > >>> if (!res) > >>> @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct > >> platform_device *pdev) > >>> if (!ls_epc) > >>> return -ENOMEM; > >>> > >>> - dbi_base = platform_get_resource_byname(pdev, > IORESOURCE_MEM, > >> "regs"); > >>> - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > >>> - if (IS_ERR(pci->dbi_base)) > >>> - return PTR_ERR(pci->dbi_base); > >>> + pcie->drvdata = of_device_get_match_data(dev); > >>> > >>> - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > >>> pci->dev = dev; > >>> - pci->ops = &ls_pcie_ep_ops; > >>> + pci->ops = pcie->drvdata->dw_pcie_ops; > >>> + > >>> pcie->pci = pci; > >>> > >>> ls_epc->linkup_notifier = false, > >>> @@ -152,6 +185,13 @@ static int __init ls_pcie_ep_probe(struct > >>> platform_device *pdev) > >>> > >>> pcie->ls_epc = ls_epc; > >>> > >>> + dbi_base = platform_get_resource_byname(pdev, > IORESOURCE_MEM, > >> "regs"); > >>> + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > >>> + if (IS_ERR(pci->dbi_base)) > >>> + return PTR_ERR(pci->dbi_base); > >>> + > >>> + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > >>> + > >>> platform_set_drvdata(pdev, pcie); > >>> > >>> ret = ls_add_pcie_ep(pcie, pdev); > >>> -- > >>> 2.9.5 > >>> > > --- > L'absence de virus dans ce courrier électronique a été vérifiée par le logiciel > antivirus Avast. > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww. > avast.com%2Fantivirus&data=02%7C01%7Cxiaowei.bao%40nxp.com%7 > C1befe9a67c8046f9535e08d7285eaab6%7C686ea1d3bc2b4c6fa92cd99c5c3 > 01635%7C0%7C0%7C637022259387139020&sdata=JAYds7X%2FHVxgtrg > e%2F%2FvnP84zdb2yReXcctQUiSLC11I%3D&reserved=0
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月23日 22:28 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support for > ls1088a and ls2088a > > On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: > > Add PCIe EP mode support for ls1088a and ls2088a, there are some > > difference between LS1 and LS2 platform, so refactor the code of the > > EP driver. > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > --- > > v2: > > - New mechanism for layerscape EP driver. > > Was there a v1 of this patch? > > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 > > ++++++++++++++++++++------ > > 1 file changed, 58 insertions(+), 18 deletions(-) > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > index 7ca5fe8..2a66f07 100644 > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > @@ -20,27 +20,29 @@ > > > > #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ > > > > -struct ls_pcie_ep { > > - struct dw_pcie *pci; > > - struct pci_epc_features *ls_epc; > > +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > + > > +struct ls_pcie_ep_drvdata { > > + u32 func_offset; > > + const struct dw_pcie_ep_ops *ops; > > + const struct dw_pcie_ops *dw_pcie_ops; > > }; > > > > -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > +struct ls_pcie_ep { > > + struct dw_pcie *pci; > > + struct pci_epc_features *ls_epc; > > + const struct ls_pcie_ep_drvdata *drvdata; }; > > > > static int ls_pcie_establish_link(struct dw_pcie *pci) { > > return 0; > > } > > > > -static const struct dw_pcie_ops ls_pcie_ep_ops = { > > +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { > > .start_link = ls_pcie_establish_link, }; > > > > -static const struct of_device_id ls_pcie_ep_of_match[] = { > > - { .compatible = "fsl,ls-pcie-ep",}, > > - { }, > > -}; > > - > > static const struct pci_epc_features* ls_pcie_ep_get_features(struct > > dw_pcie_ep *ep) { @@ -82,10 +84,44 @@ static int > > ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > > } > > } > > > > -static const struct dw_pcie_ep_ops pcie_ep_ops = { > > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > > + u8 func_no) > > +{ > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > + u8 header_type; > > + > > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > + > > + if (header_type & (1 << 7)) > > + return pcie->drvdata->func_offset * func_no; > > + else > > + return 0; > > It looks like there isn't a PCI define for multi function, the nearest I could find > was PCI_HEADER_TYPE_MULTIDEVICE in hotplug/ibmphp.h. A comment > above the test might be helpful to explain the test. OK, I will add a comment above this code. > > As the ls_pcie_ep_drvdata structures are static, the unset .func_offset will be > initialised to 0, so you could just drop the test above. Due to the different PCIe controller have different property, e.g. PCIe controller1 support multiple function feature, but PCIe controller2 don't support this feature, so I need to check which controller support it and return the correct offset value, but each board only have one ls_pcie_ep_drvdata, ^_^. > > However something to the effect of the following may help spot > misconfiguration: > > WARN_ON(func_no && !pcie->drvdata->func_offset); return > pcie->drvdata->func_offset * func_no; > > The WARN is probably quite useful as if you are attempting to use non-zero > functions and func_offset isn't set - then things may appear to work normally > but actually will break horribly. As discussion before, I think the func_offset should not depends on the function number, even if other platforms of NXP may be use write registers way to access the different function config space. I have added the comments above the code, as follow, do you have any advice? +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, + u8 func_no) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); + u8 header_type; + + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); + + /* + * Read the Header Type register of config space to check + * whether this PCI device support the multiple function. + */ + if (header_type & (1 << 7)) + return pcie->drvdata->func_offset * func_no; + + return 0; +} Thanks a lot for your detail comments. > > Thanks, > > Andrew Murray > > > +} > > + > > +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { > > .ep_init = ls_pcie_ep_init, > > .raise_irq = ls_pcie_ep_raise_irq, > > .get_features = ls_pcie_ep_get_features, > > + .func_conf_select = ls_pcie_ep_func_conf_select, }; > > + > > +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { > > + .ops = &ls_pcie_ep_ops, > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, > > +}; > > + > > +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { > > + .func_offset = 0x20000, > > + .ops = &ls_pcie_ep_ops, > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, > > +}; > > + > > +static const struct of_device_id ls_pcie_ep_of_match[] = { > > + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, > > + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, > > + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, > > + { }, > > }; > > > > static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ -98,7 > > +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, > > int ret; > > > > ep = &pci->ep; > > - ep->ops = &pcie_ep_ops; > > + ep->ops = pcie->drvdata->ops; > > > > res = platform_get_resource_byname(pdev, IORESOURCE_MEM, > "addr_space"); > > if (!res) > > @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct > platform_device *pdev) > > if (!ls_epc) > > return -ENOMEM; > > > > - dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, > "regs"); > > - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > - if (IS_ERR(pci->dbi_base)) > > - return PTR_ERR(pci->dbi_base); > > + pcie->drvdata = of_device_get_match_data(dev); > > > > - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > pci->dev = dev; > > - pci->ops = &ls_pcie_ep_ops; > > + pci->ops = pcie->drvdata->dw_pcie_ops; > > + > > pcie->pci = pci; > > > > ls_epc->linkup_notifier = false, > > @@ -152,6 +185,13 @@ static int __init ls_pcie_ep_probe(struct > > platform_device *pdev) > > > > pcie->ls_epc = ls_epc; > > > > + dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, > "regs"); > > + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > + if (IS_ERR(pci->dbi_base)) > > + return PTR_ERR(pci->dbi_base); > > + > > + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > + > > platform_set_drvdata(pdev, pcie); > > > > ret = ls_add_pcie_ep(pcie, pdev); > > -- > > 2.9.5 > >
On Fri, Aug 23, 2019 at 11:50:20PM +0000, Xiaowei Bao wrote: > > > > -----Original Message----- > > From: Andrew Murray <andrew.murray@arm.com> > > Sent: 2019年8月23日 21:25 > > To: Xiaowei Bao <xiaowei.bao@nxp.com> > > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > > Subject: Re: [PATCH v2 01/10] PCI: designware-ep: Add multiple PFs support > > for DWC > > > > On Thu, Aug 22, 2019 at 07:22:33PM +0800, Xiaowei Bao wrote: > > > Add multiple PFs support for DWC, different PF have different config > > > space we use pf-offset property which get from the DTS to access the > > > different pF config space. > > > > It looks like you're missing a --cover-letter again. > > > > > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > > --- > > > v2: > > > - Remove duplicate redundant code. > > > - Reimplement the PF config space access way. > > > > > > drivers/pci/controller/dwc/pcie-designware-ep.c | 122 > > ++++++++++++++++-------- > > > drivers/pci/controller/dwc/pcie-designware.c | 59 ++++++++---- > > > drivers/pci/controller/dwc/pcie-designware.h | 11 ++- > > > 3 files changed, 134 insertions(+), 58 deletions(-) > > > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c > > > b/drivers/pci/controller/dwc/pcie-designware-ep.c > > > index 2bf5a35..3e2b740 100644 > > > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > > > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > > > @@ -19,12 +19,17 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) > > > pci_epc_linkup(epc); > > > } > > > > > > -static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno > > bar, > > > - int flags) > > > +static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, > > > + enum pci_barno bar, int flags) > > > { > > > u32 reg; > > > + unsigned int func_offset = 0; > > > + struct dw_pcie_ep *ep = &pci->ep; > > > > > > - reg = PCI_BASE_ADDRESS_0 + (4 * bar); > > > + if (ep->ops->func_conf_select) > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > + > > > + reg = func_offset + PCI_BASE_ADDRESS_0 + (4 * bar); > > > > This pattern of checking if func_conf_select exists and using it to get an offset > > is repeated a lot throughout this file. You could move this functionality into a > > new function (similar to dw_pcie_read_dbi etc). Or perhaps a new variant of > > dw_pcie_writel_ should be created that writes takes a func_no argument. > > Thanks for your comments, I thought about this method before, but there is a issue > about the method of access the different func config space, due to our platform use > this method that different func have different offset from dbi_base to access the > different config space, but others platform maybe use the way that write a register > to implement different func config space access, so I think reserve a callback function My point here was really to move out duplicated code to its own small function. I wasn't making any comment about (removing) the callback, just that the test and callback could be in one function. > to different platform to implement the own method, my point is that, if use register > method they can implement the code in this function and return offset is 0, if use > offset method, they can return the offset value which can be use by dw_pcie_ep driver. By the way, I haven't looked to see how many of the dw_pcie_write_xxx functions would benefit from a func_no argument - if there were many calls to dw_pcie_write_xxx that all used a reg value originated from func_conf_select then an approach similar to the implementation of dw_pcie_write_dbi could probably be justified (i.e. rather than change the value of reg) for writing to functions. > > > > > > > > dw_pcie_dbi_ro_wr_en(pci); > > > dw_pcie_writel_dbi2(pci, reg, 0x0); > > > dw_pcie_writel_dbi(pci, reg, 0x0); > > > > > > > @@ -235,7 +257,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc > > *epc, u8 func_no, > > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > > > - ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size); > > > + ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr, size); > > > if (ret) { > > > dev_err(pci->dev, "Failed to enable address\n"); > > > return ret; > > > @@ -249,11 +271,15 @@ static int dw_pcie_ep_get_msi(struct pci_epc > > *epc, u8 func_no) > > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > u32 val, reg; > > > + unsigned int func_offset = 0; > > > + > > > + if (ep->ops->func_conf_select) > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > > > if (!ep->msi_cap) > > > return -EINVAL; > > > > > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > > > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > > > > This makes me nervous. > > > > From a PCI viewpoint, each function has it's own capability structure and > > within each function there may exist a MSI capability. Yet what we're doing > > here is using dw_pcie_ep_find_capability to get the list of capabilities for > > function 0, and then applying offsets from that for subsequent functions. I.e. > > we're applying DW specific knowledge to find the correct capability, rather > > than following the general PCI approach. > > > > I think the above hunk shouldn't be required - but instead > > dw_pcie_ep_find_capability is updated to take a func_no parameter. > > > > Have I understood this correctly? > > Yes, this is a issue, I think the different func maybe have different capability, > but the dw_pcie_ep_find_capability function is called by dw_pcie_ep_init > function, we can't add func_no parameter to dw_pcie_ep_find_capability, Why not? Given that 'struct dw_pcie' represents a controller - and thus potentially multiple functions - then the _find_capability function should be able to provide the correct offset for the given function. Surely it needs to know which function number? Unless there is some reason why we can assume that all functions share the same capability offset. Also the 'struct dw_pcie_ep' which represents an endpoint controller - this has msi_cap and msix_cap fields - given this may be a multifunction device what do these fields actually refer to? Perhaps Jungoo/Gustavo can comment. > I will try to fix it use a new patch, I think move this function to ep_init callback > function If better, thanks. > > > > > > > val = dw_pcie_readw_dbi(pci, reg); > > > if (!(val & PCI_MSI_FLAGS_ENABLE)) > > > return -EINVAL; > > > @@ -268,11 +294,15 @@ static int dw_pcie_ep_set_msi(struct pci_epc > > *epc, u8 func_no, u8 interrupts) > > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > u32 val, reg; > > > + unsigned int func_offset = 0; > > > + > > > + if (ep->ops->func_conf_select) > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > > > if (!ep->msi_cap) > > > return -EINVAL; > > > > > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > > > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > > > val = dw_pcie_readw_dbi(pci, reg); > > > val &= ~PCI_MSI_FLAGS_QMASK; > > > val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; @@ -288,11 +318,15 > > > @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no) > > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > u32 val, reg; > > > + unsigned int func_offset = 0; > > > + > > > + if (ep->ops->func_conf_select) > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > > > if (!ep->msix_cap) > > > return -EINVAL; > > > > > > - reg = ep->msix_cap + PCI_MSIX_FLAGS; > > > + reg = ep->msix_cap + func_offset + PCI_MSIX_FLAGS; > > > > Same for MSIX. > > Yes. > > > > > > val = dw_pcie_readw_dbi(pci, reg); > > > if (!(val & PCI_MSIX_FLAGS_ENABLE)) > > > return -EINVAL; > > > @@ -307,11 +341,15 @@ static int dw_pcie_ep_set_msix(struct pci_epc > > *epc, u8 func_no, u16 interrupts) > > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > u32 val, reg; > > > + unsigned int func_offset = 0; > > > + > > > + if (ep->ops->func_conf_select) > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > > > if (!ep->msix_cap) > > > return -EINVAL; > > > > > > - reg = ep->msix_cap + PCI_MSIX_FLAGS; > > > + reg = ep->msix_cap + func_offset + PCI_MSIX_FLAGS; > > > val = dw_pcie_readw_dbi(pci, reg); > > > val &= ~PCI_MSIX_FLAGS_QSIZE; > > > val |= interrupts; > > > @@ -398,29 +436,33 @@ int dw_pcie_ep_raise_msi_irq(struct > > dw_pcie_ep *ep, u8 func_no, > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > struct pci_epc *epc = ep->epc; > > > unsigned int aligned_offset; > > > + unsigned int func_offset = 0; > > > u16 msg_ctrl, msg_data; > > > u32 msg_addr_lower, msg_addr_upper, reg; > > > u64 msg_addr; > > > bool has_upper; > > > int ret; > > > > > > + if (ep->ops->func_conf_select) > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > + > > > > You could probably move this hunk below the test for msi_cap to save some > > cycles. > > Sorry, I didn't understand the means, please explain it detailly, thanks a lot, ^_^ If you insert the call to func_conf_select *after* the test for !msi_cap below - then in the case where msi_cap is NULL then you will save some CPU cycles by not bothering to call func_conf_select. > > > > > if (!ep->msi_cap) > > > return -EINVAL; > > > > > > /* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */ > > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > > > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > > > msg_ctrl = dw_pcie_readw_dbi(pci, reg); > > > has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT); > > > - reg = ep->msi_cap + PCI_MSI_ADDRESS_LO; > > > + reg = ep->msi_cap + func_offset + PCI_MSI_ADDRESS_LO; > > > msg_addr_lower = dw_pcie_readl_dbi(pci, reg); > > > if (has_upper) { > > > - reg = ep->msi_cap + PCI_MSI_ADDRESS_HI; > > > + reg = ep->msi_cap + func_offset + PCI_MSI_ADDRESS_HI; > > > msg_addr_upper = dw_pcie_readl_dbi(pci, reg); > > > - reg = ep->msi_cap + PCI_MSI_DATA_64; > > > + reg = ep->msi_cap + func_offset + PCI_MSI_DATA_64; > > > msg_data = dw_pcie_readw_dbi(pci, reg); > > > } else { > > > msg_addr_upper = 0; > > > - reg = ep->msi_cap + PCI_MSI_DATA_32; > > > + reg = ep->msi_cap + func_offset + PCI_MSI_DATA_32; > > > msg_data = dw_pcie_readw_dbi(pci, reg); > > > } > > > aligned_offset = msg_addr_lower & (epc->mem->page_size - 1); > > > > > > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware.c > > > b/drivers/pci/controller/dwc/pcie-designware.c > > > index 7d25102..305e73d 100644 > > > --- a/drivers/pci/controller/dwc/pcie-designware.c > > > +++ b/drivers/pci/controller/dwc/pcie-designware.c > > > @@ -158,9 +158,10 @@ static void dw_pcie_writel_ob_unroll(struct > > dw_pcie *pci, u32 index, u32 reg, > > > dw_pcie_writel_atu(pci, offset + reg, val); } > > > > > > -static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int > > index, > > > - int type, u64 cpu_addr, > > > - u64 pci_addr, u32 size) > > > +static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, u8 > > func_no, > > > + int index, int type, > > > + u64 cpu_addr, u64 pci_addr, > > > + u32 size) > > > { > > > u32 retries, val; > > > > > > @@ -175,7 +176,7 @@ static void > > dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, > > > dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_UPPER_TARGET, > > > upper_32_bits(pci_addr)); > > > dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL1, > > > - type); > > > + type | PCIE_ATU_FUNC_NUM(func_no)); > > > > Much better :) > > Do you mean that use the expression "a? b:c" > > > > > > dw_pcie_writel_ob_unroll(pci, index, PCIE_ATU_UNR_REGION_CTRL2, > > > PCIE_ATU_ENABLE); > > > > > > @@ -194,8 +195,9 @@ static void > > dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, > > > dev_err(pci->dev, "Outbound iATU is not being enabled\n"); } > > > > > > -void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int type, > > > - u64 cpu_addr, u64 pci_addr, u32 size) > > > +static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 > > func_no, > > > + int index, int type, u64 cpu_addr, > > > + u64 pci_addr, u32 size) > > > { > > > u32 retries, val; > > > > > > @@ -203,8 +205,8 @@ void dw_pcie_prog_outbound_atu(struct dw_pcie > > *pci, int index, int type, > > > cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr); > > > > > > if (pci->iatu_unroll_enabled) { > > > - dw_pcie_prog_outbound_atu_unroll(pci, index, type, cpu_addr, > > > - pci_addr, size); > > > + dw_pcie_prog_outbound_atu_unroll(pci, func_no, index, type, > > > + cpu_addr, pci_addr, size); > > > return; > > > } > > > > > > > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware.h > > > b/drivers/pci/controller/dwc/pcie-designware.h > > > index ffed084..a0fdbf7 100644 > > > --- a/drivers/pci/controller/dwc/pcie-designware.h > > > +++ b/drivers/pci/controller/dwc/pcie-designware.h > > > @@ -71,9 +71,11 @@ > > > #define PCIE_ATU_TYPE_IO 0x2 > > > #define PCIE_ATU_TYPE_CFG0 0x4 > > > #define PCIE_ATU_TYPE_CFG1 0x5 > > > +#define PCIE_ATU_FUNC_NUM(pf) (pf << 20) > > > > "Macro argument 'pf' may be better as '(pf)' to avoid precedence issues" > > > > > #define PCIE_ATU_CR2 0x908 > > > #define PCIE_ATU_ENABLE BIT(31) > > > #define PCIE_ATU_BAR_MODE_ENABLE BIT(30) > > > +#define PCIE_ATU_FUNC_NUM_MATCH_EN BIT(19) > > > #define PCIE_ATU_LOWER_BASE 0x90C > > > #define PCIE_ATU_UPPER_BASE 0x910 > > > #define PCIE_ATU_LIMIT 0x914 > > > @@ -197,6 +199,7 @@ struct dw_pcie_ep_ops { > > > int (*raise_irq)(struct dw_pcie_ep *ep, u8 func_no, > > > enum pci_epc_irq_type type, u16 interrupt_num); > > > const struct pci_epc_features* (*get_features)(struct dw_pcie_ep > > > *ep); > > > + unsigned int (*func_conf_select)(struct dw_pcie_ep *ep, u8 func_no); > > > > Given that this function will return an offset, I'm not sure the name you have > > is suitable. Something like get_pf_offset or similar is more descriptive. > > As above explain, my initial view is that this function can return 0 or offset depends on > the platform implement mechanism, so I named it func_conf_select, I think add a > comment for this function, like this: > /* > * provide a method to implement the method of different func config space access, > * if use offset method, return the offset from dbi_base, if your register method, implement > * the code in this callback function and return 0. > */ > How about it? This means that func_conf_select can never (easily) indicate an error to the caller as this would change the offset. Where func_conf_select doesn't change the offset there probably isn't much else it can do instead (unless it was responsible for doing the write as well). So I'm not sure how well this approach works. Thanks, Andrew Murray > > > > > Thanks, > > > > Andrew Murray > > > > > }; > > > > > > struct dw_pcie_ep { > > > @@ -265,8 +268,12 @@ int dw_pcie_wait_for_link(struct dw_pcie *pci); > > > void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, > > > int type, u64 cpu_addr, u64 pci_addr, > > > u32 size); > > > -int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar, > > > - u64 cpu_addr, enum dw_pcie_as_type as_type); > > > +void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 func_no, int > > index, > > > + int type, u64 cpu_addr, u64 pci_addr, > > > + u32 size); > > > +int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, > > > + int bar, u64 cpu_addr, > > > + enum dw_pcie_as_type as_type); > > > void dw_pcie_disable_atu(struct dw_pcie *pci, int index, > > > enum dw_pcie_region_type type); > > > void dw_pcie_setup(struct dw_pcie *pci); > > > -- > > > 2.9.5 > > >
On Sat, Aug 24, 2019 at 12:08:40AM +0000, Xiaowei Bao wrote: > > > > -----Original Message----- > > From: Andrew Murray <andrew.murray@arm.com> > > Sent: 2019年8月23日 21:58 > > To: Xiaowei Bao <xiaowei.bao@nxp.com> > > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > > Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to the > > doorbell way > > > > On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote: > > > The layerscape platform use the doorbell way to trigger MSIX interrupt > > > in EP mode. > > > > > > > I have no problems with this patch, however... > > > > Are you able to add to this message a reason for why you are making this > > change? Did dw_pcie_ep_raise_msix_irq not work when func_no != 0? Or did > > it work yet dw_pcie_ep_raise_msix_irq_doorbell is more efficient? > > The fact is that, this driver is verified in ls1046a platform of NXP before, and ls1046a don't > support MSIX feature, so I set the msix_capable of pci_epc_features struct is false, > but in other platform, e.g. ls1088a, it support the MSIX feature, I verified the MSIX > feature in ls1088a, it is not OK, so I changed to another way. Thanks. Right, so the existing pci-layerscape-ep.c driver never supported MSIX yet it erroneously had a switch case statement to call dw_pcie_ep_raise_msix_irq which would never get used. Now that we're adding a platform with MSIX support the existing dw_pcie_ep_raise_msix_irq doesn't work (for this platform) so we are adding a different method. Given that dw_pcie_ep_raise_msix_irq is used by pcie-designware-plat.c we can assume this function at least works for it's use case. Please update the commit message - It would be helpful to suggest that dw_pcie_ep_raise_msix_irq was never called in the exisitng driver because msix_capable was always set to false. Thanks, Andrew Murray > > > > > Thanks, > > > > Andrew Murray > > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > > --- > > > v2: > > > - No change. > > > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 3 ++- > > > 1 file changed, 2 insertions(+), 1 deletion(-) > > > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > index 8461f62..7ca5fe8 100644 > > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > @@ -74,7 +74,8 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, > > u8 func_no, > > > case PCI_EPC_IRQ_MSI: > > > return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); > > > case PCI_EPC_IRQ_MSIX: > > > - return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); > > > + return dw_pcie_ep_raise_msix_irq_doorbell(ep, func_no, > > > + interrupt_num); > > > default: > > > dev_err(pci->dev, "UNKNOWN IRQ type\n"); > > > return -EINVAL; > > > -- > > > 2.9.5 > > >
On Mon, Aug 26, 2019 at 09:49:35AM +0000, Xiaowei Bao wrote: > > > > -----Original Message----- > > From: Andrew Murray <andrew.murray@arm.com> > > Sent: 2019年8月23日 22:28 > > To: Xiaowei Bao <xiaowei.bao@nxp.com> > > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support for > > ls1088a and ls2088a > > > > On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: > > > Add PCIe EP mode support for ls1088a and ls2088a, there are some > > > difference between LS1 and LS2 platform, so refactor the code of the > > > EP driver. > > > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > > --- > > > v2: > > > - New mechanism for layerscape EP driver. > > > > Was there a v1 of this patch? > > > > > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 > > > ++++++++++++++++++++------ > > > 1 file changed, 58 insertions(+), 18 deletions(-) > > > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > index 7ca5fe8..2a66f07 100644 > > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > @@ -20,27 +20,29 @@ > > > > > > #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ > > > > > > -struct ls_pcie_ep { > > > - struct dw_pcie *pci; > > > - struct pci_epc_features *ls_epc; > > > +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > + > > > +struct ls_pcie_ep_drvdata { > > > + u32 func_offset; > > > + const struct dw_pcie_ep_ops *ops; > > > + const struct dw_pcie_ops *dw_pcie_ops; > > > }; > > > > > > -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > +struct ls_pcie_ep { > > > + struct dw_pcie *pci; > > > + struct pci_epc_features *ls_epc; > > > + const struct ls_pcie_ep_drvdata *drvdata; }; > > > > > > static int ls_pcie_establish_link(struct dw_pcie *pci) { > > > return 0; > > > } > > > > > > -static const struct dw_pcie_ops ls_pcie_ep_ops = { > > > +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { > > > .start_link = ls_pcie_establish_link, }; > > > > > > -static const struct of_device_id ls_pcie_ep_of_match[] = { > > > - { .compatible = "fsl,ls-pcie-ep",}, > > > - { }, > > > -}; > > > - > > > static const struct pci_epc_features* ls_pcie_ep_get_features(struct > > > dw_pcie_ep *ep) { @@ -82,10 +84,44 @@ static int > > > ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > > > } > > > } > > > > > > -static const struct dw_pcie_ep_ops pcie_ep_ops = { > > > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > > > + u8 func_no) > > > +{ > > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > > + u8 header_type; > > > + > > > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > > + > > > + if (header_type & (1 << 7)) > > > + return pcie->drvdata->func_offset * func_no; > > > + else > > > + return 0; > > > > It looks like there isn't a PCI define for multi function, the nearest I could find > > was PCI_HEADER_TYPE_MULTIDEVICE in hotplug/ibmphp.h. A comment > > above the test might be helpful to explain the test. > > OK, I will add a comment above this code. > > > > > As the ls_pcie_ep_drvdata structures are static, the unset .func_offset will be > > initialised to 0, so you could just drop the test above. > > Due to the different PCIe controller have different property, e.g. PCIe controller1 support > multiple function feature, but PCIe controller2 don't support this feature, so I need to check > which controller support it and return the correct offset value, but each board only have one > ls_pcie_ep_drvdata, ^_^. Yes but if they don't support the feature then func_offset will be 0. > > > > > However something to the effect of the following may help spot > > misconfiguration: > > > > WARN_ON(func_no && !pcie->drvdata->func_offset); return > > pcie->drvdata->func_offset * func_no; > > > > The WARN is probably quite useful as if you are attempting to use non-zero > > functions and func_offset isn't set - then things may appear to work normally > > but actually will break horribly. > > As discussion before, I think the func_offset should not depends on the function > number, even if other platforms of NXP may be use write registers way to access > the different function config space. I agree that func_offset is an optional parameter. But if you are attempting to determine the offset of a function and you are given a non-zero function number - then something has gone wrong if func_offset is 0. Thanks, Andrew Murray > > I have added the comments above the code, as follow, do you have any advice? > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > + u8 func_no) > +{ > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > + u8 header_type; > + > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > + > + /* > + * Read the Header Type register of config space to check > + * whether this PCI device support the multiple function. > + */ > + if (header_type & (1 << 7)) > + return pcie->drvdata->func_offset * func_no; > + > + return 0; > +} > > Thanks a lot for your detail comments. > > > > > Thanks, > > > > Andrew Murray > > > > > +} > > > + > > > +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { > > > .ep_init = ls_pcie_ep_init, > > > .raise_irq = ls_pcie_ep_raise_irq, > > > .get_features = ls_pcie_ep_get_features, > > > + .func_conf_select = ls_pcie_ep_func_conf_select, }; > > > + > > > +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { > > > + .ops = &ls_pcie_ep_ops, > > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, > > > +}; > > > + > > > +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { > > > + .func_offset = 0x20000, > > > + .ops = &ls_pcie_ep_ops, > > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, > > > +}; > > > + > > > +static const struct of_device_id ls_pcie_ep_of_match[] = { > > > + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, > > > + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, > > > + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, > > > + { }, > > > }; > > > > > > static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ -98,7 > > > +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, > > > int ret; > > > > > > ep = &pci->ep; > > > - ep->ops = &pcie_ep_ops; > > > + ep->ops = pcie->drvdata->ops; > > > > > > res = platform_get_resource_byname(pdev, IORESOURCE_MEM, > > "addr_space"); > > > if (!res) > > > @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct > > platform_device *pdev) > > > if (!ls_epc) > > > return -ENOMEM; > > > > > > - dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, > > "regs"); > > > - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > > - if (IS_ERR(pci->dbi_base)) > > > - return PTR_ERR(pci->dbi_base); > > > + pcie->drvdata = of_device_get_match_data(dev); > > > > > > - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > > pci->dev = dev; > > > - pci->ops = &ls_pcie_ep_ops; > > > + pci->ops = pcie->drvdata->dw_pcie_ops; > > > + > > > pcie->pci = pci; > > > > > > ls_epc->linkup_notifier = false, > > > @@ -152,6 +185,13 @@ static int __init ls_pcie_ep_probe(struct > > > platform_device *pdev) > > > > > > pcie->ls_epc = ls_epc; > > > > > > + dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, > > "regs"); > > > + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > > + if (IS_ERR(pci->dbi_base)) > > > + return PTR_ERR(pci->dbi_base); > > > + > > > + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > > + > > > platform_set_drvdata(pdev, pcie); > > > > > > ret = ls_add_pcie_ep(pcie, pdev); > > > -- > > > 2.9.5 > > >
On Sun, Aug 25, 2019 at 03:07:32AM +0000, Xiaowei Bao wrote: > > > > -----Original Message----- > > From: christophe leroy <christophe.leroy@c-s.fr> > > Sent: 2019年8月24日 14:45 > > To: Xiaowei Bao <xiaowei.bao@nxp.com>; Andrew Murray > > <andrew.murray@arm.com> > > Cc: mark.rutland@arm.com; Roy Zang <roy.zang@nxp.com>; > > lorenzo.pieralisi@arm.co; arnd@arndb.de; devicetree@vger.kernel.org; > > gregkh@linuxfoundation.org; linuxppc-dev@lists.ozlabs.org; > > linux-pci@vger.kernel.org; linux-kernel@vger.kernel.org; kishon@ti.com; M.h. > > Lian <minghuan.lian@nxp.com>; robh+dt@kernel.org; > > gustavo.pimentel@synopsys.com; jingoohan1@gmail.com; > > bhelgaas@google.com; Leo Li <leoyang.li@nxp.com>; shawnguo@kernel.org; > > Mingkai Hu <mingkai.hu@nxp.com>; linux-arm-kernel@lists.infradead.org > > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support for > > ls1088a and ls2088a > > > > > > > > Le 24/08/2019 à 02:18, Xiaowei Bao a écrit : > > > > > > > > >> -----Original Message----- > > >> From: Andrew Murray <andrew.murray@arm.com> > > >> Sent: 2019年8月23日 22:28 > > >> To: Xiaowei Bao <xiaowei.bao@nxp.com> > > >> Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > > >> shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > >> lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; > > M.h. > > >> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > > >> Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > >> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > >> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > >> linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > > >> Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support > > >> for ls1088a and ls2088a > > >> > > >> On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: > > >>> Add PCIe EP mode support for ls1088a and ls2088a, there are some > > >>> difference between LS1 and LS2 platform, so refactor the code of the > > >>> EP driver. > > >>> > > >>> Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > >>> --- > > >>> v2: > > >>> - New mechanism for layerscape EP driver. > > >> > > >> Was there a v1 of this patch? > > > > > > Yes, but I don't know how to comments, ^_^ > > > > As far as I can see, in the previous version of the series > > (https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpatch > > work.ozlabs.org%2Fproject%2Flinuxppc-dev%2Flist%2F%3Fseries%3D125315 > > %26state%3D*&data=02%7C01%7Cxiaowei.bao%40nxp.com%7C1befe9 > > a67c8046f9535e08d7285eaab6%7C686ea1d3bc2b4c6fa92cd99c5c301635% > > 7C0%7C0%7C637022259387139020&sdata=p4wbycd04Z7qRUfAoZtwc > > UP7pR%2FuA3%2FjVcWMz6YyQVQ%3D&reserved=0), > > the 8/10 was something completely different, and I can't find any other patch > > in the series that could have been the v1 of this patch. > > Thanks, I will correct it to v1 in next version patch. I think you numbered it correctly (so please leave it as v2, referring to the patch series revision) - I got confused trying to find a previous version of this patch. Perhaps in the future when new patches are introduced in a series you can indicate that in the description patch revision history (e.g. introduced in v2). Thanks, Andrew Murray > > > > > Christophe > > > > > > > >> > > >>> > > >>> drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 > > >>> ++++++++++++++++++++------ > > >>> 1 file changed, 58 insertions(+), 18 deletions(-) > > >>> > > >>> diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > >>> b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > >>> index 7ca5fe8..2a66f07 100644 > > >>> --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > >>> +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > >>> @@ -20,27 +20,29 @@ > > >>> > > >>> #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ > > >>> > > >>> -struct ls_pcie_ep { > > >>> - struct dw_pcie *pci; > > >>> - struct pci_epc_features *ls_epc; > > >>> +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > >>> + > > >>> +struct ls_pcie_ep_drvdata { > > >>> + u32 func_offset; > > >>> + const struct dw_pcie_ep_ops *ops; > > >>> + const struct dw_pcie_ops *dw_pcie_ops; > > >>> }; > > >>> > > >>> -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > >>> +struct ls_pcie_ep { > > >>> + struct dw_pcie *pci; > > >>> + struct pci_epc_features *ls_epc; > > >>> + const struct ls_pcie_ep_drvdata *drvdata; }; > > >>> > > >>> static int ls_pcie_establish_link(struct dw_pcie *pci) { > > >>> return 0; > > >>> } > > >>> > > >>> -static const struct dw_pcie_ops ls_pcie_ep_ops = { > > >>> +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { > > >>> .start_link = ls_pcie_establish_link, }; > > >>> > > >>> -static const struct of_device_id ls_pcie_ep_of_match[] = { > > >>> - { .compatible = "fsl,ls-pcie-ep",}, > > >>> - { }, > > >>> -}; > > >>> - > > >>> static const struct pci_epc_features* > > >>> ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { @@ -82,10 +84,44 > > >>> @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > > >>> } > > >>> } > > >>> > > >>> -static const struct dw_pcie_ep_ops pcie_ep_ops = { > > >>> +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > > >>> + u8 func_no) > > >>> +{ > > >>> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > >>> + u8 header_type; > > >>> + > > >>> + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > >>> + > > >>> + if (header_type & (1 << 7)) > > >>> + return pcie->drvdata->func_offset * func_no; > > >>> + else > > >>> + return 0; > > >> > > >> It looks like there isn't a PCI define for multi function, the > > >> nearest I could find was PCI_HEADER_TYPE_MULTIDEVICE in > > >> hotplug/ibmphp.h. A comment above the test might be helpful to explain > > the test. > > > > > > Yes, I have not find the PCI_HEADER_TYPE_MULTIDEVICE define. OK, I > > > will add The comments in next version patch. > > > > > >> > > >> As the ls_pcie_ep_drvdata structures are static, the unset > > >> .func_offset will be initialised to 0, so you could just drop the test above. > > > > > > OK, thanks > > > > > >> > > >> However something to the effect of the following may help spot > > >> misconfiguration: > > >> > > >> WARN_ON(func_no && !pcie->drvdata->func_offset); return > > >> pcie->drvdata->func_offset * func_no; > > > > > > Thanks a lot, this looks better. > > > > > >> > > >> The WARN is probably quite useful as if you are attempting to use > > >> non-zero functions and func_offset isn't set - then things may appear > > >> to work normally but actually will break horribly. > > > > > > got it, thanks. > > > > > >> > > >> Thanks, > > >> > > >> Andrew Murray > > >> > > >>> +} > > >>> + > > >>> +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { > > >>> .ep_init = ls_pcie_ep_init, > > >>> .raise_irq = ls_pcie_ep_raise_irq, > > >>> .get_features = ls_pcie_ep_get_features, > > >>> + .func_conf_select = ls_pcie_ep_func_conf_select, }; > > >>> + > > >>> +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { > > >>> + .ops = &ls_pcie_ep_ops, > > >>> + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > > >>> + > > >>> +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { > > >>> + .func_offset = 0x20000, > > >>> + .ops = &ls_pcie_ep_ops, > > >>> + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > > >>> + > > >>> +static const struct of_device_id ls_pcie_ep_of_match[] = { > > >>> + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, > > >>> + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, > > >>> + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, > > >>> + { }, > > >>> }; > > >>> > > >>> static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ -98,7 > > >>> +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, > > >>> int ret; > > >>> > > >>> ep = &pci->ep; > > >>> - ep->ops = &pcie_ep_ops; > > >>> + ep->ops = pcie->drvdata->ops; > > >>> > > >>> res = platform_get_resource_byname(pdev, IORESOURCE_MEM, > > >> "addr_space"); > > >>> if (!res) > > >>> @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct > > >> platform_device *pdev) > > >>> if (!ls_epc) > > >>> return -ENOMEM; > > >>> > > >>> - dbi_base = platform_get_resource_byname(pdev, > > IORESOURCE_MEM, > > >> "regs"); > > >>> - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > >>> - if (IS_ERR(pci->dbi_base)) > > >>> - return PTR_ERR(pci->dbi_base); > > >>> + pcie->drvdata = of_device_get_match_data(dev); > > >>> > > >>> - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > >>> pci->dev = dev; > > >>> - pci->ops = &ls_pcie_ep_ops; > > >>> + pci->ops = pcie->drvdata->dw_pcie_ops; > > >>> + > > >>> pcie->pci = pci; > > >>> > > >>> ls_epc->linkup_notifier = false, > > >>> @@ -152,6 +185,13 @@ static int __init ls_pcie_ep_probe(struct > > >>> platform_device *pdev) > > >>> > > >>> pcie->ls_epc = ls_epc; > > >>> > > >>> + dbi_base = platform_get_resource_byname(pdev, > > IORESOURCE_MEM, > > >> "regs"); > > >>> + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > >>> + if (IS_ERR(pci->dbi_base)) > > >>> + return PTR_ERR(pci->dbi_base); > > >>> + > > >>> + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > >>> + > > >>> platform_set_drvdata(pdev, pcie); > > >>> > > >>> ret = ls_add_pcie_ep(pcie, pdev); > > >>> -- > > >>> 2.9.5 > > >>> > > > > --- > > L'absence de virus dans ce courrier électronique a été vérifiée par le logiciel > > antivirus Avast. > > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%2Fwww. > > avast.com%2Fantivirus&data=02%7C01%7Cxiaowei.bao%40nxp.com%7 > > C1befe9a67c8046f9535e08d7285eaab6%7C686ea1d3bc2b4c6fa92cd99c5c3 > > 01635%7C0%7C0%7C637022259387139020&sdata=JAYds7X%2FHVxgtrg > > e%2F%2FvnP84zdb2yReXcctQUiSLC11I%3D&reserved=0 >
On Thu, Aug 22, 2019 at 07:22:36PM +0800, Xiaowei Bao wrote: > Add compatible strings for ls1088a and ls2088a. > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > --- > v2: > - No change. > > Documentation/devicetree/bindings/pci/layerscape-pci.txt | 5 ++++- > 1 file changed, 4 insertions(+), 1 deletion(-) > > diff --git a/Documentation/devicetree/bindings/pci/layerscape-pci.txt b/Documentation/devicetree/bindings/pci/layerscape-pci.txt > index e20ceaa..16f592e 100644 > --- a/Documentation/devicetree/bindings/pci/layerscape-pci.txt > +++ b/Documentation/devicetree/bindings/pci/layerscape-pci.txt > @@ -22,7 +22,10 @@ Required properties: > "fsl,ls1043a-pcie" > "fsl,ls1012a-pcie" > EP mode: > - "fsl,ls1046a-pcie-ep", "fsl,ls-pcie-ep" > + "fsl,ls-pcie-ep" Wasn't this a fallback? Each line should be one valid combination of compatible strings. > + "fsl,ls1046a-pcie-ep" > + "fsl,ls1088a-pcie-ep" > + "fsl,ls2088a-pcie-ep" > - reg: base addresses and lengths of the PCIe controller register blocks. > - interrupts: A list of interrupt outputs of the controller. Must contain an > entry for each entry in the interrupt-names property. > -- > 2.9.5 >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月27日 21:25 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to the > doorbell way > > On Sat, Aug 24, 2019 at 12:08:40AM +0000, Xiaowei Bao wrote: > > > > > > > -----Original Message----- > > > From: Andrew Murray <andrew.murray@arm.com> > > > Sent: 2019年8月23日 21:58 > > > To: Xiaowei Bao <xiaowei.bao@nxp.com> > > > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > > > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; > M.h. > > > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > > > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > > > Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to > > > the doorbell way > > > > > > On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote: > > > > The layerscape platform use the doorbell way to trigger MSIX > > > > interrupt in EP mode. > > > > > > > > > > I have no problems with this patch, however... > > > > > > Are you able to add to this message a reason for why you are making > > > this change? Did dw_pcie_ep_raise_msix_irq not work when func_no != > > > 0? Or did it work yet dw_pcie_ep_raise_msix_irq_doorbell is more > efficient? > > > > The fact is that, this driver is verified in ls1046a platform of NXP > > before, and ls1046a don't support MSIX feature, so I set the > > msix_capable of pci_epc_features struct is false, but in other > > platform, e.g. ls1088a, it support the MSIX feature, I verified the MSIX > feature in ls1088a, it is not OK, so I changed to another way. Thanks. > > Right, so the existing pci-layerscape-ep.c driver never supported MSIX yet it > erroneously had a switch case statement to call dw_pcie_ep_raise_msix_irq > which would never get used. > > Now that we're adding a platform with MSIX support the existing > dw_pcie_ep_raise_msix_irq doesn't work (for this platform) so we are adding > a different method. > > Given that dw_pcie_ep_raise_msix_irq is used by pcie-designware-plat.c we > can assume this function at least works for it's use case. > > Please update the commit message - It would be helpful to suggest that > dw_pcie_ep_raise_msix_irq was never called in the exisitng driver because > msix_capable was always set to false. Agree, this is much clearer, I will modify the commit message in the next version patch, thanks a lot. > > Thanks, > > Andrew Murray > > > > > > > > > Thanks, > > > > > > Andrew Murray > > > > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > > > --- > > > > v2: > > > > - No change. > > > > > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 3 ++- > > > > 1 file changed, 2 insertions(+), 1 deletion(-) > > > > > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > index 8461f62..7ca5fe8 100644 > > > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > @@ -74,7 +74,8 @@ static int ls_pcie_ep_raise_irq(struct > > > > dw_pcie_ep *ep, > > > u8 func_no, > > > > case PCI_EPC_IRQ_MSI: > > > > return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); > > > > case PCI_EPC_IRQ_MSIX: > > > > - return dw_pcie_ep_raise_msix_irq(ep, func_no, > interrupt_num); > > > > + return dw_pcie_ep_raise_msix_irq_doorbell(ep, func_no, > > > > + interrupt_num); > > > > default: > > > > dev_err(pci->dev, "UNKNOWN IRQ type\n"); > > > > return -EINVAL; > > > > -- > > > > 2.9.5 > > > >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月27日 22:49 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: christophe leroy <christophe.leroy@c-s.fr>; mark.rutland@arm.com; Roy > Zang <roy.zang@nxp.com>; lorenzo.pieralisi@arm.co; arnd@arndb.de; > devicetree@vger.kernel.org; gregkh@linuxfoundation.org; > linuxppc-dev@lists.ozlabs.org; linux-pci@vger.kernel.org; > linux-kernel@vger.kernel.org; kishon@ti.com; M.h. Lian > <minghuan.lian@nxp.com>; robh+dt@kernel.org; > gustavo.pimentel@synopsys.com; jingoohan1@gmail.com; > bhelgaas@google.com; Leo Li <leoyang.li@nxp.com>; shawnguo@kernel.org; > Mingkai Hu <mingkai.hu@nxp.com>; linux-arm-kernel@lists.infradead.org > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support for > ls1088a and ls2088a > > On Sun, Aug 25, 2019 at 03:07:32AM +0000, Xiaowei Bao wrote: > > > > > > > -----Original Message----- > > > From: christophe leroy <christophe.leroy@c-s.fr> > > > Sent: 2019年8月24日 14:45 > > > To: Xiaowei Bao <xiaowei.bao@nxp.com>; Andrew Murray > > > <andrew.murray@arm.com> > > > Cc: mark.rutland@arm.com; Roy Zang <roy.zang@nxp.com>; > > > lorenzo.pieralisi@arm.co; arnd@arndb.de; devicetree@vger.kernel.org; > > > gregkh@linuxfoundation.org; linuxppc-dev@lists.ozlabs.org; > > > linux-pci@vger.kernel.org; linux-kernel@vger.kernel.org; kishon@ti.com; > M.h. > > > Lian <minghuan.lian@nxp.com>; robh+dt@kernel.org; > > > gustavo.pimentel@synopsys.com; jingoohan1@gmail.com; > > > bhelgaas@google.com; Leo Li <leoyang.li@nxp.com>; > > > shawnguo@kernel.org; Mingkai Hu <mingkai.hu@nxp.com>; > > > linux-arm-kernel@lists.infradead.org > > > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support > > > for ls1088a and ls2088a > > > > > > > > > > > > Le 24/08/2019 à 02:18, Xiaowei Bao a écrit : > > > > > > > > > > > >> -----Original Message----- > > > >> From: Andrew Murray <andrew.murray@arm.com> > > > >> Sent: 2019年8月23日 22:28 > > > >> To: Xiaowei Bao <xiaowei.bao@nxp.com> > > > >> Cc: bhelgaas@google.com; robh+dt@kernel.org; > > > >> mark.rutland@arm.com; shawnguo@kernel.org; Leo Li > > > >> <leoyang.li@nxp.com>; kishon@ti.com; lorenzo.pieralisi@arm.co; > > > >> arnd@arndb.de; gregkh@linuxfoundation.org; > > > M.h. > > > >> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; > > > >> Roy Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > > >> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > > >> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > > >> linux-arm-kernel@lists.infradead.org; > > > >> linuxppc-dev@lists.ozlabs.org > > > >> Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode > > > >> support for ls1088a and ls2088a > > > >> > > > >> On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: > > > >>> Add PCIe EP mode support for ls1088a and ls2088a, there are some > > > >>> difference between LS1 and LS2 platform, so refactor the code of > > > >>> the EP driver. > > > >>> > > > >>> Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > > >>> --- > > > >>> v2: > > > >>> - New mechanism for layerscape EP driver. > > > >> > > > >> Was there a v1 of this patch? > > > > > > > > Yes, but I don't know how to comments, ^_^ > > > > > > As far as I can see, in the previous version of the series > > > (https://patch > > > > work.ozlabs.org%2Fproject%2Flinuxppc-dev%2Flist%2F%3Fseries%3D125315 > > > %26state%3D*&data=02%7C01%7Cxiaowei.bao%40nxp.com%7C1b > efe9 > > > > a67c8046f9535e08d7285eaab6%7C686ea1d3bc2b4c6fa92cd99c5c301635% > > > > 7C0%7C0%7C637022259387139020&sdata=p4wbycd04Z7qRUfAoZtwc > > > UP7pR%2FuA3%2FjVcWMz6YyQVQ%3D&reserved=0), > > > the 8/10 was something completely different, and I can't find any > > > other patch in the series that could have been the v1 of this patch. > > > > Thanks, I will correct it to v1 in next version patch. > > I think you numbered it correctly (so please leave it as v2, referring to the > patch series revision) - I got confused trying to find a previous version of this > patch. > > Perhaps in the future when new patches are introduced in a series you can > indicate that in the description patch revision history (e.g. introduced in v2). OK, thanks for your help, I will update it in the next version patch. Thanks Xiaowei > > Thanks, > > Andrew Murray > > > > > > > > > Christophe > > > > > > > > > > >> > > > >>> > > > >>> drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 > > > >>> ++++++++++++++++++++------ > > > >>> 1 file changed, 58 insertions(+), 18 deletions(-) > > > >>> > > > >>> diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > >>> b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > >>> index 7ca5fe8..2a66f07 100644 > > > >>> --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > >>> +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > >>> @@ -20,27 +20,29 @@ > > > >>> > > > >>> #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ > > > >>> > > > >>> -struct ls_pcie_ep { > > > >>> - struct dw_pcie *pci; > > > >>> - struct pci_epc_features *ls_epc; > > > >>> +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > >>> + > > > >>> +struct ls_pcie_ep_drvdata { > > > >>> + u32 func_offset; > > > >>> + const struct dw_pcie_ep_ops *ops; > > > >>> + const struct dw_pcie_ops *dw_pcie_ops; > > > >>> }; > > > >>> > > > >>> -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > >>> +struct ls_pcie_ep { > > > >>> + struct dw_pcie *pci; > > > >>> + struct pci_epc_features *ls_epc; > > > >>> + const struct ls_pcie_ep_drvdata *drvdata; }; > > > >>> > > > >>> static int ls_pcie_establish_link(struct dw_pcie *pci) { > > > >>> return 0; > > > >>> } > > > >>> > > > >>> -static const struct dw_pcie_ops ls_pcie_ep_ops = { > > > >>> +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { > > > >>> .start_link = ls_pcie_establish_link, }; > > > >>> > > > >>> -static const struct of_device_id ls_pcie_ep_of_match[] = { > > > >>> - { .compatible = "fsl,ls-pcie-ep",}, > > > >>> - { }, > > > >>> -}; > > > >>> - > > > >>> static const struct pci_epc_features* > > > >>> ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { @@ -82,10 > > > >>> +84,44 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 > func_no, > > > >>> } > > > >>> } > > > >>> > > > >>> -static const struct dw_pcie_ep_ops pcie_ep_ops = { > > > >>> +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep > *ep, > > > >>> + u8 func_no) > > > >>> +{ > > > >>> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > > >>> + u8 header_type; > > > >>> + > > > >>> + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > > >>> + > > > >>> + if (header_type & (1 << 7)) > > > >>> + return pcie->drvdata->func_offset * func_no; > > > >>> + else > > > >>> + return 0; > > > >> > > > >> It looks like there isn't a PCI define for multi function, the > > > >> nearest I could find was PCI_HEADER_TYPE_MULTIDEVICE in > > > >> hotplug/ibmphp.h. A comment above the test might be helpful to > > > >> explain > > > the test. > > > > > > > > Yes, I have not find the PCI_HEADER_TYPE_MULTIDEVICE define. OK, I > > > > will add The comments in next version patch. > > > > > > > >> > > > >> As the ls_pcie_ep_drvdata structures are static, the unset > > > >> .func_offset will be initialised to 0, so you could just drop the test > above. > > > > > > > > OK, thanks > > > > > > > >> > > > >> However something to the effect of the following may help spot > > > >> misconfiguration: > > > >> > > > >> WARN_ON(func_no && !pcie->drvdata->func_offset); return > > > >> pcie->drvdata->func_offset * func_no; > > > > > > > > Thanks a lot, this looks better. > > > > > > > >> > > > >> The WARN is probably quite useful as if you are attempting to use > > > >> non-zero functions and func_offset isn't set - then things may > > > >> appear to work normally but actually will break horribly. > > > > > > > > got it, thanks. > > > > > > > >> > > > >> Thanks, > > > >> > > > >> Andrew Murray > > > >> > > > >>> +} > > > >>> + > > > >>> +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { > > > >>> .ep_init = ls_pcie_ep_init, > > > >>> .raise_irq = ls_pcie_ep_raise_irq, > > > >>> .get_features = ls_pcie_ep_get_features, > > > >>> + .func_conf_select = ls_pcie_ep_func_conf_select, }; > > > >>> + > > > >>> +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { > > > >>> + .ops = &ls_pcie_ep_ops, > > > >>> + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > > > >>> + > > > >>> +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { > > > >>> + .func_offset = 0x20000, > > > >>> + .ops = &ls_pcie_ep_ops, > > > >>> + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > > > >>> + > > > >>> +static const struct of_device_id ls_pcie_ep_of_match[] = { > > > >>> + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, > > > >>> + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, > > > >>> + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, > > > >>> + { }, > > > >>> }; > > > >>> > > > >>> static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ > > > >>> -98,7 > > > >>> +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep > > > >>> +*pcie, > > > >>> int ret; > > > >>> > > > >>> ep = &pci->ep; > > > >>> - ep->ops = &pcie_ep_ops; > > > >>> + ep->ops = pcie->drvdata->ops; > > > >>> > > > >>> res = platform_get_resource_byname(pdev, > IORESOURCE_MEM, > > > >> "addr_space"); > > > >>> if (!res) > > > >>> @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct > > > >> platform_device *pdev) > > > >>> if (!ls_epc) > > > >>> return -ENOMEM; > > > >>> > > > >>> - dbi_base = platform_get_resource_byname(pdev, > > > IORESOURCE_MEM, > > > >> "regs"); > > > >>> - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > > >>> - if (IS_ERR(pci->dbi_base)) > > > >>> - return PTR_ERR(pci->dbi_base); > > > >>> + pcie->drvdata = of_device_get_match_data(dev); > > > >>> > > > >>> - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > > >>> pci->dev = dev; > > > >>> - pci->ops = &ls_pcie_ep_ops; > > > >>> + pci->ops = pcie->drvdata->dw_pcie_ops; > > > >>> + > > > >>> pcie->pci = pci; > > > >>> > > > >>> ls_epc->linkup_notifier = false, @@ -152,6 +185,13 @@ static > > > >>> int __init ls_pcie_ep_probe(struct platform_device *pdev) > > > >>> > > > >>> pcie->ls_epc = ls_epc; > > > >>> > > > >>> + dbi_base = platform_get_resource_byname(pdev, > > > IORESOURCE_MEM, > > > >> "regs"); > > > >>> + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > > >>> + if (IS_ERR(pci->dbi_base)) > > > >>> + return PTR_ERR(pci->dbi_base); > > > >>> + > > > >>> + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > > >>> + > > > >>> platform_set_drvdata(pdev, pcie); > > > >>> > > > >>> ret = ls_add_pcie_ep(pcie, pdev); > > > >>> -- > > > >>> 2.9.5 > > > >>> > > > > > > --- > > > L'absence de virus dans ce courrier électronique a été vérifiée par > > > le logiciel antivirus Avast. > > > https://www. > > > > avast.com%2Fantivirus&data=02%7C01%7Cxiaowei.bao%40nxp.com%7 > > > > C1befe9a67c8046f9535e08d7285eaab6%7C686ea1d3bc2b4c6fa92cd99c5c3 > > > > 01635%7C0%7C0%7C637022259387139020&sdata=JAYds7X%2FHVxgtrg > > > e%2F%2FvnP84zdb2yReXcctQUiSLC11I%3D&reserved=0 > >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月27日 21:34 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support for > ls1088a and ls2088a > > On Mon, Aug 26, 2019 at 09:49:35AM +0000, Xiaowei Bao wrote: > > > > > > > -----Original Message----- > > > From: Andrew Murray <andrew.murray@arm.com> > > > Sent: 2019年8月23日 22:28 > > > To: Xiaowei Bao <xiaowei.bao@nxp.com> > > > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > > > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; > M.h. > > > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > > > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > > > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support > > > for ls1088a and ls2088a > > > > > > On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: > > > > Add PCIe EP mode support for ls1088a and ls2088a, there are some > > > > difference between LS1 and LS2 platform, so refactor the code of > > > > the EP driver. > > > > > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > > > --- > > > > v2: > > > > - New mechanism for layerscape EP driver. > > > > > > Was there a v1 of this patch? > > > > > > > > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 > > > > ++++++++++++++++++++------ > > > > 1 file changed, 58 insertions(+), 18 deletions(-) > > > > > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > index 7ca5fe8..2a66f07 100644 > > > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > @@ -20,27 +20,29 @@ > > > > > > > > #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ > > > > > > > > -struct ls_pcie_ep { > > > > - struct dw_pcie *pci; > > > > - struct pci_epc_features *ls_epc; > > > > +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > > + > > > > +struct ls_pcie_ep_drvdata { > > > > + u32 func_offset; > > > > + const struct dw_pcie_ep_ops *ops; > > > > + const struct dw_pcie_ops *dw_pcie_ops; > > > > }; > > > > > > > > -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > > +struct ls_pcie_ep { > > > > + struct dw_pcie *pci; > > > > + struct pci_epc_features *ls_epc; > > > > + const struct ls_pcie_ep_drvdata *drvdata; }; > > > > > > > > static int ls_pcie_establish_link(struct dw_pcie *pci) { > > > > return 0; > > > > } > > > > > > > > -static const struct dw_pcie_ops ls_pcie_ep_ops = { > > > > +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { > > > > .start_link = ls_pcie_establish_link, }; > > > > > > > > -static const struct of_device_id ls_pcie_ep_of_match[] = { > > > > - { .compatible = "fsl,ls-pcie-ep",}, > > > > - { }, > > > > -}; > > > > - > > > > static const struct pci_epc_features* > > > > ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { @@ -82,10 +84,44 > > > > @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > > > > } > > > > } > > > > > > > > -static const struct dw_pcie_ep_ops pcie_ep_ops = { > > > > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep > *ep, > > > > + u8 func_no) > > > > +{ > > > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > > > + u8 header_type; > > > > + > > > > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > > > + > > > > + if (header_type & (1 << 7)) > > > > + return pcie->drvdata->func_offset * func_no; > > > > + else > > > > + return 0; > > > > > > It looks like there isn't a PCI define for multi function, the > > > nearest I could find was PCI_HEADER_TYPE_MULTIDEVICE in > > > hotplug/ibmphp.h. A comment above the test might be helpful to explain > the test. > > > > OK, I will add a comment above this code. > > > > > > > > As the ls_pcie_ep_drvdata structures are static, the unset > > > .func_offset will be initialised to 0, so you could just drop the test above. > > > > Due to the different PCIe controller have different property, e.g. > > PCIe controller1 support multiple function feature, but PCIe > > controller2 don't support this feature, so I need to check which > > controller support it and return the correct offset value, but each board only > have one ls_pcie_ep_drvdata, ^_^. > > Yes but if they don't support the feature then func_offset will be 0. > > > > > > > > > However something to the effect of the following may help spot > > > misconfiguration: > > > > > > WARN_ON(func_no && !pcie->drvdata->func_offset); return > > > pcie->drvdata->func_offset * func_no; > > > > > > The WARN is probably quite useful as if you are attempting to use > > > non-zero functions and func_offset isn't set - then things may > > > appear to work normally but actually will break horribly. > > > > As discussion before, I think the func_offset should not depends on > > the function number, even if other platforms of NXP may be use write > > registers way to access the different function config space. > > I agree that func_offset is an optional parameter. But if you are attempting to > determine the offset of a function and you are given a non-zero function > number - then something has gone wrong if func_offset is 0. I have understood you means, maybe I need to set a flag in the driver_data struct, because I may add other platform of NXP, these platform use the write register method to access different function, e.g. write func_num to register, then we can access this func_num config space. I will modify the code like this? Do you have better advice? Case1: diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c index 004a7e8..8a0d6df 100644 --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -23,6 +23,7 @@ #define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) struct ls_pcie_ep_drvdata { + u8 func_config_flag; u32 func_offset; const struct dw_pcie_ep_ops *ops; const struct dw_pcie_ops *dw_pcie_ops; @@ -97,8 +98,14 @@ static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, * Read the Header Type register of config space to check * whether this PCI device support the multiple function. */ - if (header_type & (1 << 7)) - return pcie->drvdata->func_offset * func_no; + if (header_type & (1 << 7)) { + if (pcie->drvdata->func_config_flag) { + iowrite32((func_num << n), pci->dbi_base + PCI_XXXX_XXX); + } else { + WARN_ON(func_no && !pcie->drvdata->func_offset); + return pcie->drvdata->func_offset * func_no; + } + } return 0; } Of course, I don't need to set the flag this time, because I don't use the second method(write register method), so the code like this: case2: +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, u8 func_no) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); u8 header_type; of course, this code is not requied, due to the pcie->drvdata->func_offset is 0, but I think this is more clear if use this code. header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); /* * Read the Header Type register of config space to check * whether this PCI device support the multiple function. */ if (header_type & (1 << 7)) { WARN_ON(func_no && !pcie->drvdata->func_offset); return pcie->drvdata->func_offset * func_no; } return 0; } Or like this: Case3: +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, u8 func_no) { struct dw_pcie *pci = to_dw_pcie_from_ep(ep); struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); WARN_ON(func_no && !pcie->drvdata->func_offset); return pcie->drvdata->func_offset * func_no; } Of course, we can return a -1 by adjuring the (func_no && !pcie->drvdata->func_offset) Valu in case1 Thanks Xiaowei > > Thanks, > > Andrew Murray > > > > > I have added the comments above the code, as follow, do you have any > advice? > > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > > + u8 func_no) { > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > + u8 header_type; > > + > > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > + > > + /* > > + * Read the Header Type register of config space to check > > + * whether this PCI device support the multiple function. > > + */ > > + if (header_type & (1 << 7)) > > + return pcie->drvdata->func_offset * func_no; > > + > > + return 0; > > +} > > > > Thanks a lot for your detail comments. > > > > > > > > Thanks, > > > > > > Andrew Murray > > > > > > > +} > > > > + > > > > +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { > > > > .ep_init = ls_pcie_ep_init, > > > > .raise_irq = ls_pcie_ep_raise_irq, > > > > .get_features = ls_pcie_ep_get_features, > > > > + .func_conf_select = ls_pcie_ep_func_conf_select, }; > > > > + > > > > +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { > > > > + .ops = &ls_pcie_ep_ops, > > > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > > > > + > > > > +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { > > > > + .func_offset = 0x20000, > > > > + .ops = &ls_pcie_ep_ops, > > > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > > > > + > > > > +static const struct of_device_id ls_pcie_ep_of_match[] = { > > > > + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, > > > > + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, > > > > + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, > > > > + { }, > > > > }; > > > > > > > > static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ > > > > -98,7 > > > > +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep > > > > +*pcie, > > > > int ret; > > > > > > > > ep = &pci->ep; > > > > - ep->ops = &pcie_ep_ops; > > > > + ep->ops = pcie->drvdata->ops; > > > > > > > > res = platform_get_resource_byname(pdev, IORESOURCE_MEM, > > > "addr_space"); > > > > if (!res) > > > > @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct > > > platform_device *pdev) > > > > if (!ls_epc) > > > > return -ENOMEM; > > > > > > > > - dbi_base = platform_get_resource_byname(pdev, > IORESOURCE_MEM, > > > "regs"); > > > > - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > > > - if (IS_ERR(pci->dbi_base)) > > > > - return PTR_ERR(pci->dbi_base); > > > > + pcie->drvdata = of_device_get_match_data(dev); > > > > > > > > - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > > > pci->dev = dev; > > > > - pci->ops = &ls_pcie_ep_ops; > > > > + pci->ops = pcie->drvdata->dw_pcie_ops; > > > > + > > > > pcie->pci = pci; > > > > > > > > ls_epc->linkup_notifier = false, @@ -152,6 +185,13 @@ static int > > > > __init ls_pcie_ep_probe(struct platform_device *pdev) > > > > > > > > pcie->ls_epc = ls_epc; > > > > > > > > + dbi_base = platform_get_resource_byname(pdev, > IORESOURCE_MEM, > > > "regs"); > > > > + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > > > + if (IS_ERR(pci->dbi_base)) > > > > + return PTR_ERR(pci->dbi_base); > > > > + > > > > + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > > > + > > > > platform_set_drvdata(pdev, pcie); > > > > > > > > ret = ls_add_pcie_ep(pcie, pdev); > > > > -- > > > > 2.9.5 > > > >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月27日 21:11 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 01/10] PCI: designware-ep: Add multiple PFs support > for DWC > > On Fri, Aug 23, 2019 at 11:50:20PM +0000, Xiaowei Bao wrote: > > > > > > > -----Original Message----- > > > From: Andrew Murray <andrew.murray@arm.com> > > > Sent: 2019年8月23日 21:25 > > > To: Xiaowei Bao <xiaowei.bao@nxp.com> > > > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > > > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; > M.h. > > > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > > > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > > > Subject: Re: [PATCH v2 01/10] PCI: designware-ep: Add multiple PFs > > > support for DWC > > > > > > On Thu, Aug 22, 2019 at 07:22:33PM +0800, Xiaowei Bao wrote: > > > > Add multiple PFs support for DWC, different PF have different > > > > config space we use pf-offset property which get from the DTS to > > > > access the different pF config space. > > > > > > It looks like you're missing a --cover-letter again. > > > > > > > > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > > > --- > > > > v2: > > > > - Remove duplicate redundant code. > > > > - Reimplement the PF config space access way. > > > > > > > > drivers/pci/controller/dwc/pcie-designware-ep.c | 122 > > > ++++++++++++++++-------- > > > > drivers/pci/controller/dwc/pcie-designware.c | 59 > ++++++++---- > > > > drivers/pci/controller/dwc/pcie-designware.h | 11 ++- > > > > 3 files changed, 134 insertions(+), 58 deletions(-) > > > > > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c > > > > b/drivers/pci/controller/dwc/pcie-designware-ep.c > > > > index 2bf5a35..3e2b740 100644 > > > > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > > > > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > > > > @@ -19,12 +19,17 @@ void dw_pcie_ep_linkup(struct dw_pcie_ep > *ep) > > > > pci_epc_linkup(epc); > > > > } > > > > > > > > -static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum > > > > pci_barno > > > bar, > > > > - int flags) > > > > +static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, > > > > + enum pci_barno bar, int flags) > > > > { > > > > u32 reg; > > > > + unsigned int func_offset = 0; > > > > + struct dw_pcie_ep *ep = &pci->ep; > > > > > > > > - reg = PCI_BASE_ADDRESS_0 + (4 * bar); > > > > + if (ep->ops->func_conf_select) > > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > + > > > > + reg = func_offset + PCI_BASE_ADDRESS_0 + (4 * bar); > > > > > > This pattern of checking if func_conf_select exists and using it to > > > get an offset is repeated a lot throughout this file. You could move > > > this functionality into a new function (similar to dw_pcie_read_dbi > > > etc). Or perhaps a new variant of dw_pcie_writel_ should be created that > writes takes a func_no argument. > > > > Thanks for your comments, I thought about this method before, but > > there is a issue about the method of access the different func config > > space, due to our platform use this method that different func have > > different offset from dbi_base to access the different config space, > > but others platform maybe use the way that write a register to > > implement different func config space access, so I think reserve a > > callback function > > My point here was really to move out duplicated code to its own small > function. > I wasn't making any comment about (removing) the callback, just that the test > and callback could be in one function. > > > to different platform to implement the own method, my point is that, > > if use register method they can implement the code in this function > > and return offset is 0, if use offset method, they can return the offset value > which can be use by dw_pcie_ep driver. > > By the way, I haven't looked to see how many of the dw_pcie_write_xxx > functions would benefit from a func_no argument - if there were many calls to > dw_pcie_write_xxx that all used a reg value originated from func_conf_select > then an approach similar to the implementation of dw_pcie_write_dbi could > probably be justified (i.e. rather than change the value of reg) for writing to > functions. I think you mean that move the if (ep->ops->func_conf_select) func_offset = ep->ops->func_conf_select(ep, func_no); to a new function, I will modify it in next version patch, thanks a lot. Thanks Xiaowei > > > > > > > > > > > > > dw_pcie_dbi_ro_wr_en(pci); > > > > dw_pcie_writel_dbi2(pci, reg, 0x0); > > > > dw_pcie_writel_dbi(pci, reg, 0x0); > > > > > > > > > > @@ -235,7 +257,7 @@ static int dw_pcie_ep_map_addr(struct pci_epc > > > *epc, u8 func_no, > > > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > > > > > - ret = dw_pcie_ep_outbound_atu(ep, addr, pci_addr, size); > > > > + ret = dw_pcie_ep_outbound_atu(ep, func_no, addr, pci_addr, > > > > +size); > > > > if (ret) { > > > > dev_err(pci->dev, "Failed to enable address\n"); > > > > return ret; > > > > @@ -249,11 +271,15 @@ static int dw_pcie_ep_get_msi(struct pci_epc > > > *epc, u8 func_no) > > > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > u32 val, reg; > > > > + unsigned int func_offset = 0; > > > > + > > > > + if (ep->ops->func_conf_select) > > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > > > > > if (!ep->msi_cap) > > > > return -EINVAL; > > > > > > > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > > > > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > > > > > > This makes me nervous. > > > > > > From a PCI viewpoint, each function has it's own capability > > > structure and within each function there may exist a MSI capability. > > > Yet what we're doing here is using dw_pcie_ep_find_capability to get > > > the list of capabilities for function 0, and then applying offsets from that for > subsequent functions. I.e. > > > we're applying DW specific knowledge to find the correct capability, > > > rather than following the general PCI approach. > > > > > > I think the above hunk shouldn't be required - but instead > > > dw_pcie_ep_find_capability is updated to take a func_no parameter. > > > > > > Have I understood this correctly? > > > > Yes, this is a issue, I think the different func maybe have different > > capability, but the dw_pcie_ep_find_capability function is called by > > dw_pcie_ep_init function, we can't add func_no parameter to > > dw_pcie_ep_find_capability, > > Why not? > > Given that 'struct dw_pcie' represents a controller - and thus potentially > multiple functions - then the _find_capability function should be able to > provide the correct offset for the given function. Surely it needs to know > which function number? Unless there is some reason why we can assume that > all functions share the same capability offset. > > Also the 'struct dw_pcie_ep' which represents an endpoint controller - this > has msi_cap and msix_cap fields - given this may be a multifunction device > what do these fields actually refer to? > > Perhaps Jungoo/Gustavo can comment. I have two method to fix this issue, define the msi_cap to *msi_cap, msi_cap[0] indicate func0, msi_cap[1] indicate func1..., like this: + for (func_no = 0; func_no < epc->max_functions; func_no++) { + ep->msi_cap[func_no] = + dw_pcie_ep_find_capability(pci, func_no, + PCI_CAP_ID_MSI); + ep->msix_cap[func_no] = + dw_pcie_ep_find_capability(pci, func_no, + PCI_CAP_ID_MSIX); + } But in Layerscape EP driver, we can't set the msi_capable of pci_epc_features struct by ep->msix_cap, this is not correct, unless we assume if msi_cap[0] is 1, all function will support the MSI feature. and we can return error from the get_msi or set_msi function. I think this is the simplest way in current EP framework. Another method is that add a callback function in pci_epc_ops, don't use the pci_epc_features mode, but that's a big change, we need to implement the other platform callback function like rickchip and so on. Thanks Xiaowei > > > > I will try to fix it use a new patch, I think move this function to > > ep_init callback function If better, thanks. > > > > > > > > > > > val = dw_pcie_readw_dbi(pci, reg); > > > > if (!(val & PCI_MSI_FLAGS_ENABLE)) > > > > return -EINVAL; > > > > @@ -268,11 +294,15 @@ static int dw_pcie_ep_set_msi(struct pci_epc > > > *epc, u8 func_no, u8 interrupts) > > > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > u32 val, reg; > > > > + unsigned int func_offset = 0; > > > > + > > > > + if (ep->ops->func_conf_select) > > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > > > > > if (!ep->msi_cap) > > > > return -EINVAL; > > > > > > > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > > > > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > > > > val = dw_pcie_readw_dbi(pci, reg); > > > > val &= ~PCI_MSI_FLAGS_QMASK; > > > > val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK; @@ -288,11 > > > > +318,15 @@ static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 > func_no) > > > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > u32 val, reg; > > > > + unsigned int func_offset = 0; > > > > + > > > > + if (ep->ops->func_conf_select) > > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > > > > > if (!ep->msix_cap) > > > > return -EINVAL; > > > > > > > > - reg = ep->msix_cap + PCI_MSIX_FLAGS; > > > > + reg = ep->msix_cap + func_offset + PCI_MSIX_FLAGS; > > > > > > Same for MSIX. > > > > Yes. > > > > > > > > > val = dw_pcie_readw_dbi(pci, reg); > > > > if (!(val & PCI_MSIX_FLAGS_ENABLE)) > > > > return -EINVAL; > > > > @@ -307,11 +341,15 @@ static int dw_pcie_ep_set_msix(struct > > > > pci_epc > > > *epc, u8 func_no, u16 interrupts) > > > > struct dw_pcie_ep *ep = epc_get_drvdata(epc); > > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > u32 val, reg; > > > > + unsigned int func_offset = 0; > > > > + > > > > + if (ep->ops->func_conf_select) > > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > > > > > if (!ep->msix_cap) > > > > return -EINVAL; > > > > > > > > - reg = ep->msix_cap + PCI_MSIX_FLAGS; > > > > + reg = ep->msix_cap + func_offset + PCI_MSIX_FLAGS; > > > > val = dw_pcie_readw_dbi(pci, reg); > > > > val &= ~PCI_MSIX_FLAGS_QSIZE; > > > > val |= interrupts; > > > > @@ -398,29 +436,33 @@ int dw_pcie_ep_raise_msi_irq(struct > > > dw_pcie_ep *ep, u8 func_no, > > > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > struct pci_epc *epc = ep->epc; > > > > unsigned int aligned_offset; > > > > + unsigned int func_offset = 0; > > > > u16 msg_ctrl, msg_data; > > > > u32 msg_addr_lower, msg_addr_upper, reg; > > > > u64 msg_addr; > > > > bool has_upper; > > > > int ret; > > > > > > > > + if (ep->ops->func_conf_select) > > > > + func_offset = ep->ops->func_conf_select(ep, func_no); > > > > + > > > > > > You could probably move this hunk below the test for msi_cap to save > > > some cycles. > > > > Sorry, I didn't understand the means, please explain it detailly, > > thanks a lot, ^_^ > > If you insert the call to func_conf_select *after* the test for !msi_cap below - > then in the case where msi_cap is NULL then you will save some CPU cycles > by not bothering to call func_conf_select. Got it, thanks a lot. ^_^ Thanks Xiaowei > > > > > > > > > if (!ep->msi_cap) > > > > return -EINVAL; > > > > > > > > /* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. > */ > > > > - reg = ep->msi_cap + PCI_MSI_FLAGS; > > > > + reg = ep->msi_cap + func_offset + PCI_MSI_FLAGS; > > > > msg_ctrl = dw_pcie_readw_dbi(pci, reg); > > > > has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT); > > > > - reg = ep->msi_cap + PCI_MSI_ADDRESS_LO; > > > > + reg = ep->msi_cap + func_offset + PCI_MSI_ADDRESS_LO; > > > > msg_addr_lower = dw_pcie_readl_dbi(pci, reg); > > > > if (has_upper) { > > > > - reg = ep->msi_cap + PCI_MSI_ADDRESS_HI; > > > > + reg = ep->msi_cap + func_offset + PCI_MSI_ADDRESS_HI; > > > > msg_addr_upper = dw_pcie_readl_dbi(pci, reg); > > > > - reg = ep->msi_cap + PCI_MSI_DATA_64; > > > > + reg = ep->msi_cap + func_offset + PCI_MSI_DATA_64; > > > > msg_data = dw_pcie_readw_dbi(pci, reg); > > > > } else { > > > > msg_addr_upper = 0; > > > > - reg = ep->msi_cap + PCI_MSI_DATA_32; > > > > + reg = ep->msi_cap + func_offset + PCI_MSI_DATA_32; > > > > msg_data = dw_pcie_readw_dbi(pci, reg); > > > > } > > > > aligned_offset = msg_addr_lower & (epc->mem->page_size - 1); > > > > > > > > > > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware.c > > > > b/drivers/pci/controller/dwc/pcie-designware.c > > > > index 7d25102..305e73d 100644 > > > > --- a/drivers/pci/controller/dwc/pcie-designware.c > > > > +++ b/drivers/pci/controller/dwc/pcie-designware.c > > > > @@ -158,9 +158,10 @@ static void dw_pcie_writel_ob_unroll(struct > > > dw_pcie *pci, u32 index, u32 reg, > > > > dw_pcie_writel_atu(pci, offset + reg, val); } > > > > > > > > -static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, > > > > int > > > index, > > > > - int type, u64 cpu_addr, > > > > - u64 pci_addr, u32 size) > > > > +static void dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, > > > > +u8 > > > func_no, > > > > + int index, int type, > > > > + u64 cpu_addr, u64 pci_addr, > > > > + u32 size) > > > > { > > > > u32 retries, val; > > > > > > > > @@ -175,7 +176,7 @@ static void > > > dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, > > > > dw_pcie_writel_ob_unroll(pci, index, > PCIE_ATU_UNR_UPPER_TARGET, > > > > upper_32_bits(pci_addr)); > > > > dw_pcie_writel_ob_unroll(pci, index, > PCIE_ATU_UNR_REGION_CTRL1, > > > > - type); > > > > + type | PCIE_ATU_FUNC_NUM(func_no)); > > > > > > Much better :) > > > > Do you mean that use the expression "a? b:c" > > > > > > > > > dw_pcie_writel_ob_unroll(pci, index, > PCIE_ATU_UNR_REGION_CTRL2, > > > > PCIE_ATU_ENABLE); > > > > > > > > @@ -194,8 +195,9 @@ static void > > > dw_pcie_prog_outbound_atu_unroll(struct dw_pcie *pci, int index, > > > > dev_err(pci->dev, "Outbound iATU is not being enabled\n"); } > > > > > > > > -void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, int > type, > > > > - u64 cpu_addr, u64 pci_addr, u32 size) > > > > +static void __dw_pcie_prog_outbound_atu(struct dw_pcie *pci, u8 > > > func_no, > > > > + int index, int type, u64 cpu_addr, > > > > + u64 pci_addr, u32 size) > > > > { > > > > u32 retries, val; > > > > > > > > @@ -203,8 +205,8 @@ void dw_pcie_prog_outbound_atu(struct > dw_pcie > > > *pci, int index, int type, > > > > cpu_addr = pci->ops->cpu_addr_fixup(pci, cpu_addr); > > > > > > > > if (pci->iatu_unroll_enabled) { > > > > - dw_pcie_prog_outbound_atu_unroll(pci, index, type, cpu_addr, > > > > - pci_addr, size); > > > > + dw_pcie_prog_outbound_atu_unroll(pci, func_no, index, type, > > > > + cpu_addr, pci_addr, size); > > > > return; > > > > } > > > > > > > > > > > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware.h > > > > b/drivers/pci/controller/dwc/pcie-designware.h > > > > index ffed084..a0fdbf7 100644 > > > > --- a/drivers/pci/controller/dwc/pcie-designware.h > > > > +++ b/drivers/pci/controller/dwc/pcie-designware.h > > > > @@ -71,9 +71,11 @@ > > > > #define PCIE_ATU_TYPE_IO 0x2 > > > > #define PCIE_ATU_TYPE_CFG0 0x4 > > > > #define PCIE_ATU_TYPE_CFG1 0x5 > > > > +#define PCIE_ATU_FUNC_NUM(pf) (pf << 20) > > > > > > "Macro argument 'pf' may be better as '(pf)' to avoid precedence issues" > > > > > > > #define PCIE_ATU_CR2 0x908 > > > > #define PCIE_ATU_ENABLE BIT(31) > > > > #define PCIE_ATU_BAR_MODE_ENABLE BIT(30) > > > > +#define PCIE_ATU_FUNC_NUM_MATCH_EN BIT(19) > > > > #define PCIE_ATU_LOWER_BASE 0x90C > > > > #define PCIE_ATU_UPPER_BASE 0x910 > > > > #define PCIE_ATU_LIMIT 0x914 > > > > @@ -197,6 +199,7 @@ struct dw_pcie_ep_ops { > > > > int (*raise_irq)(struct dw_pcie_ep *ep, u8 func_no, > > > > enum pci_epc_irq_type type, u16 interrupt_num); > > > > const struct pci_epc_features* (*get_features)(struct dw_pcie_ep > > > > *ep); > > > > + unsigned int (*func_conf_select)(struct dw_pcie_ep *ep, u8 > > > > +func_no); > > > > > > Given that this function will return an offset, I'm not sure the > > > name you have is suitable. Something like get_pf_offset or similar is more > descriptive. > > > > As above explain, my initial view is that this function can return 0 > > or offset depends on the platform implement mechanism, so I named it > > func_conf_select, I think add a comment for this function, like this: > > /* > > * provide a method to implement the method of different func config > > space access, > > * if use offset method, return the offset from dbi_base, if your > > register method, implement > > * the code in this callback function and return 0. > > */ > > How about it? > > This means that func_conf_select can never (easily) indicate an error to the > caller as this would change the offset. Where func_conf_select doesn't > change the offset there probably isn't much else it can do instead (unless it > was responsible for doing the write as well). So I'm not sure how well this > approach works. We can use int type of this function and return a negative value when a error occurred, that is to say: if(func_no && !pcie->drvdata->func_offset) return -1; but we need to set a flag to differentiate the method of config space access, that is we can return -1 in func_offset method, another method we only return 0. Thanks Xiaowei > > Thanks, > > Andrew Murray > > > > > > > > > Thanks, > > > > > > Andrew Murray > > > > > > > }; > > > > > > > > struct dw_pcie_ep { > > > > @@ -265,8 +268,12 @@ int dw_pcie_wait_for_link(struct dw_pcie > > > > *pci); void dw_pcie_prog_outbound_atu(struct dw_pcie *pci, int index, > > > > int type, u64 cpu_addr, u64 pci_addr, > > > > u32 size); > > > > -int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int bar, > > > > - u64 cpu_addr, enum dw_pcie_as_type as_type); > > > > +void dw_pcie_prog_ep_outbound_atu(struct dw_pcie *pci, u8 > > > > +func_no, int > > > index, > > > > + int type, u64 cpu_addr, u64 pci_addr, > > > > + u32 size); > > > > +int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, u8 func_no, int > index, > > > > + int bar, u64 cpu_addr, > > > > + enum dw_pcie_as_type as_type); > > > > void dw_pcie_disable_atu(struct dw_pcie *pci, int index, > > > > enum dw_pcie_region_type type); void > dw_pcie_setup(struct > > > > dw_pcie *pci); > > > > -- > > > > 2.9.5 > > > >
On Wed, Aug 28, 2019 at 04:29:32AM +0000, Xiaowei Bao wrote: > > > > -----Original Message----- > > From: Andrew Murray <andrew.murray@arm.com> > > Sent: 2019年8月27日 21:34 > > To: Xiaowei Bao <xiaowei.bao@nxp.com> > > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support for > > ls1088a and ls2088a > > > > On Mon, Aug 26, 2019 at 09:49:35AM +0000, Xiaowei Bao wrote: > > > > > > > > > > -----Original Message----- > > > > From: Andrew Murray <andrew.murray@arm.com> > > > > Sent: 2019年8月23日 22:28 > > > > To: Xiaowei Bao <xiaowei.bao@nxp.com> > > > > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > > > > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > > > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; > > M.h. > > > > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > > > > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > > > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > > > > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support > > > > for ls1088a and ls2088a > > > > > > > > On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: > > > > > Add PCIe EP mode support for ls1088a and ls2088a, there are some > > > > > difference between LS1 and LS2 platform, so refactor the code of > > > > > the EP driver. > > > > > > > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > > > > --- > > > > > v2: > > > > > - New mechanism for layerscape EP driver. > > > > > > > > Was there a v1 of this patch? > > > > > > > > > > > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 > > > > > ++++++++++++++++++++------ > > > > > 1 file changed, 58 insertions(+), 18 deletions(-) > > > > > > > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > > index 7ca5fe8..2a66f07 100644 > > > > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > > @@ -20,27 +20,29 @@ > > > > > > > > > > #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ > > > > > > > > > > -struct ls_pcie_ep { > > > > > - struct dw_pcie *pci; > > > > > - struct pci_epc_features *ls_epc; > > > > > +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > > > + > > > > > +struct ls_pcie_ep_drvdata { > > > > > + u32 func_offset; > > > > > + const struct dw_pcie_ep_ops *ops; > > > > > + const struct dw_pcie_ops *dw_pcie_ops; > > > > > }; > > > > > > > > > > -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > > > +struct ls_pcie_ep { > > > > > + struct dw_pcie *pci; > > > > > + struct pci_epc_features *ls_epc; > > > > > + const struct ls_pcie_ep_drvdata *drvdata; }; > > > > > > > > > > static int ls_pcie_establish_link(struct dw_pcie *pci) { > > > > > return 0; > > > > > } > > > > > > > > > > -static const struct dw_pcie_ops ls_pcie_ep_ops = { > > > > > +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { > > > > > .start_link = ls_pcie_establish_link, }; > > > > > > > > > > -static const struct of_device_id ls_pcie_ep_of_match[] = { > > > > > - { .compatible = "fsl,ls-pcie-ep",}, > > > > > - { }, > > > > > -}; > > > > > - > > > > > static const struct pci_epc_features* > > > > > ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { @@ -82,10 +84,44 > > > > > @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > > > > > } > > > > > } > > > > > > > > > > -static const struct dw_pcie_ep_ops pcie_ep_ops = { > > > > > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep > > *ep, > > > > > + u8 func_no) > > > > > +{ > > > > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > > > > + u8 header_type; > > > > > + > > > > > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > > > > + > > > > > + if (header_type & (1 << 7)) > > > > > + return pcie->drvdata->func_offset * func_no; > > > > > + else > > > > > + return 0; > > > > > > > > It looks like there isn't a PCI define for multi function, the > > > > nearest I could find was PCI_HEADER_TYPE_MULTIDEVICE in > > > > hotplug/ibmphp.h. A comment above the test might be helpful to explain > > the test. > > > > > > OK, I will add a comment above this code. > > > > > > > > > > > As the ls_pcie_ep_drvdata structures are static, the unset > > > > .func_offset will be initialised to 0, so you could just drop the test above. > > > > > > Due to the different PCIe controller have different property, e.g. > > > PCIe controller1 support multiple function feature, but PCIe > > > controller2 don't support this feature, so I need to check which > > > controller support it and return the correct offset value, but each board only > > have one ls_pcie_ep_drvdata, ^_^. > > > > Yes but if they don't support the feature then func_offset will be 0. > > > > > > > > > > > > > However something to the effect of the following may help spot > > > > misconfiguration: > > > > > > > > WARN_ON(func_no && !pcie->drvdata->func_offset); return > > > > pcie->drvdata->func_offset * func_no; > > > > > > > > The WARN is probably quite useful as if you are attempting to use > > > > non-zero functions and func_offset isn't set - then things may > > > > appear to work normally but actually will break horribly. > > > > > > As discussion before, I think the func_offset should not depends on > > > the function number, even if other platforms of NXP may be use write > > > registers way to access the different function config space. > > > > I agree that func_offset is an optional parameter. But if you are attempting to > > determine the offset of a function and you are given a non-zero function > > number - then something has gone wrong if func_offset is 0. > > I have understood you means, maybe I need to set a flag in the driver_data struct, > because I may add other platform of NXP, these platform use the write register > method to access different function, e.g. > write func_num to register, then we can access this func_num config space. > > I will modify the code like this? Do you have better advice? > Case1: > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c > index 004a7e8..8a0d6df 100644 > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > @@ -23,6 +23,7 @@ > #define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > struct ls_pcie_ep_drvdata { > + u8 func_config_flag; > u32 func_offset; > const struct dw_pcie_ep_ops *ops; > const struct dw_pcie_ops *dw_pcie_ops; > @@ -97,8 +98,14 @@ static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > * Read the Header Type register of config space to check > * whether this PCI device support the multiple function. > */ > - if (header_type & (1 << 7)) > - return pcie->drvdata->func_offset * func_no; > + if (header_type & (1 << 7)) { > + if (pcie->drvdata->func_config_flag) { > + iowrite32((func_num << n), pci->dbi_base + PCI_XXXX_XXX); > + } else { > + WARN_ON(func_no && !pcie->drvdata->func_offset); > + return pcie->drvdata->func_offset * func_no; > + } > + } > > return 0; > } > > Of course, I don't need to set the flag this time, because I don't use the second method(write > register method), so the code like this: > case2: > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > u8 func_no) { > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > u8 header_type; > > of course, this code is not requied, due to the > pcie->drvdata->func_offset is 0, but I think this is more clear > if use this code. > header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > /* > * Read the Header Type register of config space to check > * whether this PCI device support the multiple function. > */ > if (header_type & (1 << 7)) { > WARN_ON(func_no && !pcie->drvdata->func_offset); > return pcie->drvdata->func_offset * func_no; > } > > return 0; > } > > Or like this: > Case3: > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > u8 func_no) { > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > WARN_ON(func_no && !pcie->drvdata->func_offset); > return pcie->drvdata->func_offset * func_no; This is better. Given there is only currently one method of calculating an offset for layerscape, I'd recommend you add additional methods when the need arises. Thanks, Andrew Murray > > } > Of course, we can return a -1 by adjuring the (func_no && !pcie->drvdata->func_offset) > Valu in case1 > > Thanks > Xiaowei > > > > > Thanks, > > > > Andrew Murray > > > > > > > > I have added the comments above the code, as follow, do you have any > > advice? > > > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > > > + u8 func_no) { > > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > > + u8 header_type; > > > + > > > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > > + > > > + /* > > > + * Read the Header Type register of config space to check > > > + * whether this PCI device support the multiple function. > > > + */ > > > + if (header_type & (1 << 7)) > > > + return pcie->drvdata->func_offset * func_no; > > > + > > > + return 0; > > > +} > > > > > > Thanks a lot for your detail comments. > > > > > > > > > > > Thanks, > > > > > > > > Andrew Murray > > > > > > > > > +} > > > > > + > > > > > +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { > > > > > .ep_init = ls_pcie_ep_init, > > > > > .raise_irq = ls_pcie_ep_raise_irq, > > > > > .get_features = ls_pcie_ep_get_features, > > > > > + .func_conf_select = ls_pcie_ep_func_conf_select, }; > > > > > + > > > > > +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { > > > > > + .ops = &ls_pcie_ep_ops, > > > > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > > > > > + > > > > > +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { > > > > > + .func_offset = 0x20000, > > > > > + .ops = &ls_pcie_ep_ops, > > > > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > > > > > + > > > > > +static const struct of_device_id ls_pcie_ep_of_match[] = { > > > > > + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, > > > > > + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, > > > > > + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, > > > > > + { }, > > > > > }; > > > > > > > > > > static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ > > > > > -98,7 > > > > > +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep > > > > > +*pcie, > > > > > int ret; > > > > > > > > > > ep = &pci->ep; > > > > > - ep->ops = &pcie_ep_ops; > > > > > + ep->ops = pcie->drvdata->ops; > > > > > > > > > > res = platform_get_resource_byname(pdev, IORESOURCE_MEM, > > > > "addr_space"); > > > > > if (!res) > > > > > @@ -137,14 +173,11 @@ static int __init ls_pcie_ep_probe(struct > > > > platform_device *pdev) > > > > > if (!ls_epc) > > > > > return -ENOMEM; > > > > > > > > > > - dbi_base = platform_get_resource_byname(pdev, > > IORESOURCE_MEM, > > > > "regs"); > > > > > - pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > > > > - if (IS_ERR(pci->dbi_base)) > > > > > - return PTR_ERR(pci->dbi_base); > > > > > + pcie->drvdata = of_device_get_match_data(dev); > > > > > > > > > > - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > > > > pci->dev = dev; > > > > > - pci->ops = &ls_pcie_ep_ops; > > > > > + pci->ops = pcie->drvdata->dw_pcie_ops; > > > > > + > > > > > pcie->pci = pci; > > > > > > > > > > ls_epc->linkup_notifier = false, @@ -152,6 +185,13 @@ static int > > > > > __init ls_pcie_ep_probe(struct platform_device *pdev) > > > > > > > > > > pcie->ls_epc = ls_epc; > > > > > > > > > > + dbi_base = platform_get_resource_byname(pdev, > > IORESOURCE_MEM, > > > > "regs"); > > > > > + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > > > > + if (IS_ERR(pci->dbi_base)) > > > > > + return PTR_ERR(pci->dbi_base); > > > > > + > > > > > + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > > > > + > > > > > platform_set_drvdata(pdev, pcie); > > > > > > > > > > ret = ls_add_pcie_ep(pcie, pdev); > > > > > -- > > > > > 2.9.5 > > > > >
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年8月28日 17:01 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support for > ls1088a and ls2088a > > On Wed, Aug 28, 2019 at 04:29:32AM +0000, Xiaowei Bao wrote: > > > > > > > -----Original Message----- > > > From: Andrew Murray <andrew.murray@arm.com> > > > Sent: 2019年8月27日 21:34 > > > To: Xiaowei Bao <xiaowei.bao@nxp.com> > > > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > > > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; > M.h. > > > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > > > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > > > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode support > > > for ls1088a and ls2088a > > > > > > On Mon, Aug 26, 2019 at 09:49:35AM +0000, Xiaowei Bao wrote: > > > > > > > > > > > > > -----Original Message----- > > > > > From: Andrew Murray <andrew.murray@arm.com> > > > > > Sent: 2019年8月23日 22:28 > > > > > To: Xiaowei Bao <xiaowei.bao@nxp.com> > > > > > Cc: bhelgaas@google.com; robh+dt@kernel.org; > > > > > mark.rutland@arm.com; shawnguo@kernel.org; Leo Li > > > > > <leoyang.li@nxp.com>; kishon@ti.com; lorenzo.pieralisi@arm.co; > > > > > arnd@arndb.de; gregkh@linuxfoundation.org; > > > M.h. > > > > > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; > > > > > Roy Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > > > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > > > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > > > > linux-arm-kernel@lists.infradead.org; > > > > > linuxppc-dev@lists.ozlabs.org > > > > > Subject: Re: [PATCH v2 08/10] PCI: layerscape: Add EP mode > > > > > support for ls1088a and ls2088a > > > > > > > > > > On Thu, Aug 22, 2019 at 07:22:40PM +0800, Xiaowei Bao wrote: > > > > > > Add PCIe EP mode support for ls1088a and ls2088a, there are > > > > > > some difference between LS1 and LS2 platform, so refactor the > > > > > > code of the EP driver. > > > > > > > > > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > > > > > --- > > > > > > v2: > > > > > > - New mechanism for layerscape EP driver. > > > > > > > > > > Was there a v1 of this patch? > > > > > > > > > > > > > > > > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 76 > > > > > > ++++++++++++++++++++------ > > > > > > 1 file changed, 58 insertions(+), 18 deletions(-) > > > > > > > > > > > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > > > index 7ca5fe8..2a66f07 100644 > > > > > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > > > > @@ -20,27 +20,29 @@ > > > > > > > > > > > > #define PCIE_DBI2_OFFSET 0x1000 /* DBI2 base address*/ > > > > > > > > > > > > -struct ls_pcie_ep { > > > > > > - struct dw_pcie *pci; > > > > > > - struct pci_epc_features *ls_epc; > > > > > > +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > > > > + > > > > > > +struct ls_pcie_ep_drvdata { > > > > > > + u32 func_offset; > > > > > > + const struct dw_pcie_ep_ops *ops; > > > > > > + const struct dw_pcie_ops *dw_pcie_ops; > > > > > > }; > > > > > > > > > > > > -#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > > > > +struct ls_pcie_ep { > > > > > > + struct dw_pcie *pci; > > > > > > + struct pci_epc_features *ls_epc; > > > > > > + const struct ls_pcie_ep_drvdata *drvdata; }; > > > > > > > > > > > > static int ls_pcie_establish_link(struct dw_pcie *pci) { > > > > > > return 0; > > > > > > } > > > > > > > > > > > > -static const struct dw_pcie_ops ls_pcie_ep_ops = { > > > > > > +static const struct dw_pcie_ops dw_ls_pcie_ep_ops = { > > > > > > .start_link = ls_pcie_establish_link, }; > > > > > > > > > > > > -static const struct of_device_id ls_pcie_ep_of_match[] = { > > > > > > - { .compatible = "fsl,ls-pcie-ep",}, > > > > > > - { }, > > > > > > -}; > > > > > > - > > > > > > static const struct pci_epc_features* > > > > > > ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { @@ -82,10 > > > > > > +84,44 @@ static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, > u8 func_no, > > > > > > } > > > > > > } > > > > > > > > > > > > -static const struct dw_pcie_ep_ops pcie_ep_ops = { > > > > > > +static unsigned int ls_pcie_ep_func_conf_select(struct > > > > > > +dw_pcie_ep > > > *ep, > > > > > > + u8 func_no) > > > > > > +{ > > > > > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > > > > > + u8 header_type; > > > > > > + > > > > > > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > > > > > + > > > > > > + if (header_type & (1 << 7)) > > > > > > + return pcie->drvdata->func_offset * func_no; > > > > > > + else > > > > > > + return 0; > > > > > > > > > > It looks like there isn't a PCI define for multi function, the > > > > > nearest I could find was PCI_HEADER_TYPE_MULTIDEVICE in > > > > > hotplug/ibmphp.h. A comment above the test might be helpful to > > > > > explain > > > the test. > > > > > > > > OK, I will add a comment above this code. > > > > > > > > > > > > > > As the ls_pcie_ep_drvdata structures are static, the unset > > > > > .func_offset will be initialised to 0, so you could just drop the test > above. > > > > > > > > Due to the different PCIe controller have different property, e.g. > > > > PCIe controller1 support multiple function feature, but PCIe > > > > controller2 don't support this feature, so I need to check which > > > > controller support it and return the correct offset value, but > > > > each board only > > > have one ls_pcie_ep_drvdata, ^_^. > > > > > > Yes but if they don't support the feature then func_offset will be 0. > > > > > > > > > > > > > > > > > However something to the effect of the following may help spot > > > > > misconfiguration: > > > > > > > > > > WARN_ON(func_no && !pcie->drvdata->func_offset); return > > > > > pcie->drvdata->func_offset * func_no; > > > > > > > > > > The WARN is probably quite useful as if you are attempting to > > > > > use non-zero functions and func_offset isn't set - then things > > > > > may appear to work normally but actually will break horribly. > > > > > > > > As discussion before, I think the func_offset should not depends > > > > on the function number, even if other platforms of NXP may be use > > > > write registers way to access the different function config space. > > > > > > I agree that func_offset is an optional parameter. But if you are > > > attempting to determine the offset of a function and you are given a > > > non-zero function number - then something has gone wrong if func_offset > is 0. > > > > I have understood you means, maybe I need to set a flag in the > > driver_data struct, because I may add other platform of NXP, these > > platform use the write register method to access different function, e.g. > > write func_num to register, then we can access this func_num config space. > > > > I will modify the code like this? Do you have better advice? > > Case1: > > diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > index 004a7e8..8a0d6df 100644 > > --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > @@ -23,6 +23,7 @@ > > #define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > > struct ls_pcie_ep_drvdata { > > + u8 func_config_flag; > > u32 func_offset; > > const struct dw_pcie_ep_ops *ops; > > const struct dw_pcie_ops *dw_pcie_ops; > > @@ -97,8 +98,14 @@ static unsigned int > ls_pcie_ep_func_conf_select(struct dw_pcie_ep *ep, > > * Read the Header Type register of config space to check > > * whether this PCI device support the multiple function. > > */ > > - if (header_type & (1 << 7)) > > - return pcie->drvdata->func_offset * func_no; > > + if (header_type & (1 << 7)) { > > + if (pcie->drvdata->func_config_flag) { > > + iowrite32((func_num << n), pci->dbi_base + > PCI_XXXX_XXX); > > + } else { > > + WARN_ON(func_no > && !pcie->drvdata->func_offset); > > + return pcie->drvdata->func_offset * func_no; > > + } > > + } > > > > return 0; > > } > > > > Of course, I don't need to set the flag this time, because I don't use > > the second method(write register method), so the code like this: > > case2: > > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep > > +*ep, > > u8 func_no) { > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > u8 header_type; > > > > of course, this code is not requied, due to the > > pcie->drvdata->func_offset is 0, but I think this is more clear > > if use this code. > > header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > > > /* > > * Read the Header Type register of config space to check > > * whether this PCI device support the multiple function. > > */ > > if (header_type & (1 << 7)) { > > WARN_ON(func_no && !pcie->drvdata->func_offset); > > return pcie->drvdata->func_offset * func_no; > > } > > > > return 0; > > } > > > > Or like this: > > Case3: > > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep > > +*ep, > > u8 func_no) { > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > > > WARN_ON(func_no && !pcie->drvdata->func_offset); > > return pcie->drvdata->func_offset * func_no; > > This is better. Given there is only currently one method of calculating an offset > for layerscape, I'd recommend you add additional methods when the need > arises. OK, thanks > > Thanks, > > Andrew Murray > > > > > } > > Of course, we can return a -1 by adjuring the (func_no && > > !pcie->drvdata->func_offset) Valu in case1 > > > > Thanks > > Xiaowei > > > > > > > > Thanks, > > > > > > Andrew Murray > > > > > > > > > > > I have added the comments above the code, as follow, do you have > > > > any > > > advice? > > > > +static unsigned int ls_pcie_ep_func_conf_select(struct dw_pcie_ep > *ep, > > > > + u8 func_no) > { > > > > + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > > + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > > > + u8 header_type; > > > > + > > > > + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); > > > > + > > > > + /* > > > > + * Read the Header Type register of config space to check > > > > + * whether this PCI device support the multiple function. > > > > + */ > > > > + if (header_type & (1 << 7)) > > > > + return pcie->drvdata->func_offset * func_no; > > > > + > > > > + return 0; > > > > +} > > > > > > > > Thanks a lot for your detail comments. > > > > > > > > > > > > > > Thanks, > > > > > > > > > > Andrew Murray > > > > > > > > > > > +} > > > > > > + > > > > > > +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { > > > > > > .ep_init = ls_pcie_ep_init, > > > > > > .raise_irq = ls_pcie_ep_raise_irq, > > > > > > .get_features = ls_pcie_ep_get_features, > > > > > > + .func_conf_select = ls_pcie_ep_func_conf_select, }; > > > > > > + > > > > > > +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { > > > > > > + .ops = &ls_pcie_ep_ops, > > > > > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > > > > > > + > > > > > > +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { > > > > > > + .func_offset = 0x20000, > > > > > > + .ops = &ls_pcie_ep_ops, > > > > > > + .dw_pcie_ops = &dw_ls_pcie_ep_ops, }; > > > > > > + > > > > > > +static const struct of_device_id ls_pcie_ep_of_match[] = { > > > > > > + { .compatible = "fsl,ls1046a-pcie-ep", .data = > &ls1_ep_drvdata }, > > > > > > + { .compatible = "fsl,ls1088a-pcie-ep", .data = > &ls2_ep_drvdata }, > > > > > > + { .compatible = "fsl,ls2088a-pcie-ep", .data = > &ls2_ep_drvdata }, > > > > > > + { }, > > > > > > }; > > > > > > > > > > > > static int __init ls_add_pcie_ep(struct ls_pcie_ep *pcie, @@ > > > > > > -98,7 > > > > > > +134,7 @@ static int __init ls_add_pcie_ep(struct ls_pcie_ep > > > > > > +*pcie, > > > > > > int ret; > > > > > > > > > > > > ep = &pci->ep; > > > > > > - ep->ops = &pcie_ep_ops; > > > > > > + ep->ops = pcie->drvdata->ops; > > > > > > > > > > > > res = platform_get_resource_byname(pdev, > IORESOURCE_MEM, > > > > > "addr_space"); > > > > > > if (!res) > > > > > > @@ -137,14 +173,11 @@ static int __init > > > > > > ls_pcie_ep_probe(struct > > > > > platform_device *pdev) > > > > > > if (!ls_epc) > > > > > > return -ENOMEM; > > > > > > > > > > > > - dbi_base = platform_get_resource_byname(pdev, > > > IORESOURCE_MEM, > > > > > "regs"); > > > > > > - pci->dbi_base = devm_pci_remap_cfg_resource(dev, > dbi_base); > > > > > > - if (IS_ERR(pci->dbi_base)) > > > > > > - return PTR_ERR(pci->dbi_base); > > > > > > + pcie->drvdata = of_device_get_match_data(dev); > > > > > > > > > > > > - pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > > > > > pci->dev = dev; > > > > > > - pci->ops = &ls_pcie_ep_ops; > > > > > > + pci->ops = pcie->drvdata->dw_pcie_ops; > > > > > > + > > > > > > pcie->pci = pci; > > > > > > > > > > > > ls_epc->linkup_notifier = false, @@ -152,6 +185,13 @@ static > > > > > > int __init ls_pcie_ep_probe(struct platform_device *pdev) > > > > > > > > > > > > pcie->ls_epc = ls_epc; > > > > > > > > > > > > + dbi_base = platform_get_resource_byname(pdev, > > > IORESOURCE_MEM, > > > > > "regs"); > > > > > > + pci->dbi_base = devm_pci_remap_cfg_resource(dev, > dbi_base); > > > > > > + if (IS_ERR(pci->dbi_base)) > > > > > > + return PTR_ERR(pci->dbi_base); > > > > > > + > > > > > > + pci->dbi_base2 = pci->dbi_base + PCIE_DBI2_OFFSET; > > > > > > + > > > > > > platform_set_drvdata(pdev, pcie); > > > > > > > > > > > > ret = ls_add_pcie_ep(pcie, pdev); > > > > > > -- > > > > > > 2.9.5 > > > > > >
Gustavo,
On 27/08/19 6:55 PM, Andrew Murray wrote:
> On Sat, Aug 24, 2019 at 12:08:40AM +0000, Xiaowei Bao wrote:
>>
>>
>>> -----Original Message-----
>>> From: Andrew Murray <andrew.murray@arm.com>
>>> Sent: 2019年8月23日 21:58
>>> To: Xiaowei Bao <xiaowei.bao@nxp.com>
>>> Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
>>> shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com;
>>> lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h.
>>> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy
>>> Zang <roy.zang@nxp.com>; jingoohan1@gmail.com;
>>> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org;
>>> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
>>> linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org
>>> Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to the
>>> doorbell way
>>>
>>> On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote:
>>>> The layerscape platform use the doorbell way to trigger MSIX interrupt
>>>> in EP mode.
>>>>
>>>
>>> I have no problems with this patch, however...
>>>
>>> Are you able to add to this message a reason for why you are making this
>>> change? Did dw_pcie_ep_raise_msix_irq not work when func_no != 0? Or did
>>> it work yet dw_pcie_ep_raise_msix_irq_doorbell is more efficient?
>>
>> The fact is that, this driver is verified in ls1046a platform of NXP before, and ls1046a don't
>> support MSIX feature, so I set the msix_capable of pci_epc_features struct is false,
>> but in other platform, e.g. ls1088a, it support the MSIX feature, I verified the MSIX
>> feature in ls1088a, it is not OK, so I changed to another way. Thanks.
>
> Right, so the existing pci-layerscape-ep.c driver never supported MSIX yet it
> erroneously had a switch case statement to call dw_pcie_ep_raise_msix_irq which
> would never get used.
>
> Now that we're adding a platform with MSIX support the existing
> dw_pcie_ep_raise_msix_irq doesn't work (for this platform) so we are adding a
> different method.
Gustavo, can you confirm dw_pcie_ep_raise_msix_irq() works for designware as it
didn't work for both me and Xiaowei?
Thanks
Kishon
> -----Original Message----- > From: Rob Herring <robh@kernel.org> > Sent: 2019年8月28日 6:26 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: bhelgaas@google.com; mark.rutland@arm.com; shawnguo@kernel.org; > Leo Li <leoyang.li@nxp.com>; kishon@ti.com; lorenzo.pieralisi@arm.co; > arnd@arndb.de; gregkh@linuxfoundation.org; M.h. Lian > <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy Zang > <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org; > andrew.murray@arm.com > Subject: Re: [PATCH v2 04/10] dt-bindings: pci: layerscape-pci: add compatible > strings for ls1088a and ls2088a > > On Thu, Aug 22, 2019 at 07:22:36PM +0800, Xiaowei Bao wrote: > > Add compatible strings for ls1088a and ls2088a. > > > > Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > --- > > v2: > > - No change. > > > > Documentation/devicetree/bindings/pci/layerscape-pci.txt | 5 ++++- > > 1 file changed, 4 insertions(+), 1 deletion(-) > > > > diff --git a/Documentation/devicetree/bindings/pci/layerscape-pci.txt > > b/Documentation/devicetree/bindings/pci/layerscape-pci.txt > > index e20ceaa..16f592e 100644 > > --- a/Documentation/devicetree/bindings/pci/layerscape-pci.txt > > +++ b/Documentation/devicetree/bindings/pci/layerscape-pci.txt > > @@ -22,7 +22,10 @@ Required properties: > > "fsl,ls1043a-pcie" > > "fsl,ls1012a-pcie" > > EP mode: > > - "fsl,ls1046a-pcie-ep", "fsl,ls-pcie-ep" > > + "fsl,ls-pcie-ep" > > Wasn't this a fallback? Each line should be one valid combination of > compatible strings. Thanks, got it, I will modify it in next version patch. Thanks Xiaowei > > > + "fsl,ls1046a-pcie-ep" > > + "fsl,ls1088a-pcie-ep" > > + "fsl,ls2088a-pcie-ep" > > - reg: base addresses and lengths of the PCIe controller register blocks. > > - interrupts: A list of interrupt outputs of the controller. Must contain an > > entry for each entry in the interrupt-names property. > > -- > > 2.9.5 > >
On Fri, Aug 23, 2019 at 04:13:30AM +0000, Xiaowei Bao wrote: > > > > -----Original Message----- > > From: Kishon Vijay Abraham I <kishon@ti.com> > > Sent: 2019年8月23日 11:40 > > To: Xiaowei Bao <xiaowei.bao@nxp.com>; bhelgaas@google.com; > > robh+dt@kernel.org; mark.rutland@arm.com; shawnguo@kernel.org; Leo Li > > <leoyang.li@nxp.com>; lorenzo.pieralisi@arm.co > > <lorenzo.pieralisi@arm.com>; arnd@arndb.de; gregkh@linuxfoundation.org; > > M.h. Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; > > Roy Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org; > > andrew.murray@arm.com > > Subject: Re: [PATCH v2 06/10] PCI: layerscape: Modify the way of getting > > capability with different PEX > > > > Hi, > > > > (Fixed Lorenzo's email address. All the patches in the series have wrong email > > id) > > > > On 23/08/19 8:09 AM, Xiaowei Bao wrote: > > > > > > > > >> -----Original Message----- > > >> From: Kishon Vijay Abraham I <kishon@ti.com> > > >> Sent: 2019年8月22日 19:44 > > >> To: Xiaowei Bao <xiaowei.bao@nxp.com>; bhelgaas@google.com; > > >> robh+dt@kernel.org; mark.rutland@arm.com; shawnguo@kernel.org; Leo > > Li > > >> <leoyang.li@nxp.com>; lorenzo.pieralisi@arm.co; arnd@arndb.de; > > >> gregkh@linuxfoundation.org; M.h. Lian <minghuan.lian@nxp.com>; > > >> Mingkai Hu <mingkai.hu@nxp.com>; Roy Zang <roy.zang@nxp.com>; > > >> jingoohan1@gmail.com; gustavo.pimentel@synopsys.com; > > >> linux-pci@vger.kernel.org; devicetree@vger.kernel.org; > > >> linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org; > > >> linuxppc-dev@lists.ozlabs.org; andrew.murray@arm.com > > >> Subject: Re: [PATCH v2 06/10] PCI: layerscape: Modify the way of > > >> getting capability with different PEX > > >> > > >> Hi, > > >> > > >> On 22/08/19 4:52 PM, Xiaowei Bao wrote: > > >>> The different PCIe controller in one board may be have different > > >>> capability of MSI or MSIX, so change the way of getting the MSI > > >>> capability, make it more flexible. > > >> > > >> please use different pci_epc_features table for different boards. > > > Thanks, I think that it will be more flexible to dynamically get MSI > > > or MSIX capability, Thus, we will not need to define the pci_epc_feature for > > different boards. > > > > Is the restriction because you cannot have different compatible for different > > boards? > Sorry, I am not very clear what your mean, I think even if I use the same compatible > with different boards, each boards will enter the probe function, in there I will get > the MSI or MSIX PCIe capability of the current controller in this board. Why do I need > to define the pci_epc_feature for different boards? At present you determine how to set the [msi,msix]_capable flags of pci_epc_features based on reading the function capabilities at probe time. Instead of doing this, is it possible that you can determine the flags based on the compatible type alone? For example, is the MSI/MSIX capability the same for all fsl,ls2088a-pcie-ep devices? If it isn't *necessary* to probe for this information at probe time, then you could instead create a static pci_epc_features structure and assign it to something in your drvdata. This may provide some benefits. The dw_pcie_ep_get_features function would then look like: static const struct pci_epc_features* ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { struct dw_pcie *pci = to_dw_pcie_from_pp(ep); struct ls_pcie_ep *pcie = dev_get_drvdata(pci->dev); return pcie->epc_features; } This also means you can revert "[v3,03/11] PCI: designware-ep: Move the". Is this what you had in mind Kishon? Thanks, Andrew Murray > > > > Thanks > > Kishon > > > > >> > > >> Thanks > > >> Kishon > > >>> > > >>> Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > >>> --- > > >>> v2: > > >>> - Remove the repeated assignment code. > > >>> > > >>> drivers/pci/controller/dwc/pci-layerscape-ep.c | 26 > > >>> +++++++++++++++++++------- > > >>> 1 file changed, 19 insertions(+), 7 deletions(-) > > >>> > > >>> diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > >>> b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > >>> index 4e92a95..8461f62 100644 > > >>> --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > >>> +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > >>> @@ -22,6 +22,7 @@ > > >>> > > >>> struct ls_pcie_ep { > > >>> struct dw_pcie *pci; > > >>> + struct pci_epc_features *ls_epc; > > >>> }; > > >>> > > >>> #define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > >>> @@ -40,25 +41,26 @@ static const struct of_device_id > > >> ls_pcie_ep_of_match[] = { > > >>> { }, > > >>> }; > > >>> > > >>> -static const struct pci_epc_features ls_pcie_epc_features = { > > >>> - .linkup_notifier = false, > > >>> - .msi_capable = true, > > >>> - .msix_capable = false, > > >>> -}; > > >>> - > > >>> static const struct pci_epc_features* > > >>> ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { > > >>> - return &ls_pcie_epc_features; > > >>> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > >>> + > > >>> + return pcie->ls_epc; > > >>> } > > >>> > > >>> static void ls_pcie_ep_init(struct dw_pcie_ep *ep) { > > >>> struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > >>> enum pci_barno bar; > > >>> > > >>> for (bar = BAR_0; bar <= BAR_5; bar++) > > >>> dw_pcie_ep_reset_bar(pci, bar); > > >>> + > > >>> + pcie->ls_epc->msi_capable = ep->msi_cap ? true : false; > > >>> + pcie->ls_epc->msix_capable = ep->msix_cap ? true : false; > > >>> } > > >>> > > >>> static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, > > >>> @@ > > >>> -118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct > > >>> platform_device > > >> *pdev) > > >>> struct device *dev = &pdev->dev; > > >>> struct dw_pcie *pci; > > >>> struct ls_pcie_ep *pcie; > > >>> + struct pci_epc_features *ls_epc; > > >>> struct resource *dbi_base; > > >>> int ret; > > >>> > > >>> @@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct > > >> platform_device *pdev) > > >>> if (!pci) > > >>> return -ENOMEM; > > >>> > > >>> + ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL); > > >>> + if (!ls_epc) > > >>> + return -ENOMEM; > > >>> + > > >>> dbi_base = platform_get_resource_byname(pdev, > > IORESOURCE_MEM, > > >> "regs"); > > >>> pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); > > >>> if (IS_ERR(pci->dbi_base)) > > >>> @@ -139,6 +146,11 @@ static int __init ls_pcie_ep_probe(struct > > >> platform_device *pdev) > > >>> pci->ops = &ls_pcie_ep_ops; > > >>> pcie->pci = pci; > > >>> > > >>> + ls_epc->linkup_notifier = false, > > >>> + ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4), > > >>> + > > >>> + pcie->ls_epc = ls_epc; > > >>> + > > >>> platform_set_drvdata(pdev, pcie); > > >>> > > >>> ret = ls_add_pcie_ep(pcie, pdev); > > >>>
> -----Original Message----- > From: Andrew Murray <andrew.murray@arm.com> > Sent: 2019年9月2日 21:37 > To: Xiaowei Bao <xiaowei.bao@nxp.com> > Cc: Kishon Vijay Abraham I <kishon@ti.com>; bhelgaas@google.com; > robh+dt@kernel.org; mark.rutland@arm.com; shawnguo@kernel.org; Leo Li > <leoyang.li@nxp.com>; lorenzo.pieralisi@arm.co > <lorenzo.pieralisi@arm.com>; arnd@arndb.de; gregkh@linuxfoundation.org; > M.h. Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; > Roy Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 06/10] PCI: layerscape: Modify the way of getting > capability with different PEX > > On Fri, Aug 23, 2019 at 04:13:30AM +0000, Xiaowei Bao wrote: > > > > > > > -----Original Message----- > > > From: Kishon Vijay Abraham I <kishon@ti.com> > > > Sent: 2019年8月23日 11:40 > > > To: Xiaowei Bao <xiaowei.bao@nxp.com>; bhelgaas@google.com; > > > robh+dt@kernel.org; mark.rutland@arm.com; shawnguo@kernel.org; Leo > > > robh+Li > > > <leoyang.li@nxp.com>; lorenzo.pieralisi@arm.co > > > <lorenzo.pieralisi@arm.com>; arnd@arndb.de; > > > gregkh@linuxfoundation.org; M.h. Lian <minghuan.lian@nxp.com>; > > > Mingkai Hu <mingkai.hu@nxp.com>; Roy Zang <roy.zang@nxp.com>; > > > jingoohan1@gmail.com; gustavo.pimentel@synopsys.com; > > > linux-pci@vger.kernel.org; devicetree@vger.kernel.org; > > > linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org; > > > linuxppc-dev@lists.ozlabs.org; andrew.murray@arm.com > > > Subject: Re: [PATCH v2 06/10] PCI: layerscape: Modify the way of > > > getting capability with different PEX > > > > > > Hi, > > > > > > (Fixed Lorenzo's email address. All the patches in the series have > > > wrong email > > > id) > > > > > > On 23/08/19 8:09 AM, Xiaowei Bao wrote: > > > > > > > > > > > >> -----Original Message----- > > > >> From: Kishon Vijay Abraham I <kishon@ti.com> > > > >> Sent: 2019年8月22日 19:44 > > > >> To: Xiaowei Bao <xiaowei.bao@nxp.com>; bhelgaas@google.com; > > > >> robh+dt@kernel.org; mark.rutland@arm.com; shawnguo@kernel.org; > > > >> robh+Leo > > > Li > > > >> <leoyang.li@nxp.com>; lorenzo.pieralisi@arm.co; arnd@arndb.de; > > > >> gregkh@linuxfoundation.org; M.h. Lian <minghuan.lian@nxp.com>; > > > >> Mingkai Hu <mingkai.hu@nxp.com>; Roy Zang <roy.zang@nxp.com>; > > > >> jingoohan1@gmail.com; gustavo.pimentel@synopsys.com; > > > >> linux-pci@vger.kernel.org; devicetree@vger.kernel.org; > > > >> linux-kernel@vger.kernel.org; > > > >> linux-arm-kernel@lists.infradead.org; > > > >> linuxppc-dev@lists.ozlabs.org; andrew.murray@arm.com > > > >> Subject: Re: [PATCH v2 06/10] PCI: layerscape: Modify the way of > > > >> getting capability with different PEX > > > >> > > > >> Hi, > > > >> > > > >> On 22/08/19 4:52 PM, Xiaowei Bao wrote: > > > >>> The different PCIe controller in one board may be have different > > > >>> capability of MSI or MSIX, so change the way of getting the MSI > > > >>> capability, make it more flexible. > > > >> > > > >> please use different pci_epc_features table for different boards. > > > > Thanks, I think that it will be more flexible to dynamically get > > > > MSI or MSIX capability, Thus, we will not need to define the > > > > pci_epc_feature for > > > different boards. > > > > > > Is the restriction because you cannot have different compatible for > > > different boards? > > Sorry, I am not very clear what your mean, I think even if I use the > > same compatible with different boards, each boards will enter the > > probe function, in there I will get the MSI or MSIX PCIe capability of > > the current controller in this board. Why do I need to define the > pci_epc_feature for different boards? > > At present you determine how to set the [msi,msix]_capable flags of > pci_epc_features based on reading the function capabilities at probe time. > Instead of doing this, is it possible that you can determine the flags based on > the compatible type alone? For example, is the MSI/MSIX capability the same > for all fsl,ls2088a-pcie-ep devices? > > If it isn't *necessary* to probe for this information at probe time, then you > could instead create a static pci_epc_features structure and assign it to > something in your drvdata. This may provide some benefits. > > The dw_pcie_ep_get_features function would then look like: > > static const struct pci_epc_features* > ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { > struct dw_pcie *pci = to_dw_pcie_from_pp(ep); > struct ls_pcie_ep *pcie = dev_get_drvdata(pci->dev); > return pcie->epc_features; > } > > This also means you can revert "[v3,03/11] PCI: designware-ep: Move the". > > Is this what you had in mind Kishon? Yes, I consider this scheme, but there is a issue with my board, e.g. my board have three PCIE controllers, but only two controllers support MSI, I can't said that the board support the MSI feature, so I only set the msi_capabitily by reading the MSI capability struct the current PCIE controller, I am also very entangled in this issue. so, do you have better advice? Thanks a lot. Thanks Xiaowei > > Thanks, > > Andrew Murray > > > > > > > Thanks > > > Kishon > > > > > > >> > > > >> Thanks > > > >> Kishon > > > >>> > > > >>> Signed-off-by: Xiaowei Bao <xiaowei.bao@nxp.com> > > > >>> --- > > > >>> v2: > > > >>> - Remove the repeated assignment code. > > > >>> > > > >>> drivers/pci/controller/dwc/pci-layerscape-ep.c | 26 > > > >>> +++++++++++++++++++------- > > > >>> 1 file changed, 19 insertions(+), 7 deletions(-) > > > >>> > > > >>> diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > >>> b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > >>> index 4e92a95..8461f62 100644 > > > >>> --- a/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > >>> +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c > > > >>> @@ -22,6 +22,7 @@ > > > >>> > > > >>> struct ls_pcie_ep { > > > >>> struct dw_pcie *pci; > > > >>> + struct pci_epc_features *ls_epc; > > > >>> }; > > > >>> > > > >>> #define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) > > > >>> @@ -40,25 +41,26 @@ static const struct of_device_id > > > >> ls_pcie_ep_of_match[] = { > > > >>> { }, > > > >>> }; > > > >>> > > > >>> -static const struct pci_epc_features ls_pcie_epc_features = { > > > >>> - .linkup_notifier = false, > > > >>> - .msi_capable = true, > > > >>> - .msix_capable = false, > > > >>> -}; > > > >>> - > > > >>> static const struct pci_epc_features* > > > >>> ls_pcie_ep_get_features(struct dw_pcie_ep *ep) { > > > >>> - return &ls_pcie_epc_features; > > > >>> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > > >>> + > > > >>> + return pcie->ls_epc; > > > >>> } > > > >>> > > > >>> static void ls_pcie_ep_init(struct dw_pcie_ep *ep) { > > > >>> struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > > >>> + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); > > > >>> enum pci_barno bar; > > > >>> > > > >>> for (bar = BAR_0; bar <= BAR_5; bar++) > > > >>> dw_pcie_ep_reset_bar(pci, bar); > > > >>> + > > > >>> + pcie->ls_epc->msi_capable = ep->msi_cap ? true : false; > > > >>> + pcie->ls_epc->msix_capable = ep->msix_cap ? true : false; > > > >>> } > > > >>> > > > >>> static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 > > > >>> func_no, @@ > > > >>> -118,6 +120,7 @@ static int __init ls_pcie_ep_probe(struct > > > >>> platform_device > > > >> *pdev) > > > >>> struct device *dev = &pdev->dev; > > > >>> struct dw_pcie *pci; > > > >>> struct ls_pcie_ep *pcie; > > > >>> + struct pci_epc_features *ls_epc; > > > >>> struct resource *dbi_base; > > > >>> int ret; > > > >>> > > > >>> @@ -129,6 +132,10 @@ static int __init ls_pcie_ep_probe(struct > > > >> platform_device *pdev) > > > >>> if (!pci) > > > >>> return -ENOMEM; > > > >>> > > > >>> + ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL); > > > >>> + if (!ls_epc) > > > >>> + return -ENOMEM; > > > >>> + > > > >>> dbi_base = platform_get_resource_byname(pdev, > > > IORESOURCE_MEM, > > > >> "regs"); > > > >>> pci->dbi_base = devm_pci_remap_cfg_resource(dev, > dbi_base); > > > >>> if (IS_ERR(pci->dbi_base)) > > > >>> @@ -139,6 +146,11 @@ static int __init ls_pcie_ep_probe(struct > > > >> platform_device *pdev) > > > >>> pci->ops = &ls_pcie_ep_ops; > > > >>> pcie->pci = pci; > > > >>> > > > >>> + ls_epc->linkup_notifier = false, > > > >>> + ls_epc->bar_fixed_64bit = (1 << BAR_2) | (1 << BAR_4), > > > >>> + > > > >>> + pcie->ls_epc = ls_epc; > > > >>> + > > > >>> platform_set_drvdata(pdev, pcie); > > > >>> > > > >>> ret = ls_add_pcie_ep(pcie, pdev); > > > >>>
On Thu, Aug 29, 2019 at 10:43:18AM +0530, Kishon Vijay Abraham I wrote:
> Gustavo,
>
> On 27/08/19 6:55 PM, Andrew Murray wrote:
> > On Sat, Aug 24, 2019 at 12:08:40AM +0000, Xiaowei Bao wrote:
> >>
> >>
> >>> -----Original Message-----
> >>> From: Andrew Murray <andrew.murray@arm.com>
> >>> Sent: 2019年8月23日 21:58
> >>> To: Xiaowei Bao <xiaowei.bao@nxp.com>
> >>> Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com;
> >>> shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com;
> >>> lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h.
> >>> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy
> >>> Zang <roy.zang@nxp.com>; jingoohan1@gmail.com;
> >>> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org;
> >>> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org;
> >>> linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org
> >>> Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to the
> >>> doorbell way
> >>>
> >>> On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote:
> >>>> The layerscape platform use the doorbell way to trigger MSIX interrupt
> >>>> in EP mode.
> >>>>
> >>>
> >>> I have no problems with this patch, however...
> >>>
> >>> Are you able to add to this message a reason for why you are making this
> >>> change? Did dw_pcie_ep_raise_msix_irq not work when func_no != 0? Or did
> >>> it work yet dw_pcie_ep_raise_msix_irq_doorbell is more efficient?
> >>
> >> The fact is that, this driver is verified in ls1046a platform of NXP before, and ls1046a don't
> >> support MSIX feature, so I set the msix_capable of pci_epc_features struct is false,
> >> but in other platform, e.g. ls1088a, it support the MSIX feature, I verified the MSIX
> >> feature in ls1088a, it is not OK, so I changed to another way. Thanks.
> >
> > Right, so the existing pci-layerscape-ep.c driver never supported MSIX yet it
> > erroneously had a switch case statement to call dw_pcie_ep_raise_msix_irq which
> > would never get used.
> >
> > Now that we're adding a platform with MSIX support the existing
> > dw_pcie_ep_raise_msix_irq doesn't work (for this platform) so we are adding a
> > different method.
>
> Gustavo, can you confirm dw_pcie_ep_raise_msix_irq() works for
> designware as it didn't work for both me and Xiaowei?
This question needs an answer.
Thanks,
Lorenzo
> -----Original Message----- > From: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com> > Sent: 2019年11月5日 20:38 > To: Kishon Vijay Abraham I <kishon@ti.com> > Cc: Andrew Murray <andrew.murray@arm.com>; Xiaowei Bao > <xiaowei.bao@nxp.com>; gustavo.pimentel@synopsys.com; > bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > linux-pci@vger.kernel.org; devicetree@vger.kernel.org; > linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org; > linuxppc-dev@lists.ozlabs.org > Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to the > doorbell way > > On Thu, Aug 29, 2019 at 10:43:18AM +0530, Kishon Vijay Abraham I wrote: > > Gustavo, > > > > On 27/08/19 6:55 PM, Andrew Murray wrote: > > > On Sat, Aug 24, 2019 at 12:08:40AM +0000, Xiaowei Bao wrote: > > >> > > >> > > >>> -----Original Message----- > > >>> From: Andrew Murray <andrew.murray@arm.com> > > >>> Sent: 2019年8月23日 21:58 > > >>> To: Xiaowei Bao <xiaowei.bao@nxp.com> > > >>> Cc: bhelgaas@google.com; robh+dt@kernel.org; > mark.rutland@arm.com; > > >>> shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > >>> lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; > M.h. > > >>> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; > Roy > > >>> Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > >>> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > >>> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > >>> linux-arm-kernel@lists.infradead.org; > > >>> linuxppc-dev@lists.ozlabs.org > > >>> Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to > > >>> the doorbell way > > >>> > > >>> On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote: > > >>>> The layerscape platform use the doorbell way to trigger MSIX > > >>>> interrupt in EP mode. > > >>>> > > >>> > > >>> I have no problems with this patch, however... > > >>> > > >>> Are you able to add to this message a reason for why you are > > >>> making this change? Did dw_pcie_ep_raise_msix_irq not work when > > >>> func_no != 0? Or did it work yet dw_pcie_ep_raise_msix_irq_doorbell is > more efficient? > > >> > > >> The fact is that, this driver is verified in ls1046a platform of > > >> NXP before, and ls1046a don't support MSIX feature, so I set the > > >> msix_capable of pci_epc_features struct is false, but in other > > >> platform, e.g. ls1088a, it support the MSIX feature, I verified the MSIX > feature in ls1088a, it is not OK, so I changed to another way. Thanks. > > > > > > Right, so the existing pci-layerscape-ep.c driver never supported > > > MSIX yet it erroneously had a switch case statement to call > > > dw_pcie_ep_raise_msix_irq which would never get used. > > > > > > Now that we're adding a platform with MSIX support the existing > > > dw_pcie_ep_raise_msix_irq doesn't work (for this platform) so we are > > > adding a different method. > > > > Gustavo, can you confirm dw_pcie_ep_raise_msix_irq() works for > > designware as it didn't work for both me and Xiaowei? > > This question needs an answer. This question have answered by Gustavo in "[PATCH v3 08/11] PCI: layerscape: Modify the MSIX to the doorbell mode", please refer to the comments, due to add a new patch for this sets of patch, this patch number changed from 07/10 of v2 to 08/11 of v3, sorry for causing your confuse. Thanks Xiaowei > > Thanks, > Lorenzo
On Thu, Aug 29, 2019 at 6:13:18, Kishon Vijay Abraham I <kishon@ti.com> wrote: Hi, this email slip away from my attention... > Gustavo, > > On 27/08/19 6:55 PM, Andrew Murray wrote: > > On Sat, Aug 24, 2019 at 12:08:40AM +0000, Xiaowei Bao wrote: > >> > >> > >>> -----Original Message----- > >>> From: Andrew Murray <andrew.murray@arm.com> > >>> Sent: 2019年8月23日 21:58 > >>> To: Xiaowei Bao <xiaowei.bao@nxp.com> > >>> Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > >>> shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > >>> lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > >>> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > >>> Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > >>> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > >>> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > >>> linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > >>> Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to the > >>> doorbell way > >>> > >>> On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote: > >>>> The layerscape platform use the doorbell way to trigger MSIX interrupt > >>>> in EP mode. > >>>> > >>> > >>> I have no problems with this patch, however... > >>> > >>> Are you able to add to this message a reason for why you are making this > >>> change? Did dw_pcie_ep_raise_msix_irq not work when func_no != 0? Or did > >>> it work yet dw_pcie_ep_raise_msix_irq_doorbell is more efficient? > >> > >> The fact is that, this driver is verified in ls1046a platform of NXP before, and ls1046a don't > >> support MSIX feature, so I set the msix_capable of pci_epc_features struct is false, > >> but in other platform, e.g. ls1088a, it support the MSIX feature, I verified the MSIX > >> feature in ls1088a, it is not OK, so I changed to another way. Thanks. > > > > Right, so the existing pci-layerscape-ep.c driver never supported MSIX yet it > > erroneously had a switch case statement to call dw_pcie_ep_raise_msix_irq which > > would never get used. > > > > Now that we're adding a platform with MSIX support the existing > > dw_pcie_ep_raise_msix_irq doesn't work (for this platform) so we are adding a > > different method. > > Gustavo, can you confirm dw_pcie_ep_raise_msix_irq() works for designware as it > didn't work for both me and Xiaowei? When I implemented the dw_pcie_ep_raise_msix_irq(), the implementation was working quite fine on DesignWare solution. Otherwise, I wouldn't submit it to the kernel. From what I have seen and if I recall well, Xiaowei implementation was done having PF's configurated on his solution, which is a configuration that I don't have in my solution, I believe this could be the missing piece that differs between our 2 implementations. Since patch submission into the kernel related to msix feature on pcitest tool, I didn't touch or re-tested the msix feature by lack of time (other projects requires my full attention for now). However is on my roadmap to came back to add some other features on DesignWare eDMA driver and I can do at that time some tests to see if the dw_pcie_ep_raise_msix_irq_doorbell() is compatible or not with my solution. If so, I can do some patch to simplify and use the dw_pcie_ep_raise_msix_irq_doorbell() if it still works as expected like on dw_pcie_ep_raise_msix_irq(). Agree? Gustavo > > Thanks > Kishon
> -----Original Message----- > From: Gustavo Pimentel <Gustavo.Pimentel@synopsys.com> > Sent: 2019年11月6日 17:40 > To: Kishon Vijay Abraham I <kishon@ti.com>; Andrew Murray > <andrew.murray@arm.com>; Xiaowei Bao <xiaowei.bao@nxp.com>; > gustavo.pimentel@synopsys.com > Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; > lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > linux-pci@vger.kernel.org; devicetree@vger.kernel.org; > linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org; > linuxppc-dev@lists.ozlabs.org > Subject: RE: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to the > doorbell way > > On Thu, Aug 29, 2019 at 6:13:18, Kishon Vijay Abraham I <kishon@ti.com> > wrote: > > Hi, this email slip away from my attention... > > > Gustavo, > > > > On 27/08/19 6:55 PM, Andrew Murray wrote: > > > On Sat, Aug 24, 2019 at 12:08:40AM +0000, Xiaowei Bao wrote: > > >> > > >> > > >>> -----Original Message----- > > >>> From: Andrew Murray <andrew.murray@arm.com> > > >>> Sent: 2019年8月23日 21:58 > > >>> To: Xiaowei Bao <xiaowei.bao@nxp.com> > > >>> Cc: bhelgaas@google.com; robh+dt@kernel.org; > mark.rutland@arm.com; > > >>> shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > > >>> lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; > M.h. > > >>> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; > Roy > > >>> Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > > >>> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > > >>> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > > >>> linux-arm-kernel@lists.infradead.org; > > >>> linuxppc-dev@lists.ozlabs.org > > >>> Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to > > >>> the doorbell way > > >>> > > >>> On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote: > > >>>> The layerscape platform use the doorbell way to trigger MSIX > > >>>> interrupt in EP mode. > > >>>> > > >>> > > >>> I have no problems with this patch, however... > > >>> > > >>> Are you able to add to this message a reason for why you are > > >>> making this change? Did dw_pcie_ep_raise_msix_irq not work when > > >>> func_no != 0? Or did it work yet dw_pcie_ep_raise_msix_irq_doorbell is > more efficient? > > >> > > >> The fact is that, this driver is verified in ls1046a platform of > > >> NXP before, and ls1046a don't support MSIX feature, so I set the > > >> msix_capable of pci_epc_features struct is false, but in other > > >> platform, e.g. ls1088a, it support the MSIX feature, I verified the MSIX > feature in ls1088a, it is not OK, so I changed to another way. Thanks. > > > > > > Right, so the existing pci-layerscape-ep.c driver never supported > > > MSIX yet it erroneously had a switch case statement to call > > > dw_pcie_ep_raise_msix_irq which would never get used. > > > > > > Now that we're adding a platform with MSIX support the existing > > > dw_pcie_ep_raise_msix_irq doesn't work (for this platform) so we are > > > adding a different method. > > > > Gustavo, can you confirm dw_pcie_ep_raise_msix_irq() works for > > designware as it didn't work for both me and Xiaowei? > > When I implemented the dw_pcie_ep_raise_msix_irq(), the implementation > was working quite fine on DesignWare solution. Otherwise, I wouldn't submit > it to the kernel. > From what I have seen and if I recall well, Xiaowei implementation was done > having PF's configurated on his solution, which is a configuration that I don't > have in my solution, I believe this could be the missing piece that differs > between our 2 implementations. > > Since patch submission into the kernel related to msix feature on pcitest tool, I > didn't touch or re-tested the msix feature by lack of time (other projects > requires my full attention for now). However is on my roadmap to came back > to add some other features on DesignWare eDMA driver and I can do at that > time some tests to see if the > dw_pcie_ep_raise_msix_irq_doorbell() is compatible or not with my solution. > If so, I can do some patch to simplify and use the > dw_pcie_ep_raise_msix_irq_doorbell() if it still works as expected like on > dw_pcie_ep_raise_msix_irq(). Agree? > Thanks Gustavo, it looks well for me. Thanks Xiaowei > Gustavo > > > > > Thanks > > Kishon >
Gustavo, On 06/11/19 3:10 PM, Gustavo Pimentel wrote: > On Thu, Aug 29, 2019 at 6:13:18, Kishon Vijay Abraham I <kishon@ti.com> > wrote: > > Hi, this email slip away from my attention... > >> Gustavo, >> >> On 27/08/19 6:55 PM, Andrew Murray wrote: >>> On Sat, Aug 24, 2019 at 12:08:40AM +0000, Xiaowei Bao wrote: >>>> >>>> >>>>> -----Original Message----- >>>>> From: Andrew Murray <andrew.murray@arm.com> >>>>> Sent: 2019年8月23日 21:58 >>>>> To: Xiaowei Bao <xiaowei.bao@nxp.com> >>>>> Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; >>>>> shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; >>>>> lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. >>>>> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy >>>>> Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; >>>>> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; >>>>> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; >>>>> linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org >>>>> Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to the >>>>> doorbell way >>>>> >>>>> On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote: >>>>>> The layerscape platform use the doorbell way to trigger MSIX interrupt >>>>>> in EP mode. >>>>>> >>>>> >>>>> I have no problems with this patch, however... >>>>> >>>>> Are you able to add to this message a reason for why you are making this >>>>> change? Did dw_pcie_ep_raise_msix_irq not work when func_no != 0? Or did >>>>> it work yet dw_pcie_ep_raise_msix_irq_doorbell is more efficient? >>>> >>>> The fact is that, this driver is verified in ls1046a platform of NXP before, and ls1046a don't >>>> support MSIX feature, so I set the msix_capable of pci_epc_features struct is false, >>>> but in other platform, e.g. ls1088a, it support the MSIX feature, I verified the MSIX >>>> feature in ls1088a, it is not OK, so I changed to another way. Thanks. >>> >>> Right, so the existing pci-layerscape-ep.c driver never supported MSIX yet it >>> erroneously had a switch case statement to call dw_pcie_ep_raise_msix_irq which >>> would never get used. >>> >>> Now that we're adding a platform with MSIX support the existing >>> dw_pcie_ep_raise_msix_irq doesn't work (for this platform) so we are adding a >>> different method. >> >> Gustavo, can you confirm dw_pcie_ep_raise_msix_irq() works for designware as it >> didn't work for both me and Xiaowei? > > When I implemented the dw_pcie_ep_raise_msix_irq(), the implementation > was working quite fine on DesignWare solution. Otherwise, I wouldn't > submit it to the kernel. > From what I have seen and if I recall well, Xiaowei implementation was > done having PF's configurated on his solution, which is a configuration > that I don't have in my solution, I believe this could be the missing > piece that differs between our 2 implementations. I haven't debugged the issue yet but in my understanding the MSI-X table should be in the memory (DDR) of EP system. This table will be populated by RC while configuring MSI-X (with msg address and msg data). The EP will use the populated msg address and msg data for raising MSI-X interrupt. From the dw_pcie_ep_raise_msix_irq() (copied below), nowhere the MSI-X table is being read from the memory of EP system. I've given my comments below. int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, u16 interrupt_num) { . . reg = PCI_BASE_ADDRESS_0 + (4 * bir); bar_addr_upper = 0; bar_addr_lower = dw_pcie_readl_dbi(pci, reg); BAR register will hold the "PCI address" programmed by the host. So "bar_addr_lower" will have PCI address. reg_u64 = (bar_addr_lower & PCI_BASE_ADDRESS_MEM_TYPE_MASK); if (reg_u64 == PCI_BASE_ADDRESS_MEM_TYPE_64) bar_addr_upper = dw_pcie_readl_dbi(pci, reg + 4); tbl_addr = ((u64) bar_addr_upper) << 32 | bar_addr_lower; The "tbl_addr" now has the PCI address programmed by the host. tbl_addr += (tbl_offset + ((interrupt_num - 1) * PCI_MSIX_ENTRY_SIZE)); tbl_addr &= PCI_BASE_ADDRESS_MEM_MASK; msix_tbl = ioremap_nocache(ep->phys_base + tbl_addr, PCI_MSIX_ENTRY_SIZE); "ep->phys_base" will have EPs outbound memory address and "tbl_addr" will have PCI address. So msix_tbl points to the EPs outbound memory region. if (!msix_tbl) return -EINVAL; msg_addr_lower = readl(msix_tbl + PCI_MSIX_ENTRY_LOWER_ADDR); msg_addr_upper = readl(msix_tbl + PCI_MSIX_ENTRY_UPPER_ADDR); Here an access to the EP outbound region is made (and the transaction will be based on ATU configuration). The message address should ideally be obtained from the MSI-X table present in the EP system. There need not be any access to the OB region for getting data from MSI-X table. msg_addr = ((u64) msg_addr_upper) << 32 | msg_addr_lower; msg_data = readl(msix_tbl + PCI_MSIX_ENTRY_DATA); vec_ctrl = readl(msix_tbl + PCI_MSIX_ENTRY_VECTOR_CTRL); All this should be obtained from the memory of EP. . . } I'm not sure how this worked for you. Thanks Kishon > > Since patch submission into the kernel related to msix feature on pcitest > tool, I didn't touch or re-tested the msix feature by lack of time (other > projects requires my full attention for now). However is on my roadmap to > came back to add some other features on DesignWare eDMA driver and I can > do at that time some tests to see if the > dw_pcie_ep_raise_msix_irq_doorbell() is compatible or not with my > solution. If so, I can do some patch to simplify and use the > dw_pcie_ep_raise_msix_irq_doorbell() if it still works as expected like > on dw_pcie_ep_raise_msix_irq(). Agree? > > Gustavo > >> >> Thanks >> Kishon > >
On Wed, Nov 6, 2019 at 13:39:5, Kishon Vijay Abraham I <kishon@ti.com> wrote: Hi Kishon, > Gustavo, > > On 06/11/19 3:10 PM, Gustavo Pimentel wrote: > > On Thu, Aug 29, 2019 at 6:13:18, Kishon Vijay Abraham I <kishon@ti.com> > > wrote: > > > > Hi, this email slip away from my attention... > > > >> Gustavo, > >> > >> On 27/08/19 6:55 PM, Andrew Murray wrote: > >>> On Sat, Aug 24, 2019 at 12:08:40AM +0000, Xiaowei Bao wrote: > >>>> > >>>> > >>>>> -----Original Message----- > >>>>> From: Andrew Murray <andrew.murray@arm.com> > >>>>> Sent: 2019年8月23日 21:58 > >>>>> To: Xiaowei Bao <xiaowei.bao@nxp.com> > >>>>> Cc: bhelgaas@google.com; robh+dt@kernel.org; mark.rutland@arm.com; > >>>>> shawnguo@kernel.org; Leo Li <leoyang.li@nxp.com>; kishon@ti.com; > >>>>> lorenzo.pieralisi@arm.co; arnd@arndb.de; gregkh@linuxfoundation.org; M.h. > >>>>> Lian <minghuan.lian@nxp.com>; Mingkai Hu <mingkai.hu@nxp.com>; Roy > >>>>> Zang <roy.zang@nxp.com>; jingoohan1@gmail.com; > >>>>> gustavo.pimentel@synopsys.com; linux-pci@vger.kernel.org; > >>>>> devicetree@vger.kernel.org; linux-kernel@vger.kernel.org; > >>>>> linux-arm-kernel@lists.infradead.org; linuxppc-dev@lists.ozlabs.org > >>>>> Subject: Re: [PATCH v2 07/10] PCI: layerscape: Modify the MSIX to the > >>>>> doorbell way > >>>>> > >>>>> On Thu, Aug 22, 2019 at 07:22:39PM +0800, Xiaowei Bao wrote: > >>>>>> The layerscape platform use the doorbell way to trigger MSIX interrupt > >>>>>> in EP mode. > >>>>>> > >>>>> > >>>>> I have no problems with this patch, however... > >>>>> > >>>>> Are you able to add to this message a reason for why you are making this > >>>>> change? Did dw_pcie_ep_raise_msix_irq not work when func_no != 0? Or did > >>>>> it work yet dw_pcie_ep_raise_msix_irq_doorbell is more efficient? > >>>> > >>>> The fact is that, this driver is verified in ls1046a platform of NXP before, and ls1046a don't > >>>> support MSIX feature, so I set the msix_capable of pci_epc_features struct is false, > >>>> but in other platform, e.g. ls1088a, it support the MSIX feature, I verified the MSIX > >>>> feature in ls1088a, it is not OK, so I changed to another way. Thanks. > >>> > >>> Right, so the existing pci-layerscape-ep.c driver never supported MSIX yet it > >>> erroneously had a switch case statement to call dw_pcie_ep_raise_msix_irq which > >>> would never get used. > >>> > >>> Now that we're adding a platform with MSIX support the existing > >>> dw_pcie_ep_raise_msix_irq doesn't work (for this platform) so we are adding a > >>> different method. > >> > >> Gustavo, can you confirm dw_pcie_ep_raise_msix_irq() works for designware as it > >> didn't work for both me and Xiaowei? > > > > When I implemented the dw_pcie_ep_raise_msix_irq(), the implementation > > was working quite fine on DesignWare solution. Otherwise, I wouldn't > > submit it to the kernel. > > From what I have seen and if I recall well, Xiaowei implementation was > > done having PF's configurated on his solution, which is a configuration > > that I don't have in my solution, I believe this could be the missing > > piece that differs between our 2 implementations. > > I haven't debugged the issue yet but in my understanding the MSI-X table should > be in the memory (DDR) of EP system. This table will be populated by RC while > configuring MSI-X (with msg address and msg data). The EP will use the > populated msg address and msg data for raising MSI-X interrupt. Right. > > From the dw_pcie_ep_raise_msix_irq() (copied below), nowhere the MSI-X table is > being read from the memory of EP system. I've given my comments below. > > int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, > u16 interrupt_num) > { > . > . > reg = PCI_BASE_ADDRESS_0 + (4 * bir); > bar_addr_upper = 0; > bar_addr_lower = dw_pcie_readl_dbi(pci, reg); > > BAR register will hold the "PCI address" programmed by the host. So > "bar_addr_lower" will have PCI address. > > reg_u64 = (bar_addr_lower & PCI_BASE_ADDRESS_MEM_TYPE_MASK); > if (reg_u64 == PCI_BASE_ADDRESS_MEM_TYPE_64) > bar_addr_upper = dw_pcie_readl_dbi(pci, reg + 4); > > tbl_addr = ((u64) bar_addr_upper) << 32 | bar_addr_lower; > > The "tbl_addr" now has the PCI address programmed by the host. > > tbl_addr += (tbl_offset + ((interrupt_num - 1) * PCI_MSIX_ENTRY_SIZE)); > tbl_addr &= PCI_BASE_ADDRESS_MEM_MASK; > > msix_tbl = ioremap_nocache(ep->phys_base + tbl_addr, > PCI_MSIX_ENTRY_SIZE); > > "ep->phys_base" will have EPs outbound memory address and "tbl_addr" will have > PCI address. So msix_tbl points to the EPs outbound memory region. > if (!msix_tbl) > return -EINVAL; > > msg_addr_lower = readl(msix_tbl + PCI_MSIX_ENTRY_LOWER_ADDR); > msg_addr_upper = readl(msix_tbl + PCI_MSIX_ENTRY_UPPER_ADDR); > > Here an access to the EP outbound region is made (and the transaction will be > based on ATU configuration). > The message address should ideally be obtained from the MSI-X table present in > the EP system. There need not be any access to the OB region for getting data > from MSI-X table. > > msg_addr = ((u64) msg_addr_upper) << 32 | msg_addr_lower; > msg_data = readl(msix_tbl + PCI_MSIX_ENTRY_DATA); > vec_ctrl = readl(msix_tbl + PCI_MSIX_ENTRY_VECTOR_CTRL); > > All this should be obtained from the memory of EP. > . > . > } > > I'm not sure how this worked for you. Hum, it was a very time ago that I implemented this. I've to debug it to give you an accurate answer. However, it very likely that my DesignWare prototype solution might have some peculiarities that could allow this type of access. At the time, nobody had the msix feature enabled that was working with pcitest EP in their SOCs that could be used to test my implementation against, at least that I know of. As soon as I get back to work on PCI, I will retest this with the other function, if it works I'll make a patch to use that function instead, I'm assuming that function is working for other SOCs, right? Gustavo > > Thanks > Kishon > > > > > Since patch submission into the kernel related to msix feature on pcitest > > tool, I didn't touch or re-tested the msix feature by lack of time (other > > projects requires my full attention for now). However is on my roadmap to > > came back to add some other features on DesignWare eDMA driver and I can > > do at that time some tests to see if the > > dw_pcie_ep_raise_msix_irq_doorbell() is compatible or not with my > > solution. If so, I can do some patch to simplify and use the > > dw_pcie_ep_raise_msix_irq_doorbell() if it still works as expected like > > on dw_pcie_ep_raise_msix_irq(). Agree? > > > > Gustavo > > > >> > >> Thanks > >> Kishon > > > >