Re: [PATCH v3 18/26] PCI: endpoint: Add support to allocate aligned buffers to be mapped in BARs

[Kishon Vijay Abraham I <kishon@ti.com>]

Hi Lorenzo,

On 11/04/19 9:02 PM, Lorenzo Pieralisi wrote:
> On Mon, Mar 25, 2019 at 03:09:39PM +0530, Kishon Vijay Abraham I wrote:
>> Modify pci_epf_alloc_space API to take alignment size as argument in
>> order to argument in order to allocate aligned buffers to be mapped to
>> BARs.
>>
>> Add 'align' parameter to epc_features which can be used by platform
>> drivers to specifiy the BAR allocation alignment requirements and use
>> this while invoking pci_epf_alloc_space.
>>
>> This is mainly required for Synopsys Designware PCIe core which masks
>> the lower bits based on the BAR size (See "I/O and MEM Match Modes"
>> section in DesignWare Cores PCI Express Controller Databook version
>> 4.90a).
> 
> Hi Kishon,
> 
> isn't this a generic PCI BAR requirement rather than a Designware
> specific one ?
> 
> The patch is fine I just would like to understand the statement above.

The address that is allocated using pci_epf_alloc_space would be directly
written to the target address of the Inbound address translation unit.
Designware IP has a configuration parameter (CX_ATU_MIN_REGION_SIZE [1]) which
has 64KB as default value and the lower 16 bits of the Base, Limit and Target
registers of the Inbound ATU are fixed to zero.  So if the programmed memory
address is not aligned to 64 KB boundary there will be memory corruption.

PCI spec has restrictions on the minimum memory address range requested by BAR
as 128 Bytes and the size to be a power of 2, however it doesn't have any
restriction on the local endpoint memory addresses that are mapped to BAR. That
is an endpoint controller restriction.

[1] -> http://www.ti.com/lit/ug/spruid7c/spruid7c.pdf

Thanks
Kishon

> Thanks,
> Lorenzo
> 
>> Signed-off-by: Kishon Vijay Abraham I <kishon@ti.com>
>> ---
>>  drivers/pci/endpoint/functions/pci-epf-test.c |  5 +++--
>>  drivers/pci/endpoint/pci-epf-core.c           | 10 ++++++++--
>>  include/linux/pci-epc.h                       |  2 ++
>>  include/linux/pci-epf.h                       |  3 ++-
>>  4 files changed, 15 insertions(+), 5 deletions(-)
>>
>> diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
>> index d0b91da49bf4..c0786ca74312 100644
>> --- a/drivers/pci/endpoint/functions/pci-epf-test.c
>> +++ b/drivers/pci/endpoint/functions/pci-epf-test.c
>> @@ -438,7 +438,7 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
>>  	epc_features = epf_test->epc_features;
>>  
>>  	base = pci_epf_alloc_space(epf, sizeof(struct pci_epf_test_reg),
>> -				   test_reg_bar);
>> +				   test_reg_bar, epc_features->align);
>>  	if (!base) {
>>  		dev_err(dev, "Failed to allocated register space\n");
>>  		return -ENOMEM;
>> @@ -453,7 +453,8 @@ static int pci_epf_test_alloc_space(struct pci_epf *epf)
>>  		if (!!(epc_features->reserved_bar & (1 << bar)))
>>  			continue;
>>  
>> -		base = pci_epf_alloc_space(epf, bar_size[bar], bar);
>> +		base = pci_epf_alloc_space(epf, bar_size[bar], bar,
>> +					   epc_features->align);
>>  		if (!base)
>>  			dev_err(dev, "Failed to allocate space for BAR%d\n",
>>  				bar);
>> diff --git a/drivers/pci/endpoint/pci-epf-core.c b/drivers/pci/endpoint/pci-epf-core.c
>> index 8bfdcd291196..fb1306de8f40 100644
>> --- a/drivers/pci/endpoint/pci-epf-core.c
>> +++ b/drivers/pci/endpoint/pci-epf-core.c
>> @@ -109,10 +109,12 @@ EXPORT_SYMBOL_GPL(pci_epf_free_space);
>>   * pci_epf_alloc_space() - allocate memory for the PCI EPF register space
>>   * @size: the size of the memory that has to be allocated
>>   * @bar: the BAR number corresponding to the allocated register space
>> + * @align: alignment size for the allocation region
>>   *
>>   * Invoke to allocate memory for the PCI EPF register space.
>>   */
>> -void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar)
>> +void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
>> +			  size_t align)
>>  {
>>  	void *space;
>>  	struct device *dev = epf->epc->dev.parent;
>> @@ -120,7 +122,11 @@ void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar)
>>  
>>  	if (size < 128)
>>  		size = 128;
>> -	size = roundup_pow_of_two(size);
>> +
>> +	if (align)
>> +		size = ALIGN(size, align);
>> +	else
>> +		size = roundup_pow_of_two(size);
>>  
>>  	space = dma_alloc_coherent(dev, size, &phys_addr, GFP_KERNEL);
>>  	if (!space) {
>> diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h
>> index c3ffa3917f88..f641badc2c61 100644
>> --- a/include/linux/pci-epc.h
>> +++ b/include/linux/pci-epc.h
>> @@ -109,6 +109,7 @@ struct pci_epc {
>>   * @reserved_bar: bitmap to indicate reserved BAR unavailable to function driver
>>   * @bar_fixed_64bit: bitmap to indicate fixed 64bit BARs
>>   * @bar_fixed_size: Array specifying the size supported by each BAR
>> + * @align: alignment size required for BAR buffer allocation
>>   */
>>  struct pci_epc_features {
>>  	unsigned int	linkup_notifier : 1;
>> @@ -117,6 +118,7 @@ struct pci_epc_features {
>>  	u8	reserved_bar;
>>  	u8	bar_fixed_64bit;
>>  	u64	bar_fixed_size[BAR_5 + 1];
>> +	size_t	align;
>>  };
>>  
>>  #define to_pci_epc(device) container_of((device), struct pci_epc, dev)
>> diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h
>> index ec02f58758c8..2d6f07556682 100644
>> --- a/include/linux/pci-epf.h
>> +++ b/include/linux/pci-epf.h
>> @@ -149,7 +149,8 @@ void pci_epf_destroy(struct pci_epf *epf);
>>  int __pci_epf_register_driver(struct pci_epf_driver *driver,
>>  			      struct module *owner);
>>  void pci_epf_unregister_driver(struct pci_epf_driver *driver);
>> -void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar);
>> +void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
>> +			  size_t align);
>>  void pci_epf_free_space(struct pci_epf *epf, void *addr, enum pci_barno bar);
>>  int pci_epf_bind(struct pci_epf *epf);
>>  void pci_epf_unbind(struct pci_epf *epf);
>> -- 
>> 2.17.1
>>