[PATCH V3 0/2] vDPA/ifcvf: implement doorbell mapping feature

[PATCH V3 0/2] vDPA/ifcvf: implement doorbell mapping feature

[Zhu Lingshan]

This series implements doorbell mapping feature for ifcvf.

Please help review

Thanks!
Changes from V2:
assign notify_off_multiplier to notification.size first than
use PAGE_SIZE directly(Jason)

Chagnes from V1:
calculate the doorbell address per vq than per device(Jason)
let upper layer driver decide how to use the non page_size
aligned doorbell(Jason)

Zhu Lingshan (2):
  vDPA/ifcvf: record virtio notify base
  vDPA/ifcvf: implement doorbell mapping for ifcvf

 drivers/vdpa/ifcvf/ifcvf_base.c |  4 ++++
 drivers/vdpa/ifcvf/ifcvf_base.h |  2 ++
 drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
 3 files changed, 27 insertions(+)

-- 
2.27.0

[PATCH V3 1/2] vDPA/ifcvf: record virtio notify base

[Zhu Lingshan]

This commit records virtio notify base physical addr and
calculate doorbell physical address for vqs.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/ifcvf/ifcvf_base.c | 4 ++++
 drivers/vdpa/ifcvf/ifcvf_base.h | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
index 1a661ab45af5..6e197fe0fcf9 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.c
+++ b/drivers/vdpa/ifcvf/ifcvf_base.c
@@ -133,6 +133,8 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
 					      &hw->notify_off_multiplier);
 			hw->notify_bar = cap.bar;
 			hw->notify_base = get_cap_addr(hw, &cap);
+			hw->notify_base_pa = pci_resource_start(pdev, cap.bar) +
+					le32_to_cpu(cap.offset);
 			IFCVF_DBG(pdev, "hw->notify_base = %p\n",
 				  hw->notify_base);
 			break;
@@ -161,6 +163,8 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
 		notify_off = ifc_ioread16(&hw->common_cfg->queue_notify_off);
 		hw->vring[i].notify_addr = hw->notify_base +
 			notify_off * hw->notify_off_multiplier;
+		hw->vring[i].notify_pa = hw->notify_base_pa +
+			notify_off * hw->notify_off_multiplier;
 	}
 
 	hw->lm_cfg = hw->base[IFCVF_LM_BAR];
diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
index 0111bfdeb342..447f4ad9c0bf 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.h
+++ b/drivers/vdpa/ifcvf/ifcvf_base.h
@@ -73,6 +73,7 @@ struct vring_info {
 	u16 last_avail_idx;
 	bool ready;
 	void __iomem *notify_addr;
+	phys_addr_t notify_pa;
 	u32 irq;
 	struct vdpa_callback cb;
 	char msix_name[256];
@@ -87,6 +88,7 @@ struct ifcvf_hw {
 	u8 notify_bar;
 	/* Notificaiton bar address */
 	void __iomem *notify_base;
+	phys_addr_t notify_base_pa;
 	u32 notify_off_multiplier;
 	u64 req_features;
 	u64 hw_features;
-- 
2.27.0

[PATCH V3 2/2] vDPA/ifcvf: implement doorbell mapping for ifcvf

[Zhu Lingshan]

This commit implements doorbell mapping feature for ifcvf.
This feature maps the notify page to userspace, to eliminate
vmexit when kick a vq.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index ab0ab5cf0f6e..d41db042612c 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,
 	return vf->vring[qid].irq;
 }
 
+static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+							       u16 idx)
+{
+	struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+	struct pci_dev *pdev = adapter->pdev;
+	struct vdpa_notification_area area;
+
+	area.addr = vf->vring[idx].notify_pa;
+	if (!vf->notify_off_multiplier)
+		area.size = PAGE_SIZE;
+	else
+		area.size = vf->notify_off_multiplier;
+
+	if (area.addr % PAGE_SIZE)
+		IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE aligned\n", idx);
+
+	return area;
+}
+
 /*
  * IFCVF currently does't have on-chip IOMMU, so not
  * implemented set_map()/dma_map()/dma_unmap()
@@ -440,6 +460,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
 	.get_config	= ifcvf_vdpa_get_config,
 	.set_config	= ifcvf_vdpa_set_config,
 	.set_config_cb  = ifcvf_vdpa_set_config_cb,
+	.get_vq_notification = ifcvf_get_vq_notification,
 };
 
 static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
-- 
2.27.0

Re: [PATCH V3 2/2] vDPA/ifcvf: implement doorbell mapping for ifcvf

[Jason Wang]

在 2021/6/1 下午2:28, Zhu Lingshan 写道:
> This commit implements doorbell mapping feature for ifcvf.
> This feature maps the notify page to userspace, to eliminate
> vmexit when kick a vq.
>
> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
> ---
>   drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
>   1 file changed, 21 insertions(+)
>
> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
> index ab0ab5cf0f6e..d41db042612c 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
> @@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,
>   	return vf->vring[qid].irq;
>   }
>   
> +static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
> +							       u16 idx)
> +{
> +	struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
> +	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
> +	struct pci_dev *pdev = adapter->pdev;
> +	struct vdpa_notification_area area;
> +
> +	area.addr = vf->vring[idx].notify_pa;
> +	if (!vf->notify_off_multiplier)
> +		area.size = PAGE_SIZE;
> +	else
> +		area.size = vf->notify_off_multiplier;
> +
> +	if (area.addr % PAGE_SIZE)
> +		IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE aligned\n", idx);


I don't see the reason to keep this, or get_notification is not the 
proper place to do this kind of warning.

Thanks


> +
> +	return area;
> +}
> +
>   /*
>    * IFCVF currently does't have on-chip IOMMU, so not
>    * implemented set_map()/dma_map()/dma_unmap()
> @@ -440,6 +460,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
>   	.get_config	= ifcvf_vdpa_get_config,
>   	.set_config	= ifcvf_vdpa_set_config,
>   	.set_config_cb  = ifcvf_vdpa_set_config_cb,
> +	.get_vq_notification = ifcvf_get_vq_notification,
>   };
>   
>   static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)

Re: [PATCH V3 1/2] vDPA/ifcvf: record virtio notify base

[Jason Wang]

在 2021/6/1 下午2:28, Zhu Lingshan 写道:
> This commit records virtio notify base physical addr and
> calculate doorbell physical address for vqs.
>
> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>


Acked-by: Jason Wang <jasowang@redhat.com>


> ---
>   drivers/vdpa/ifcvf/ifcvf_base.c | 4 ++++
>   drivers/vdpa/ifcvf/ifcvf_base.h | 2 ++
>   2 files changed, 6 insertions(+)
>
> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
> index 1a661ab45af5..6e197fe0fcf9 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_base.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_base.c
> @@ -133,6 +133,8 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
>   					      &hw->notify_off_multiplier);
>   			hw->notify_bar = cap.bar;
>   			hw->notify_base = get_cap_addr(hw, &cap);
> +			hw->notify_base_pa = pci_resource_start(pdev, cap.bar) +
> +					le32_to_cpu(cap.offset);
>   			IFCVF_DBG(pdev, "hw->notify_base = %p\n",
>   				  hw->notify_base);
>   			break;
> @@ -161,6 +163,8 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
>   		notify_off = ifc_ioread16(&hw->common_cfg->queue_notify_off);
>   		hw->vring[i].notify_addr = hw->notify_base +
>   			notify_off * hw->notify_off_multiplier;
> +		hw->vring[i].notify_pa = hw->notify_base_pa +
> +			notify_off * hw->notify_off_multiplier;
>   	}
>   
>   	hw->lm_cfg = hw->base[IFCVF_LM_BAR];
> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
> index 0111bfdeb342..447f4ad9c0bf 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_base.h
> +++ b/drivers/vdpa/ifcvf/ifcvf_base.h
> @@ -73,6 +73,7 @@ struct vring_info {
>   	u16 last_avail_idx;
>   	bool ready;
>   	void __iomem *notify_addr;
> +	phys_addr_t notify_pa;
>   	u32 irq;
>   	struct vdpa_callback cb;
>   	char msix_name[256];
> @@ -87,6 +88,7 @@ struct ifcvf_hw {
>   	u8 notify_bar;
>   	/* Notificaiton bar address */
>   	void __iomem *notify_base;
> +	phys_addr_t notify_base_pa;
>   	u32 notify_off_multiplier;
>   	u64 req_features;
>   	u64 hw_features;

Re: [PATCH V3 2/2] vDPA/ifcvf: implement doorbell mapping for ifcvf

[Zhu, Lingshan]

On 6/1/2021 4:50 PM, Jason Wang wrote:
>
> 在 2021/6/1 下午2:28, Zhu Lingshan 写道:
>> This commit implements doorbell mapping feature for ifcvf.
>> This feature maps the notify page to userspace, to eliminate
>> vmexit when kick a vq.
>>
>> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
>> ---
>>   drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
>>   1 file changed, 21 insertions(+)
>>
>> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c 
>> b/drivers/vdpa/ifcvf/ifcvf_main.c
>> index ab0ab5cf0f6e..d41db042612c 100644
>> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
>> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
>> @@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct 
>> vdpa_device *vdpa_dev,
>>       return vf->vring[qid].irq;
>>   }
>>   +static struct vdpa_notification_area 
>> ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
>> +                                   u16 idx)
>> +{
>> +    struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
>> +    struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
>> +    struct pci_dev *pdev = adapter->pdev;
>> +    struct vdpa_notification_area area;
>> +
>> +    area.addr = vf->vring[idx].notify_pa;
>> +    if (!vf->notify_off_multiplier)
>> +        area.size = PAGE_SIZE;
>> +    else
>> +        area.size = vf->notify_off_multiplier;
>> +
>> +    if (area.addr % PAGE_SIZE)
>> +        IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE 
>> aligned\n", idx);
>
>
> I don't see the reason to keep this, or get_notification is not the 
> proper place to do this kind of warning.
>
> Thanks
some customers have ever complained have troubles to enable such 
features with their IP,
I think this can help them debug.

Thanks
>
>
>> +
>> +    return area;
>> +}
>> +
>>   /*
>>    * IFCVF currently does't have on-chip IOMMU, so not
>>    * implemented set_map()/dma_map()/dma_unmap()
>> @@ -440,6 +460,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
>>       .get_config    = ifcvf_vdpa_get_config,
>>       .set_config    = ifcvf_vdpa_set_config,
>>       .set_config_cb  = ifcvf_vdpa_set_config_cb,
>> +    .get_vq_notification = ifcvf_get_vq_notification,
>>   };
>>     static int ifcvf_probe(struct pci_dev *pdev, const struct 
>> pci_device_id *id)
>

Re: [PATCH V3 2/2] vDPA/ifcvf: implement doorbell mapping for ifcvf

[Jason Wang]

在 2021/6/1 下午4:56, Zhu, Lingshan 写道:
>
>
> On 6/1/2021 4:50 PM, Jason Wang wrote:
>>
>> 在 2021/6/1 下午2:28, Zhu Lingshan 写道:
>>> This commit implements doorbell mapping feature for ifcvf.
>>> This feature maps the notify page to userspace, to eliminate
>>> vmexit when kick a vq.
>>>
>>> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
>>> ---
>>>   drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
>>>   1 file changed, 21 insertions(+)
>>>
>>> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c 
>>> b/drivers/vdpa/ifcvf/ifcvf_main.c
>>> index ab0ab5cf0f6e..d41db042612c 100644
>>> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
>>> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
>>> @@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct 
>>> vdpa_device *vdpa_dev,
>>>       return vf->vring[qid].irq;
>>>   }
>>>   +static struct vdpa_notification_area 
>>> ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
>>> +                                   u16 idx)
>>> +{
>>> +    struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
>>> +    struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
>>> +    struct pci_dev *pdev = adapter->pdev;
>>> +    struct vdpa_notification_area area;
>>> +
>>> +    area.addr = vf->vring[idx].notify_pa;
>>> +    if (!vf->notify_off_multiplier)
>>> +        area.size = PAGE_SIZE;
>>> +    else
>>> +        area.size = vf->notify_off_multiplier;
>>> +
>>> +    if (area.addr % PAGE_SIZE)
>>> +        IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE 
>>> aligned\n", idx);
>>
>>
>> I don't see the reason to keep this, or get_notification is not the 
>> proper place to do this kind of warning.
>>
>> Thanks
> some customers have ever complained have troubles to enable such 
> features with their IP,
> I think this can help them debug.


If you want to do this, the ifcvf_init_hw() is the proper place.

Note that this function is called by userspace.

Thanks


>
> Thanks
>>
>>
>>> +
>>> +    return area;
>>> +}
>>> +
>>>   /*
>>>    * IFCVF currently does't have on-chip IOMMU, so not
>>>    * implemented set_map()/dma_map()/dma_unmap()
>>> @@ -440,6 +460,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops 
>>> = {
>>>       .get_config    = ifcvf_vdpa_get_config,
>>>       .set_config    = ifcvf_vdpa_set_config,
>>>       .set_config_cb  = ifcvf_vdpa_set_config_cb,
>>> +    .get_vq_notification = ifcvf_get_vq_notification,
>>>   };
>>>     static int ifcvf_probe(struct pci_dev *pdev, const struct 
>>> pci_device_id *id)
>>
>

Re: [PATCH V3 2/2] vDPA/ifcvf: implement doorbell mapping for ifcvf

[Zhu, Lingshan]

On 6/1/2021 4:57 PM, Jason Wang wrote:
>
> 在 2021/6/1 下午4:56, Zhu, Lingshan 写道:
>>
>>
>> On 6/1/2021 4:50 PM, Jason Wang wrote:
>>>
>>> 在 2021/6/1 下午2:28, Zhu Lingshan 写道:
>>>> This commit implements doorbell mapping feature for ifcvf.
>>>> This feature maps the notify page to userspace, to eliminate
>>>> vmexit when kick a vq.
>>>>
>>>> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
>>>> ---
>>>>   drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
>>>>   1 file changed, 21 insertions(+)
>>>>
>>>> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c 
>>>> b/drivers/vdpa/ifcvf/ifcvf_main.c
>>>> index ab0ab5cf0f6e..d41db042612c 100644
>>>> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
>>>> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
>>>> @@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct 
>>>> vdpa_device *vdpa_dev,
>>>>       return vf->vring[qid].irq;
>>>>   }
>>>>   +static struct vdpa_notification_area 
>>>> ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
>>>> +                                   u16 idx)
>>>> +{
>>>> +    struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
>>>> +    struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
>>>> +    struct pci_dev *pdev = adapter->pdev;
>>>> +    struct vdpa_notification_area area;
>>>> +
>>>> +    area.addr = vf->vring[idx].notify_pa;
>>>> +    if (!vf->notify_off_multiplier)
>>>> +        area.size = PAGE_SIZE;
>>>> +    else
>>>> +        area.size = vf->notify_off_multiplier;
>>>> +
>>>> +    if (area.addr % PAGE_SIZE)
>>>> +        IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE 
>>>> aligned\n", idx);
>>>
>>>
>>> I don't see the reason to keep this, or get_notification is not the 
>>> proper place to do this kind of warning.
>>>
>>> Thanks
>> some customers have ever complained have troubles to enable such 
>> features with their IP,
>> I think this can help them debug.
>
>
> If you want to do this, the ifcvf_init_hw() is the proper place.
>
> Note that this function is called by userspace.
>
> Thanks
OK, will move to there.

Thanks!
>
>
>>
>> Thanks
>>>
>>>
>>>> +
>>>> +    return area;
>>>> +}
>>>> +
>>>>   /*
>>>>    * IFCVF currently does't have on-chip IOMMU, so not
>>>>    * implemented set_map()/dma_map()/dma_unmap()
>>>> @@ -440,6 +460,7 @@ static const struct vdpa_config_ops 
>>>> ifc_vdpa_ops = {
>>>>       .get_config    = ifcvf_vdpa_get_config,
>>>>       .set_config    = ifcvf_vdpa_set_config,
>>>>       .set_config_cb  = ifcvf_vdpa_set_config_cb,
>>>> +    .get_vq_notification = ifcvf_get_vq_notification,
>>>>   };
>>>>     static int ifcvf_probe(struct pci_dev *pdev, const struct 
>>>> pci_device_id *id)
>>>
>>
>

Re: [PATCH V3 2/2] vDPA/ifcvf: implement doorbell mapping for ifcvf

[Zhu, Lingshan]

On 6/1/2021 4:58 PM, Zhu, Lingshan wrote:
>
>
> On 6/1/2021 4:57 PM, Jason Wang wrote:
>>
>> 在 2021/6/1 下午4:56, Zhu, Lingshan 写道:
>>>
>>>
>>> On 6/1/2021 4:50 PM, Jason Wang wrote:
>>>>
>>>> 在 2021/6/1 下午2:28, Zhu Lingshan 写道:
>>>>> This commit implements doorbell mapping feature for ifcvf.
>>>>> This feature maps the notify page to userspace, to eliminate
>>>>> vmexit when kick a vq.
>>>>>
>>>>> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
>>>>> ---
>>>>>   drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
>>>>>   1 file changed, 21 insertions(+)
>>>>>
>>>>> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c 
>>>>> b/drivers/vdpa/ifcvf/ifcvf_main.c
>>>>> index ab0ab5cf0f6e..d41db042612c 100644
>>>>> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
>>>>> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
>>>>> @@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct 
>>>>> vdpa_device *vdpa_dev,
>>>>>       return vf->vring[qid].irq;
>>>>>   }
>>>>>   +static struct vdpa_notification_area 
>>>>> ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
>>>>> +                                   u16 idx)
>>>>> +{
>>>>> +    struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
>>>>> +    struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
>>>>> +    struct pci_dev *pdev = adapter->pdev;
>>>>> +    struct vdpa_notification_area area;
>>>>> +
>>>>> +    area.addr = vf->vring[idx].notify_pa;
>>>>> +    if (!vf->notify_off_multiplier)
>>>>> +        area.size = PAGE_SIZE;
>>>>> +    else
>>>>> +        area.size = vf->notify_off_multiplier;
>>>>> +
>>>>> +    if (area.addr % PAGE_SIZE)
>>>>> +        IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE 
>>>>> aligned\n", idx);
>>>>
>>>>
>>>> I don't see the reason to keep this, or get_notification is not the 
>>>> proper place to do this kind of warning.
>>>>
>>>> Thanks
>>> some customers have ever complained have troubles to enable such 
>>> features with their IP,
>>> I think this can help them debug.
>>
>>
>> If you want to do this, the ifcvf_init_hw() is the proper place.
>>
>> Note that this function is called by userspace.
>>
>> Thanks
> OK, will move to there.
>
> Thanks!
oh, I see patch 1 is already has been acked, so I will remove this warning.

Thanks!
>>
>>
>>>
>>> Thanks
>>>>
>>>>
>>>>> +
>>>>> +    return area;
>>>>> +}
>>>>> +
>>>>>   /*
>>>>>    * IFCVF currently does't have on-chip IOMMU, so not
>>>>>    * implemented set_map()/dma_map()/dma_unmap()
>>>>> @@ -440,6 +460,7 @@ static const struct vdpa_config_ops 
>>>>> ifc_vdpa_ops = {
>>>>>       .get_config    = ifcvf_vdpa_get_config,
>>>>>       .set_config    = ifcvf_vdpa_set_config,
>>>>>       .set_config_cb  = ifcvf_vdpa_set_config_cb,
>>>>> +    .get_vq_notification = ifcvf_get_vq_notification,
>>>>>   };
>>>>>     static int ifcvf_probe(struct pci_dev *pdev, const struct 
>>>>> pci_device_id *id)
>>>>
>>>
>>
>