[PATCH v2 1/4] PCI/AER: Fix pci_ops return NULL when hotplug a pci bus doing aer error inject

[PATCH v2 1/4] PCI/AER: Fix pci_ops return NULL when hotplug a pci bus doing aer error inject

[Yijing Wang]

When we inject aer errors to the target pcie device by aer_inject module, the pci_ops of pci
bus which the target device is on will be assigned to pci_ops_aer.So if the target pci device
is a bridge, once we hot-remove and hot-add the bridge, the newly created child bus's pci_ops
will be assigned to pci_ops_aer too.Now every access to the child bus's devices will result to
system panic, because it get a NULL pci_ops in pci_read_aer/pci_write_aer.

Signed-off-by: Yijing Wang <wangyijing@huawei.com>
Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
Reviewed-by: Sven Dietrich <Sven.Dietrich@huawei.com>
---
 drivers/pci/pcie/aer/aer_inject.c |   23 +++++++++++++++++++++++
 1 files changed, 23 insertions(+), 0 deletions(-)

diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
index 4e24cb8..0816483 100644
--- a/drivers/pci/pcie/aer/aer_inject.c
+++ b/drivers/pci/pcie/aer/aer_inject.c
@@ -109,6 +109,19 @@ static struct aer_error *__find_aer_error_by_dev(struct pci_dev *dev)
 	return __find_aer_error((u16)domain, dev->bus->number, dev->devfn);
 }
 
+static bool pci_is_upstream_bus(struct pci_bus *bus, struct pci_bus *up_bus)
+{
+	struct pci_bus *pbus = bus->parent;
+
+	while (pbus) {
+		if (pbus == up_bus)
+			return true;
+		pbus = pbus->parent;
+	}
+
+	return false;
+}
+
 /* inject_lock must be held before calling */
 static struct pci_ops *__find_pci_bus_ops(struct pci_bus *bus)
 {
@@ -118,6 +131,13 @@ static struct pci_ops *__find_pci_bus_ops(struct pci_bus *bus)
 		if (bus_ops->bus == bus)
 			return bus_ops->ops;
 	}
+
+	/* can't find bus_ops, fall back to get bus_ops of upstream bus */
+	list_for_each_entry(bus_ops, &pci_bus_ops_list, list) {
+		if (pci_is_upstream_bus(bus, bus_ops->bus))
+			return bus_ops->ops;
+	}
+
 	return NULL;
 }
 
@@ -208,6 +228,7 @@ static int pci_read_aer(struct pci_bus *bus, unsigned int devfn, int where,
 	}
 out:
 	ops = __find_pci_bus_ops(bus);
+	BUG_ON(!ops);
 	spin_unlock_irqrestore(&inject_lock, flags);
 	return ops->read(bus, devfn, where, size, val);
 }
@@ -243,6 +264,7 @@ int pci_write_aer(struct pci_bus *bus, unsigned int devfn, int where, int size,
 	}
 out:
 	ops = __find_pci_bus_ops(bus);
+	BUG_ON(!ops);
 	spin_unlock_irqrestore(&inject_lock, flags);
 	return ops->write(bus, devfn, where, size, val);
 }
@@ -506,6 +528,7 @@ static struct miscdevice aer_inject_device = {
 	.fops = &aer_inject_fops,
 };
 
+
 static int __init aer_inject_init(void)
 {
 	return misc_register(&aer_inject_device);
-- 
1.7.1

[PATCH v2 2/4] PCI/AER: clean all untracked pci_ops_aer when rmmod aer_inject

[Yijing Wang]

Since hot plug for pci devices while doing aer inject, some newly created child buses'
pci_ops will be assigned to pci_ops_aer. Aer_inject does not track these pci_ops_aer(not
list in pci_bus_ops_list),we should clean all of these when rmmod aer_inject module.

Signed-off-by: Yijing Wang <wangyijing@huawei.com>
---
 drivers/pci/pcie/aer/aer_inject.c |   32 ++++++++++++++++++++++++++++++++
 1 files changed, 32 insertions(+), 0 deletions(-)

diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
index 0816483..9d195ae 100644
--- a/drivers/pci/pcie/aer/aer_inject.c
+++ b/drivers/pci/pcie/aer/aer_inject.c
@@ -283,6 +283,28 @@ static void pci_bus_ops_init(struct pci_bus_ops *bus_ops,
 	bus_ops->ops = ops;
 }
 
+static void pci_clean_child_aer_ops(struct pci_bus *bus)
+{
+	struct pci_bus *child;
+
+	list_for_each_entry(child, &bus->children, node) {
+		if (child->ops == &pci_ops_aer)
+			pci_bus_set_ops(child, bus->ops);
+		pci_clean_child_aer_ops(child);
+	}
+}
+
+/* find pci_ops_aer from root bus, and replace it by parent bus's pci_ops.
+ * pci_ops of root bus won't be pci_ops_aer here*/
+static void clean_untracked_pci_ops_aer(void)
+{
+	struct pci_bus *bus = NULL;
+
+	while ((bus = pci_find_next_bus(bus)) != NULL)
+		pci_clean_child_aer_ops(bus);
+}
+
+
 static int pci_bus_set_aer_ops(struct pci_bus *bus)
 {
 	struct pci_ops *ops;
@@ -546,6 +568,8 @@ static void __exit aer_inject_exit(void)
 		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
 		kfree(bus_ops);
 	}
+
+	clean_untracked_pci_aer_ops();
 
 	spin_lock_irqsave(&inject_lock, flags);
 	list_for_each_entry_safe(err, err_next, &einjected, list) {
@@ -553,6 +577,14 @@ static void __exit aer_inject_exit(void)
 		kfree(err);
 	}
 	spin_unlock_irqrestore(&inject_lock, flags);
+
+	/* Inject aer errors and hotplug the same pcie device
+	 * maybe assign some newly created buses' pci_ops pci_ops_aer.
+	 * Since these pci_ops_aer are not tracked in pci_bus_ops_list,
+	 * we need to find and clean untracked pci_ops_aer before aer_inject
+	 * module exit
+	 */
+	clean_untracked_pci_ops_aer();
 }
 
 module_init(aer_inject_init);
-- 
1.7.1

Re: [PATCH v2 2/4] PCI/AER: clean all untracked pci_ops_aer when rmmod aer_inject

[Huang Ying]

On Wed, 2012-09-12 at 20:33 +0800, Yijing Wang wrote:
> Since hot plug for pci devices while doing aer inject, some newly created child buses'
> pci_ops will be assigned to pci_ops_aer. Aer_inject does not track these pci_ops_aer(not
> list in pci_bus_ops_list),we should clean all of these when rmmod aer_inject module.
> 
> Signed-off-by: Yijing Wang <wangyijing@huawei.com>
> ---
>  drivers/pci/pcie/aer/aer_inject.c |   32 ++++++++++++++++++++++++++++++++
>  1 files changed, 32 insertions(+), 0 deletions(-)
> 
> diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
> index 0816483..9d195ae 100644
> --- a/drivers/pci/pcie/aer/aer_inject.c
> +++ b/drivers/pci/pcie/aer/aer_inject.c
> @@ -283,6 +283,28 @@ static void pci_bus_ops_init(struct pci_bus_ops *bus_ops,
>  	bus_ops->ops = ops;
>  }
>  
> +static void pci_clean_child_aer_ops(struct pci_bus *bus)
> +{
> +	struct pci_bus *child;
> +
> +	list_for_each_entry(child, &bus->children, node) {
> +		if (child->ops == &pci_ops_aer)
> +			pci_bus_set_ops(child, bus->ops);
> +		pci_clean_child_aer_ops(child);
> +	}
> +}
> +
> +/* find pci_ops_aer from root bus, and replace it by parent bus's pci_ops.
> + * pci_ops of root bus won't be pci_ops_aer here*/

Why scan all pci buses?  Is it possible just to check buses in
pci_bus_ops_list.

Best Regards,
Huang Ying

> +static void clean_untracked_pci_ops_aer(void)
> +{
> +	struct pci_bus *bus = NULL;
> +
> +	while ((bus = pci_find_next_bus(bus)) != NULL)
> +		pci_clean_child_aer_ops(bus);
> +}
> +
> +
>  static int pci_bus_set_aer_ops(struct pci_bus *bus)
>  {
>  	struct pci_ops *ops;
> @@ -546,6 +568,8 @@ static void __exit aer_inject_exit(void)
>  		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
>  		kfree(bus_ops);
>  	}
> +
> +	clean_untracked_pci_aer_ops();
>  
>  	spin_lock_irqsave(&inject_lock, flags);
>  	list_for_each_entry_safe(err, err_next, &einjected, list) {
> @@ -553,6 +577,14 @@ static void __exit aer_inject_exit(void)
>  		kfree(err);
>  	}
>  	spin_unlock_irqrestore(&inject_lock, flags);
> +
> +	/* Inject aer errors and hotplug the same pcie device
> +	 * maybe assign some newly created buses' pci_ops pci_ops_aer.
> +	 * Since these pci_ops_aer are not tracked in pci_bus_ops_list,
> +	 * we need to find and clean untracked pci_ops_aer before aer_inject
> +	 * module exit
> +	 */
> +	clean_untracked_pci_ops_aer();
>  }
>  
>  module_init(aer_inject_init);

Re: [PATCH v2 2/4] PCI/AER: clean all untracked pci_ops_aer when rmmod aer_inject

[Yijing Wang]

On 2012/9/13 9:43, Huang Ying wrote:
> On Wed, 2012-09-12 at 20:33 +0800, Yijing Wang wrote:
>> Since hot plug for pci devices while doing aer inject, some newly created child buses'
>> pci_ops will be assigned to pci_ops_aer. Aer_inject does not track these pci_ops_aer(not
>> list in pci_bus_ops_list),we should clean all of these when rmmod aer_inject module.
>>
>> Signed-off-by: Yijing Wang <wangyijing@huawei.com>
>> ---
>>  drivers/pci/pcie/aer/aer_inject.c |   32 ++++++++++++++++++++++++++++++++
>>  1 files changed, 32 insertions(+), 0 deletions(-)
>>
>> diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
>> index 0816483..9d195ae 100644
>> --- a/drivers/pci/pcie/aer/aer_inject.c
>> +++ b/drivers/pci/pcie/aer/aer_inject.c
>> @@ -283,6 +283,28 @@ static void pci_bus_ops_init(struct pci_bus_ops *bus_ops,
>>  	bus_ops->ops = ops;
>>  }
>>  
>> +static void pci_clean_child_aer_ops(struct pci_bus *bus)
>> +{
>> +	struct pci_bus *child;
>> +
>> +	list_for_each_entry(child, &bus->children, node) {
>> +		if (child->ops == &pci_ops_aer)
>> +			pci_bus_set_ops(child, bus->ops);
>> +		pci_clean_child_aer_ops(child);
>> +	}
>> +}
>> +
>> +/* find pci_ops_aer from root bus, and replace it by parent bus's pci_ops.
>> + * pci_ops of root bus won't be pci_ops_aer here*/
> 
> Why scan all pci buses?  Is it possible just to check buses in
> pci_bus_ops_list.
> 

Hi Huang Ying,
   Some buses' pci_ops_aer may be not assigned by pci_bus_set_aer_ops() function,
As the first patch [PATCH v2 1/4] said. Some newly created buses' pci_ops assigned to
pci_ops_aer because their parent just doing aer inject, parent bus's pci_ops was pci_ops_aer.
So when we rmmod aer_inject module, these buses' pci_ops_aer will be invalid.

1、Tracked pci_ops_aer will be cleaned and pci_ops will reassign to their original pci_ops saved in pci_bus_ops_list.
2、Un-tracked pci_ops_aer won't be cleaned.so we need to fix it.

> Best Regards,
> Huang Ying
> 
>> +static void clean_untracked_pci_ops_aer(void)
>> +{
>> +	struct pci_bus *bus = NULL;
>> +
>> +	while ((bus = pci_find_next_bus(bus)) != NULL)
>> +		pci_clean_child_aer_ops(bus);
>> +}
>> +
>> +
>>  static int pci_bus_set_aer_ops(struct pci_bus *bus)
>>  {
>>  	struct pci_ops *ops;
>> @@ -546,6 +568,8 @@ static void __exit aer_inject_exit(void)
>>  		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
>>  		kfree(bus_ops);
>>  	}
>> +
>> +	clean_untracked_pci_aer_ops();
>>  
>>  	spin_lock_irqsave(&inject_lock, flags);
>>  	list_for_each_entry_safe(err, err_next, &einjected, list) {
>> @@ -553,6 +577,14 @@ static void __exit aer_inject_exit(void)
>>  		kfree(err);
>>  	}
>>  	spin_unlock_irqrestore(&inject_lock, flags);
>> +
>> +	/* Inject aer errors and hotplug the same pcie device
>> +	 * maybe assign some newly created buses' pci_ops pci_ops_aer.
>> +	 * Since these pci_ops_aer are not tracked in pci_bus_ops_list,
>> +	 * we need to find and clean untracked pci_ops_aer before aer_inject
>> +	 * module exit
>> +	 */
>> +	clean_untracked_pci_ops_aer();
>>  }
>>  
>>  module_init(aer_inject_init);
> 
> 
> 
> .
> 


-- 
Thanks!
Yijing

Re: [PATCH v2 2/4] PCI/AER: clean all untracked pci_ops_aer when rmmod aer_inject

[Huang Ying]

On Thu, 2012-09-13 at 18:54 +0800, Yijing Wang wrote:
> On 2012/9/13 9:43, Huang Ying wrote:
> > On Wed, 2012-09-12 at 20:33 +0800, Yijing Wang wrote:
> >> Since hot plug for pci devices while doing aer inject, some newly created child buses'
> >> pci_ops will be assigned to pci_ops_aer. Aer_inject does not track these pci_ops_aer(not
> >> list in pci_bus_ops_list),we should clean all of these when rmmod aer_inject module.
> >>
> >> Signed-off-by: Yijing Wang <wangyijing@huawei.com>
> >> ---
> >>  drivers/pci/pcie/aer/aer_inject.c |   32 ++++++++++++++++++++++++++++++++
> >>  1 files changed, 32 insertions(+), 0 deletions(-)
> >>
> >> diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
> >> index 0816483..9d195ae 100644
> >> --- a/drivers/pci/pcie/aer/aer_inject.c
> >> +++ b/drivers/pci/pcie/aer/aer_inject.c
> >> @@ -283,6 +283,28 @@ static void pci_bus_ops_init(struct pci_bus_ops *bus_ops,
> >>  	bus_ops->ops = ops;
> >>  }
> >>  
> >> +static void pci_clean_child_aer_ops(struct pci_bus *bus)
> >> +{
> >> +	struct pci_bus *child;
> >> +
> >> +	list_for_each_entry(child, &bus->children, node) {
> >> +		if (child->ops == &pci_ops_aer)
> >> +			pci_bus_set_ops(child, bus->ops);
> >> +		pci_clean_child_aer_ops(child);
> >> +	}
> >> +}
> >> +
> >> +/* find pci_ops_aer from root bus, and replace it by parent bus's pci_ops.
> >> + * pci_ops of root bus won't be pci_ops_aer here*/
> > 
> > Why scan all pci buses?  Is it possible just to check buses in
> > pci_bus_ops_list.
> > 
> 
> Hi Huang Ying,
>    Some buses' pci_ops_aer may be not assigned by pci_bus_set_aer_ops() function,
> As the first patch [PATCH v2 1/4] said. Some newly created buses' pci_ops assigned to
> pci_ops_aer because their parent just doing aer inject, parent bus's pci_ops was pci_ops_aer.
> So when we rmmod aer_inject module, these buses' pci_ops_aer will be invalid.
> 
> 1、Tracked pci_ops_aer will be cleaned and pci_ops will reassign to their original pci_ops saved in pci_bus_ops_list.
> 2、Un-tracked pci_ops_aer won't be cleaned.so we need to fix it.

Can we just check all bus and its children in pci_bus_ops_list?  If my
understanding were correct, bus itself or some of its parents should be
saved in pci_bus_ops_list?

Best Regards,
Huang Ying

> > 
> >> +static void clean_untracked_pci_ops_aer(void)
> >> +{
> >> +	struct pci_bus *bus = NULL;
> >> +
> >> +	while ((bus = pci_find_next_bus(bus)) != NULL)
> >> +		pci_clean_child_aer_ops(bus);
> >> +}
> >> +
> >> +
> >>  static int pci_bus_set_aer_ops(struct pci_bus *bus)
> >>  {
> >>  	struct pci_ops *ops;
> >> @@ -546,6 +568,8 @@ static void __exit aer_inject_exit(void)
> >>  		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
> >>  		kfree(bus_ops);
> >>  	}
> >> +
> >> +	clean_untracked_pci_aer_ops();
> >>  
> >>  	spin_lock_irqsave(&inject_lock, flags);
> >>  	list_for_each_entry_safe(err, err_next, &einjected, list) {
> >> @@ -553,6 +577,14 @@ static void __exit aer_inject_exit(void)
> >>  		kfree(err);
> >>  	}
> >>  	spin_unlock_irqrestore(&inject_lock, flags);
> >> +
> >> +	/* Inject aer errors and hotplug the same pcie device
> >> +	 * maybe assign some newly created buses' pci_ops pci_ops_aer.
> >> +	 * Since these pci_ops_aer are not tracked in pci_bus_ops_list,
> >> +	 * we need to find and clean untracked pci_ops_aer before aer_inject
> >> +	 * module exit
> >> +	 */
> >> +	clean_untracked_pci_ops_aer();
> >>  }
> >>  
> >>  module_init(aer_inject_init);
> > 
> > 
> > 
> > .
> > 
> 
> 

Re: [PATCH v2 2/4] PCI/AER: clean all untracked pci_ops_aer when rmmod aer_inject

[Yijing Wang]

On 2012/9/14 8:56, Huang Ying wrote:
> On Thu, 2012-09-13 at 18:54 +0800, Yijing Wang wrote:
>> On 2012/9/13 9:43, Huang Ying wrote:
>>> On Wed, 2012-09-12 at 20:33 +0800, Yijing Wang wrote:
>>>> Since hot plug for pci devices while doing aer inject, some newly created child buses'
>>>> pci_ops will be assigned to pci_ops_aer. Aer_inject does not track these pci_ops_aer(not
>>>> list in pci_bus_ops_list),we should clean all of these when rmmod aer_inject module.
>>>>
>>>> Signed-off-by: Yijing Wang <wangyijing@huawei.com>
>>>> ---
>>>>  drivers/pci/pcie/aer/aer_inject.c |   32 ++++++++++++++++++++++++++++++++
>>>>  1 files changed, 32 insertions(+), 0 deletions(-)
>>>>
>>>> diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
>>>> index 0816483..9d195ae 100644
>>>> --- a/drivers/pci/pcie/aer/aer_inject.c
>>>> +++ b/drivers/pci/pcie/aer/aer_inject.c
>>>> @@ -283,6 +283,28 @@ static void pci_bus_ops_init(struct pci_bus_ops *bus_ops,
>>>>  	bus_ops->ops = ops;
>>>>  }
>>>>  
>>>> +static void pci_clean_child_aer_ops(struct pci_bus *bus)
>>>> +{
>>>> +	struct pci_bus *child;
>>>> +
>>>> +	list_for_each_entry(child, &bus->children, node) {
>>>> +		if (child->ops == &pci_ops_aer)
>>>> +			pci_bus_set_ops(child, bus->ops);
>>>> +		pci_clean_child_aer_ops(child);
>>>> +	}
>>>> +}
>>>> +
>>>> +/* find pci_ops_aer from root bus, and replace it by parent bus's pci_ops.
>>>> + * pci_ops of root bus won't be pci_ops_aer here*/
>>>
>>> Why scan all pci buses?  Is it possible just to check buses in
>>> pci_bus_ops_list.
>>>
>>
>> Hi Huang Ying,
>>    Some buses' pci_ops_aer may be not assigned by pci_bus_set_aer_ops() function,
>> As the first patch [PATCH v2 1/4] said. Some newly created buses' pci_ops assigned to
>> pci_ops_aer because their parent just doing aer inject, parent bus's pci_ops was pci_ops_aer.
>> So when we rmmod aer_inject module, these buses' pci_ops_aer will be invalid.
>>
>> 1、Tracked pci_ops_aer will be cleaned and pci_ops will reassign to their original pci_ops saved in pci_bus_ops_list.
>> 2、Un-tracked pci_ops_aer won't be cleaned.so we need to fix it.
> 
> Can we just check all bus and its children in pci_bus_ops_list?  If my
> understanding were correct, bus itself or some of its parents should be
> saved in pci_bus_ops_list?
> 

Yes, good idea! I think the buses that owned untracked pci_ops_aer always are child buses of the root buses
tracked in the pci_bus_ops_list. I will improve this patch in this way.

Thanks
Yijing

> Best Regards,
> Huang Ying
> 
>>>
>>>> +static void clean_untracked_pci_ops_aer(void)
>>>> +{
>>>> +	struct pci_bus *bus = NULL;
>>>> +
>>>> +	while ((bus = pci_find_next_bus(bus)) != NULL)
>>>> +		pci_clean_child_aer_ops(bus);
>>>> +}
>>>> +
>>>> +
>>>>  static int pci_bus_set_aer_ops(struct pci_bus *bus)
>>>>  {
>>>>  	struct pci_ops *ops;
>>>> @@ -546,6 +568,8 @@ static void __exit aer_inject_exit(void)
>>>>  		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
>>>>  		kfree(bus_ops);
>>>>  	}
>>>> +
>>>> +	clean_untracked_pci_aer_ops();
>>>>  
>>>>  	spin_lock_irqsave(&inject_lock, flags);
>>>>  	list_for_each_entry_safe(err, err_next, &einjected, list) {
>>>> @@ -553,6 +577,14 @@ static void __exit aer_inject_exit(void)
>>>>  		kfree(err);
>>>>  	}
>>>>  	spin_unlock_irqrestore(&inject_lock, flags);
>>>> +
>>>> +	/* Inject aer errors and hotplug the same pcie device
>>>> +	 * maybe assign some newly created buses' pci_ops pci_ops_aer.
>>>> +	 * Since these pci_ops_aer are not tracked in pci_bus_ops_list,
>>>> +	 * we need to find and clean untracked pci_ops_aer before aer_inject
>>>> +	 * module exit
>>>> +	 */
>>>> +	clean_untracked_pci_ops_aer();
>>>>  }
>>>>  
>>>>  module_init(aer_inject_init);
>>>
>>>
>>>
>>> .
>>>
>>
>>
> 
> 
> 
> .
> 


-- 
Thanks!
Yijing

[PATCH v2 3/4] PCI/AER: Clean pci_bus_ops when related pci bus was removed

[Yijing Wang]

When Inject aer errors to the target pci device, a pci_bus_ops will
be allocated for the pci device's pci bus.When the pci bus was removed,
we should also release the pci_bus_ops.

Signed-off-by: Yijing Wang <wangyijing@huawei.com>
Reviewed-by: Sven Dietrich <Sven.Dietrich@huawei.com>
---
 drivers/pci/pcie/aer/aer_inject.c |   48 ++++++++++++++++++++++++++++++++++++-
 1 files changed, 47 insertions(+), 1 deletions(-)

diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
index 9d195ae..bd043db 100644
--- a/drivers/pci/pcie/aer/aer_inject.c
+++ b/drivers/pci/pcie/aer/aer_inject.c
@@ -550,10 +550,55 @@ static struct miscdevice aer_inject_device = {
 	.fops = &aer_inject_fops,
 };
 
+static void aer_clean_pci_bus_ops(struct pci_dev *dev)
+{
+	unsigned long flags;
+	struct pci_bus_ops *bus_ops, *tmp_ops;
+	struct pci_bus *bus;
+	bus = dev->subordinate;
+	if (!bus)
+		return;
+
+	spin_lock_irqsave(&inject_lock, flags);
+	list_for_each_entry_safe(bus_ops, tmp_ops, &pci_bus_ops_list, list)
+		if (bus_ops->bus == bus) {
+			list_del(&bus_ops->list);
+			kfree(bus_ops);
+			break;
+		}
+	spin_unlock_irqrestore(&inject_lock, flags);
+}
+
+static int aer_hp_notify_fn(struct notifier_block *nb,
+		unsigned long event, void *data)
+{
+	switch (event) {
+	case BUS_NOTIFY_DEL_DEVICE:
+		aer_clean_pci_bus_ops(to_pci_dev(data));
+		break;
+	default:
+		return NOTIFY_DONE;
+	}
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block aerinj_hp_notifier = {
+	.notifier_call = &aer_hp_notify_fn,
+};
 
 static int __init aer_inject_init(void)
 {
-	return misc_register(&aer_inject_device);
+	int ret;
+	ret = misc_register(&aer_inject_device);
+	if (ret)
+		goto out;
+
+	ret = bus_register_notifier(&pci_bus_type, &aerinj_hp_notifier);
+	if (ret)
+		misc_deregister(&aer_inject_device);
+out:
+	return ret;
 }
 
 static void __exit aer_inject_exit(void)
@@ -562,6 +607,7 @@ static void __exit aer_inject_exit(void)
 	unsigned long flags;
 	struct pci_bus_ops *bus_ops;
 
+	bus_unregister_notifier(&pci_bus_type, &aerinj_hp_notifier);
 	misc_deregister(&aer_inject_device);
 
 	while ((bus_ops = pci_bus_ops_pop())) {
-- 
1.7.1

[PATCH v2 4/4] PCI/AER: fix a small race condition window when rmmod aer_inject

[Yijing Wang]

Fix a small race condition window between pci_bus_ops_pop() and
pci_bus_set_ops() functions.

Signed-off-by: Yijing Wang <wangyijing@huawei.com>
Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
---
 drivers/pci/pcie/aer/aer_inject.c |    8 ++------
 1 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
index bd043db..69c2d77 100644
--- a/drivers/pci/pcie/aer/aer_inject.c
+++ b/drivers/pci/pcie/aer/aer_inject.c
@@ -141,12 +141,11 @@ static struct pci_ops *__find_pci_bus_ops(struct pci_bus *bus)
 	return NULL;
 }
 
+/* inject_lock must be held before calling */
 static struct pci_bus_ops *pci_bus_ops_pop(void)
 {
-	unsigned long flags;
 	struct pci_bus_ops *bus_ops = NULL;
 
-	spin_lock_irqsave(&inject_lock, flags);
 	if (list_empty(&pci_bus_ops_list))
 		bus_ops = NULL;
 	else {
@@ -154,7 +153,6 @@ static struct pci_bus_ops *pci_bus_ops_pop(void)
 		list_del(lh);
 		bus_ops = list_entry(lh, struct pci_bus_ops, list);
 	}
-	spin_unlock_irqrestore(&inject_lock, flags);
 	return bus_ops;
 }
 
@@ -610,14 +608,12 @@ static void __exit aer_inject_exit(void)
 	bus_unregister_notifier(&pci_bus_type, &aerinj_hp_notifier);
 	misc_deregister(&aer_inject_device);
 
+	spin_lock_irqsave(&inject_lock, flags);
 	while ((bus_ops = pci_bus_ops_pop())) {
 		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
 		kfree(bus_ops);
 	}
 	
-	clean_untracked_pci_aer_ops();
-
-	spin_lock_irqsave(&inject_lock, flags);
 	list_for_each_entry_safe(err, err_next, &einjected, list) {
 		list_del(&err->list);
 		kfree(err);
-- 
1.7.1

Re: [PATCH v2 4/4] PCI/AER: fix a small race condition window when rmmod aer_inject

[Huang Ying]

On Wed, 2012-09-12 at 20:33 +0800, Yijing Wang wrote:
> Fix a small race condition window between pci_bus_ops_pop() and
> pci_bus_set_ops() functions.
> 
> Signed-off-by: Yijing Wang <wangyijing@huawei.com>
> Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
> ---
>  drivers/pci/pcie/aer/aer_inject.c |    8 ++------
>  1 files changed, 2 insertions(+), 6 deletions(-)
> 
> diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
> index bd043db..69c2d77 100644
> --- a/drivers/pci/pcie/aer/aer_inject.c
> +++ b/drivers/pci/pcie/aer/aer_inject.c
> @@ -141,12 +141,11 @@ static struct pci_ops *__find_pci_bus_ops(struct pci_bus *bus)
>  	return NULL;
>  }
>  
> +/* inject_lock must be held before calling */
>  static struct pci_bus_ops *pci_bus_ops_pop(void)
>  {
> -	unsigned long flags;
>  	struct pci_bus_ops *bus_ops = NULL;
>  
> -	spin_lock_irqsave(&inject_lock, flags);
>  	if (list_empty(&pci_bus_ops_list))
>  		bus_ops = NULL;
>  	else {
> @@ -154,7 +153,6 @@ static struct pci_bus_ops *pci_bus_ops_pop(void)
>  		list_del(lh);
>  		bus_ops = list_entry(lh, struct pci_bus_ops, list);
>  	}
> -	spin_unlock_irqrestore(&inject_lock, flags);
>  	return bus_ops;
>  }
>  
> @@ -610,14 +608,12 @@ static void __exit aer_inject_exit(void)
>  	bus_unregister_notifier(&pci_bus_type, &aerinj_hp_notifier);
>  	misc_deregister(&aer_inject_device);
>  
> +	spin_lock_irqsave(&inject_lock, flags);
>  	while ((bus_ops = pci_bus_ops_pop())) {
>  		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
>  		kfree(bus_ops);
>  	}
>  	
> -	clean_untracked_pci_aer_ops();
> -
> -	spin_lock_irqsave(&inject_lock, flags);
>  	list_for_each_entry_safe(err, err_next, &einjected, list) {
>  		list_del(&err->list);
>  		kfree(err);

This may trigger a AB BA dead lock.  In aer_inject_exit, the lock
sequence is:

inject_lock -> pci_lock

In pci config read/write path, the lock sequence is:

pci_lock -> inject_lock

With lockdep enabled, you may found warning on that.

Best Regards,
Huang YIng

Re: [PATCH v2 4/4] PCI/AER: fix a small race condition window when rmmod aer_inject

[Yijing Wang]

On 2012/9/13 9:29, Huang Ying wrote:
> On Wed, 2012-09-12 at 20:33 +0800, Yijing Wang wrote:
>> Fix a small race condition window between pci_bus_ops_pop() and
>> pci_bus_set_ops() functions.
>>
>> Signed-off-by: Yijing Wang <wangyijing@huawei.com>
>> Signed-off-by: Jiang Liu <jiang.liu@huawei.com>
>> ---
>>  drivers/pci/pcie/aer/aer_inject.c |    8 ++------
>>  1 files changed, 2 insertions(+), 6 deletions(-)
>>
>> diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c
>> index bd043db..69c2d77 100644
>> --- a/drivers/pci/pcie/aer/aer_inject.c
>> +++ b/drivers/pci/pcie/aer/aer_inject.c
>> @@ -141,12 +141,11 @@ static struct pci_ops *__find_pci_bus_ops(struct pci_bus *bus)
>>  	return NULL;
>>  }
>>  
>> +/* inject_lock must be held before calling */
>>  static struct pci_bus_ops *pci_bus_ops_pop(void)
>>  {
>> -	unsigned long flags;
>>  	struct pci_bus_ops *bus_ops = NULL;
>>  
>> -	spin_lock_irqsave(&inject_lock, flags);
>>  	if (list_empty(&pci_bus_ops_list))
>>  		bus_ops = NULL;
>>  	else {
>> @@ -154,7 +153,6 @@ static struct pci_bus_ops *pci_bus_ops_pop(void)
>>  		list_del(lh);
>>  		bus_ops = list_entry(lh, struct pci_bus_ops, list);
>>  	}
>> -	spin_unlock_irqrestore(&inject_lock, flags);
>>  	return bus_ops;
>>  }
>>  
>> @@ -610,14 +608,12 @@ static void __exit aer_inject_exit(void)
>>  	bus_unregister_notifier(&pci_bus_type, &aerinj_hp_notifier);
>>  	misc_deregister(&aer_inject_device);
>>  
>> +	spin_lock_irqsave(&inject_lock, flags);
>>  	while ((bus_ops = pci_bus_ops_pop())) {
>>  		pci_bus_set_ops(bus_ops->bus, bus_ops->ops);
>>  		kfree(bus_ops);
>>  	}
>>  	
>> -	clean_untracked_pci_aer_ops();
>> -
>> -	spin_lock_irqsave(&inject_lock, flags);
>>  	list_for_each_entry_safe(err, err_next, &einjected, list) {
>>  		list_del(&err->list);
>>  		kfree(err);
> 
> This may trigger a AB BA dead lock.  In aer_inject_exit, the lock
> sequence is:
> 
> inject_lock -> pci_lock
> 
> In pci config read/write path, the lock sequence is:
> 
> pci_lock -> inject_lock
> 
> With lockdep enabled, you may found warning on that.

OK, I will carry out a detailed test with lockdep enabled, and try to improve this patch.
Thanks for review and comments!

------
Thanks
Yijing

> 
> Best Regards,
> Huang YIng
> 
> 
> 
> .
> 


-- 
Thanks!
Yijing