[PATCH] android: fix warning when releasing active sync point

[PATCH] android: fix warning when releasing active sync point

[Dmitry Torokhov]

Userspace can close the sync device while there are still active fence
points, in which case kernel produces the following warning:

[   43.853176] ------------[ cut here ]------------
[   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
[   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
[   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
[   43.885834] Call trace:
[   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
[   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
[   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
[   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
[   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
[   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
[   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
[   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
[   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
[   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
[   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
[   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
[   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
[   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---

Let's fix it by introducing a new optional callback (disable_signaling)
to fence operations so that drivers can do proper clean ups when we
remove last callback for given fence.

Reviewed-by: Andrew Bresticker <abrestic@chromium.org>
Signed-off-by: Dmitry Torokhov <dtor@chromium.org>
---
 drivers/dma-buf/fence.c        | 6 +++++-
 drivers/staging/android/sync.c | 8 ++++++++
 include/linux/fence.h          | 2 ++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/fence.c
index 7b05dbe..0ed73ad 100644
--- a/drivers/dma-buf/fence.c
+++ b/drivers/dma-buf/fence.c
@@ -304,8 +304,12 @@ fence_remove_callback(struct fence *fence, struct fence_cb *cb)
 	spin_lock_irqsave(fence->lock, flags);
 
 	ret = !list_empty(&cb->node);
-	if (ret)
+	if (ret) {
 		list_del_init(&cb->node);
+		if (list_empty(&fence->cb_list))
+			if (fence->ops->disable_signaling)
+				fence->ops->disable_signaling(fence);
+	}
 
 	spin_unlock_irqrestore(fence->lock, flags);
 
diff --git a/drivers/staging/android/sync.c b/drivers/staging/android/sync.c
index e0c1acb..f8566c1 100644
--- a/drivers/staging/android/sync.c
+++ b/drivers/staging/android/sync.c
@@ -465,6 +465,13 @@ static bool android_fence_enable_signaling(struct fence *fence)
 	return true;
 }
 
+static void android_fence_disable_signaling(struct fence *fence)
+{
+	struct sync_pt *pt = container_of(fence, struct sync_pt, base);
+
+	list_del_init(&pt->active_list);
+}
+
 static int android_fence_fill_driver_data(struct fence *fence,
 					  void *data, int size)
 {
@@ -508,6 +515,7 @@ static const struct fence_ops android_fence_ops = {
 	.get_driver_name = android_fence_get_driver_name,
 	.get_timeline_name = android_fence_get_timeline_name,
 	.enable_signaling = android_fence_enable_signaling,
+	.disable_signaling = android_fence_disable_signaling,
 	.signaled = android_fence_signaled,
 	.wait = fence_default_wait,
 	.release = android_fence_release,
diff --git a/include/linux/fence.h b/include/linux/fence.h
index bb52201..ce44348 100644
--- a/include/linux/fence.h
+++ b/include/linux/fence.h
@@ -107,6 +107,7 @@ struct fence_cb {
  * @get_driver_name: returns the driver name.
  * @get_timeline_name: return the name of the context this fence belongs to.
  * @enable_signaling: enable software signaling of fence.
+ * @disable_signaling: disable software signaling of fence (optional).
  * @signaled: [optional] peek whether the fence is signaled, can be null.
  * @wait: custom wait implementation, or fence_default_wait.
  * @release: [optional] called on destruction of fence, can be null
@@ -166,6 +167,7 @@ struct fence_ops {
 	const char * (*get_driver_name)(struct fence *fence);
 	const char * (*get_timeline_name)(struct fence *fence);
 	bool (*enable_signaling)(struct fence *fence);
+	void (*disable_signaling)(struct fence *fence);
 	bool (*signaled)(struct fence *fence);
 	signed long (*wait)(struct fence *fence, bool intr, signed long timeout);
 	void (*release)(struct fence *fence);
-- 
2.6.0.rc2.230.g3dd15c0


-- 
Dmitry

Re: [PATCH] android: fix warning when releasing active sync point

[Daniel Vetter]

On Mon, Dec 14, 2015 at 05:29:55PM -0800, Dmitry Torokhov wrote:
> Userspace can close the sync device while there are still active fence
> points, in which case kernel produces the following warning:
> 
> [   43.853176] ------------[ cut here ]------------
> [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
> [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
> [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
> [   43.885834] Call trace:
> [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
> [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
> [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
> [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
> [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
> [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
> [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
> [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
> [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
> [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
> [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
> [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
> [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
> [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
> 
> Let's fix it by introducing a new optional callback (disable_signaling)
> to fence operations so that drivers can do proper clean ups when we
> remove last callback for given fence.
> 
> Reviewed-by: Andrew Bresticker <abrestic@chromium.org>
> Signed-off-by: Dmitry Torokhov <dtor@chromium.org>
> ---
>  drivers/dma-buf/fence.c        | 6 +++++-
>  drivers/staging/android/sync.c | 8 ++++++++
>  include/linux/fence.h          | 2 ++
>  3 files changed, 15 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/fence.c
> index 7b05dbe..0ed73ad 100644
> --- a/drivers/dma-buf/fence.c
> +++ b/drivers/dma-buf/fence.c
> @@ -304,8 +304,12 @@ fence_remove_callback(struct fence *fence, struct fence_cb *cb)
>  	spin_lock_irqsave(fence->lock, flags);
>  
>  	ret = !list_empty(&cb->node);
> -	if (ret)
> +	if (ret) {
>  		list_del_init(&cb->node);
> +		if (list_empty(&fence->cb_list))
> +			if (fence->ops->disable_signaling)
> +				fence->ops->disable_signaling(fence);

What exactly is the bug here? A fence with no callbacks registered any
more shouldn't have any problem. Why exactly does this blow up?

I guess I don't really understand the bug ... we do seem to remove the
callback already.

Thanks, Daniel


> +	}
>  
>  	spin_unlock_irqrestore(fence->lock, flags);
>  
> diff --git a/drivers/staging/android/sync.c b/drivers/staging/android/sync.c
> index e0c1acb..f8566c1 100644
> --- a/drivers/staging/android/sync.c
> +++ b/drivers/staging/android/sync.c
> @@ -465,6 +465,13 @@ static bool android_fence_enable_signaling(struct fence *fence)
>  	return true;
>  }
>  
> +static void android_fence_disable_signaling(struct fence *fence)
> +{
> +	struct sync_pt *pt = container_of(fence, struct sync_pt, base);
> +
> +	list_del_init(&pt->active_list);
> +}
> +
>  static int android_fence_fill_driver_data(struct fence *fence,
>  					  void *data, int size)
>  {
> @@ -508,6 +515,7 @@ static const struct fence_ops android_fence_ops = {
>  	.get_driver_name = android_fence_get_driver_name,
>  	.get_timeline_name = android_fence_get_timeline_name,
>  	.enable_signaling = android_fence_enable_signaling,
> +	.disable_signaling = android_fence_disable_signaling,
>  	.signaled = android_fence_signaled,
>  	.wait = fence_default_wait,
>  	.release = android_fence_release,
> diff --git a/include/linux/fence.h b/include/linux/fence.h
> index bb52201..ce44348 100644
> --- a/include/linux/fence.h
> +++ b/include/linux/fence.h
> @@ -107,6 +107,7 @@ struct fence_cb {
>   * @get_driver_name: returns the driver name.
>   * @get_timeline_name: return the name of the context this fence belongs to.
>   * @enable_signaling: enable software signaling of fence.
> + * @disable_signaling: disable software signaling of fence (optional).
>   * @signaled: [optional] peek whether the fence is signaled, can be null.
>   * @wait: custom wait implementation, or fence_default_wait.
>   * @release: [optional] called on destruction of fence, can be null
> @@ -166,6 +167,7 @@ struct fence_ops {
>  	const char * (*get_driver_name)(struct fence *fence);
>  	const char * (*get_timeline_name)(struct fence *fence);
>  	bool (*enable_signaling)(struct fence *fence);
> +	void (*disable_signaling)(struct fence *fence);
>  	bool (*signaled)(struct fence *fence);
>  	signed long (*wait)(struct fence *fence, bool intr, signed long timeout);
>  	void (*release)(struct fence *fence);
> -- 
> 2.6.0.rc2.230.g3dd15c0
> 
> 
> -- 
> Dmitry
> _______________________________________________
> dri-devel mailing list
> dri-devel@lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/dri-devel

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

Re: [PATCH] android: fix warning when releasing active sync point

[Maarten Lankhorst]

Op 15-12-15 om 02:29 schreef Dmitry Torokhov:
> Userspace can close the sync device while there are still active fence
> points, in which case kernel produces the following warning:
>
> [   43.853176] ------------[ cut here ]------------
> [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
> [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
> [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
> [   43.885834] Call trace:
> [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
> [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
> [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
> [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
> [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
> [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
> [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
> [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
> [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
> [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
> [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
> [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
> [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
> [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
>
> Let's fix it by introducing a new optional callback (disable_signaling)
> to fence operations so that drivers can do proper clean ups when we
> remove last callback for given fence.
>
> Reviewed-by: Andrew Bresticker <abrestic@chromium.org>
> Signed-off-by: Dmitry Torokhov <dtor@chromium.org>
>
NACK! There's no way to do this race free.
The driver should hold a refcount until fence is signaled.

Re: [PATCH] android: fix warning when releasing active sync point

[Gustavo Padovan]

2015-12-14 Dmitry Torokhov <dtor@chromium.org>:

> Userspace can close the sync device while there are still active fence
> points, in which case kernel produces the following warning:
> 
> [   43.853176] ------------[ cut here ]------------
> [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
> [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
> [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
> [   43.885834] Call trace:
> [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
> [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
> [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
> [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
> [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
> [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
> [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
> [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
> [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
> [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
> [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
> [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
> [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
> [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---

This crash report seems to be for a 3.18 kernel. Can you reproduce it
on upstream kernel as well?

	Gustavo

Re: [PATCH] android: fix warning when releasing active sync point

[Frank Binns]

Is this not the issue fixed by 8e43c9c75?

Thanks
Frank

On 15/12/15 13:30, Gustavo Padovan wrote:
> 2015-12-14 Dmitry Torokhov <dtor@chromium.org>:
>
>> Userspace can close the sync device while there are still active fence
>> points, in which case kernel produces the following warning:
>>
>> [   43.853176] ------------[ cut here ]------------
>> [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
>> [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
>> [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
>> [   43.885834] Call trace:
>> [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
>> [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
>> [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
>> [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
>> [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
>> [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
>> [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
>> [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
>> [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
>> [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
>> [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
>> [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
>> [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
>> [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
> This crash report seems to be for a 3.18 kernel. Can you reproduce it
> on upstream kernel as well?
>
> 	Gustavo
> _______________________________________________
> dri-devel mailing list
> dri-devel@lists.freedesktop.org
> http://lists.freedesktop.org/mailman/listinfo/dri-devel

Re: [PATCH] android: fix warning when releasing active sync point

[Dmitry Torokhov]

On Tue, Dec 15, 2015 at 1:26 AM, Daniel Vetter <daniel@ffwll.ch> wrote:
> On Mon, Dec 14, 2015 at 05:29:55PM -0800, Dmitry Torokhov wrote:
>> Userspace can close the sync device while there are still active fence
>> points, in which case kernel produces the following warning:
>>
>> [   43.853176] ------------[ cut here ]------------
>> [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
>> [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
>> [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
>> [   43.885834] Call trace:
>> [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
>> [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
>> [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
>> [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
>> [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
>> [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
>> [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
>> [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
>> [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
>> [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
>> [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
>> [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
>> [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
>> [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
>>
>> Let's fix it by introducing a new optional callback (disable_signaling)
>> to fence operations so that drivers can do proper clean ups when we
>> remove last callback for given fence.
>>
>> Reviewed-by: Andrew Bresticker <abrestic@chromium.org>
>> Signed-off-by: Dmitry Torokhov <dtor@chromium.org>
>> ---
>>  drivers/dma-buf/fence.c        | 6 +++++-
>>  drivers/staging/android/sync.c | 8 ++++++++
>>  include/linux/fence.h          | 2 ++
>>  3 files changed, 15 insertions(+), 1 deletion(-)
>>
>> diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/fence.c
>> index 7b05dbe..0ed73ad 100644
>> --- a/drivers/dma-buf/fence.c
>> +++ b/drivers/dma-buf/fence.c
>> @@ -304,8 +304,12 @@ fence_remove_callback(struct fence *fence, struct fence_cb *cb)
>>       spin_lock_irqsave(fence->lock, flags);
>>
>>       ret = !list_empty(&cb->node);
>> -     if (ret)
>> +     if (ret) {
>>               list_del_init(&cb->node);
>> +             if (list_empty(&fence->cb_list))
>> +                     if (fence->ops->disable_signaling)
>> +                             fence->ops->disable_signaling(fence);
>
> What exactly is the bug here? A fence with no callbacks registered any
> more shouldn't have any problem. Why exactly does this blow up?
>
> I guess I don't really understand the bug ... we do seem to remove the
> callback already.
>

The issue is that when enabling signalling in sync driver we put fence
on an internal list in the driver and there is no way of taking it off
this list, except when it is signalled. The driver, when destroying
the fence, checks if the fence is not on this list (as a sanity
measure) and that produces the backtrace in the commit log.

IOW for some drivers we need an "undo" for enable_signaling() callback
so that drivers can maintain consistent internal state.

Thanks.

-- 
Dmitry

Re: [PATCH] android: fix warning when releasing active sync point

[Dmitry Torokhov]

On Tue, Dec 15, 2015 at 2:01 AM, Maarten Lankhorst
<maarten.lankhorst@linux.intel.com> wrote:
> Op 15-12-15 om 02:29 schreef Dmitry Torokhov:
>> Userspace can close the sync device while there are still active fence
>> points, in which case kernel produces the following warning:
>>
>> [   43.853176] ------------[ cut here ]------------
>> [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
>> [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
>> [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
>> [   43.885834] Call trace:
>> [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
>> [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
>> [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
>> [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
>> [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
>> [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
>> [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
>> [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
>> [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
>> [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
>> [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
>> [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
>> [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
>> [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
>>
>> Let's fix it by introducing a new optional callback (disable_signaling)
>> to fence operations so that drivers can do proper clean ups when we
>> remove last callback for given fence.
>>
>> Reviewed-by: Andrew Bresticker <abrestic@chromium.org>
>> Signed-off-by: Dmitry Torokhov <dtor@chromium.org>
>>
> NACK! There's no way to do this race free.

Can you please explain the race because as far as I can see there is not one.

> The driver should hold a refcount until fence is signaled.

If we are no longer interested in fence why do we need to wait for the
fence to be signaled?

Thanks.

-- 
Dmitry

Re: [PATCH] android: fix warning when releasing active sync point

[Dmitry Torokhov]

On Tue, Dec 15, 2015 at 5:50 AM, Frank Binns <frank.binns@imgtec.com> wrote:
> Is this not the issue fixed by 8e43c9c75?

No because if we start teardown without waiting for the fence to be
signaled it will still be on the active_list.

Thanks.

-- 
Dmitry

Re: [PATCH] android: fix warning when releasing active sync point

[Dmitry Torokhov]

On Tue, Dec 15, 2015 at 5:30 AM, Gustavo Padovan <gustavo@padovan.org> wrote:
> 2015-12-14 Dmitry Torokhov <dtor@chromium.org>:
>
>> Userspace can close the sync device while there are still active fence
>> points, in which case kernel produces the following warning:
>>
>> [   43.853176] ------------[ cut here ]------------
>> [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
>> [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
>> [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
>> [   43.885834] Call trace:
>> [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
>> [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
>> [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
>> [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
>> [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
>> [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
>> [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
>> [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
>> [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
>> [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
>> [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
>> [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
>> [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
>> [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
>
> This crash report seems to be for a 3.18 kernel. Can you reproduce it
> on upstream kernel as well?

Unfortunately this board does not run upsrteam just yet, but looking
at the sync driver and fence code we are pretty much in sync with
upstream.

Thanks.

-- 
Dmitry

Re: [PATCH] android: fix warning when releasing active sync point

[Gustavo Padovan]

2015-12-15 Daniel Vetter <daniel@ffwll.ch>:

> On Mon, Dec 14, 2015 at 05:29:55PM -0800, Dmitry Torokhov wrote:
> > Userspace can close the sync device while there are still active fence
> > points, in which case kernel produces the following warning:
> > 
> > [   43.853176] ------------[ cut here ]------------
> > [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
> > [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
> > [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
> > [   43.885834] Call trace:
> > [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
> > [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
> > [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
> > [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
> > [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
> > [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
> > [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
> > [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
> > [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
> > [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
> > [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
> > [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
> > [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
> > [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
> > 
> > Let's fix it by introducing a new optional callback (disable_signaling)
> > to fence operations so that drivers can do proper clean ups when we
> > remove last callback for given fence.
> > 
> > Reviewed-by: Andrew Bresticker <abrestic@chromium.org>
> > Signed-off-by: Dmitry Torokhov <dtor@chromium.org>
> > ---
> >  drivers/dma-buf/fence.c        | 6 +++++-
> >  drivers/staging/android/sync.c | 8 ++++++++
> >  include/linux/fence.h          | 2 ++
> >  3 files changed, 15 insertions(+), 1 deletion(-)
> > 
> > diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/fence.c
> > index 7b05dbe..0ed73ad 100644
> > --- a/drivers/dma-buf/fence.c
> > +++ b/drivers/dma-buf/fence.c
> > @@ -304,8 +304,12 @@ fence_remove_callback(struct fence *fence, struct fence_cb *cb)
> >  	spin_lock_irqsave(fence->lock, flags);
> >  
> >  	ret = !list_empty(&cb->node);
> > -	if (ret)
> > +	if (ret) {
> >  		list_del_init(&cb->node);
> > +		if (list_empty(&fence->cb_list))
> > +			if (fence->ops->disable_signaling)
> > +				fence->ops->disable_signaling(fence);
> 
> What exactly is the bug here? A fence with no callbacks registered any
> more shouldn't have any problem. Why exactly does this blow up?

The WARN_ON is probably this one:
https://android.googlesource.com/kernel/common/+/android-3.18/drivers/staging/android/sync.c#433

I've been wondering in the last few days if this warning is really
necessary. If the user is closing a sync_timeline that has unsignalled
fences it should probably be aware of that already. Then I think it is
okay to remove the the sync_pt from the active_list at the release-time.
In fact I've already prepared a patch doing that. Thoughts?  

	Gustavo

Re: [PATCH] android: fix warning when releasing active sync point

[Dmitry Torokhov]

On Tue, Dec 15, 2015 at 11:00 AM, Gustavo Padovan <gustavo@padovan.org> wrote:
> 2015-12-15 Daniel Vetter <daniel@ffwll.ch>:
>
>> On Mon, Dec 14, 2015 at 05:29:55PM -0800, Dmitry Torokhov wrote:
>> > Userspace can close the sync device while there are still active fence
>> > points, in which case kernel produces the following warning:
>> >
>> > [   43.853176] ------------[ cut here ]------------
>> > [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
>> > [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
>> > [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
>> > [   43.885834] Call trace:
>> > [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
>> > [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
>> > [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
>> > [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
>> > [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
>> > [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
>> > [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
>> > [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
>> > [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
>> > [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
>> > [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
>> > [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
>> > [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
>> > [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
>> >
>> > Let's fix it by introducing a new optional callback (disable_signaling)
>> > to fence operations so that drivers can do proper clean ups when we
>> > remove last callback for given fence.
>> >
>> > Reviewed-by: Andrew Bresticker <abrestic@chromium.org>
>> > Signed-off-by: Dmitry Torokhov <dtor@chromium.org>
>> > ---
>> >  drivers/dma-buf/fence.c        | 6 +++++-
>> >  drivers/staging/android/sync.c | 8 ++++++++
>> >  include/linux/fence.h          | 2 ++
>> >  3 files changed, 15 insertions(+), 1 deletion(-)
>> >
>> > diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/fence.c
>> > index 7b05dbe..0ed73ad 100644
>> > --- a/drivers/dma-buf/fence.c
>> > +++ b/drivers/dma-buf/fence.c
>> > @@ -304,8 +304,12 @@ fence_remove_callback(struct fence *fence, struct fence_cb *cb)
>> >     spin_lock_irqsave(fence->lock, flags);
>> >
>> >     ret = !list_empty(&cb->node);
>> > -   if (ret)
>> > +   if (ret) {
>> >             list_del_init(&cb->node);
>> > +           if (list_empty(&fence->cb_list))
>> > +                   if (fence->ops->disable_signaling)
>> > +                           fence->ops->disable_signaling(fence);
>>
>> What exactly is the bug here? A fence with no callbacks registered any
>> more shouldn't have any problem. Why exactly does this blow up?
>
> The WARN_ON is probably this one:
> https://android.googlesource.com/kernel/common/+/android-3.18/drivers/staging/android/sync.c#433
>
> I've been wondering in the last few days if this warning is really
> necessary. If the user is closing a sync_timeline that has unsignalled
> fences it should probably be aware of that already. Then I think it is
> okay to remove the the sync_pt from the active_list at the release-time.
> In fact I've already prepared a patch doing that. Thoughts?
>

Maybe, but you need to make sure that you only affecting your fences.

My main objection is that still leaves fence_remove_callback() being
not mirror image of fence_add_callback().

-- 
Dmitry

Re: [PATCH] android: fix warning when releasing active sync point

[Maarten Lankhorst]

Op 15-12-15 om 18:19 schreef Dmitry Torokhov:
> On Tue, Dec 15, 2015 at 2:01 AM, Maarten Lankhorst
> <maarten.lankhorst@linux.intel.com> wrote:
>> Op 15-12-15 om 02:29 schreef Dmitry Torokhov:
>>> Userspace can close the sync device while there are still active fence
>>> points, in which case kernel produces the following warning:
>>>
>>> [   43.853176] ------------[ cut here ]------------
>>> [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
>>> [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
>>> [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
>>> [   43.885834] Call trace:
>>> [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
>>> [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
>>> [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
>>> [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
>>> [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
>>> [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
>>> [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
>>> [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
>>> [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
>>> [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
>>> [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
>>> [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
>>> [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
>>> [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
>>>
>>> Let's fix it by introducing a new optional callback (disable_signaling)
>>> to fence operations so that drivers can do proper clean ups when we
>>> remove last callback for given fence.
>>>
>>> Reviewed-by: Andrew Bresticker <abrestic@chromium.org>
>>> Signed-off-by: Dmitry Torokhov <dtor@chromium.org>
>>>
>> NACK! There's no way to do this race free.
> Can you please explain the race because as far as I can see there is not one.\
The entire code in fence.c assumes that a fence can only go from not enable_signaling to .enable_signaling. .enable_signaling is not refcounted so 2 calls to .enable_disabling and 1 to .disable_signaling would mess up.
Furthermore we try to make sure that fence_signal doesn't take locks if its unneeded. With a disable_signaling callback you would always need locks.

To get rid of these warnings make sure that there's a refcount on the fence until it's signaled.
>> The driver should hold a refcount until fence is signaled.
> If we are no longer interested in fence why do we need to wait for the
> fence to be signaled?
It's the part of the design. A driver tracks its outstanding requests and submissions, and every submission has its own fence. Before the driver releases its final ref the request should be completed or aborted. In either case it must call fence_signal.

Re: [PATCH] android: fix warning when releasing active sync point

[Daniel Vetter]

On Tue, Dec 15, 2015 at 11:08:01AM -0800, Dmitry Torokhov wrote:
> On Tue, Dec 15, 2015 at 11:00 AM, Gustavo Padovan <gustavo@padovan.org> wrote:
> > 2015-12-15 Daniel Vetter <daniel@ffwll.ch>:
> >
> >> On Mon, Dec 14, 2015 at 05:29:55PM -0800, Dmitry Torokhov wrote:
> >> > Userspace can close the sync device while there are still active fence
> >> > points, in which case kernel produces the following warning:
> >> >
> >> > [   43.853176] ------------[ cut here ]------------
> >> > [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
> >> > [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
> >> > [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
> >> > [   43.885834] Call trace:
> >> > [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
> >> > [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
> >> > [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
> >> > [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
> >> > [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
> >> > [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
> >> > [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
> >> > [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
> >> > [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
> >> > [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
> >> > [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
> >> > [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
> >> > [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
> >> > [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
> >> >
> >> > Let's fix it by introducing a new optional callback (disable_signaling)
> >> > to fence operations so that drivers can do proper clean ups when we
> >> > remove last callback for given fence.
> >> >
> >> > Reviewed-by: Andrew Bresticker <abrestic@chromium.org>
> >> > Signed-off-by: Dmitry Torokhov <dtor@chromium.org>
> >> > ---
> >> >  drivers/dma-buf/fence.c        | 6 +++++-
> >> >  drivers/staging/android/sync.c | 8 ++++++++
> >> >  include/linux/fence.h          | 2 ++
> >> >  3 files changed, 15 insertions(+), 1 deletion(-)
> >> >
> >> > diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/fence.c
> >> > index 7b05dbe..0ed73ad 100644
> >> > --- a/drivers/dma-buf/fence.c
> >> > +++ b/drivers/dma-buf/fence.c
> >> > @@ -304,8 +304,12 @@ fence_remove_callback(struct fence *fence, struct fence_cb *cb)
> >> >     spin_lock_irqsave(fence->lock, flags);
> >> >
> >> >     ret = !list_empty(&cb->node);
> >> > -   if (ret)
> >> > +   if (ret) {
> >> >             list_del_init(&cb->node);
> >> > +           if (list_empty(&fence->cb_list))
> >> > +                   if (fence->ops->disable_signaling)
> >> > +                           fence->ops->disable_signaling(fence);
> >>
> >> What exactly is the bug here? A fence with no callbacks registered any
> >> more shouldn't have any problem. Why exactly does this blow up?
> >
> > The WARN_ON is probably this one:
> > https://android.googlesource.com/kernel/common/+/android-3.18/drivers/staging/android/sync.c#433
> >
> > I've been wondering in the last few days if this warning is really
> > necessary. If the user is closing a sync_timeline that has unsignalled
> > fences it should probably be aware of that already. Then I think it is
> > okay to remove the the sync_pt from the active_list at the release-time.
> > In fact I've already prepared a patch doing that. Thoughts?
> >
> 
> Maybe, but you need to make sure that you only affecting your fences.
> 
> My main objection is that still leaves fence_remove_callback() being
> not mirror image of fence_add_callback().

That's 100% intentional. I looked at the sync.c code a bit more and it
duplicates a bunch of the fence stuff still. We need to either merge that
code into the mainline struct fence logic, or remove it. There shouldn't
really be any need for the userspace ABI layer to keep track of active
fences at all. Worse, it means that you must use the sync_pt struct to be
able to export it to userspace, and can't just export any normal struct
fence object. That breaks the abstraction we're aiming for.

Imo just remove that WARN_ON for now.
-Daniel
-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch

Re: [PATCH] android: fix warning when releasing active sync point

[Daniel Vetter]

On Tue, Dec 15, 2015 at 09:22:56AM -0800, Dmitry Torokhov wrote:
> On Tue, Dec 15, 2015 at 5:30 AM, Gustavo Padovan <gustavo@padovan.org> wrote:
> > 2015-12-14 Dmitry Torokhov <dtor@chromium.org>:
> >
> >> Userspace can close the sync device while there are still active fence
> >> points, in which case kernel produces the following warning:
> >>
> >> [   43.853176] ------------[ cut here ]------------
> >> [   43.857834] WARNING: CPU: 0 PID: 892 at /mnt/host/source/src/third_party/kernel/v3.18/drivers/staging/android/sync.c:439 android_fence_release+0x88/0x104()
> >> [   43.871741] CPU: 0 PID: 892 Comm: Binder_5 Tainted: G     U 3.18.0-07661-g0550ce9 #1
> >> [   43.880176] Hardware name: Google Tegra210 Smaug Rev 1+ (DT)
> >> [   43.885834] Call trace:
> >> [   43.888294] [<ffffffc000207464>] dump_backtrace+0x0/0x10c
> >> [   43.893697] [<ffffffc000207580>] show_stack+0x10/0x1c
> >> [   43.898756] [<ffffffc000ab1258>] dump_stack+0x74/0xb8
> >> [   43.903814] [<ffffffc00021d414>] warn_slowpath_common+0x84/0xb0
> >> [   43.909736] [<ffffffc00021d530>] warn_slowpath_null+0x14/0x20
> >> [   43.915482] [<ffffffc00088aefc>] android_fence_release+0x84/0x104
> >> [   43.921582] [<ffffffc000671cc4>] fence_release+0x104/0x134
> >> [   43.927066] [<ffffffc00088b0cc>] sync_fence_free+0x74/0x9c
> >> [   43.932552] [<ffffffc00088b128>] sync_fence_release+0x34/0x48
> >> [   43.938304] [<ffffffc000317bbc>] __fput+0x100/0x1b8
> >> [   43.943185] [<ffffffc000317cc8>] ____fput+0x8/0x14
> >> [   43.947982] [<ffffffc000237f38>] task_work_run+0xb0/0xe4
> >> [   43.953297] [<ffffffc000207074>] do_notify_resume+0x44/0x5c
> >> [   43.958867] ---[ end trace 5a2aa4027cc5d171 ]---
> >
> > This crash report seems to be for a 3.18 kernel. Can you reproduce it
> > on upstream kernel as well?
> 
> Unfortunately this board does not run upsrteam just yet, but looking
> at the sync driver and fence code we are pretty much in sync with
> upstream.

Just to check: Is that with a proper hw driver, or using SW_SYNC? The
later will get removed in upstream since it's a debug/validation only
interface. Well, removed for drivers and production systems, the
kselftests will use it.
-Daniel
-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch