[PATCH] PM-Trace: add pm-trace support for suspending phase

[PATCH] PM-Trace: add pm-trace support for suspending phase

[Fu, Zhonghui]

>From ec9c0f24e800461361eaf8b280abe4272b00772d Mon Sep 17 00:00:00 2001
From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
Date: Mon, 9 Feb 2015 11:09:22 +0800
Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase

Occasinally, the system can't come back up after suspend/resume
due to problems of device suspending phase. This patch make
PM_TRACE infrastructure cover device suspending phase of
suspend/resume process, and the information in RTC can tell
develpers which device suspending function make system hang.

Signed-off-by: Zhonghui Fu <zhonghui.fu@linux.intel.com>
---
 arch/x86/include/asm/resume-trace.h |    2 +-
 drivers/base/power/main.c           |   30 +++++++++++++++++++++---------
 include/linux/resume-trace.h        |    2 +-
 3 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/arch/x86/include/asm/resume-trace.h b/arch/x86/include/asm/resume-trace.h
index 3ff1c2c..0ca2215 100644
--- a/arch/x86/include/asm/resume-trace.h
+++ b/arch/x86/include/asm/resume-trace.h
@@ -3,7 +3,7 @@
 
 #include <asm/asm.h>
 
-#define TRACE_RESUME(user)					\
+#define TRACE_SUSPEND_RESUME(user)				\
 do {								\
 	if (pm_trace_enabled) {					\
 		const void *tracedata;				\
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index 9717d5f..d2070b5 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -477,7 +477,7 @@ static int device_resume_noirq(struct device *dev, pm_message_t state, bool asyn
 	int error = 0;
 
 	TRACE_DEVICE(dev);
-	TRACE_RESUME(0);
+	TRACE_SUSPEND_RESUME(0);
 
 	if (dev->power.syscore || dev->power.direct_complete)
 		goto Out;
@@ -511,7 +511,7 @@ static int device_resume_noirq(struct device *dev, pm_message_t state, bool asyn
 
  Out:
 	complete_all(&dev->power.completion);
-	TRACE_RESUME(error);
+	TRACE_SUSPEND_RESUME(error);
 	return error;
 }
 
@@ -606,7 +606,7 @@ static int device_resume_early(struct device *dev, pm_message_t state, bool asyn
 	int error = 0;
 
 	TRACE_DEVICE(dev);
-	TRACE_RESUME(0);
+	TRACE_SUSPEND_RESUME(0);
 
 	if (dev->power.syscore || dev->power.direct_complete)
 		goto Out;
@@ -639,7 +639,7 @@ static int device_resume_early(struct device *dev, pm_message_t state, bool asyn
 	dev->power.is_late_suspended = false;
 
  Out:
-	TRACE_RESUME(error);
+	TRACE_SUSPEND_RESUME(error);
 
 	pm_runtime_enable(dev);
 	complete_all(&dev->power.completion);
@@ -735,7 +735,7 @@ static int device_resume(struct device *dev, pm_message_t state, bool async)
 	DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 
 	TRACE_DEVICE(dev);
-	TRACE_RESUME(0);
+	TRACE_SUSPEND_RESUME(0);
 
 	if (dev->power.syscore)
 		goto Complete;
@@ -811,7 +811,7 @@ static int device_resume(struct device *dev, pm_message_t state, bool async)
  Complete:
 	complete_all(&dev->power.completion);
 
-	TRACE_RESUME(error);
+	TRACE_SUSPEND_RESUME(error);
 
 	return error;
 }
@@ -1017,6 +1017,9 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
 	char *info = NULL;
 	int error = 0;
 
+	TRACE_DEVICE(dev);
+	TRACE_SUSPEND_RESUME(0);
+
 	if (async_error)
 		goto Complete;
 
@@ -1057,6 +1060,7 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
 
 Complete:
 	complete_all(&dev->power.completion);
+	TRACE_SUSPEND_RESUME(error);
 	return error;
 }
 
@@ -1078,7 +1082,7 @@ static int device_suspend_noirq(struct device *dev)
 {
 	reinit_completion(&dev->power.completion);
 
-	if (pm_async_enabled && dev->power.async_suspend) {
+	if (is_async(dev)) {
 		get_device(dev);
 		async_schedule(async_suspend_noirq, dev);
 		return 0;
@@ -1157,6 +1161,9 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as
 	char *info = NULL;
 	int error = 0;
 
+	TRACE_DEVICE(dev);
+	TRACE_SUSPEND_RESUME(0);
+
 	__pm_runtime_disable(dev, false);
 
 	if (async_error)
@@ -1198,6 +1205,7 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as
 		async_error = error;
 
 Complete:
+	TRACE_SUSPEND_RESUME(error);
 	complete_all(&dev->power.completion);
 	return error;
 }
@@ -1219,7 +1227,7 @@ static int device_suspend_late(struct device *dev)
 {
 	reinit_completion(&dev->power.completion);
 
-	if (pm_async_enabled && dev->power.async_suspend) {
+	if (is_async(dev)) {
 		get_device(dev);
 		async_schedule(async_suspend_late, dev);
 		return 0;
@@ -1338,6 +1346,9 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 	int error = 0;
 	DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 
+	TRACE_DEVICE(dev);
+	TRACE_SUSPEND_RESUME(0);
+
 	dpm_wait_for_children(dev, async);
 
 	if (async_error)
@@ -1444,6 +1455,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 	if (error)
 		async_error = error;
 
+	TRACE_SUSPEND_RESUME(error);
 	return error;
 }
 
@@ -1465,7 +1477,7 @@ static int device_suspend(struct device *dev)
 {
 	reinit_completion(&dev->power.completion);
 
-	if (pm_async_enabled && dev->power.async_suspend) {
+	if (is_async(dev)) {
 		get_device(dev);
 		async_schedule(async_suspend, dev);
 		return 0;
diff --git a/include/linux/resume-trace.h b/include/linux/resume-trace.h
index f31db23..9080f25 100644
--- a/include/linux/resume-trace.h
+++ b/include/linux/resume-trace.h
@@ -27,7 +27,7 @@ extern int show_trace_dev_match(char *buf, size_t size);
 static inline int pm_trace_is_enabled(void) { return 0; }
 
 #define TRACE_DEVICE(dev) do { } while (0)
-#define TRACE_RESUME(dev) do { } while (0)
+#define TRACE_SUSPEND_RESUME(dev) do { } while (0)
 
 #endif
 
-- 1.7.1

Re: [PATCH] PM-Trace: add pm-trace support for suspending phase

[Greg Kroah-Hartman]

On Mon, Feb 09, 2015 at 12:03:23PM +0800, Fu, Zhonghui wrote:
> >From ec9c0f24e800461361eaf8b280abe4272b00772d Mon Sep 17 00:00:00 2001
> From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
> Date: Mon, 9 Feb 2015 11:09:22 +0800
> Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase

Why is this crud here?

> 
> Occasinally, the system can't come back up after suspend/resume
> due to problems of device suspending phase. This patch make
> PM_TRACE infrastructure cover device suspending phase of
> suspend/resume process, and the information in RTC can tell
> develpers which device suspending function make system hang.
> 
> Signed-off-by: Zhonghui Fu <zhonghui.fu@linux.intel.com>
> ---
>  arch/x86/include/asm/resume-trace.h |    2 +-
>  drivers/base/power/main.c           |   30 +++++++++++++++++++++---------
>  include/linux/resume-trace.h        |    2 +-
>  3 files changed, 23 insertions(+), 11 deletions(-)
> 
> diff --git a/arch/x86/include/asm/resume-trace.h b/arch/x86/include/asm/resume-trace.h
> index 3ff1c2c..0ca2215 100644
> --- a/arch/x86/include/asm/resume-trace.h
> +++ b/arch/x86/include/asm/resume-trace.h
> @@ -3,7 +3,7 @@
>  
>  #include <asm/asm.h>
>  
> -#define TRACE_RESUME(user)					\
> +#define TRACE_SUSPEND_RESUME(user)				\
>  do {								\
>  	if (pm_trace_enabled) {					\
>  		const void *tracedata;				\

Why rename this?

If you want to do that, do it in 2 different functions (rename first,
with justification, add new functionality in a second patch)

> @@ -1078,7 +1082,7 @@ static int device_suspend_noirq(struct device *dev)
>  {
>  	reinit_completion(&dev->power.completion);
>  
> -	if (pm_async_enabled && dev->power.async_suspend) {
> +	if (is_async(dev)) {

Why change this logic?

greg k-h

Re: [PATCH] PM-Trace: add pm-trace support for suspending phase

[Fu, Zhonghui]

On 2015/2/10 5:55, Greg Kroah-Hartman wrote:
> On Mon, Feb 09, 2015 at 12:03:23PM +0800, Fu, Zhonghui wrote:
>> >From ec9c0f24e800461361eaf8b280abe4272b00772d Mon Sep 17 00:00:00 2001
>> From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
>> Date: Mon, 9 Feb 2015 11:09:22 +0800
>> Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase
> Why is this crud here?
I will fix this in new patch.
>
>> Occasinally, the system can't come back up after suspend/resume
>> due to problems of device suspending phase. This patch make
>> PM_TRACE infrastructure cover device suspending phase of
>> suspend/resume process, and the information in RTC can tell
>> develpers which device suspending function make system hang.
>>
>> Signed-off-by: Zhonghui Fu <zhonghui.fu@linux.intel.com>
>> ---
>>  arch/x86/include/asm/resume-trace.h |    2 +-
>>  drivers/base/power/main.c           |   30 +++++++++++++++++++++---------
>>  include/linux/resume-trace.h        |    2 +-
>>  3 files changed, 23 insertions(+), 11 deletions(-)
>>
>> diff --git a/arch/x86/include/asm/resume-trace.h b/arch/x86/include/asm/resume-trace.h
>> index 3ff1c2c..0ca2215 100644
>> --- a/arch/x86/include/asm/resume-trace.h
>> +++ b/arch/x86/include/asm/resume-trace.h
>> @@ -3,7 +3,7 @@
>>  
>>  #include <asm/asm.h>
>>  
>> -#define TRACE_RESUME(user)					\
>> +#define TRACE_SUSPEND_RESUME(user)				\
>>  do {								\
>>  	if (pm_trace_enabled) {					\
>>  		const void *tracedata;				\
> Why rename this?
>
> If you want to do that, do it in 2 different functions (rename first,
> with justification, add new functionality in a second patch)
Good comments, I will sent a new patch.
>
>> @@ -1078,7 +1082,7 @@ static int device_suspend_noirq(struct device *dev)
>>  {
>>  	reinit_completion(&dev->power.completion);
>>  
>> -	if (pm_async_enabled && dev->power.async_suspend) {
>> +	if (is_async(dev)) {
> Why change this logic?
Now, PM Trace does not support asynchronous suspend/resume mode. is_async() function can inhibit async mode during using PM Trace.

Thanks,
Zhonghui
>
> greg k-h
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [PATCH] PM-Trace: add pm-trace support for suspending phase

[Fu, Zhonghui]

I have sent a new patch with the same subject in another mail. Any comments are welcome.

Thanks,
Zhonghui

On 2015/2/11 10:50, Fu, Zhonghui wrote:
> On 2015/2/10 5:55, Greg Kroah-Hartman wrote:
>> On Mon, Feb 09, 2015 at 12:03:23PM +0800, Fu, Zhonghui wrote:
>>> >From ec9c0f24e800461361eaf8b280abe4272b00772d Mon Sep 17 00:00:00 2001
>>> From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
>>> Date: Mon, 9 Feb 2015 11:09:22 +0800
>>> Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase
>> Why is this crud here?
> I will fix this in new patch.
>>> Occasinally, the system can't come back up after suspend/resume
>>> due to problems of device suspending phase. This patch make
>>> PM_TRACE infrastructure cover device suspending phase of
>>> suspend/resume process, and the information in RTC can tell
>>> develpers which device suspending function make system hang.
>>>
>>> Signed-off-by: Zhonghui Fu <zhonghui.fu@linux.intel.com>
>>> ---
>>>  arch/x86/include/asm/resume-trace.h |    2 +-
>>>  drivers/base/power/main.c           |   30 +++++++++++++++++++++---------
>>>  include/linux/resume-trace.h        |    2 +-
>>>  3 files changed, 23 insertions(+), 11 deletions(-)
>>>
>>> diff --git a/arch/x86/include/asm/resume-trace.h b/arch/x86/include/asm/resume-trace.h
>>> index 3ff1c2c..0ca2215 100644
>>> --- a/arch/x86/include/asm/resume-trace.h
>>> +++ b/arch/x86/include/asm/resume-trace.h
>>> @@ -3,7 +3,7 @@
>>>  
>>>  #include <asm/asm.h>
>>>  
>>> -#define TRACE_RESUME(user)					\
>>> +#define TRACE_SUSPEND_RESUME(user)				\
>>>  do {								\
>>>  	if (pm_trace_enabled) {					\
>>>  		const void *tracedata;				\
>> Why rename this?
>>
>> If you want to do that, do it in 2 different functions (rename first,
>> with justification, add new functionality in a second patch)
> Good comments, I will sent a new patch.
>>> @@ -1078,7 +1082,7 @@ static int device_suspend_noirq(struct device *dev)
>>>  {
>>>  	reinit_completion(&dev->power.completion);
>>>  
>>> -	if (pm_async_enabled && dev->power.async_suspend) {
>>> +	if (is_async(dev)) {
>> Why change this logic?
> Now, PM Trace does not support asynchronous suspend/resume mode. is_async() function can inhibit async mode during using PM Trace.
>
> Thanks,
> Zhonghui
>> greg k-h
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>> Please read the FAQ at  http://www.tux.org/lkml/

Re: [PATCH] PM-Trace: add pm-trace support for suspending phase

[Fu, Zhonghui]

On 2015/2/13 6:24, Rafael J. Wysocki wrote:
> On Thursday, February 12, 2015 12:48:06 PM Fu, Zhonghui wrote:
>> On 2015/2/11 23:25, Rafael J. Wysocki wrote:
>>> On Wednesday, February 11, 2015 10:43:13 PM Fu, Zhonghui wrote:
>>>> From 2626594c03ca3b9884dd44073385c36f99a3651d Mon Sep 17 00:00:00 2001
>>>> From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
>>>> Date: Wed, 11 Feb 2015 16:20:21 +0800
>>>> Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase
>>> Can you please avoid putting the above header lines into your patches?
>>>
>>> The only one that may be necessary is the From: line, but that's only
>>> needed if you submit from a different address.
>>>
>>> Also please don't send To: mailing lists.  Use CC instead.  In fact, there
>>> should be *one* address in the To: field when you're submitting a patch:
>>> The address of the maintainer you want to apply the patch for you.
>> Thanks for your guidance, and I will re-send this patch.
>>
>>>> Occasionally, the system can't come back up after suspend/resume
>>>> due to problems of device suspending phase. This patch make
>>>> PM_TRACE infrastructure cover device suspending phase of
>>>> suspend/resume process, and the information in RTC can tell
>>>> developers which device suspending function make system hang.
>>> Do you have any examples of when this helped and no other debug methods did?
>>>
>>> It seems odd to me that no one has ever asked for this for several years and
>>> now you need it.
>> We ever run into some system hang during suspending some devices on BayTrail-T
>> platform such as DMA, audio, etc. So, extending PM-Trace to suspend phase should
>> be helpful.
> Well, did you actually use this particular patch for debugging that?
Yes, I ever used this method to debug suspending process of some devices.

BTW, I have re-sent this patch with -M so that the rename and changes can be reviewed much easier. The subject is "[PATCH v2] PM-Trace: add pm-trace support for suspending phase".


Thanks,
Zhonghui
>
> Rafael
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [PATCH] PM-Trace: add pm-trace support for suspending phase

[Fu, Zhonghui]

On 2015/2/12 9:23, Greg Kroah-Hartman wrote:
> On Wed, Feb 11, 2015 at 10:43:13PM +0800, Fu, Zhonghui wrote:
>> >From 2626594c03ca3b9884dd44073385c36f99a3651d Mon Sep 17 00:00:00 2001
>> From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
>> Date: Wed, 11 Feb 2015 16:20:21 +0800
>> Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase
>>
>> Occasionally, the system can't come back up after suspend/resume
>> due to problems of device suspending phase. This patch make
>> PM_TRACE infrastructure cover device suspending phase of
>> suspend/resume process, and the information in RTC can tell
>> developers which device suspending function make system hang.
>>
>> Signed-off-by: Zhonghui Fu <zhonghui.fu@linux.intel.com>
>> ---
>>  arch/x86/include/asm/pm-trace.h     |   36 +++++++++++++++++++++++++++++++++++
>>  arch/x86/include/asm/resume-trace.h |   21 --------------------
>>  drivers/base/power/main.c           |   20 +++++++++++++++---
>>  drivers/base/power/trace.c          |    6 ++--
>>  include/linux/pm-trace.h            |   35 ++++++++++++++++++++++++++++++++++
>>  include/linux/resume-trace.h        |   34 ---------------------------------
>>  kernel/power/main.c                 |    2 +-
>>  7 files changed, 91 insertions(+), 63 deletions(-)
>>  create mode 100644 arch/x86/include/asm/pm-trace.h
>>  delete mode 100644 arch/x86/include/asm/resume-trace.h
>>  create mode 100644 include/linux/pm-trace.h
>>  delete mode 100644 include/linux/resume-trace.h
> Please create patches that move files around with -M so that the rename
> and changes can be reviewed much easier.  As it is, I can't see the
> changes you have made here.
I have re-sent this patch according to your comments, and the subject is "[PATCH v2] PM-Trace: add pm-trace support for suspending phase".


Thanks,
Zhonghui
>
> thanks,
>
> greg k-h
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [PATCH] PM-Trace: add pm-trace support for suspending phase

[Rafael J. Wysocki]

On Thursday, February 12, 2015 12:48:06 PM Fu, Zhonghui wrote:
> 
> On 2015/2/11 23:25, Rafael J. Wysocki wrote:
> > On Wednesday, February 11, 2015 10:43:13 PM Fu, Zhonghui wrote:
> >> From 2626594c03ca3b9884dd44073385c36f99a3651d Mon Sep 17 00:00:00 2001
> >> From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
> >> Date: Wed, 11 Feb 2015 16:20:21 +0800
> >> Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase
> > Can you please avoid putting the above header lines into your patches?
> >
> > The only one that may be necessary is the From: line, but that's only
> > needed if you submit from a different address.
> >
> > Also please don't send To: mailing lists.  Use CC instead.  In fact, there
> > should be *one* address in the To: field when you're submitting a patch:
> > The address of the maintainer you want to apply the patch for you.
> Thanks for your guidance, and I will re-send this patch.
> 
> >
> >> Occasionally, the system can't come back up after suspend/resume
> >> due to problems of device suspending phase. This patch make
> >> PM_TRACE infrastructure cover device suspending phase of
> >> suspend/resume process, and the information in RTC can tell
> >> developers which device suspending function make system hang.
> > Do you have any examples of when this helped and no other debug methods did?
> >
> > It seems odd to me that no one has ever asked for this for several years and
> > now you need it.
> We ever run into some system hang during suspending some devices on BayTrail-T
> platform such as DMA, audio, etc. So, extending PM-Trace to suspend phase should
> be helpful.

Well, did you actually use this particular patch for debugging that?

Rafael

[PATCH] PM-Trace: add pm-trace support for suspending phase

[Fu, Zhonghui]

Occasionally, the system can't come back up after suspend/resume
due to problems of device suspending phase. This patch make
PM_TRACE infrastructure cover device suspending phase of
suspend/resume process, and the information in RTC can tell
developers which device suspending function make system hang.

Signed-off-by: Zhonghui Fu <zhonghui.fu@linux.intel.com>
---
 arch/x86/include/asm/pm-trace.h     |   36 +++++++++++++++++++++++++++++++++++
 arch/x86/include/asm/resume-trace.h |   21 --------------------
 drivers/base/power/main.c           |   20 +++++++++++++++---
 drivers/base/power/trace.c          |    6 ++--
 include/linux/pm-trace.h            |   35 ++++++++++++++++++++++++++++++++++
 include/linux/resume-trace.h        |   34 ---------------------------------
 kernel/power/main.c                 |    2 +-
 7 files changed, 91 insertions(+), 63 deletions(-)
 create mode 100644 arch/x86/include/asm/pm-trace.h
 delete mode 100644 arch/x86/include/asm/resume-trace.h
 create mode 100644 include/linux/pm-trace.h
 delete mode 100644 include/linux/resume-trace.h

diff --git a/arch/x86/include/asm/pm-trace.h b/arch/x86/include/asm/pm-trace.h
new file mode 100644
index 0000000..09bd918
--- /dev/null
+++ b/arch/x86/include/asm/pm-trace.h
@@ -0,0 +1,36 @@
+#ifndef _ASM_X86_PM_TRACE_H
+#define _ASM_X86_PM_TRACE_H
+
+#include <asm/asm.h>
+
+#define TRACE_RESUME(user)					\
+do {								\
+	if (pm_trace_enabled) {					\
+		const void *tracedata;				\
+		asm volatile(_ASM_MOV " $1f,%0\n"		\
+			     ".section .tracedata,\"a\"\n"	\
+			     "1:\t.word %c1\n\t"		\
+			     _ASM_PTR " %c2\n"			\
+			     ".previous"			\
+			     :"=r" (tracedata)			\
+			     : "i" (__LINE__), "i" (__FILE__));	\
+		generate_pm_trace(tracedata, user);		\
+	}							\
+} while (0)
+
+#define TRACE_SUSPEND(user)					\
+do {								\
+	if (pm_trace_enabled) {					\
+		const void *tracedata;				\
+		asm volatile(_ASM_MOV " $1f,%0\n"		\
+			     ".section .tracedata,\"a\"\n"	\
+			     "1:\t.word %c1\n\t"		\
+			     _ASM_PTR " %c2\n"			\
+			     ".previous"			\
+			     :"=r" (tracedata)			\
+			     : "i" (__LINE__), "i" (__FILE__));	\
+		generate_pm_trace(tracedata, user);		\
+	}							\
+} while (0)
+
+#endif /* _ASM_X86_PM_TRACE_H */
diff --git a/arch/x86/include/asm/resume-trace.h b/arch/x86/include/asm/resume-trace.h
deleted file mode 100644
index 3ff1c2c..0000000
--- a/arch/x86/include/asm/resume-trace.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#ifndef _ASM_X86_RESUME_TRACE_H
-#define _ASM_X86_RESUME_TRACE_H
-
-#include <asm/asm.h>
-
-#define TRACE_RESUME(user)					\
-do {								\
-	if (pm_trace_enabled) {					\
-		const void *tracedata;				\
-		asm volatile(_ASM_MOV " $1f,%0\n"		\
-			     ".section .tracedata,\"a\"\n"	\
-			     "1:\t.word %c1\n\t"		\
-			     _ASM_PTR " %c2\n"			\
-			     ".previous"			\
-			     :"=r" (tracedata)			\
-			     : "i" (__LINE__), "i" (__FILE__));	\
-		generate_resume_trace(tracedata, user);		\
-	}							\
-} while (0)
-
-#endif /* _ASM_X86_RESUME_TRACE_H */
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index 9717d5f..3d874ec 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -23,7 +23,7 @@
 #include <linux/mutex.h>
 #include <linux/pm.h>
 #include <linux/pm_runtime.h>
-#include <linux/resume-trace.h>
+#include <linux/pm-trace.h>
 #include <linux/interrupt.h>
 #include <linux/sched.h>
 #include <linux/async.h>
@@ -1017,6 +1017,9 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
 	char *info = NULL;
 	int error = 0;
 
+	TRACE_DEVICE(dev);
+	TRACE_SUSPEND(0);
+
 	if (async_error)
 		goto Complete;
 
@@ -1057,6 +1060,7 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
 
 Complete:
 	complete_all(&dev->power.completion);
+	TRACE_SUSPEND(error);
 	return error;
 }
 
@@ -1078,7 +1082,7 @@ static int device_suspend_noirq(struct device *dev)
 {
 	reinit_completion(&dev->power.completion);
 
-	if (pm_async_enabled && dev->power.async_suspend) {
+	if (is_async(dev)) {
 		get_device(dev);
 		async_schedule(async_suspend_noirq, dev);
 		return 0;
@@ -1157,6 +1161,9 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as
 	char *info = NULL;
 	int error = 0;
 
+	TRACE_DEVICE(dev);
+	TRACE_SUSPEND(0);
+
 	__pm_runtime_disable(dev, false);
 
 	if (async_error)
@@ -1198,6 +1205,7 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as
 		async_error = error;
 
 Complete:
+	TRACE_SUSPEND(error);
 	complete_all(&dev->power.completion);
 	return error;
 }
@@ -1219,7 +1227,7 @@ static int device_suspend_late(struct device *dev)
 {
 	reinit_completion(&dev->power.completion);
 
-	if (pm_async_enabled && dev->power.async_suspend) {
+	if (is_async(dev)) {
 		get_device(dev);
 		async_schedule(async_suspend_late, dev);
 		return 0;
@@ -1338,6 +1346,9 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 	int error = 0;
 	DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 
+	TRACE_DEVICE(dev);
+	TRACE_SUSPEND(0);
+
 	dpm_wait_for_children(dev, async);
 
 	if (async_error)
@@ -1444,6 +1455,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 	if (error)
 		async_error = error;
 
+	TRACE_SUSPEND(error);
 	return error;
 }
 
@@ -1465,7 +1477,7 @@ static int device_suspend(struct device *dev)
 {
 	reinit_completion(&dev->power.completion);
 
-	if (pm_async_enabled && dev->power.async_suspend) {
+	if (is_async(dev)) {
 		get_device(dev);
 		async_schedule(async_suspend, dev);
 		return 0;
diff --git a/drivers/base/power/trace.c b/drivers/base/power/trace.c
index d94a1f5..a311cfa 100644
--- a/drivers/base/power/trace.c
+++ b/drivers/base/power/trace.c
@@ -7,7 +7,7 @@
  * devices may be working.
  */
 
-#include <linux/resume-trace.h>
+#include <linux/pm-trace.h>
 #include <linux/export.h>
 #include <linux/rtc.h>
 
@@ -154,7 +154,7 @@ EXPORT_SYMBOL(set_trace_device);
  * it's not any guarantee, but it's a high _likelihood_ that
  * the match is valid).
  */
-void generate_resume_trace(const void *tracedata, unsigned int user)
+void generate_pm_trace(const void *tracedata, unsigned int user)
 {
 	unsigned short lineno = *(unsigned short *)tracedata;
 	const char *file = *(const char **)(tracedata + 2);
@@ -164,7 +164,7 @@ void generate_resume_trace(const void *tracedata, unsigned int user)
 	file_hash_value = hash_string(lineno, file, FILEHASH);
 	set_magic_time(user_hash_value, file_hash_value, dev_hash_value);
 }
-EXPORT_SYMBOL(generate_resume_trace);
+EXPORT_SYMBOL(generate_pm_trace);
 
 extern char __tracedata_start, __tracedata_end;
 static int show_file_hash(unsigned int value)
diff --git a/include/linux/pm-trace.h b/include/linux/pm-trace.h
new file mode 100644
index 0000000..ecbde7a
--- /dev/null
+++ b/include/linux/pm-trace.h
@@ -0,0 +1,35 @@
+#ifndef PM_TRACE_H
+#define PM_TRACE_H
+
+#ifdef CONFIG_PM_TRACE
+#include <asm/pm-trace.h>
+#include <linux/types.h>
+
+extern int pm_trace_enabled;
+
+static inline int pm_trace_is_enabled(void)
+{
+       return pm_trace_enabled;
+}
+
+struct device;
+extern void set_trace_device(struct device *);
+extern void generate_pm_trace(const void *tracedata, unsigned int user);
+extern int show_trace_dev_match(char *buf, size_t size);
+
+#define TRACE_DEVICE(dev) do { \
+	if (pm_trace_enabled) \
+		set_trace_device(dev); \
+	} while(0)
+
+#else
+
+static inline int pm_trace_is_enabled(void) { return 0; }
+
+#define TRACE_DEVICE(dev) do { } while (0)
+#define TRACE_RESUME(dev) do { } while (0)
+#define TRACE_SUSPEND(dev) do { } while (0)
+
+#endif
+
+#endif
diff --git a/include/linux/resume-trace.h b/include/linux/resume-trace.h
deleted file mode 100644
index f31db23..0000000
--- a/include/linux/resume-trace.h
+++ /dev/null
@@ -1,34 +0,0 @@
-#ifndef RESUME_TRACE_H
-#define RESUME_TRACE_H
-
-#ifdef CONFIG_PM_TRACE
-#include <asm/resume-trace.h>
-#include <linux/types.h>
-
-extern int pm_trace_enabled;
-
-static inline int pm_trace_is_enabled(void)
-{
-       return pm_trace_enabled;
-}
-
-struct device;
-extern void set_trace_device(struct device *);
-extern void generate_resume_trace(const void *tracedata, unsigned int user);
-extern int show_trace_dev_match(char *buf, size_t size);
-
-#define TRACE_DEVICE(dev) do { \
-	if (pm_trace_enabled) \
-		set_trace_device(dev); \
-	} while(0)
-
-#else
-
-static inline int pm_trace_is_enabled(void) { return 0; }
-
-#define TRACE_DEVICE(dev) do { } while (0)
-#define TRACE_RESUME(dev) do { } while (0)
-
-#endif
-
-#endif
diff --git a/kernel/power/main.c b/kernel/power/main.c
index 9a59d04..86e8157 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -11,7 +11,7 @@
 #include <linux/export.h>
 #include <linux/kobject.h>
 #include <linux/string.h>
-#include <linux/resume-trace.h>
+#include <linux/pm-trace.h>
 #include <linux/workqueue.h>
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
-- 1.7.1

Re: [PATCH] PM-Trace: add pm-trace support for suspending phase

[Fu, Zhonghui]

On 2015/2/11 23:25, Rafael J. Wysocki wrote:
> On Wednesday, February 11, 2015 10:43:13 PM Fu, Zhonghui wrote:
>> From 2626594c03ca3b9884dd44073385c36f99a3651d Mon Sep 17 00:00:00 2001
>> From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
>> Date: Wed, 11 Feb 2015 16:20:21 +0800
>> Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase
> Can you please avoid putting the above header lines into your patches?
>
> The only one that may be necessary is the From: line, but that's only
> needed if you submit from a different address.
>
> Also please don't send To: mailing lists.  Use CC instead.  In fact, there
> should be *one* address in the To: field when you're submitting a patch:
> The address of the maintainer you want to apply the patch for you.
Thanks for your guidance, and I will re-send this patch.

>
>> Occasionally, the system can't come back up after suspend/resume
>> due to problems of device suspending phase. This patch make
>> PM_TRACE infrastructure cover device suspending phase of
>> suspend/resume process, and the information in RTC can tell
>> developers which device suspending function make system hang.
> Do you have any examples of when this helped and no other debug methods did?
>
> It seems odd to me that no one has ever asked for this for several years and
> now you need it.
We ever run into some system hang during suspending some devices on BayTrail-T platform such as DMA, audio, etc. So, extending PM-Trace to suspend phase should be helpful.

Thanks,
Zhonghui
>
>

Re: [PATCH] PM-Trace: add pm-trace support for suspending phase

[Greg Kroah-Hartman]

On Wed, Feb 11, 2015 at 10:43:13PM +0800, Fu, Zhonghui wrote:
> >From 2626594c03ca3b9884dd44073385c36f99a3651d Mon Sep 17 00:00:00 2001
> From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
> Date: Wed, 11 Feb 2015 16:20:21 +0800
> Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase
> 
> Occasionally, the system can't come back up after suspend/resume
> due to problems of device suspending phase. This patch make
> PM_TRACE infrastructure cover device suspending phase of
> suspend/resume process, and the information in RTC can tell
> developers which device suspending function make system hang.
> 
> Signed-off-by: Zhonghui Fu <zhonghui.fu@linux.intel.com>
> ---
>  arch/x86/include/asm/pm-trace.h     |   36 +++++++++++++++++++++++++++++++++++
>  arch/x86/include/asm/resume-trace.h |   21 --------------------
>  drivers/base/power/main.c           |   20 +++++++++++++++---
>  drivers/base/power/trace.c          |    6 ++--
>  include/linux/pm-trace.h            |   35 ++++++++++++++++++++++++++++++++++
>  include/linux/resume-trace.h        |   34 ---------------------------------
>  kernel/power/main.c                 |    2 +-
>  7 files changed, 91 insertions(+), 63 deletions(-)
>  create mode 100644 arch/x86/include/asm/pm-trace.h
>  delete mode 100644 arch/x86/include/asm/resume-trace.h
>  create mode 100644 include/linux/pm-trace.h
>  delete mode 100644 include/linux/resume-trace.h

Please create patches that move files around with -M so that the rename
and changes can be reviewed much easier.  As it is, I can't see the
changes you have made here.

thanks,

greg k-h

Re: [PATCH] PM-Trace: add pm-trace support for suspending phase

[Rafael J. Wysocki]

On Wednesday, February 11, 2015 10:43:13 PM Fu, Zhonghui wrote:
> From 2626594c03ca3b9884dd44073385c36f99a3651d Mon Sep 17 00:00:00 2001
> From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
> Date: Wed, 11 Feb 2015 16:20:21 +0800
> Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase

Can you please avoid putting the above header lines into your patches?

The only one that may be necessary is the From: line, but that's only
needed if you submit from a different address.

Also please don't send To: mailing lists.  Use CC instead.  In fact, there
should be *one* address in the To: field when you're submitting a patch:
The address of the maintainer you want to apply the patch for you.

> Occasionally, the system can't come back up after suspend/resume
> due to problems of device suspending phase. This patch make
> PM_TRACE infrastructure cover device suspending phase of
> suspend/resume process, and the information in RTC can tell
> developers which device suspending function make system hang.

Do you have any examples of when this helped and no other debug methods did?

It seems odd to me that no one has ever asked for this for several years and
now you need it.


-- 
I speak only for myself.
Rafael J. Wysocki, Intel Open Source Technology Center.

[PATCH] PM-Trace: add pm-trace support for suspending phase

[Fu, Zhonghui]

>From 2626594c03ca3b9884dd44073385c36f99a3651d Mon Sep 17 00:00:00 2001
From: Zhonghui Fu <zhonghui.fu@linux.intel.com>
Date: Wed, 11 Feb 2015 16:20:21 +0800
Subject: [PATCH] PM-Trace: add pm-trace support for suspending phase

Occasionally, the system can't come back up after suspend/resume
due to problems of device suspending phase. This patch make
PM_TRACE infrastructure cover device suspending phase of
suspend/resume process, and the information in RTC can tell
developers which device suspending function make system hang.

Signed-off-by: Zhonghui Fu <zhonghui.fu@linux.intel.com>
---
 arch/x86/include/asm/pm-trace.h     |   36 +++++++++++++++++++++++++++++++++++
 arch/x86/include/asm/resume-trace.h |   21 --------------------
 drivers/base/power/main.c           |   20 +++++++++++++++---
 drivers/base/power/trace.c          |    6 ++--
 include/linux/pm-trace.h            |   35 ++++++++++++++++++++++++++++++++++
 include/linux/resume-trace.h        |   34 ---------------------------------
 kernel/power/main.c                 |    2 +-
 7 files changed, 91 insertions(+), 63 deletions(-)
 create mode 100644 arch/x86/include/asm/pm-trace.h
 delete mode 100644 arch/x86/include/asm/resume-trace.h
 create mode 100644 include/linux/pm-trace.h
 delete mode 100644 include/linux/resume-trace.h

diff --git a/arch/x86/include/asm/pm-trace.h b/arch/x86/include/asm/pm-trace.h
new file mode 100644
index 0000000..09bd918
--- /dev/null
+++ b/arch/x86/include/asm/pm-trace.h
@@ -0,0 +1,36 @@
+#ifndef _ASM_X86_PM_TRACE_H
+#define _ASM_X86_PM_TRACE_H
+
+#include <asm/asm.h>
+
+#define TRACE_RESUME(user)					\
+do {								\
+	if (pm_trace_enabled) {					\
+		const void *tracedata;				\
+		asm volatile(_ASM_MOV " $1f,%0\n"		\
+			     ".section .tracedata,\"a\"\n"	\
+			     "1:\t.word %c1\n\t"		\
+			     _ASM_PTR " %c2\n"			\
+			     ".previous"			\
+			     :"=r" (tracedata)			\
+			     : "i" (__LINE__), "i" (__FILE__));	\
+		generate_pm_trace(tracedata, user);		\
+	}							\
+} while (0)
+
+#define TRACE_SUSPEND(user)					\
+do {								\
+	if (pm_trace_enabled) {					\
+		const void *tracedata;				\
+		asm volatile(_ASM_MOV " $1f,%0\n"		\
+			     ".section .tracedata,\"a\"\n"	\
+			     "1:\t.word %c1\n\t"		\
+			     _ASM_PTR " %c2\n"			\
+			     ".previous"			\
+			     :"=r" (tracedata)			\
+			     : "i" (__LINE__), "i" (__FILE__));	\
+		generate_pm_trace(tracedata, user);		\
+	}							\
+} while (0)
+
+#endif /* _ASM_X86_PM_TRACE_H */
diff --git a/arch/x86/include/asm/resume-trace.h b/arch/x86/include/asm/resume-trace.h
deleted file mode 100644
index 3ff1c2c..0000000
--- a/arch/x86/include/asm/resume-trace.h
+++ /dev/null
@@ -1,21 +0,0 @@
-#ifndef _ASM_X86_RESUME_TRACE_H
-#define _ASM_X86_RESUME_TRACE_H
-
-#include <asm/asm.h>
-
-#define TRACE_RESUME(user)					\
-do {								\
-	if (pm_trace_enabled) {					\
-		const void *tracedata;				\
-		asm volatile(_ASM_MOV " $1f,%0\n"		\
-			     ".section .tracedata,\"a\"\n"	\
-			     "1:\t.word %c1\n\t"		\
-			     _ASM_PTR " %c2\n"			\
-			     ".previous"			\
-			     :"=r" (tracedata)			\
-			     : "i" (__LINE__), "i" (__FILE__));	\
-		generate_resume_trace(tracedata, user);		\
-	}							\
-} while (0)
-
-#endif /* _ASM_X86_RESUME_TRACE_H */
diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c
index 9717d5f..3d874ec 100644
--- a/drivers/base/power/main.c
+++ b/drivers/base/power/main.c
@@ -23,7 +23,7 @@
 #include <linux/mutex.h>
 #include <linux/pm.h>
 #include <linux/pm_runtime.h>
-#include <linux/resume-trace.h>
+#include <linux/pm-trace.h>
 #include <linux/interrupt.h>
 #include <linux/sched.h>
 #include <linux/async.h>
@@ -1017,6 +1017,9 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
 	char *info = NULL;
 	int error = 0;
 
+	TRACE_DEVICE(dev);
+	TRACE_SUSPEND(0);
+
 	if (async_error)
 		goto Complete;
 
@@ -1057,6 +1060,7 @@ static int __device_suspend_noirq(struct device *dev, pm_message_t state, bool a
 
 Complete:
 	complete_all(&dev->power.completion);
+	TRACE_SUSPEND(error);
 	return error;
 }
 
@@ -1078,7 +1082,7 @@ static int device_suspend_noirq(struct device *dev)
 {
 	reinit_completion(&dev->power.completion);
 
-	if (pm_async_enabled && dev->power.async_suspend) {
+	if (is_async(dev)) {
 		get_device(dev);
 		async_schedule(async_suspend_noirq, dev);
 		return 0;
@@ -1157,6 +1161,9 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as
 	char *info = NULL;
 	int error = 0;
 
+	TRACE_DEVICE(dev);
+	TRACE_SUSPEND(0);
+
 	__pm_runtime_disable(dev, false);
 
 	if (async_error)
@@ -1198,6 +1205,7 @@ static int __device_suspend_late(struct device *dev, pm_message_t state, bool as
 		async_error = error;
 
 Complete:
+	TRACE_SUSPEND(error);
 	complete_all(&dev->power.completion);
 	return error;
 }
@@ -1219,7 +1227,7 @@ static int device_suspend_late(struct device *dev)
 {
 	reinit_completion(&dev->power.completion);
 
-	if (pm_async_enabled && dev->power.async_suspend) {
+	if (is_async(dev)) {
 		get_device(dev);
 		async_schedule(async_suspend_late, dev);
 		return 0;
@@ -1338,6 +1346,9 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 	int error = 0;
 	DECLARE_DPM_WATCHDOG_ON_STACK(wd);
 
+	TRACE_DEVICE(dev);
+	TRACE_SUSPEND(0);
+
 	dpm_wait_for_children(dev, async);
 
 	if (async_error)
@@ -1444,6 +1455,7 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async)
 	if (error)
 		async_error = error;
 
+	TRACE_SUSPEND(error);
 	return error;
 }
 
@@ -1465,7 +1477,7 @@ static int device_suspend(struct device *dev)
 {
 	reinit_completion(&dev->power.completion);
 
-	if (pm_async_enabled && dev->power.async_suspend) {
+	if (is_async(dev)) {
 		get_device(dev);
 		async_schedule(async_suspend, dev);
 		return 0;
diff --git a/drivers/base/power/trace.c b/drivers/base/power/trace.c
index d94a1f5..a311cfa 100644
--- a/drivers/base/power/trace.c
+++ b/drivers/base/power/trace.c
@@ -7,7 +7,7 @@
  * devices may be working.
  */
 
-#include <linux/resume-trace.h>
+#include <linux/pm-trace.h>
 #include <linux/export.h>
 #include <linux/rtc.h>
 
@@ -154,7 +154,7 @@ EXPORT_SYMBOL(set_trace_device);
  * it's not any guarantee, but it's a high _likelihood_ that
  * the match is valid).
  */
-void generate_resume_trace(const void *tracedata, unsigned int user)
+void generate_pm_trace(const void *tracedata, unsigned int user)
 {
 	unsigned short lineno = *(unsigned short *)tracedata;
 	const char *file = *(const char **)(tracedata + 2);
@@ -164,7 +164,7 @@ void generate_resume_trace(const void *tracedata, unsigned int user)
 	file_hash_value = hash_string(lineno, file, FILEHASH);
 	set_magic_time(user_hash_value, file_hash_value, dev_hash_value);
 }
-EXPORT_SYMBOL(generate_resume_trace);
+EXPORT_SYMBOL(generate_pm_trace);
 
 extern char __tracedata_start, __tracedata_end;
 static int show_file_hash(unsigned int value)
diff --git a/include/linux/pm-trace.h b/include/linux/pm-trace.h
new file mode 100644
index 0000000..ecbde7a
--- /dev/null
+++ b/include/linux/pm-trace.h
@@ -0,0 +1,35 @@
+#ifndef PM_TRACE_H
+#define PM_TRACE_H
+
+#ifdef CONFIG_PM_TRACE
+#include <asm/pm-trace.h>
+#include <linux/types.h>
+
+extern int pm_trace_enabled;
+
+static inline int pm_trace_is_enabled(void)
+{
+       return pm_trace_enabled;
+}
+
+struct device;
+extern void set_trace_device(struct device *);
+extern void generate_pm_trace(const void *tracedata, unsigned int user);
+extern int show_trace_dev_match(char *buf, size_t size);
+
+#define TRACE_DEVICE(dev) do { \
+	if (pm_trace_enabled) \
+		set_trace_device(dev); \
+	} while(0)
+
+#else
+
+static inline int pm_trace_is_enabled(void) { return 0; }
+
+#define TRACE_DEVICE(dev) do { } while (0)
+#define TRACE_RESUME(dev) do { } while (0)
+#define TRACE_SUSPEND(dev) do { } while (0)
+
+#endif
+
+#endif
diff --git a/include/linux/resume-trace.h b/include/linux/resume-trace.h
deleted file mode 100644
index f31db23..0000000
--- a/include/linux/resume-trace.h
+++ /dev/null
@@ -1,34 +0,0 @@
-#ifndef RESUME_TRACE_H
-#define RESUME_TRACE_H
-
-#ifdef CONFIG_PM_TRACE
-#include <asm/resume-trace.h>
-#include <linux/types.h>
-
-extern int pm_trace_enabled;
-
-static inline int pm_trace_is_enabled(void)
-{
-       return pm_trace_enabled;
-}
-
-struct device;
-extern void set_trace_device(struct device *);
-extern void generate_resume_trace(const void *tracedata, unsigned int user);
-extern int show_trace_dev_match(char *buf, size_t size);
-
-#define TRACE_DEVICE(dev) do { \
-	if (pm_trace_enabled) \
-		set_trace_device(dev); \
-	} while(0)
-
-#else
-
-static inline int pm_trace_is_enabled(void) { return 0; }
-
-#define TRACE_DEVICE(dev) do { } while (0)
-#define TRACE_RESUME(dev) do { } while (0)
-
-#endif
-
-#endif
diff --git a/kernel/power/main.c b/kernel/power/main.c
index 9a59d04..86e8157 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -11,7 +11,7 @@
 #include <linux/export.h>
 #include <linux/kobject.h>
 #include <linux/string.h>
-#include <linux/resume-trace.h>
+#include <linux/pm-trace.h>
 #include <linux/workqueue.h>
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
-- 1.7.1