preempt-timing-2.6.8-rc1

preempt-timing-2.6.8-rc1

[William Lee Irwin III]

This patch uses the preemption counter increments and decrements to time
non-preemptible critical sections.

This is an instrumentation patch intended to help determine the causes of
scheduling latency related to long non-preemptible critical sections.

Changes from 2.6.7-based patch:
(1) fix unmap_vmas() check correctly this time
(2) add touch_preempt_timing() to cond_resched_lock()
(3) depend on preempt until it's worked out wtf goes wrong without it


-- wli

Index: timing-2.6.8-rc1/include/linux/preempt.h
===================================================================
--- timing-2.6.8-rc1.orig/include/linux/preempt.h	2004-07-11 10:34:18.000000000 -0700
+++ timing-2.6.8-rc1/include/linux/preempt.h	2004-07-13 03:56:37.887605624 -0700
@@ -9,17 +9,17 @@
 #include <linux/config.h>
 #include <linux/linkage.h>
 
-#define preempt_count()	(current_thread_info()->preempt_count)
-
-#define inc_preempt_count() \
-do { \
-	preempt_count()++; \
-} while (0)
+#ifdef CONFIG_PREEMPT_TIMING
+void inc_preempt_count(void);
+void dec_preempt_count(void);
+void touch_preempt_timing(void);
+#else
+#define touch_preempt_timing()	do { } while (0)
+#define inc_preempt_count()	do { preempt_count()++; } while (0)
+#define dec_preempt_count()	do { preempt_count()--; } while (0)
+#endif
 
-#define dec_preempt_count() \
-do { \
-	preempt_count()--; \
-} while (0)
+#define preempt_count()	(current_thread_info()->preempt_count)
 
 #ifdef CONFIG_PREEMPT
 
@@ -51,9 +51,15 @@
 
 #else
 
+#ifdef CONFIG_PREEMPT_TIMING
+#define preempt_disable()		inc_preempt_count()
+#define preempt_enable_no_resched()	dec_preempt_count()
+#define preempt_enable()		dec_preempt_count()
+#else
 #define preempt_disable()		do { } while (0)
 #define preempt_enable_no_resched()	do { } while (0)
 #define preempt_enable()		do { } while (0)
+#endif
 #define preempt_check_resched()		do { } while (0)
 
 #endif
Index: timing-2.6.8-rc1/include/linux/sched.h
===================================================================
--- timing-2.6.8-rc1.orig/include/linux/sched.h	2004-07-11 10:33:55.000000000 -0700
+++ timing-2.6.8-rc1/include/linux/sched.h	2004-07-13 03:56:37.895604408 -0700
@@ -1021,6 +1021,7 @@
 extern void __cond_resched(void);
 static inline void cond_resched(void)
 {
+	touch_preempt_timing();
 	if (need_resched())
 		__cond_resched();
 }
@@ -1040,7 +1041,8 @@
 		preempt_enable_no_resched();
 		__cond_resched();
 		spin_lock(lock);
-	}
+	} else
+		touch_preempt_timing();
 }
 
 /* Reevaluate whether the task has signals pending delivery.
Index: timing-2.6.8-rc1/init/Kconfig
===================================================================
--- timing-2.6.8-rc1.orig/init/Kconfig	2004-07-11 10:35:08.000000000 -0700
+++ timing-2.6.8-rc1/init/Kconfig	2004-07-13 03:56:37.898603952 -0700
@@ -279,6 +279,15 @@
 	  Disabling this option will cause the kernel to be built without
 	  support for epoll family of system calls.
 
+config PREEMPT_TIMING
+	bool "Non-preemptible critical section timing"
+	default n
+	depends on PREEMPT
+	help
+	  This option measures the time spent in non-preemptible critical
+	  sections and reports warnings when a boot-time configurable
+	  latency threshold is exceeded.
+
 source "drivers/block/Kconfig.iosched"
 
 config CC_OPTIMIZE_FOR_SIZE
Index: timing-2.6.8-rc1/kernel/printk.c
===================================================================
--- timing-2.6.8-rc1.orig/kernel/printk.c	2004-07-11 10:35:31.000000000 -0700
+++ timing-2.6.8-rc1/kernel/printk.c	2004-07-13 03:56:37.901603496 -0700
@@ -650,10 +650,8 @@
  */
 void console_conditional_schedule(void)
 {
-	if (console_may_schedule && need_resched()) {
-		set_current_state(TASK_RUNNING);
-		schedule();
-	}
+	if (console_may_schedule)
+		cond_resched();
 }
 EXPORT_SYMBOL(console_conditional_schedule);
 
Index: timing-2.6.8-rc1/kernel/sched.c
===================================================================
--- timing-2.6.8-rc1.orig/kernel/sched.c	2004-07-11 10:35:08.000000000 -0700
+++ timing-2.6.8-rc1/kernel/sched.c	2004-07-13 03:56:37.914601520 -0700
@@ -4040,3 +4040,74 @@
 
 EXPORT_SYMBOL(__preempt_write_lock);
 #endif /* defined(CONFIG_SMP) && defined(CONFIG_PREEMPT) */
+
+#ifdef CONFIG_PREEMPT_TIMING
+#include <linux/kallsyms.h>
+
+static int preempt_thresh;
+static DEFINE_PER_CPU(u64, preempt_timings);
+static DEFINE_PER_CPU(unsigned long, preempt_entry);
+
+static int setup_preempt_thresh(char *s)
+{
+	int thresh;
+
+	get_option(&s, &thresh);
+	if (thresh > 0) {
+		preempt_thresh = thresh;
+		printk("Preemption threshold = %dms\n", preempt_thresh);
+	}
+	return 1;
+}
+__setup("preempt_thresh=", setup_preempt_thresh);
+
+static void __touch_preempt_timing(void *addr)
+{
+	__get_cpu_var(preempt_timings) = sched_clock();
+	__get_cpu_var(preempt_entry) = (unsigned long)addr;
+}
+
+void touch_preempt_timing(void)
+{
+	if (preempt_count() > 0 && system_state == SYSTEM_RUNNING &&
+						__get_cpu_var(preempt_entry))
+		__touch_preempt_timing(__builtin_return_address(0));
+}
+EXPORT_SYMBOL(touch_preempt_timing);
+
+void inc_preempt_count(void)
+{
+	preempt_count()++;
+	if (preempt_count() == 1 && system_state == SYSTEM_RUNNING)
+		__touch_preempt_timing(__builtin_return_address(0));
+}
+EXPORT_SYMBOL(inc_preempt_count);
+
+void dec_preempt_count(void)
+{
+	if (preempt_count() == 1 && system_state == SYSTEM_RUNNING &&
+					__get_cpu_var(preempt_entry)) {
+		u64 hold;
+		unsigned long preempt_exit
+				= (unsigned long)__builtin_return_address(0);
+		hold = sched_clock() - __get_cpu_var(preempt_timings) + 999999;
+		do_div(hold, 1000000);
+		if (preempt_thresh && hold > preempt_thresh &&
+							printk_ratelimit()) {
+			printk("%lums non-preemptible critical section "
+				"violated %d ms preempt threshold "
+				"starting at ",
+				(unsigned long)hold,
+				preempt_thresh);
+			print_symbol("%s", __get_cpu_var(preempt_entry));
+			printk(" and ending at ");
+			print_symbol("%s", preempt_exit);
+			printk("\n");
+			dump_stack();
+		}
+		__get_cpu_var(preempt_entry) = 0;
+	}
+	preempt_count()--;
+}
+EXPORT_SYMBOL(dec_preempt_count);
+#endif
Index: timing-2.6.8-rc1/mm/memory.c
===================================================================
--- timing-2.6.8-rc1.orig/mm/memory.c	2004-07-11 10:34:37.000000000 -0700
+++ timing-2.6.8-rc1/mm/memory.c	2004-07-13 03:56:37.920600608 -0700
@@ -567,14 +567,17 @@
 			zap_bytes -= block;
 			if ((long)zap_bytes > 0)
 				continue;
-			if (!atomic && need_resched()) {
+			zap_bytes = ZAP_BLOCK_SIZE;
+			if (atomic)
+				continue;
+			if (need_resched()) {
 				int fullmm = tlb_is_full_mm(*tlbp);
 				tlb_finish_mmu(*tlbp, tlb_start, start);
 				cond_resched_lock(&mm->page_table_lock);
 				*tlbp = tlb_gather_mmu(mm, fullmm);
 				tlb_start_valid = 0;
-			}
-			zap_bytes = ZAP_BLOCK_SIZE;
+			} else
+				touch_preempt_timing();
 		}
 	}
 	return ret;

Re: preempt-timing-2.6.8-rc1

[Nick Piggin]

William Lee Irwin III wrote:
> This patch uses the preemption counter increments and decrements to time
> non-preemptible critical sections.
> 
> This is an instrumentation patch intended to help determine the causes of
> scheduling latency related to long non-preemptible critical sections.
> 
> Changes from 2.6.7-based patch:
> (1) fix unmap_vmas() check correctly this time
> (2) add touch_preempt_timing() to cond_resched_lock()
> (3) depend on preempt until it's worked out wtf goes wrong without it
> 
> --- timing-2.6.8-rc1.orig/kernel/printk.c	2004-07-11 10:35:31.000000000 -0700
> +++ timing-2.6.8-rc1/kernel/printk.c	2004-07-13 03:56:37.901603496 -0700
> @@ -650,10 +650,8 @@
>   */
>  void console_conditional_schedule(void)
>  {
> -	if (console_may_schedule && need_resched()) {
> -		set_current_state(TASK_RUNNING);
> -		schedule();
> -	}
> +	if (console_may_schedule)
> +		cond_resched();
>  }

You should send that one in

> +void dec_preempt_count(void)
> +{
> +	if (preempt_count() == 1 && system_state == SYSTEM_RUNNING &&
> +					__get_cpu_var(preempt_entry)) {
> +		u64 hold;
> +		unsigned long preempt_exit
> +				= (unsigned long)__builtin_return_address(0);
> +		hold = sched_clock() - __get_cpu_var(preempt_timings) + 999999;
> +		do_div(hold, 1000000);
> +		if (preempt_thresh && hold > preempt_thresh &&
> +							printk_ratelimit()) {

This looks wrong. This means hold times of 1ns to 1000000ns trigger the
exceeded 1ms threshold, 1000001 to 2000000 trigger the 2ms one, etc.

Removing the + 999999 gives the correct result:
1000000 - 1999999ns triggers the 1ms threshold
2000000 - 2999999ns triggers the 2ms threshold
etc

Or have I missed something?

Re: preempt-timing-2.6.8-rc1

[William Lee Irwin III]

William Lee Irwin III wrote:
>>+		unsigned long preempt_exit
>>+				= (unsigned long)__builtin_return_address(0);
>>+		hold = sched_clock() - __get_cpu_var(preempt_timings) + 
>>999999;
>>+		do_div(hold, 1000000);
>>+		if (preempt_thresh && hold > preempt_thresh &&
>>+							printk_ratelimit()) {

On Tue, Jul 13, 2004 at 10:51:24PM +1000, Nick Piggin wrote:
> This looks wrong. This means hold times of 1ns to 1000000ns trigger the
> exceeded 1ms threshold, 1000001 to 2000000 trigger the 2ms one, etc.
> Removing the + 999999 gives the correct result:
> 1000000 - 1999999ns triggers the 1ms threshold
> 2000000 - 2999999ns triggers the 2ms threshold
> etc
> Or have I missed something?

AFAICT this is nothing more than rounding up.


-- wli

Re: preempt-timing-2.6.8-rc1

[Nick Piggin]

William Lee Irwin III wrote:
> William Lee Irwin III wrote:
> 
>>>+		unsigned long preempt_exit
>>>+				= (unsigned long)__builtin_return_address(0);
>>>+		hold = sched_clock() - __get_cpu_var(preempt_timings) + 
>>>999999;
>>>+		do_div(hold, 1000000);
>>>+		if (preempt_thresh && hold > preempt_thresh &&
>>>+							printk_ratelimit()) {
> 
> 
> On Tue, Jul 13, 2004 at 10:51:24PM +1000, Nick Piggin wrote:
> 
>>This looks wrong. This means hold times of 1ns to 1000000ns trigger the
>>exceeded 1ms threshold, 1000001 to 2000000 trigger the 2ms one, etc.
>>Removing the + 999999 gives the correct result:
>>1000000 - 1999999ns triggers the 1ms threshold
>>2000000 - 2999999ns triggers the 2ms threshold
>>etc
>>Or have I missed something?
> 
> 
> AFAICT this is nothing more than rounding up.
> 

But you want to round down by definition of preempt_thresh, don't you?

preempt_thresh = 1ms = 1000000us
ie. warn me if the lock hold goes _to or above_ 1000000us

Re: preempt-timing-2.6.8-rc1

[William Lee Irwin III]

William Lee Irwin III wrote:
>> AFAICT this is nothing more than rounding up.

On Tue, Jul 13, 2004 at 10:57:54PM +1000, Nick Piggin wrote:
> But you want to round down by definition of preempt_thresh, don't you?
> preempt_thresh = 1ms = 1000000us
> ie. warn me if the lock hold goes _to or above_ 1000000us

The semantics I implemented are warning for strictly above the
preempt_thresh. Whether those semantics are ideal is irrelevant; it's
faithful to those semantics. Given that people are asking for sub-
millisecond latencies, maybe I should increase the precision.


-- wli

Re: preempt-timing-2.6.8-rc1

[Nick Piggin]

William Lee Irwin III wrote:
> William Lee Irwin III wrote:
> 
>>>AFAICT this is nothing more than rounding up.
> 
> 
> On Tue, Jul 13, 2004 at 10:57:54PM +1000, Nick Piggin wrote:
> 
>>But you want to round down by definition of preempt_thresh, don't you?
>>preempt_thresh = 1ms = 1000000us
>>ie. warn me if the lock hold goes _to or above_ 1000000us
> 
> 
> The semantics I implemented are warning for strictly above the
> preempt_thresh. Whether those semantics are ideal is irrelevant; it's
> faithful to those semantics.

You are right - I misread it, sorry.

> Given that people are asking for sub-
> millisecond latencies, maybe I should increase the precision.
> 

Would soon be useful I think.

Re: preempt-timing-2.6.8-rc1

[William Lee Irwin III]

William Lee Irwin III wrote:
>> The semantics I implemented are warning for strictly above the
>> preempt_thresh. Whether those semantics are ideal is irrelevant; it's
>> faithful to those semantics.

On Tue, Jul 13, 2004 at 11:10:09PM +1000, Nick Piggin wrote:
> You are right - I misread it, sorry.

William Lee Irwin III wrote:
>> Given that people are asking for sub-
>> millisecond latencies, maybe I should increase the precision.

On Tue, Jul 13, 2004 at 11:10:09PM +1000, Nick Piggin wrote:
> Would soon be useful I think.

Let me spin up the CONFIG_PREEMPT=n fix and then I'll move on that.


-- wli

Re: preempt-timing-2.6.8-rc1

[William Lee Irwin III]

On Tue, Jul 13, 2004 at 06:19:44AM -0700, William Lee Irwin III wrote:
> Let me spin up the CONFIG_PREEMPT=n fix and then I'll move on that.

smp_lock.h and hardirq.h for all arches -- some of which may not have
CONFIG_PREEMPT. -EWONTFIX. So the "workaround" stays.


-- wli

Re: preempt-timing-2.6.8-rc1

[Lenar Lõhmus]

Hi,

My first results with 2.6.8-rc1 + preempt-timing:

Boot-time:

2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at buffered_rmqueue+0xea/0x190 and ending at 
buffered_rmqueue+0x144/0x190
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at search_by_key+0xe3/0xf70 and ending at do_IRQ+0xec/0x130
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at copy_mm+0x2e6/0x3b0 and ending at copy_mm+0x331/0x3b0
11ms non-preemptible critical section violated 1 ms preempt threshold 
starting at ohci_hub_resume+0x3c/0x350 [ohci_hcd] and ending at 
ohci_hub_resume+0x79/0x350 [ohci_hcd]
14ms non-preemptible critical section violated 1 ms preempt threshold 
starting at schedule+0x36/0x480 and ending at schedule+0x291/0x480
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at new_inode+0x1a/0x80 and ending at new_inode+0x58/0x80
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20

Normal use (logging in, browsing, playing video with mplayer, moving 
windows around, etc):

2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at search_by_key+0xe3/0xf70 and ending at 
smp_apic_timer_interrupt+0x9a/0xe0
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at do_munmap+0xd2/0x140 and ending at do_munmap+0xeb/0x140
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at __d_lookup+0x66/0x170 and ending at __d_lookup+0x9c/0x170
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at copy_mm+0x2e6/0x3b0 and ending at copy_mm+0x331/0x3b0
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at find_get_page+0x14/0x60 and ending at find_get_page+0x2f/0x60
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at dev_queue_xmit+0x7f/0x320 and ending at 
dev_queue_xmit+0x27f/0x320
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at dev_queue_xmit+0x7f/0x320 and ending at 
dev_queue_xmit+0x27f/0x320
49ms non-preemptible critical section violated 1 ms preempt threshold 
starting at snd_pcm_action_lock_irq+0x1b/0x1d0 [snd_pcm] and ending at 
snd_pcm_action_lock_irq+0x65/0x1d0 [snd_pcm]
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at do_no_page+0xd5/0x310 and ending at do_no_page+0x178/0x310
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at fget+0x28/0x70 and ending at fget+0x41/0x70
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at sys_ioctl+0x42/0x270 and ending at do_IRQ+0xec/0x130
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at dnotify_parent+0x27/0xc0 and ending at dnotify_parent+0x85/0xc0
2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at buffered_rmqueue+0xea/0x190 and ending at 
buffered_rmqueue+0x144/0x190

What I've excluded (happens all the time):

1) 2ms non-preemptible critical section violated 1 ms preempt threshold 
starting at schedule+0x36/0x480 and ending at do_IRQ+0xec/0x130
it's 2ms 98%. This really happens all the time. Bogus?

2) 49ms non-preemptible critical section violated 1 ms preempt threshold 
starting at sys_ioctl+0x42/0x270 and ending at sys_ioctl+0xbd/0x270
40-50 ms most of the time, 12 ms couple of times.

Let me now if you need those traces for some of these (I've built kernel 
with 8K stacks).

Lenar

William Lee Irwin III wrote:

>This patch uses the preemption counter increments and decrements to time
>non-preemptible critical sections.
>  
>

Re: preempt-timing-2.6.8-rc1

[Joe Korty]

On Tue, Jul 13, 2004 at 05:28:05AM -0700, William Lee Irwin III wrote:
> This patch uses the preemption counter increments and decrements to time
> non-preemptible critical sections.
> 
> This is an instrumentation patch intended to help determine the causes of
> scheduling latency related to long non-preemptible critical sections.
> 
> Changes from 2.6.7-based patch:
> (1) fix unmap_vmas() check correctly this time
> (2) add touch_preempt_timing() to cond_resched_lock()
> (3) depend on preempt until it's worked out wtf goes wrong without it

You preemption-block hold times will improve *enormously* if you move all
softirq processing down to the daemon (and possibly raise the daemon to
one of the higher SCHED_RR priorities, to compensate for softirq processing
no longer happening at interrupt level).

Joe

Re: preempt-timing-2.6.8-rc1

[William Lee Irwin III]

On Tue, Jul 13, 2004 at 05:24:32PM +0300, Lenar L?hmus wrote:
> My first results with 2.6.8-rc1 + preempt-timing:
> Boot-time:
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at buffered_rmqueue+0xea/0x190 and ending at 
> buffered_rmqueue+0x144/0x190
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at search_by_key+0xe3/0xf70 and ending at do_IRQ+0xec/0x130
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at copy_mm+0x2e6/0x3b0 and ending at copy_mm+0x331/0x3b0
> 11ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at ohci_hub_resume+0x3c/0x350 [ohci_hcd] and ending at 
> ohci_hub_resume+0x79/0x350 [ohci_hcd]
> 14ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at schedule+0x36/0x480 and ending at schedule+0x291/0x480
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at new_inode+0x1a/0x80 and ending at new_inode+0x58/0x80
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20

Boot-time stuff is unlikely to have much done about it. I highly suspect
this is the kmap_atomic() trick for pagecache writes. 1ms is probably
more than your box can handle if so.


On Tue, Jul 13, 2004 at 05:24:32PM +0300, Lenar L?hmus wrote:
> Normal use (logging in, browsing, playing video with mplayer, moving 
> windows around, etc):
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at search_by_key+0xe3/0xf70 and ending at 
> smp_apic_timer_interrupt+0x9a/0xe0
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at do_munmap+0xd2/0x140 and ending at do_munmap+0xeb/0x140
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at __d_lookup+0x66/0x170 and ending at __d_lookup+0x9c/0x170
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at copy_mm+0x2e6/0x3b0 and ending at copy_mm+0x331/0x3b0
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at find_get_page+0x14/0x60 and ending at find_get_page+0x2f/0x60
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at dev_queue_xmit+0x7f/0x320 and ending at 
> dev_queue_xmit+0x27f/0x320
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at dev_queue_xmit+0x7f/0x320 and ending at 
> dev_queue_xmit+0x27f/0x320
> 49ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at snd_pcm_action_lock_irq+0x1b/0x1d0 [snd_pcm] and ending at 
> snd_pcm_action_lock_irq+0x65/0x1d0 [snd_pcm]
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at do_no_page+0xd5/0x310 and ending at do_no_page+0x178/0x310
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at fget+0x28/0x70 and ending at fget+0x41/0x70
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at sys_ioctl+0x42/0x270 and ending at do_IRQ+0xec/0x130
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at dnotify_parent+0x27/0xc0 and ending at dnotify_parent+0x85/0xc0
> 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at buffered_rmqueue+0xea/0x190 and ending at 
> buffered_rmqueue+0x144/0x190

If you're getting these in find_get_page() and buffered_rmqueue() either
your box is definitely too slow to handle 1ms or sched_clock()'s results
are questionable on your machine. cpufreq?

On Tue, Jul 13, 2004 at 05:24:32PM +0300, Lenar L?hmus wrote:
> What I've excluded (happens all the time):
> 1) 2ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at schedule+0x36/0x480 and ending at do_IRQ+0xec/0x130
> it's 2ms 98%. This really happens all the time. Bogus?

Wild guess is that you took an IRQ in dec_preempt_count() and that threw
your results off. Let me know if the patch below helps at all. My guess
is it'll cause more apparent problems than it solves.


On Tue, Jul 13, 2004 at 05:24:32PM +0300, Lenar L?hmus wrote:
> 2) 49ms non-preemptible critical section violated 1 ms preempt threshold 
> starting at sys_ioctl+0x42/0x270 and ending at sys_ioctl+0xbd/0x270
> 40-50 ms most of the time, 12 ms couple of times.
> Let me now if you need those traces for some of these (I've built kernel 
> with 8K stacks).

ioctl() is typically grossly inefficient and even involves the BKL.


-- wli

Index: timing-2.6.8-rc1/kernel/sched.c
===================================================================
--- timing-2.6.8-rc1.orig/kernel/sched.c
+++ timing-2.6.8-rc1/kernel/sched.c
@@ -4069,22 +4069,33 @@
 
 void touch_preempt_timing(void)
 {
+	unsigned long flags;
+
+	local_irq_save(flags);
 	if (preempt_count() > 0 && system_state == SYSTEM_RUNNING &&
 						__get_cpu_var(preempt_entry))
 		__touch_preempt_timing(__builtin_return_address(0));
+	local_irq_restore(flags);
 }
 EXPORT_SYMBOL(touch_preempt_timing);
 
 void inc_preempt_count(void)
 {
+	unsigned long flags;
+
+	local_irq_save(flags);
 	preempt_count()++;
 	if (preempt_count() == 1 && system_state == SYSTEM_RUNNING)
 		__touch_preempt_timing(__builtin_return_address(0));
+	local_irq_restore(flags);
 }
 EXPORT_SYMBOL(inc_preempt_count);
 
 void dec_preempt_count(void)
 {
+	unsigned long flags;
+
+	local_irq_save(flags);
 	if (preempt_count() == 1 && system_state == SYSTEM_RUNNING &&
 					__get_cpu_var(preempt_entry)) {
 		u64 hold;
@@ -4108,6 +4119,7 @@
 		__get_cpu_var(preempt_entry) = 0;
 	}
 	preempt_count()--;
+	local_irq_restore(flags);
 }
 EXPORT_SYMBOL(dec_preempt_count);
 #endif

Re: preempt-timing-2.6.8-rc1

[William Lee Irwin III]

On Tue, Jul 13, 2004 at 05:28:05AM -0700, William Lee Irwin III wrote:
>> This patch uses the preemption counter increments and decrements to time
>> non-preemptible critical sections.
>> This is an instrumentation patch intended to help determine the causes of
>> scheduling latency related to long non-preemptible critical sections.
>> Changes from 2.6.7-based patch:
>> (1) fix unmap_vmas() check correctly this time
>> (2) add touch_preempt_timing() to cond_resched_lock()
>> (3) depend on preempt until it's worked out wtf goes wrong without it

On Tue, Jul 13, 2004 at 10:36:00AM -0400, Joe Korty wrote:
> You preemption-block hold times will improve *enormously* if you move all
> softirq processing down to the daemon (and possibly raise the daemon to
> one of the higher SCHED_RR priorities, to compensate for softirq processing
> no longer happening at interrupt level).

Plausible. Got a patch?


-- wli

Re: preempt-timing-2.6.8-rc1

[Nick Piggin]

William Lee Irwin III wrote:
> On Tue, Jul 13, 2004 at 05:28:05AM -0700, William Lee Irwin III wrote:
> 
>>>This patch uses the preemption counter increments and decrements to time
>>>non-preemptible critical sections.
>>>This is an instrumentation patch intended to help determine the causes of
>>>scheduling latency related to long non-preemptible critical sections.
>>>Changes from 2.6.7-based patch:
>>>(1) fix unmap_vmas() check correctly this time
>>>(2) add touch_preempt_timing() to cond_resched_lock()
>>>(3) depend on preempt until it's worked out wtf goes wrong without it
> 
> 
> On Tue, Jul 13, 2004 at 10:36:00AM -0400, Joe Korty wrote:
> 
>>You preemption-block hold times will improve *enormously* if you move all
>>softirq processing down to the daemon (and possibly raise the daemon to
>>one of the higher SCHED_RR priorities, to compensate for softirq processing
>>no longer happening at interrupt level).
> 
> 
> Plausible. Got a patch?
> 

Make MAX_SOFTIRQ_RESTART 1?

I don't think you should make ksoftirq a realtime task, because that
defeats the purpose of having it to prevent livelocking userspace
doesn't it?

However, you may want to increase it from nice +19. Probably just to
nice 0 would be an idea.

Re: preempt-timing-2.6.8-rc1

[bert hubert]

On Tue, Jul 13, 2004 at 07:39:47AM -0700, William Lee Irwin III wrote:

> > 49ms non-preemptible critical section violated 1 ms preempt threshold 
> > starting at snd_pcm_action_lock_irq+0x1b/0x1d0 [snd_pcm] and ending at 
> > snd_pcm_action_lock_irq+0x65/0x1d0 [snd_pcm]

woa

> > 2) 49ms non-preemptible critical section violated 1 ms preempt threshold 
> > starting at sys_ioctl+0x42/0x270 and ending at sys_ioctl+0xbd/0x270
> > 40-50 ms most of the time, 12 ms couple of times.
> > Let me now if you need those traces for some of these (I've built kernel 
> > with 8K stacks).
> 
> ioctl() is typically grossly inefficient and even involves the BKL.

Indeed - but 49ms is stunning and worthy of investigation. The interesting
thing is that sys_ioctl blankly locks the kernel, even if the systems below
it don't need it. Would be a big change to fix.

In this case, how about adding

	printk(KERN_DEBUG "ioctl cmd=%d\n", cmd);

here in fs/ioctl.c:

	unlock_kernel();
	fput(filp);

out:
		return error;
}

Or something else to instrument ioctl?

-- 
http://www.PowerDNS.com      Open source, database driven DNS Software 
http://lartc.org           Linux Advanced Routing & Traffic Control HOWTO

Re: preempt-timing-2.6.8-rc1

[La Monte H.P. Yarroll]

[-- Attachment #1: Type: text/plain, Size: 1277 bytes --]

William Lee Irwin III wrote:

>On Tue, Jul 13, 2004 at 05:28:05AM -0700, William Lee Irwin III wrote:
>  
>
>>>This patch uses the preemption counter increments and decrements to time
>>>non-preemptible critical sections.
>>>This is an instrumentation patch intended to help determine the causes of
>>>scheduling latency related to long non-preemptible critical sections.
>>>Changes from 2.6.7-based patch:
>>>(1) fix unmap_vmas() check correctly this time
>>>(2) add touch_preempt_timing() to cond_resched_lock()
>>>(3) depend on preempt until it's worked out wtf goes wrong without it
>>>      
>>>
>
>On Tue, Jul 13, 2004 at 10:36:00AM -0400, Joe Korty wrote:
>  
>
>>You preemption-block hold times will improve *enormously* if you move all
>>softirq processing down to the daemon (and possibly raise the daemon to
>>one of the higher SCHED_RR priorities, to compensate for softirq processing
>>no longer happening at interrupt level).
>>    
>>
>
>Plausible. Got a patch?
>  
>

How about this?  It moves softirqs to a thread but keeps the old spill 
thread.

This is a patch against 2.6.6 written primarily by Scott Wood.

Signed-off-by: La Monte H.P. Yarroll <piggy@timesys.com> under TS0058

-- 
  Anyone who quotes me in their sig is an idiot. -- Rusty Russell's sig


[-- Attachment #2: softirq.patch --]
[-- Type: text/x-patch, Size: 18604 bytes --]

--- linux-orig/include/asm-alpha/hardirq.h
+++ linux/include/asm-alpha/hardirq.h
@@ -86,14 +86,18 @@
 #define in_atomic()	(preempt_count() != 0)
 #define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 # endif
-#define irq_exit()						\
-do {								\
-		preempt_count() -= IRQ_EXIT_OFFSET;		\
-		if (!in_interrupt() &&				\
-		    softirq_pending(smp_processor_id()))	\
-			do_softirq();				\
-		preempt_enable_no_resched();			\
+
+#ifndef CONFIG_SOFTIRQ_THREADS
+#define irq_exit()							\
+do {									\
+		preempt_count() -= IRQ_EXIT_OFFSET;			\
+		if (!in_interrupt() && softirq_pending(smp_processor_id())) \
+			do_softirq();					\
+		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-arm/hardirq.h
+++ linux/include/asm-arm/hardirq.h
@@ -81,6 +81,7 @@
 
 extern asmlinkage void __do_softirq(void);
 
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 	do {								\
 		preempt_count() -= IRQ_EXIT_OFFSET;			\
@@ -88,6 +89,9 @@
 			__do_softirq();					\
 		preempt_enable_no_resched();				\
 	} while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #define synchronize_irq(irq)	barrier()
 #else
--- linux-orig/include/asm-arm26/hardirq.h
+++ linux/include/asm-arm26/hardirq.h
@@ -76,6 +76,8 @@
 #endif
 
 #ifndef CONFIG_SMP
+
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 	do {								\
 		preempt_count() -= HARDIRQ_OFFSET;			\
@@ -83,6 +85,9 @@
 			__asm__("bl%? __do_softirq": : : "lr");/* out of line */\
 		preempt_enable_no_resched();				\
 	} while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #define synchronize_irq(irq)	barrier()
 #else
--- linux-orig/include/asm-cris/hardirq.h
+++ linux/include/asm-cris/hardirq.h
@@ -84,6 +84,9 @@
 # define in_atomic()	(preempt_count() != 0)
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
+
+
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 do {									\
 		preempt_count() -= IRQ_EXIT_OFFSET;			\
@@ -91,6 +94,9 @@
 			do_softirq();					\
 		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #define synchronize_irq(irq)	barrier()
 
--- linux-orig/include/asm-h8300/hardirq.h
+++ linux/include/asm-h8300/hardirq.h
@@ -82,6 +82,8 @@
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
 
+
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 do {									\
 		preempt_count() -= IRQ_EXIT_OFFSET;			\
@@ -89,6 +91,9 @@
 			do_softirq();					\
 		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-i386/hardirq.h
+++ linux/include/asm-i386/hardirq.h
@@ -84,6 +84,8 @@
 # define in_atomic()	(preempt_count() != 0)
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
+
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 do {									\
 		preempt_count() -= IRQ_EXIT_OFFSET;			\
@@ -91,6 +93,9 @@
 			do_softirq();					\
 		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-m68k/hardirq.h
+++ linux/include/asm-m68k/hardirq.h
@@ -78,6 +78,8 @@
 # define in_atomic()	(preempt_count() != 0)
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
+
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 do {									\
 		preempt_count() -= IRQ_EXIT_OFFSET;			\
@@ -85,6 +87,9 @@
 			do_softirq();					\
 		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #define synchronize_irq(irq)	barrier()
 
--- linux-orig/include/asm-m68knommu/hardirq.h
+++ linux/include/asm-m68knommu/hardirq.h
@@ -86,6 +86,7 @@
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
 
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 do {									\
 		preempt_count() -= IRQ_EXIT_OFFSET;			\
@@ -93,6 +94,9 @@
 			do_softirq();					\
 		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-mips/hardirq.h
+++ linux/include/asm-mips/hardirq.h
@@ -87,13 +87,18 @@
 # define in_atomic()	(preempt_count() != 0)
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
-#define irq_exit()                                                     \
-do {                                                                   \
-	preempt_count() -= IRQ_EXIT_OFFSET;                     \
-	if (!in_interrupt() && softirq_pending(smp_processor_id())) \
-		do_softirq();                                   \
-	preempt_enable_no_resched();                            \
+
+#ifndef CONFIG_SOFTIRQ_THREADS
+#define irq_exit()							\
+do {									\
+	preempt_count() -= IRQ_EXIT_OFFSET;				\
+	if (!in_interrupt() && softirq_pending(smp_processor_id()))	\
+		do_softirq();						\
+	preempt_enable_no_resched();					\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-parisc/hardirq.h
+++ linux/include/asm-parisc/hardirq.h
@@ -96,13 +96,17 @@
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
 
-#define irq_exit()								\
-do {										\
-		preempt_count() -= IRQ_EXIT_OFFSET;				\
-		if (!in_interrupt() && softirq_pending(smp_processor_id()))	\
-			do_softirq();						\
-		preempt_enable_no_resched();					\
+#ifndef CONFIG_SOFTIRQ_THREADS
+#define irq_exit()							\
+do {									\
+		preempt_count() -= IRQ_EXIT_OFFSET;			\
+		if (!in_interrupt() && softirq_pending(smp_processor_id())) \
+			do_softirq();					\
+		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifdef CONFIG_SMP
   extern void synchronize_irq (unsigned int irq);
--- linux-orig/include/asm-ppc/hardirq.h
+++ linux/include/asm-ppc/hardirq.h
@@ -88,13 +88,18 @@
 # define in_atomic()	(preempt_count() != 0)
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
+
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
-do {									\
+do {                                                                    \
 	preempt_count() -= IRQ_EXIT_OFFSET;				\
 	if (!in_interrupt() && softirq_pending(smp_processor_id()))	\
 		do_softirq();						\
 	preempt_enable_no_resched();					\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-ppc64/hardirq.h
+++ linux/include/asm-ppc64/hardirq.h
@@ -87,6 +87,8 @@
 # define in_atomic()	(preempt_count() != 0)
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
+
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 do {									\
 		preempt_count() -= IRQ_EXIT_OFFSET;			\
@@ -94,6 +96,9 @@
 			do_softirq();					\
 		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-sh/hardirq.h
+++ linux/include/asm-sh/hardirq.h
@@ -81,6 +81,8 @@
 # define in_atomic()	(preempt_count() != 0)
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
+
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 do {									\
 		preempt_count() -= IRQ_EXIT_OFFSET;			\
@@ -88,6 +90,9 @@
 			do_softirq();					\
 		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-sparc/hardirq.h
+++ linux/include/asm-sparc/hardirq.h
@@ -87,13 +87,18 @@
 # define in_atomic()	(preempt_count() != 0)
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
-#define irq_exit()                                                      \
-do {                                                                    \
-                preempt_count() -= IRQ_EXIT_OFFSET;                     \
-                if (!in_interrupt() && softirq_pending(smp_processor_id())) \
-                        do_softirq();                                   \
-                preempt_enable_no_resched();                            \
+
+#ifndef CONFIG_SOFTIRQ_THREADS
+#define irq_exit()							\
+do {									\
+		preempt_count() -= IRQ_EXIT_OFFSET;			\
+		if (!in_interrupt() && softirq_pending(smp_processor_id())) \
+			do_softirq();					\
+		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-sparc64/hardirq.h
+++ linux/include/asm-sparc64/hardirq.h
@@ -86,6 +86,8 @@
 # define in_atomic()	(preempt_count() != 0)
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
+
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 do {									\
 		preempt_count() -= IRQ_EXIT_OFFSET;			\
@@ -93,6 +95,9 @@
 			do_softirq();					\
 		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-v850/hardirq.h
+++ linux/include/asm-v850/hardirq.h
@@ -80,13 +80,17 @@
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
 
-#define irq_exit()							      \
-do {									      \
-	preempt_count() -= IRQ_EXIT_OFFSET;				      \
-	if (!in_interrupt() && softirq_pending(smp_processor_id()))	      \
-		do_softirq();						      \
-	preempt_enable_no_resched();					      \
+#ifndef CONFIG_SOFTIRQ_THREADS
+#define irq_exit()							\
+do {									\
+	preempt_count() -= IRQ_EXIT_OFFSET;				\
+	if (!in_interrupt() && softirq_pending(smp_processor_id()))	\
+		do_softirq();						\
+	preempt_enable_no_resched();					\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/asm-x86_64/hardirq.h
+++ linux/include/asm-x86_64/hardirq.h
@@ -85,6 +85,8 @@
 # define in_atomic()   (preempt_count() != 0)
 # define IRQ_EXIT_OFFSET HARDIRQ_OFFSET
 #endif
+
+#ifndef CONFIG_SOFTIRQ_THREADS
 #define irq_exit()							\
 do {									\
 		preempt_count() -= IRQ_EXIT_OFFSET;			\
@@ -92,6 +94,9 @@
 			do_softirq();					\
 		preempt_enable_no_resched();				\
 } while (0)
+#else
+#define irq_exit()              (preempt_count() -= HARDIRQ_OFFSET)
+#endif
 
 #ifndef CONFIG_SMP
 # define synchronize_irq(irq)	barrier()
--- linux-orig/include/linux/interrupt.h
+++ linux/include/linux/interrupt.h
@@ -59,6 +59,8 @@
 #endif
 
 /* SoftIRQ primitives.  */
+#ifndef CONFIG_SOFTIRQ_THREADS
+
 #define local_bh_disable() \
 		do { preempt_count() += SOFTIRQ_OFFSET; barrier(); } while (0)
 #define __local_bh_enable() \
@@ -66,6 +68,27 @@
 
 extern void local_bh_enable(void);
 
+#else
+
+/* As far as I can tell, local_bh_disable() didn't stop ksoftirqd
+   from running before.  Since all softirqs now run from one of
+   the ksoftirqds, this shouldn't be necessary. */
+
+static inline void local_bh_disable(void)
+{
+}
+
+static inline void __local_bh_enable(void)
+{
+}
+
+static inline void local_bh_enable(void)
+{
+}
+
+#endif
+
+
 /* PLEASE, avoid to allocate new softirqs, if you need not _really_ high
    frequency threaded job scheduling. For almost all the purposes
    tasklets are more than enough. F.e. all serial device BHs et
--- linux-orig/include/linux/sched.h
+++ linux/include/linux/sched.h
@@ -541,6 +541,11 @@
 #define PF_LESS_THROTTLE 0x00100000	/* Throttle me less: I clean memory */
 #define PF_SYNCWRITE	0x00200000	/* I am doing a sync write */
 
+/* Thread is an IRQ handler.  This is used to determine which softirq
+   thread to wake. */
+
+#define PF_IRQHANDLER 0x10000000
+
 #ifdef CONFIG_SMP
 extern int set_cpus_allowed(task_t *p, cpumask_t new_mask);
 #else
@@ -916,6 +921,9 @@
 
 extern void signal_wake_up(struct task_struct *t, int resume_stopped);
 
+asmlinkage long sys_sched_setscheduler(pid_t pid, int policy,
+                                       struct sched_param __user *param);
+
 /*
  * Wrappers for p->thread_info->cpu access. No-op on UP.
  */
--- linux-orig/init/Kconfig
+++ linux/init/Kconfig
@@ -265,6 +265,18 @@
 
 	  If unsure, say N.
 
+config SOFTIRQ_THREADS
+	bool "Run all softirqs in threads"
+	depends on PREEMPT
+	help
+	  This option creates a second softirq thread per CPU, which
+	  runs at high real-time priority, to replace the softirqs
+	  which were previously run immediately.  This allows these
+	  softirqs to be prioritized, so as to avoid preempting
+	  very high priority real-time tasks.  This also allows
+	  certain spinlocks to be converted into sleeping mutexes,
+	  for futher reduction of scheduling latency.
+
 endmenu		# General setup
 
 
--- linux-orig/kernel/softirq.c
+++ linux/kernel/softirq.c
@@ -15,6 +15,11 @@
 #include <linux/percpu.h>
 #include <linux/cpu.h>
 #include <linux/kthread.h>
+#include <asm/uaccess.h>
+ 
+#ifdef CONFIG_SOFTIRQ_THREADS
+static const int softirq_prio = MAX_USER_RT_PRIO - 3;
+#endif
 
 #include <asm/irq.h>
 /*
@@ -44,6 +49,10 @@
 
 static DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
 
+#ifdef CONFIG_SOFTIRQ_THREADS
+static DEFINE_PER_CPU(struct task_struct *, ksoftirqd_high_prio);
+#endif
+
 /*
  * we cannot loop indefinitely here to avoid userspace starvation,
  * but we also don't want to introduce a worst case 1/HZ latency
@@ -59,6 +68,19 @@
 		wake_up_process(tsk);
 }
 
+#ifdef CONFIG_SOFTIRQ_THREADS
+
+static inline void wakeup_softirqd_high_prio(void)
+{
+	/* Interrupts are disabled: no need to stop preemption */
+	struct task_struct *tsk = __get_cpu_var(ksoftirqd_high_prio);
+
+	if (tsk && tsk->state != TASK_RUNNING)
+		wake_up_process(tsk);
+}
+
+#endif
+
 /*
  * We restart softirq processing MAX_SOFTIRQ_RESTART times,
  * and we fall back to softirqd after that.
@@ -113,8 +135,13 @@
 	__u32 pending;
 	unsigned long flags;
 
+#ifdef CONFIG_SOFTIRQ_THREADS
+	if (in_interrupt())
+		BUG();
+#else
 	if (in_interrupt())
 		return;
+#endif
 
 	local_irq_save(flags);
 
@@ -130,17 +157,20 @@
 
 #endif
 
+#ifndef CONFIG_SOFTIRQ_THREADS
+
 void local_bh_enable(void)
 {
 	__local_bh_enable();
 	WARN_ON(irqs_disabled());
-	if (unlikely(!in_interrupt() &&
-		     local_softirq_pending()))
+	if (unlikely(!in_interrupt() && local_softirq_pending()))
 		invoke_softirq();
 	preempt_check_resched();
 }
 EXPORT_SYMBOL(local_bh_enable);
 
+#endif
+
 /*
  * This function must run with irqs disabled!
  */
@@ -157,8 +187,19 @@
 	 * Otherwise we wake up ksoftirqd to make sure we
 	 * schedule the softirq soon.
 	 */
+#ifdef CONFIG_SOFTIRQ_THREADS
+
+	if (in_interrupt() || (current->flags & PF_IRQHANDLER))
+		wakeup_softirqd_high_prio();
+	else
+		wakeup_softirqd();
+
+#else
+
 	if (!in_interrupt())
 		wakeup_softirqd();
+
+#endif
 }
 
 EXPORT_SYMBOL(raise_softirq_irqoff);
@@ -320,6 +361,47 @@
 	open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
 }
 
+#ifdef CONFIG_SOFTIRQ_THREADS
+ 
+static int ksoftirqd_high_prio(void *__bind_cpu)
+{
+	int cpu = (int)(long)__bind_cpu;
+	struct sched_param param = { .sched_priority = softirq_prio };
+
+	/* Yuck.  Thanks for separating the implementation from the
+	   user API. */
+
+	set_fs(KERNEL_DS);
+	sys_sched_setscheduler(0, SCHED_FIFO, &param);
+	
+	current->flags |= PF_NOFREEZE;
+
+	/* Migrate to the right CPU */
+	set_cpus_allowed(current, cpumask_of_cpu(cpu));
+	BUG_ON(smp_processor_id() != cpu);
+
+	__set_current_state(TASK_INTERRUPTIBLE);
+	mb();
+
+	__get_cpu_var(ksoftirqd_high_prio) = current;
+
+	for (;;) {
+		if (!local_softirq_pending())
+			schedule();
+
+		__set_current_state(TASK_RUNNING);
+
+		while (local_softirq_pending()) {
+ 			do_softirq();
+			cond_resched();
+		}
+
+		__set_current_state(TASK_INTERRUPTIBLE);
+	}
+}
+
+#endif
+
 static int ksoftirqd(void * __bind_cpu)
 {
 	set_user_nice(current, 19);
@@ -425,15 +507,28 @@
 	case CPU_UP_PREPARE:
 		BUG_ON(per_cpu(tasklet_vec, hotcpu).list);
 		BUG_ON(per_cpu(tasklet_hi_vec, hotcpu).list);
-		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu);
+		p = kthread_create(ksoftirqd, hcpu, "ksoftirqd/l%d", hotcpu);
 		if (IS_ERR(p)) {
-			printk("ksoftirqd for %i failed\n", hotcpu);
+			printk("ksoftirqd/l%i failed\n", hotcpu);
 			return NOTIFY_BAD;
 		}
 		kthread_bind(p, hotcpu);
   		per_cpu(ksoftirqd, hotcpu) = p;
+#ifdef CONFIG_SOFTIRQ_THREADS
+		p = kthread_create(ksoftirqd_high_prio, hcpu, "ksoftirqd/h%d", hotcpu);
+		if (IS_ERR(p)) {
+			printk("ksoftirqd/h%i failed\n", hotcpu);
+			return NOTIFY_BAD;
+		}
+		per_cpu(ksoftirqd_high_prio, hotcpu) = p;
+		kthread_bind(p, hotcpu);
+  		per_cpu(ksoftirqd_high_prio, hotcpu) = p;
+#endif
  		break;
 	case CPU_ONLINE:
+#ifdef CONFIG_SOFTIRQ_THREADS
+		wake_up_process(per_cpu(ksoftirqd_high_prio, hotcpu));
+#endif
 		wake_up_process(per_cpu(ksoftirqd, hotcpu));
 		break;
 #ifdef CONFIG_HOTPLUG_CPU
--- linux-orig/kernel/timer.c
+++ linux/kernel/timer.c
@@ -910,12 +910,16 @@
 void do_timer(struct pt_regs *regs)
 {
 	jiffies_64++;
+	update_times();
+
 #ifndef CONFIG_SMP
 	/* SMP process accounting uses the local APIC timer */
 
+	// Do this after update_times(), so that the softirq thread
+	// is not counted as running all the time.
+
 	update_process_times(user_mode(regs));
 #endif
-	update_times();
 }
 
 #if !defined(__alpha__) && !defined(__ia64__)
--- linux-orig/mm/slab.c
+++ linux/mm/slab.c
@@ -2593,7 +2593,7 @@
 {
 	struct list_head *walk;
 
-#if DEBUG
+#if DEBUG && !defined(CONFIG_SOFTIRQ_THREADS)
 	BUG_ON(!in_interrupt());
 	BUG_ON(in_irq());
 #endif
--- linux-orig/net/ipv4/ipconfig.c
+++ linux/net/ipv4/ipconfig.c
@@ -1085,8 +1085,13 @@
 
 		jiff = jiffies + (d->next ? CONF_INTER_TIMEOUT : timeout);
 		while (time_before(jiffies, jiff) && !ic_got_reply) {
-			barrier();
-			cpu_relax();
+			// Wait queues are apparently too hard
+			// for the ipconfig people, but we
+			// need to drop the BKL here to allow
+			// preemption.
+			
+			set_current_state(TASK_INTERRUPTIBLE);
+			schedule_timeout(1);
 		}
 #ifdef IPCONFIG_DHCP
 		/* DHCP isn't done until we get a DHCPACK. */

Re: preempt-timing-2.6.8-rc1

[Joe Korty]

On Tue, Jul 13, 2004 at 07:40:28AM -0700, William Lee Irwin III wrote:
> On Tue, Jul 13, 2004 at 05:28:05AM -0700, William Lee Irwin III wrote:
> >> This patch uses the preemption counter increments and decrements to time
> >> non-preemptible critical sections.
> >> This is an instrumentation patch intended to help determine the causes of
> >> scheduling latency related to long non-preemptible critical sections.
> >> Changes from 2.6.7-based patch:
> >> (1) fix unmap_vmas() check correctly this time
> >> (2) add touch_preempt_timing() to cond_resched_lock()
> >> (3) depend on preempt until it's worked out wtf goes wrong without it
> 
> On Tue, Jul 13, 2004 at 10:36:00AM -0400, Joe Korty wrote:
> > You preemption-block hold times will improve *enormously* if you move all
> > softirq processing down to the daemon (and possibly raise the daemon to
> > one of the higher SCHED_RR priorities, to compensate for softirq processing
> > no longer happening at interrupt level).
> 
> Plausible. Got a patch?

I can scrape one up.  It'll be a day or so.
Joe

Re: preempt-timing-2.6.8-rc1

[Lenar Lõhmus]

William Lee Irwin III wrote:

>Wild guess is that you took an IRQ in dec_preempt_count() and that threw
>your results off. Let me know if the patch below helps at all. My guess
>is it'll cause more apparent problems than it solves.
>
Machine in question is XP2500+@1.84GHz (it was overlocked@2.25GHz during 
last test, now running at
official speed). Is this really slow for 1ms?

Applied your patch. Booted.

With preempt_thresh=1 I still got tons of those violations at schedule().

With preempt_thresh=2 I do not get those anymore. Apart from sys_ioctl() 
violation, getting now these:

16ms non-preemptible critical section violated 2 ms preempt threshold 
starting at exit_notify+0x1d/0x7b0 and ending at schedule+0x291/0x480
7ms non-preemptible critical section violated 2 ms preempt threshold 
starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20
6ms non-preemptible critical section violated 2 ms preempt threshold 
starting at fget+0x28/0x70 and ending at fget+0x41/0x70

No apparent side-effects noticed.

As before, when running mplayer I'm getting many sys_ioctl() things 
coupled with messages:
rtc: lost some interrupts at 1024Hz.
It happens when madly seeking around in video.

Lenar

[PATCH] fix arbitrarily long preemption lockout times [was: re: preempt-timing-2.6.8-rc1]

[Joe Korty]

On Tue, Jul 13, 2004 at 07:40:28AM -0700, William Lee Irwin III wrote:
> On Tue, Jul 13, 2004 at 10:36:00AM -0400, Joe Korty wrote:
>> You preemption-block hold times will improve *enormously* if you move all
>> softirq processing down to the daemon (and possibly raise the daemon to
>> one of the higher SCHED_RR priorities, to compensate for softirq processing
>> no longer happening at interrupt level).
> 
> Plausible. Got a patch?

Here it is.  Against 2.6.7.  Testing: compiles, boots, shutdown ok, ran
some disk and networking traffic under it.

IIRC, my first attempt had problems booting some configurations, my guess
is that some drivers try to raise softIRQs before the daemons came up.
So this version uses the global 'system_running' to switch at runtime
between the interrupt/daemon and daemon-only modes of operation.

This also has the advantage of switching back to the safer, interrupt/daemon
mode during system shutdown and while crash dumps are in progress (for
those who have applied that patch).

IMO, this patch would be better if it introduced a seperate, hi-priority
softirq daemon, as La Monte H.P. Yarroll's patch does.

Regards,
Joe

Signed-off-by: Joe Korty <joe.korty@ccur.com>


diff -ura base/arch/i386/kernel/irq.c new/arch/i386/kernel/irq.c
--- base/arch/i386/kernel/irq.c	2004-06-16 01:18:57.000000000 -0400
+++ new/arch/i386/kernel/irq.c	2004-07-13 12:52:33.000000000 -0400
@@ -1154,7 +1154,7 @@
 
 extern asmlinkage void __do_softirq(void);
 
-asmlinkage void do_softirq(void)
+asmlinkage void do_softirq_arch(void)
 {
 	unsigned long flags;
 	struct thread_info *curctx;
@@ -1189,5 +1189,5 @@
 	local_irq_restore(flags);
 }
 
-EXPORT_SYMBOL(do_softirq);
+EXPORT_SYMBOL(do_softirq_arch);
 #endif
diff -ura base/arch/ppc64/kernel/irq.c new/arch/ppc64/kernel/irq.c
--- base/arch/ppc64/kernel/irq.c	2004-06-16 01:19:22.000000000 -0400
+++ new/arch/ppc64/kernel/irq.c	2004-07-13 12:15:31.000000000 -0400
@@ -1015,7 +1015,7 @@
 	}
 }
 
-void do_softirq(void)
+void do_softirq_arch(void)
 {
 	unsigned long flags;
 	struct thread_info *curtp, *irqtp;
@@ -1035,7 +1035,7 @@
 
 	local_irq_restore(flags);
 }
-EXPORT_SYMBOL(do_softirq);
+EXPORT_SYMBOL(do_softirq_arch);
 
 #endif /* CONFIG_IRQSTACKS */
 
diff -ura base/include/linux/interrupt.h new/include/linux/interrupt.h
--- base/include/linux/interrupt.h	2004-06-16 01:19:29.000000000 -0400
+++ new/include/linux/interrupt.h	2004-07-13 12:14:14.000000000 -0400
@@ -93,6 +93,7 @@
 };
 
 asmlinkage void do_softirq(void);
+asmlinkage void do_softirq_arch(void);
 extern void open_softirq(int nr, void (*action)(struct softirq_action*), void *data);
 extern void softirq_init(void);
 #define __raise_softirq_irqoff(nr) do { local_softirq_pending() |= 1UL << (nr); } while (0)
diff -ura base/init/Kconfig new/init/Kconfig
--- base/init/Kconfig	2004-06-16 01:19:52.000000000 -0400
+++ new/init/Kconfig	2004-07-13 12:56:02.000000000 -0400
@@ -218,6 +218,43 @@
 	  This option enables access to kernel configuration file and build
 	  information through /proc/config.gz.
 
+config SOFTIRQ_PREEMPT_BLOCK
+	bool "SoftIRQs to be affected by preemption blocks"
+	default y
+	help
+	  If N then SoftIRQ interrupt handlers are to be blocked by
+	  exactly the same conditions as hard IRQ handlers.  If Y then
+	  SoftIRQ handlers are also to be blocked by preemption blocks.
+
+	  The reasons why Y is useful are too long to elaborate here.
+	  Suffice it to say that SoftIRQ handlers unlike hardIRQ handlers
+	  have no upper bounds on their execution time and allowing
+	  them to interrupt spinlocked critical regions protected only
+	  by a preemption block means that no such region can have an
+	  upper bound on the lock hold time.
+
+	  Realtime systems should say Y here, all others may say N or
+	  Y.  If unsure, say N.
+
+config SOFTIRQ_PRI
+	int "SoftIRQ daemon priority"
+	default 0
+	help
+	  Select the scheduling policy and priority for the per-cpu
+	  SoftIRQ daemons.
+
+	  If >0 then the daemons will have a policy of SCHED_FIFO and
+	  their priorities will each be set to the given value.
+
+	  If ==0 then the daemons will run with a SCHED_OTHER policy
+	  unless SOFTIRQ_PREEMPT_BLOCK is set to Y.  In that case the
+	  daemons will run under SCHED_FIFO with a priority one less
+	  than the system maximum.
+
+	  Most users should say 0 unless there is a specific priority
+	  the softIRQ daemons must run at.  If unsure, say 0.
+
+
 
 menuconfig EMBEDDED
 	bool "Configure standard kernel features (for small systems)"
diff -ura base/kernel/softirq.c new/kernel/softirq.c
--- base/kernel/softirq.c	2004-06-16 01:19:02.000000000 -0400
+++ new/kernel/softirq.c	2004-07-13 13:05:13.000000000 -0400
@@ -15,7 +15,9 @@
 #include <linux/percpu.h>
 #include <linux/cpu.h>
 #include <linux/kthread.h>
+#include <linux/syscalls.h>
 
+#include <asm/uaccess.h>
 #include <asm/irq.h>
 /*
    - No shared variables, all the data are CPU local.
@@ -108,7 +110,7 @@
 
 #ifndef __ARCH_HAS_DO_SOFTIRQ
 
-asmlinkage void do_softirq(void)
+asmlinkage void do_softirq_arch(void)
 {
 	__u32 pending;
 	unsigned long flags;
@@ -126,10 +128,34 @@
 	local_irq_restore(flags);
 }
 
-EXPORT_SYMBOL(do_softirq);
+EXPORT_SYMBOL(do_softirq_arch);
+
+#endif
+
+#ifdef CONFIG_SOFTIRQ_PREEMPT_BLOCK
+
+asmlinkage void do_softirq(void)
+{
+	int cpu;
+
+	cpu = get_cpu();
+	if (system_state == SYSTEM_RUNNING && per_cpu(ksoftirqd, cpu)) {
+		wakeup_softirqd();
+	} else 
+		do_softirq_arch();
+	put_cpu();
+}
+
+#else /* !CONFIG_SOFTIRQ_PREEMPT_BLOCK */
 
+asmlinkage void do_softirq()
+{
+	do_softirq_arch();
+}
 #endif
 
+EXPORT_SYMBOL(do_softirq);
+
 void local_bh_enable(void)
 {
 	__local_bh_enable();
@@ -325,6 +351,20 @@
 	set_user_nice(current, 19);
 	current->flags |= PF_NOFREEZE;
 
+#if CONFIG_SOFTIRQ_PRI > 0
+	{
+	struct sched_param priconfig = { .sched_priority = CONFIG_SOFTIRQ_PRI };
+	set_fs(KERNEL_DS);
+	sys_sched_setscheduler(0, SCHED_FIFO, &priconfig);
+	}
+#elif defined(CONFIG_SOFTIRQ_PREEMPT_BLOCK)
+	{
+	struct sched_param pridefault = { .sched_priority = MAX_RT_PRIO - 2 };
+	set_fs(KERNEL_DS);
+	sys_sched_setscheduler(0, SCHED_FIFO, &pridefault);
+	}
+#endif
+
 	set_current_state(TASK_INTERRUPTIBLE);
 
 	while (!kthread_should_stop()) {
@@ -340,7 +380,7 @@
 			preempt_disable();
 			if (cpu_is_offline((long)__bind_cpu))
 				goto wait_to_die;
-			do_softirq();
+			do_softirq_arch();
 			preempt_enable();
 			cond_resched();
 		}

Re: preempt-timing-2.6.8-rc1

[William Lee Irwin III]

William Lee Irwin III wrote:
>> Wild guess is that you took an IRQ in dec_preempt_count() and that threw
>> your results off. Let me know if the patch below helps at all. My guess
>> is it'll cause more apparent problems than it solves.

On Tue, Jul 13, 2004 at 06:32:16PM +0300, Lenar L?hmus wrote:
> Machine in question is XP2500+@1.84GHz (it was overlocked@2.25GHz during 
> last test, now running at
> official speed). Is this really slow for 1ms?

It should actually be fast enough. I suspect something else, maybe some
slow devices. What's /proc/interrupts look like?


On Tue, Jul 13, 2004 at 06:32:16PM +0300, Lenar L?hmus wrote:
> Applied your patch. Booted.
> With preempt_thresh=1 I still got tons of those violations at schedule().
> With preempt_thresh=2 I do not get those anymore. Apart from sys_ioctl() 
> violation, getting now these:
> 16ms non-preemptible critical section violated 2 ms preempt threshold 
> starting at exit_notify+0x1d/0x7b0 and ending at schedule+0x291/0x480
> 7ms non-preemptible critical section violated 2 ms preempt threshold 
> starting at kmap_atomic+0x13/0x70 and ending at kunmap_atomic+0x5/0x20
> 6ms non-preemptible critical section violated 2 ms preempt threshold 
> starting at fget+0x28/0x70 and ending at fget+0x41/0x70

exit_notify() isn't a huge surprise unless you're not doing things with
lots of processes. Actually, it probably is a surprise, since it should
only hurt when you're doing forkbombs and/or threadbombs.

The kmap_atomic() stuff is too consistent. Maybe you're taking an
interrupt during the copy operation.

fget() is mind-bogglingly O(1) and very short. Only plausible guess is
we're seeing interrupts taken there because it's so frequently called.


On Tue, Jul 13, 2004 at 06:32:16PM +0300, Lenar L?hmus wrote:
> No apparent side-effects noticed.
> As before, when running mplayer I'm getting many sys_ioctl() things 
> coupled with messages:
> rtc: lost some interrupts at 1024Hz.
> It happens when madly seeking around in video.

Not surprised either. There's probably enough time spent with interrupts
off the local_irq_save() hurt, and it didn't make your schedule() things
go away, so my wild guesswork thus far is it made things worse with no
tangible benefit, so best to drop that local_irq_save() change.


-- wli

Re: preempt-timing-2.6.8-rc1

[Lenar Lõhmus]

William Lee Irwin III wrote:

>On Tue, Jul 13, 2004 at 06:32:16PM +0300, Lenar L?hmus wrote:
>  
>
>>Machine in question is XP2500+@1.84GHz (it was overlocked@2.25GHz during 
>>last test, now running at
>>official speed). Is this really slow for 1ms?
>>    
>>
>
>It should actually be fast enough. I suspect something else, maybe some
>slow devices. What's /proc/interrupts look like?
>  
>
           CPU0
  0:   60521505    IO-APIC-edge  timer
  1:      11445    IO-APIC-edge  i8042
  8:      33269    IO-APIC-edge  rtc
  9:          0   IO-APIC-level  acpi
 14:     523202    IO-APIC-edge  ide0
 15:      61761    IO-APIC-edge  ide1
 16:   10376547   IO-APIC-level  eth0
 19:     522859   IO-APIC-level  ide2
 20:     131904   IO-APIC-level  NVidia nForce2, ehci_hcd
 21:      66209   IO-APIC-level  ohci_hcd
 22:    9165291   IO-APIC-level  ohci_hcd, eth1
NMI:          0
LOC:   60351703
ERR:          0
MIS:          2

> <>exit_notify() isn't a huge surprise unless you're not doing things with
> lots of processes. Actually, it probably is a surprise, since it should
> only hurt when you're doing forkbombs and/or threadbombs.

It probably happened when users where simultaneously logging in (many 
kdeinit processes created at that time).

>Not surprised either. There's probably enough time spent with interrupts
>off the local_irq_save() hurt, and it didn't make your schedule() things
>go away, so my wild guesswork thus far is it made things worse with no
>tangible benefit, so best to drop that local_irq_save() change.
>  
>
Yeah, ok.

In the meantime couple of these found their way to logs during the night:
3ms non-preemptible critical section violated 2 ms preempt threshold 
starting at search_by_key+0xe3/0xf70 and ending at do_IRQ+0xec/0x130

And this still seems to be very long and real:
50ms non-preemptible critical section violated 2 ms preempt threshold 
starting at snd_pcm_action_lock_irq+0x1b/0x1d0 [snd_pcm] and ending at 
snd_pcm_action_lock_irq+0x65/0x1d0 [snd_pcm]
Trace has this:
[<f9239b09>] snd_pcm_playback_ioctl1+0x49/0x2f0 [snd_pcm]
So maybe this too is ioctl related (non-educated guess)?

Lenar

Re: preempt-timing-2.6.8-rc1

[Takashi Iwai]

At Wed, 14 Jul 2004 10:59:53 +0300,
Lenar Lõhmus wrote:
> 
> And this still seems to be very long and real:
> 50ms non-preemptible critical section violated 2 ms preempt threshold 
> starting at snd_pcm_action_lock_irq+0x1b/0x1d0 [snd_pcm] and ending at 
> snd_pcm_action_lock_irq+0x65/0x1d0 [snd_pcm]
> Trace has this:
> [<f9239b09>] snd_pcm_playback_ioctl1+0x49/0x2f0 [snd_pcm]
> So maybe this too is ioctl related (non-educated guess)?

If it's SNDRV_PCM_IOCTL_PREPARE, I'm modifying the relevant code with
rwsem instead of rwlock.  But it's still in BKL of ioctl.  Do we need
to unlock internally?


Takashi

Re: preempt-timing-2.6.8-rc1

[William Lee Irwin III]

At Wed, 14 Jul 2004 10:59:53 +0300, Lenar L?hmus wrote:
>> And this still seems to be very long and real:
>> 50ms non-preemptible critical section violated 2 ms preempt threshold 
>> starting at snd_pcm_action_lock_irq+0x1b/0x1d0 [snd_pcm] and ending at 
>> snd_pcm_action_lock_irq+0x65/0x1d0 [snd_pcm]
>> Trace has this:
>> [<f9239b09>] snd_pcm_playback_ioctl1+0x49/0x2f0 [snd_pcm]
>> So maybe this too is ioctl related (non-educated guess)?

On Wed, Jul 14, 2004 at 12:29:43PM +0200, Takashi Iwai wrote:
> If it's SNDRV_PCM_IOCTL_PREPARE, I'm modifying the relevant code with
> rwsem instead of rwlock.  But it's still in BKL of ioctl.  Do we need
> to unlock internally?

Yes, otherwise it'll be non-preemptible.


-- wli

Re: preempt-timing-2.6.8-rc1

[Lenar Lõhmus]

Hi,

More violations found.

This time during NFS umount/mount. Since (u)mounting is not always
the operation that is only done at boot-time (at least in my case), I 
thought I report.

umount /nfs/dir
16ms non-preemptible critical section violated 2 ms preempt threshold 
starting at truncate_inode_pages+0x9d/0x280 and ending at 
generic_shutdown_super+0xd0/0x190
 [<c0153340>] generic_shutdown_super+0xd0/0x190
 [<c0117d30>] dec_preempt_count+0x110/0x120
 [<c0153340>] generic_shutdown_super+0xd0/0x190
 [<c0117c70>] dec_preempt_count+0x50/0x120
 [<c0153cf9>] kill_anon_super+0x9/0x40
 [<f935884c>] nfs_kill_super+0xc/0x20 [nfs]
 [<c015317c>] deactivate_super+0x6c/0x90
 [<c016809b>] sys_umount+0x3b/0x90
 [<c0164a85>] destroy_inode+0x35/0x40
 [<c014e645>] __fput+0xb5/0x120
 [<c0168107>] sys_oldumount+0x17/0x20
 [<c0103ee1>] sysenter_past_esp+0x52/0x71

mount /nfs/dir
nfs warning: mount version older than kernel
15ms non-preemptible critical section violated 2 ms preempt threshold 
starting at sys_mount+0x78/0x110 and ending at schedule+0x237/0x480
 [<c0279677>] schedule+0x237/0x480
 [<c0117d30>] dec_preempt_count+0x110/0x120
 [<c0279677>] schedule+0x237/0x480
 [<c0127c21>] worker_thread+0x1d1/0x240
 [<c0127c72>] worker_thread+0x222/0x240
 [<c01166f8>] activate_task+0x68/0x80
 [<c0219800>] fb_flashcursor+0x0/0x80
 [<c0116d60>] default_wake_function+0x0/0x10
 [<c0279677>] schedule+0x237/0x480
 [<c0116d60>] default_wake_function+0x0/0x10
 [<c0127a50>] worker_thread+0x0/0x240
 [<c012b2c4>] kthread+0x94/0xa0
 [<c012b230>] kthread+0x0/0xa0
 [<c010227d>] kernel_thread_helper+0x5/0x18

umount /nfs/dir
19ms non-preemptible critical section violated 2 ms preempt threshold 
starting at generic_shutdown_super+0x69/0x190 and ending at 
generic_shutdown_super+0xd0/0x190
 [<c0153340>] generic_shutdown_super+0xd0/0x190
 [<c0117d30>] dec_preempt_count+0x110/0x120
 [<c0153340>] generic_shutdown_super+0xd0/0x190
 [<c0117c70>] dec_preempt_count+0x50/0x120
 [<c0153cf9>] kill_anon_super+0x9/0x40
 [<f935884c>] nfs_kill_super+0xc/0x20 [nfs]
 [<c015317c>] deactivate_super+0x6c/0x90
 [<c016809b>] sys_umount+0x3b/0x90
 [<c0164a85>] destroy_inode+0x35/0x40
 [<c014e645>] __fput+0xb5/0x120
 [<c0168107>] sys_oldumount+0x17/0x20
 [<c0103ee1>] sysenter_past_esp+0x52/0x71

Lenar

Re: preempt-timing-2.6.8-rc1

[Lee Revell]

On Tue, 2004-07-13 at 10:39, William Lee Irwin III wrote:
> On Tue, Jul 13, 2004 at 05:24:32PM +0300, Lenar L?hmus wrote:
> > What I've excluded (happens all the time):
> > 1) 2ms non-preemptible critical section violated 1 ms preempt threshold 
> > starting at schedule+0x36/0x480 and ending at do_IRQ+0xec/0x130
> > it's 2ms 98%. This really happens all the time. Bogus?
> 
> Wild guess is that you took an IRQ in dec_preempt_count() and that threw
> your results off. Let me know if the patch below helps at all. My guess
> is it'll cause more apparent problems than it solves.

I applied the patch to 2.6.8-rc2 + voluntary-preempt-I4, and am
constantly getting these:

2ms non-preemptible critical section violated 1 ms preempt threshold starting at schedule+0x65/0x5b0 and ending at do_IRQ+0x101/0x170
 [<c01066e7>] dump_stack+0x17/0x20
 [<c011426e>] dec_preempt_count+0x10e/0x120
 [<c0107cb1>] do_IRQ+0x101/0x170
 [<c01062a8>] common_interrupt+0x18/0x20
 [<c010421d>] cpu_idle+0x2d/0x40
 [<c030a782>] start_kernel+0x1a2/0x1f0
 [<c010019f>] 0xc010019f
printk: 1 messages suppressed.
2ms non-preemptible critical section violated 1 ms preempt threshold starting at schedule+0x65/0x5b0 and ending at do_IRQ+0x101/0x170
 [<c01066e7>] dump_stack+0x17/0x20
 [<c011426e>] dec_preempt_count+0x10e/0x120
 [<c0107cb1>] do_IRQ+0x101/0x170
 [<c01062a8>] common_interrupt+0x18/0x20
 [<c010421d>] cpu_idle+0x2d/0x40
 [<c030a782>] start_kernel+0x1a2/0x1f0
 [<c010019f>] 0xc010019f
printk: 1 messages suppressed.
2ms non-preemptible critical section violated 1 ms preempt threshold starting at schedule+0x65/0x5b0 and ending at do_IRQ+0x101/0x170
 [<c01066e7>] dump_stack+0x17/0x20
 [<c011426e>] dec_preempt_count+0x10e/0x120
 [<c0107cb1>] do_IRQ+0x101/0x170
 [<c01062a8>] common_interrupt+0x18/0x20
 [<c010421d>] cpu_idle+0x2d/0x40
 [<c030a782>] start_kernel+0x1a2/0x1f0
 [<c010019f>] 0xc010019f

This seems related to keyboard input; if I keep a key pressed down they
happen regularly.

These are similar to the XRUN traces I get from ALSA, so I think there
is a real problem here.

Lee

Re: preempt-timing-2.6.8-rc1

[Lee Revell]

On Sun, 2004-07-25 at 01:15, Lee Revell wrote:
> On Tue, 2004-07-13 at 10:39, William Lee Irwin III wrote:
> > On Tue, Jul 13, 2004 at 05:24:32PM +0300, Lenar L?hmus wrote:
> > > What I've excluded (happens all the time):
> > > 1) 2ms non-preemptible critical section violated 1 ms preempt threshold 
> > > starting at schedule+0x36/0x480 and ending at do_IRQ+0xec/0x130
> > > it's 2ms 98%. This really happens all the time. Bogus?
> > 
> > Wild guess is that you took an IRQ in dec_preempt_count() and that threw
> > your results off. Let me know if the patch below helps at all. My guess
> > is it'll cause more apparent problems than it solves.
> 
> I applied the patch to 2.6.8-rc2 + voluntary-preempt-I4, and am
> constantly getting these:
> 

Here is another one:

Jul 25 18:46:18 mindpipe kernel: 11ms non-preemptible critical section violated 1 ms preempt threshold starting at kmap_atomic+0x10/0x60 and ending at kunmap_atomic+0x8/0x20
Jul 25 18:46:18 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Jul 25 18:46:18 mindpipe kernel:  [dec_preempt_count+270/288] dec_preempt_count+0x10e/0x120
Jul 25 18:46:18 mindpipe kernel:  [kunmap_atomic+8/32] kunmap_atomic+0x8/0x20
Jul 25 18:46:18 mindpipe kernel:  [do_anonymous_page+139/400] do_anonymous_page+0x8b/0x190
Jul 25 18:46:18 mindpipe kernel:  [do_no_page+78/784] do_no_page+0x4e/0x310
Jul 25 18:46:18 mindpipe kernel:  [handle_mm_fault+193/368] handle_mm_fault+0xc1/0x170
Jul 25 18:46:18 mindpipe kernel:  [get_user_pages+288/944] get_user_pages+0x120/0x3b0
Jul 25 18:46:18 mindpipe kernel:  [make_pages_present+104/144] make_pages_present+0x68/0x90
Jul 25 18:46:18 mindpipe kernel:  [do_mmap_pgoff+998/1568] do_mmap_pgoff+0x3e6/0x620
Jul 25 18:46:18 mindpipe kernel:  [sys_mmap2+118/176] sys_mmap2+0x76/0xb0
Jul 25 18:46:18 mindpipe kernel:  [syscall_call+7/11] syscall_call+0x7/0xb

Lee

Re: preempt-timing-2.6.8-rc1

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Here is another one:
> 
> Jul 25 18:46:18 mindpipe kernel: 11ms non-preemptible critical section violated 1 ms preempt threshold starting at kmap_atomic+0x10/0x60 and ending at kunmap_atomic+0x8/0x20

> Jul 25 18:46:18 mindpipe kernel:  [do_no_page+78/784] do_no_page+0x4e/0x310
> Jul 25 18:46:18 mindpipe kernel:  [handle_mm_fault+193/368] handle_mm_fault+0xc1/0x170
> Jul 25 18:46:18 mindpipe kernel:  [get_user_pages+288/944] get_user_pages+0x120/0x3b0
> Jul 25 18:46:18 mindpipe kernel:  [make_pages_present+104/144] make_pages_present+0x68/0x90
> Jul 25 18:46:18 mindpipe kernel:  [do_mmap_pgoff+998/1568] do_mmap_pgoff+0x3e6/0x620
> Jul 25 18:46:18 mindpipe kernel:  [sys_mmap2+118/176] sys_mmap2+0x76/0xb0
> Jul 25 18:46:18 mindpipe kernel:  [syscall_call+7/11] syscall_call+0x7/0xb

is this an mmap(MAP_LOCKED) instance [or a process doing an mmap() after
having done an mlockall()]? I'll take a look.

	Ingo

Re: preempt-timing-2.6.8-rc1

[Lee Revell]

On Mon, 2004-07-26 at 04:23, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Here is another one:
> > 
> > Jul 25 18:46:18 mindpipe kernel: 11ms non-preemptible critical section violated 1 ms preempt threshold starting at kmap_atomic+0x10/0x60 and ending at kunmap_atomic+0x8/0x20
> 
> > Jul 25 18:46:18 mindpipe kernel:  [do_no_page+78/784] do_no_page+0x4e/0x310
> > Jul 25 18:46:18 mindpipe kernel:  [handle_mm_fault+193/368] handle_mm_fault+0xc1/0x170
> > Jul 25 18:46:18 mindpipe kernel:  [get_user_pages+288/944] get_user_pages+0x120/0x3b0
> > Jul 25 18:46:18 mindpipe kernel:  [make_pages_present+104/144] make_pages_present+0x68/0x90
> > Jul 25 18:46:18 mindpipe kernel:  [do_mmap_pgoff+998/1568] do_mmap_pgoff+0x3e6/0x620
> > Jul 25 18:46:18 mindpipe kernel:  [sys_mmap2+118/176] sys_mmap2+0x76/0xb0
> > Jul 25 18:46:18 mindpipe kernel:  [syscall_call+7/11] syscall_call+0x7/0xb
> 
> is this an mmap(MAP_LOCKED) instance [or a process doing an mmap() after
> having done an mlockall()]? I'll take a look.
> 

Yes, jackd does exactly this, mlockall then opens the ALSA driver with
mmap.

Lee

[patch] voluntary-preempt-2.6.8-rc2-J3

[Ingo Molnar]

> Yes, jackd does exactly this, mlockall then opens the ALSA driver with
> mmap.

ok, i fixed this in -J3:

  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-J3

-J3 also includes a number of softirq latency fixes for the networking
layer.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-J3

[Lee Revell]

On Mon, 2004-07-26 at 04:35, Ingo Molnar wrote:
> > Yes, jackd does exactly this, mlockall then opens the ALSA driver with
> > mmap.
> 
> ok, i fixed this in -J3:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-J3
> 
> -J3 also includes a number of softirq latency fixes for the networking
> layer.
> 

OK, I will try this.  I have not seen any latency issues with softirqs
with -I4.  Other than the few remaining hot spots, the only thing that
triggers latencies over 100 usecs during normal operation is the IDE I/O
completion, which can be easily controlled by lowering the max SG size.
 
Here is one that I think happens when deleting a large number of files,
or a directory that had a large number of files.  Specifically, this
happens when bonnie exits.

Jul 25 20:25:36 mindpipe kernel: 16ms non-preemptible critical section violated 1 ms preempt threshold starting at select_parent+0x18/0xd0 and ending at select_parent+0x94/0xd0
Jul 25 20:25:36 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Jul 25 20:25:36 mindpipe kernel:  [dec_preempt_count+270/288] dec_preempt_count+0x10e/0x120
Jul 25 20:25:36 mindpipe kernel:  [select_parent+148/208] select_parent+0x94/0xd0
Jul 25 20:25:36 mindpipe kernel:  [shrink_dcache_parent+22/48] shrink_dcache_parent+0x16/0x30
Jul 25 20:25:36 mindpipe kernel:  [d_unhash+60/176] d_unhash+0x3c/0xb0
Jul 25 20:25:36 mindpipe kernel:  [vfs_rmdir+108/432] vfs_rmdir+0x6c/0x1b0
Jul 25 20:25:36 mindpipe kernel:  [sys_rmdir+207/240] sys_rmdir+0xcf/0xf0
Jul 25 20:25:36 mindpipe kernel:  [syscall_call+7/11] syscall_call+0x7/0xb
Jul 25 20:27:45 mindpipe kernel: ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:169: XRUN: pcmC0D2c
Jul 25 20:27:45 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Jul 25 20:27:45 mindpipe kernel:  [__crc_totalram_pages+1165/3197748] snd_pcm_period_elapsed+0x2c7/0x400 [snd_pcm]
Jul 25 20:27:45 mindpipe kernel:  [__crc_totalram_pages+65879/3197748] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
Jul 25 20:27:45 mindpipe kernel:  [handle_IRQ_event+51/96] handle_IRQ_event+0x33/0x60
Jul 25 20:27:45 mindpipe kernel:  [do_IRQ+167/368] do_IRQ+0xa7/0x170
Jul 25 20:27:45 mindpipe kernel:  [common_interrupt+24/32] common_interrupt+0x18/0x20
Jul 25 20:27:45 mindpipe kernel:  [shrink_dcache_parent+22/48] shrink_dcache_parent+0x16/0x30
Jul 25 20:27:45 mindpipe kernel:  [d_unhash+60/176] d_unhash+0x3c/0xb0
Jul 25 20:27:45 mindpipe kernel:  [vfs_rmdir+108/432] vfs_rmdir+0x6c/0x1b0
Jul 25 20:27:45 mindpipe kernel:  [sys_rmdir+207/240] sys_rmdir+0xcf/0xf0
Jul 25 20:27:45 mindpipe kernel:  [syscall_call+7/11] syscall_call+0x7/0xb

I am also seeing a lot of shorter timing violations that involve
unmap_vmas.  Not sure what triggers this one.

Jul 25 21:05:04 mindpipe kernel: 2ms non-preemptible critical section violated 1 ms preempt threshold starting at unmap_vmas+0x1ff/0x210 and ending at unmap_vmas+0x1f5/0x210
Jul 25 21:05:04 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Jul 25 21:05:04 mindpipe kernel:  [dec_preempt_count+270/288] dec_preempt_count+0x10e/0x120
Jul 25 21:05:04 mindpipe kernel:  [unmap_vmas+501/528] unmap_vmas+0x1f5/0x210
Jul 25 21:05:04 mindpipe kernel:  [exit_mmap+94/336] exit_mmap+0x5e/0x150
Jul 25 21:05:04 mindpipe kernel:  [mmput+114/160] mmput+0x72/0xa0
Jul 25 21:05:04 mindpipe kernel:  [do_exit+251/1072] do_exit+0xfb/0x430
Jul 25 21:05:04 mindpipe kernel:  [do_group_exit+50/192] do_group_exit+0x32/0xc0
Jul 25 21:05:04 mindpipe kernel:  [get_signal_to_deliver+605/880] get_signal_to_deliver+0x25d/0x370
Jul 25 21:05:04 mindpipe kernel:  [do_signal+86/208] do_signal+0x56/0xd0
Jul 25 21:05:04 mindpipe kernel:  [do_notify_resume+71/80] do_notify_resume+0x47/0x50
Jul 25 21:05:04 mindpipe kernel:  [work_notifysig+19/21] work_notifysig+0x13/0x15
Jul 25 21:05:04 mindpipe kernel: 2ms non-preemptible critical section violated 1 ms preempt threshold starting at unmap_vmas+0x1ff/0x210 and ending at unmap_vmas+0x1f5/0x210
Jul 25 21:05:04 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Jul 25 21:05:04 mindpipe kernel:  [dec_preempt_count+270/288] dec_preempt_count+0x10e/0x120
Jul 25 21:05:04 mindpipe kernel:  [unmap_vmas+501/528] unmap_vmas+0x1f5/0x210
Jul 25 21:05:04 mindpipe kernel:  [exit_mmap+94/336] exit_mmap+0x5e/0x150
Jul 25 21:05:04 mindpipe kernel:  [mmput+114/160] mmput+0x72/0xa0
Jul 25 21:05:04 mindpipe kernel:  [do_exit+251/1072] do_exit+0xfb/0x430
Jul 25 21:05:04 mindpipe kernel:  [do_group_exit+50/192] do_group_exit+0x32/0xc0
Jul 25 21:05:04 mindpipe kernel:  [get_signal_to_deliver+605/880] get_signal_to_deliver+0x25d/0x370
Jul 25 21:05:04 mindpipe kernel:  [do_signal+86/208] do_signal+0x56/0xd0
Jul 25 21:05:04 mindpipe kernel:  [do_notify_resume+71/80] do_notify_resume+0x47/0x50
Jul 25 21:05:04 mindpipe kernel:  [work_notifysig+19/21] work_notifysig+0x13/0x15
Jul 25 21:05:17 mindpipe kernel: 2ms non-preemptible critical section violated 1 ms preempt threshold starting at unmap_vmas+0x1ff/0x210 and ending at unmap_vmas+0x1f5/0x210
Jul 25 21:05:17 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Jul 25 21:05:17 mindpipe kernel:  [dec_preempt_count+270/288] dec_preempt_count+0x10e/0x120
Jul 25 21:05:17 mindpipe kernel:  [unmap_vmas+501/528] unmap_vmas+0x1f5/0x210
Jul 25 21:05:17 mindpipe kernel:  [exit_mmap+94/336] exit_mmap+0x5e/0x150
Jul 25 21:05:17 mindpipe kernel:  [mmput+114/160] mmput+0x72/0xa0
Jul 25 21:05:17 mindpipe kernel:  [do_exit+251/1072] do_exit+0xfb/0x430
Jul 25 21:05:17 mindpipe kernel:  [do_group_exit+50/192] do_group_exit+0x32/0xc0
Jul 25 21:05:17 mindpipe kernel:  [syscall_call+7/11] syscall_call+0x7/0xb

Lee

[patch] voluntary-preempt-2.6.8-rc2-J4

[Ingo Molnar]

i've uploaded the -J4 patch:

   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-J4

Changes since -J3:

 - make block device max_sectors sysfs tunable. There's a new entry
   /sys/block/*/queue/max_sectors_kb which stores the current max 
   request size in KB. You can write it to change the size.

   Jens: i've attached a standalone patch against 2.6.8-rc2 below. 
   Please apply if it looks good to you. (I've added extra locking to 
   make sure max_sectors and readahead_pages gets updated at once, for 
   the unlikely event of two CPUs updating max_sectors at once.) I've 
   tested this on IDE using the UP kernel.

 - include the unlock-tty changes suggested by akpm and tested by Lee 
   Revell. I've also unlocked the tty_read path - only the release/open 
   paths remain under the big kernel lock. Please report any tty related 
   regressions.

	Ingo

--
introduce max_sectors_kb tunable.

Signed-off-by: Ingo Molnar <mingo@elte.hu>

--- linux/drivers/block/ll_rw_blk.c.orig	
+++ linux/drivers/block/ll_rw_blk.c	
@@ -3062,13 +3063,50 @@ queue_ra_store(struct request_queue *q, 
 	unsigned long ra_kb;
 	ssize_t ret = queue_var_store(&ra_kb, page, count);
 
+	spin_lock_irq(q->queue_lock);
 	if (ra_kb > (q->max_sectors >> 1))
 		ra_kb = (q->max_sectors >> 1);
 
 	q->backing_dev_info.ra_pages = ra_kb >> (PAGE_CACHE_SHIFT - 10);
+	spin_unlock_irq(q->queue_lock);
+
+	return ret;
+}
+
+static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
+{
+	int max_sectors_kb = q->max_sectors >> 1;
+
+	return queue_var_show(max_sectors_kb, (page));
+}
+
+static ssize_t
+queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
+{
+	unsigned long max_sectors_kb;
+	ssize_t ret = queue_var_store(&max_sectors_kb, page, count);
+	int ra_kb;
+
+	/*
+	 * Take the queue lock to update the readahead and max_sectors
+	 * values synchronously:
+	 */
+	spin_lock_irq(q->queue_lock);
+	/*
+	 * Trim readahead window as well, if necessary:
+	 */
+	ra_kb = q->backing_dev_info.ra_pages << (PAGE_CACHE_SHIFT - 10);
+	if (ra_kb > max_sectors_kb)
+		q->backing_dev_info.ra_pages =
+				max_sectors_kb >> (PAGE_CACHE_SHIFT - 10);
+
+	q->max_sectors = max_sectors_kb << 1;
+	spin_unlock_irq(q->queue_lock);
+
 	return ret;
 }
 
+
 static struct queue_sysfs_entry queue_requests_entry = {
 	.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
 	.show = queue_requests_show,
@@ -3081,9 +3119,16 @@ static struct queue_sysfs_entry queue_ra
 	.store = queue_ra_store,
 };
 
+static struct queue_sysfs_entry queue_max_sectors_entry = {
+	.attr = {.name = "max_sectors_kb", .mode = S_IRUGO | S_IWUSR },
+	.show = queue_max_sectors_show,
+	.store = queue_max_sectors_store,
+};
+
 static struct attribute *default_attrs[] = {
 	&queue_requests_entry.attr,
 	&queue_ra_entry.attr,
+	&queue_max_sectors_entry.attr,
 	NULL,
 };
 

Re: [patch] voluntary-preempt-2.6.8-rc2-J4

[Ingo Molnar]

> Changes since -J3:
> 
>  - make block device max_sectors sysfs tunable. There's a new entry
>    /sys/block/*/queue/max_sectors_kb which stores the current max 
>    request size in KB. You can write it to change the size.
> 
>    Jens: i've attached a standalone patch against 2.6.8-rc2 below. 
>    Please apply if it looks good to you. (I've added extra locking to 
>    make sure max_sectors and readahead_pages gets updated at once, for 
>    the unlikely event of two CPUs updating max_sectors at once.) I've 
>    tested this on IDE using the UP kernel.

i've refined the patch (new version attached below): drivers use
blk_queue_max_sectors() to set the maximum # of sectors that the driver
or hw can handle.

so i've introduced a new queue entry called max_hw_sectors, and the new
/sys entry listens to this maximum and only updates max_sectors. This
entry is also exported to /sys as a readonly entry. (so that users can
see the maximum the driver supports.)

i've added another fix as well: do not allow users to set a max_sectors
value lower than the pagecache page size.

these changes should make the new tunable pretty fool-proof.

	Ingo

Signed-off-by: Ingo Molnar <mingo@elte.hu>

--- linux/include/linux/blkdev.h.orig	
+++ linux/include/linux/blkdev.h	
@@ -338,6 +338,7 @@ struct request_queue
 	unsigned int		nr_congestion_off;
 
 	unsigned short		max_sectors;
+	unsigned short		max_hw_sectors;
 	unsigned short		max_phys_segments;
 	unsigned short		max_hw_segments;
 	unsigned short		hardsect_size;
--- linux/drivers/block/ll_rw_blk.c.orig	
+++ linux/drivers/block/ll_rw_blk.c	
@@ -311,7 +311,7 @@ void blk_queue_max_sectors(request_queue
 		printk("%s: set to minimum %d\n", __FUNCTION__, max_sectors);
 	}
 
-	q->max_sectors = max_sectors;
+	q->max_sectors = q->max_hw_sectors = max_sectors;
 }
 
 EXPORT_SYMBOL(blk_queue_max_sectors);
@@ -3062,13 +3063,61 @@ queue_ra_store(struct request_queue *q, 
 	unsigned long ra_kb;
 	ssize_t ret = queue_var_store(&ra_kb, page, count);
 
+	spin_lock_irq(q->queue_lock);
 	if (ra_kb > (q->max_sectors >> 1))
 		ra_kb = (q->max_sectors >> 1);
 
 	q->backing_dev_info.ra_pages = ra_kb >> (PAGE_CACHE_SHIFT - 10);
+	spin_unlock_irq(q->queue_lock);
+
 	return ret;
 }
 
+static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
+{
+	int max_sectors_kb = q->max_sectors >> 1;
+
+	return queue_var_show(max_sectors_kb, (page));
+}
+
+static ssize_t
+queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
+{
+	unsigned long max_sectors_kb,
+			max_hw_sectors_kb = q->max_hw_sectors >> 1,
+			page_kb = 1 << (PAGE_CACHE_SHIFT - 10);
+	ssize_t ret = queue_var_store(&max_sectors_kb, page, count);
+	int ra_kb;
+
+	if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
+		return -EINVAL;
+	/*
+	 * Take the queue lock to update the readahead and max_sectors
+	 * values synchronously:
+	 */
+	spin_lock_irq(q->queue_lock);
+	/*
+	 * Trim readahead window as well, if necessary:
+	 */
+	ra_kb = q->backing_dev_info.ra_pages << (PAGE_CACHE_SHIFT - 10);
+	if (ra_kb > max_sectors_kb)
+		q->backing_dev_info.ra_pages =
+				max_sectors_kb >> (PAGE_CACHE_SHIFT - 10);
+
+	q->max_sectors = max_sectors_kb << 1;
+	spin_unlock_irq(q->queue_lock);
+
+	return ret;
+}
+
+static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)
+{
+	int max_hw_sectors_kb = q->max_hw_sectors >> 1;
+
+	return queue_var_show(max_hw_sectors_kb, (page));
+}
+
+
 static struct queue_sysfs_entry queue_requests_entry = {
 	.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
 	.show = queue_requests_show,
@@ -3081,9 +3130,22 @@ static struct queue_sysfs_entry queue_ra
 	.store = queue_ra_store,
 };
 
+static struct queue_sysfs_entry queue_max_sectors_entry = {
+	.attr = {.name = "max_sectors_kb", .mode = S_IRUGO | S_IWUSR },
+	.show = queue_max_sectors_show,
+	.store = queue_max_sectors_store,
+};
+
+static struct queue_sysfs_entry queue_max_hw_sectors_entry = {
+	.attr = {.name = "max_hw_sectors_kb", .mode = S_IRUGO },
+	.show = queue_max_hw_sectors_show,
+};
+
 static struct attribute *default_attrs[] = {
 	&queue_requests_entry.attr,
 	&queue_ra_entry.attr,
+	&queue_max_hw_sectors_entry.attr,
+	&queue_max_sectors_entry.attr,
 	NULL,
 };
 

Re: [patch] voluntary-preempt-2.6.8-rc2-J3

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > > Yes, jackd does exactly this, mlockall then opens the ALSA driver with
> > > mmap.
> > 
> > ok, i fixed this in -J3:
> > 
> >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-J3
> > 
> > -J3 also includes a number of softirq latency fixes for the networking
> > layer.
> 
> OK, I will try this.  I have not seen any latency issues with softirqs
> with -I4. [...]

i'm going through the subsystems systematically and i'm stressing them
beyond normal use. These networking latencies need a high number of in
flight packets, multiple TCP sockets, and a 100 mbit link or faster. 
(this is a more common workload on a corporate desktop, but not typical
on a home desktop.)

> [...]  Other than the few remaining hot spots, the only thing that
> triggers latencies over 100 usecs during normal operation is the IDE
> I/O completion, which can be easily controlled by lowering the max SG
> size.

ok, i'll take a look whether there's a way to control this without
having to artificially impact the IO patterns.

> Here is one that I think happens when deleting a large number of
> files, or a directory that had a large number of files.  Specifically,
> this happens when bonnie exits.
> 
> Jul 25 20:25:36 mindpipe kernel:
>
> [...] 16ms non-preemptible critical section violated 1 ms preempt 
> threshold starting at select_parent+0x18/0xd0 and ending at
> select_parent+0x94/0xd0

ok, this should be the dcache/icache zapping. I've done some latency
reduction in this area but apparently not enough.

> I am also seeing a lot of shorter timing violations that involve
> unmap_vmas.  Not sure what triggers this one.

it might just be a common operation, being hit by the IDE hardirq
latency. (which thus is added to the 'scheduling' latency.) Although
none of the traces show IDE hardirq leftovers [but they might be cleared
from the stack by the time we notice the latency.]

Can you see these 1-2 msec latencies even if you reduce the IDE sg-size
drastically via the max_sectors tunable? (just for testing purposes, to
eliminate hardirq latencies as much as possible)

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-J3

[Andrew Morton]

Ingo Molnar <mingo@elte.hu> wrote:
>
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-J3

The bigger this thing gets, the more worried I get.  Sometime this is going
to need to be split up into individual fixes, and they need to be based
upon an overall approach which we haven't yet settled on.

In particular your whole approach (with voluntary_need_resched()) doesn't
work on SMP.

The approach I'm using is to unconditionally drop locks on every Nth pass
around the loop to allow another CPU to grab the lock, do some work, drop
the lock, then be preempted.  eg:

@@ -773,6 +774,12 @@ int get_user_pages(struct task_struct *t
 			struct page *map = NULL;
 			int lookup_write = write;
 
+			if ((++nr_pages & 63) == 0) {
+				spin_unlock(&mm->page_table_lock);
+				cpu_relax();
+				spin_lock(&mm->page_table_lock);
+			}
+
 			/*
 			 * We don't follow pagetables for VM_IO regions - they

This is a bit nasty, because it assumes that the other CPU will be able to
get in there and acquire the lock which is in a different cpu's cache. 
Long transfer latencies may defeat this.

For voluntary preempt the above will need a cond_resched added to it.

So we need to come up with a suitable approach to covering voluntary and
involuntary preemption on both UP and SMP and set up the header file
infrastructure to support it all.  After that we need to redo all the
points-of-high-latency whcih have thus far been discovered to use that
infrastructure.

Re: [patch] voluntary-preempt-2.6.8-rc2-J3

[Ingo Molnar]

* Andrew Morton <akpm@osdl.org> wrote:

> The bigger this thing gets, the more worried I get.  Sometime this is
> going to need to be split up into individual fixes, and they need to
> be based upon an overall approach which we haven't yet settled on.

i will do that splitup. Right now i'm simply mapping how widespread the
problem is and what type of fixes we need. The situation isnt all that
bad but we might need (an optional) mechanism to make softirqs
synchronous. All of this stuff is nicely modular and i'll do a splitup
post 2.6.8 (i dont think we want to disturb 2.6.8 with any of this).

> In particular your whole approach (with voluntary_need_resched())
> doesn't work on SMP.

(for the record, voluntary_need_resched() == need_resched() - i'm only
keeping a distinction between the two to be able to track progress and
regressions between the vanilla and modified kernels.)

need_resched() indeed doesnt do a lock-break for SMP purposes.

> The approach I'm using is to unconditionally drop locks on every Nth pass
> around the loop to allow another CPU to grab the lock, do some work, drop
> the lock, then be preempted.  eg:
> 
> @@ -773,6 +774,12 @@ int get_user_pages(struct task_struct *t
>  			struct page *map = NULL;
>  			int lookup_write = write;
>  
> +			if ((++nr_pages & 63) == 0) {
> +				spin_unlock(&mm->page_table_lock);
> +				cpu_relax();
> +				spin_lock(&mm->page_table_lock);
> +			}
> +

guaranteeing latencies on SMP is hard, because the latency of a CPU
might depend on the latency of a task on another CPU - which CPU isnt
notified of the rescheduling request.

one alternative technique to yours would be to notify _all_ CPUs that a
high-prio RT task is ready to run (via a broadcast need-resched). That
way the UP latency-break techniques map to SMP in a 1:1 way.

non-RT tasks dont get this benefit, which is a difference to the UP
situation, but i dont think it would be appropriate to use the UP
behavior, due to the overhead of broadcasting.

a combination of the two techniques could be used too: a global 'break
locks from now on' flag which gets set if a (RT?) task wants to
reschedule. Normally this flag would be zero and the cacheline would be
clean and shared between all CPUs, causing no overhead. Once a task
wants to reschedule it would increase by 1 until that task has been
scheduled. This would remove the ugliness factor too, your sample code
would look:

> +			if (need_lock_break()) {
> +				spin_unlock(&mm->page_table_lock);
> +				cpu_relax();
> +				spin_lock(&mm->page_table_lock);
> +			}
> +

or a shortcut:

> +			cond_lock_break(&mm->page_table_lock);

there are two problems with the unconditional lock-break:

 - i'm not sure cpu_relax() guarantees that another CPU can grab this 
   lock. It all depends on whether the lock cacheline can bounce over 
   to the other CPU faster than cpu_relax() finishes and this CPU
   re-acquires the lock.

 - the latencies will still be higher than on UP, in a hard to predict
   way. Every time the codepath encounters a spinlock it has to take in
   order to exit for a reschedule, the latency gets re-added. Also, the 
   now scheduled high-prio task will encounter the latency every time it
   acquires a spinlock - while on UP it has the CPU for itself with no
   interaction from other CPUs.

	Ingo

no luck with max_sectors_kb (Re: voluntary-preempt-2.6.8-rc2-J4)

[Rudo Thomas]

> > Changes since -J3:
> > 
> >  - make block device max_sectors sysfs tunable. There's a new entry
> >    /sys/block/*/queue/max_sectors_kb which stores the current max 
> >    request size in KB. You can write it to change the size.
> > 
> > [...]
>
> i've refined the patch (new version attached below): drivers use
> blk_queue_max_sectors() to set the maximum # of sectors that the driver
> or hw can handle.
> 
> so i've introduced a new queue entry called max_hw_sectors, and the new
> /sys entry listens to this maximum and only updates max_sectors. This
> entry is also exported to /sys as a readonly entry. (so that users can
> see the maximum the driver supports.)

Hi there.

I do not seem to have success with tuning the max_sectors_kb value.

After setting it to 32 (the hw max is 128), userland programs fail with I/O
errors. Setting it back to 128 gets it back to working, sort of. The errors
probably get bufferred somewhere.

During the "bad" setting (32), messages like this one show up in kernel log.

bio too big device hda3 (104 > 64)

I am using J7 voluntary_preemption patch (set at 2:1), cfq io scheduler,
via82cxxx IDE driver. I will gladly provide further details.

Rudo.

[patch] voluntary-preempt-2.6.8-rc2-J7

[Ingo Molnar]

i've uploaded -J7:

   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-J7

Changes since -J4:

- fix the latency that occurs when a large number of files are deleted: 
  the guilty one is select_parent() - this should fix the Bonnie latency 
  reported by Lee Revell.

[ the ones below add conditional reschedule points that dont affect 
  users who have kernel_preemption turned on:]

- fix /proc/PID/maps latencies

- fix latencies triggered by 'df' on a large filesystem

- fix exec() latency when dealing with large environments

- add might_sleep() to lock_buffer()

	Ingo

Re: no luck with max_sectors_kb (Re: voluntary-preempt-2.6.8-rc2-J4)

[Ingo Molnar]

* Rudo Thomas <rudo@matfyz.cz> wrote:

> After setting it to 32 (the hw max is 128), userland programs fail
> with I/O errors. Setting it back to 128 gets it back to working, sort
> of. The errors probably get bufferred somewhere.
> 
> During the "bad" setting (32), messages like this one show up in kernel log.
> 
> bio too big device hda3 (104 > 64)

does the patch below, ontop of -J7, help?

	Ingo

--- drivers/block/ll_rw_blk.c.orig
+++ drivers/block/ll_rw_blk.c
@@ -2447,11 +2447,11 @@ end_io:
 			break;
 		}
 
-		if (unlikely(bio_sectors(bio) > q->max_sectors)) {
+		if (unlikely(bio_sectors(bio) > q->max_hw_sectors)) {
 			printk("bio too big device %s (%u > %u)\n", 
 				bdevname(bio->bi_bdev, b),
 				bio_sectors(bio),
-				q->max_sectors);
+				q->max_hw_sectors);
 			goto end_io;
 		}
 

Re: no luck with max_sectors_kb (Re: voluntary-preempt-2.6.8-rc2-J4)

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> does the patch below, ontop of -J7, help?

i've added this patch to -J7, so re-downloading the -J7 patch should
give you the patch too.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-J3

[Andrew Morton]

Ingo Molnar <mingo@elte.hu> wrote:
>
> ...
> guaranteeing latencies on SMP is hard, because the latency of a CPU
> might depend on the latency of a task on another CPU - which CPU isnt
> notified of the rescheduling request.
> 
> one alternative technique to yours would be to notify _all_ CPUs that a
> high-prio RT task is ready to run (via a broadcast need-resched). That
> way the UP latency-break techniques map to SMP in a 1:1 way.

I did exactly that in the 2.4 ll patch.  It works, but doesn't seem very nice.

> non-RT tasks dont get this benefit, which is a difference to the UP
> situation, but i dont think it would be appropriate to use the UP
> behavior, due to the overhead of broadcasting.
> 
> a combination of the two techniques could be used too: a global 'break
> locks from now on' flag which gets set if a (RT?) task wants to
> reschedule. Normally this flag would be zero and the cacheline would be
> clean and shared between all CPUs, causing no overhead. Once a task
> wants to reschedule it would increase by 1 until that task has been
> scheduled.

Something like that.  The obscure livelock opportunities worry me.

> This would remove the ugliness factor too, your sample code
> would look:
> 
> > +			if (need_lock_break()) {
> > +				spin_unlock(&mm->page_table_lock);
> > +				cpu_relax();
> > +				spin_lock(&mm->page_table_lock);
> > +			}
> > +
> 
> or a shortcut:
> 
> > +			cond_lock_break(&mm->page_table_lock);

The usual problem is, of course, long-running loops under locks.  These
loops usually have some sort of counter (or you add one).  The below patch
changes cond_resched_lock() so that it takes the counter, and the tuned
upper-loop-count limit.  It also does the right thing on UP.  It needs
enhancements for the situations where we already have a counter in the
loop.

I dunno if it's sufficient, but I do think that we need to settle on some
infrastructure of this sort before proceeding too far into this thing.

> there are two problems with the unconditional lock-break:
> 
>  - i'm not sure cpu_relax() guarantees that another CPU can grab this 
>    lock. It all depends on whether the lock cacheline can bounce over 
>    to the other CPU faster than cpu_relax() finishes and this CPU
>    re-acquires the lock.

precisely.


diff -puN include/linux/sched.h~cond_resched_lock-smp-fix include/linux/sched.h
--- 25/include/linux/sched.h~cond_resched_lock-smp-fix	2004-07-13 16:58:42.445155480 -0700
+++ 25-akpm/include/linux/sched.h	2004-07-13 17:02:56.194579704 -0700
@@ -1038,22 +1038,32 @@ static inline void cond_resched(void)
 }
 
 /*
- * cond_resched_lock() - if a reschedule is pending, drop the given lock,
- * call schedule, and on return reacquire the lock.
- *
  * This works OK both with and without CONFIG_PREEMPT.  We do strange low-level
  * operations here to prevent schedule() from being called twice (once via
  * spin_unlock(), once by hand).
  */
-static inline void cond_resched_lock(spinlock_t * lock)
-{
-	if (need_resched()) {
-		_raw_spin_unlock(lock);
-		preempt_enable_no_resched();
-		__cond_resched();
-		spin_lock(lock);
-	}
-}
+
+#ifdef CONFIG_SMP
+#define cond_resched_lock(lock, counter, limit)		\
+	do {						\
+		if (++(counter) >= limit) {		\
+			spin_unlock(lock);		\
+			cpu_relax();			\
+			spin_lock(lock);		\
+		}					\
+		(counter) = 0;				\
+	} while (0)
+#else
+#define cond_resched_lock(lock, counter, limit)		\
+	do {						\
+		if (need_resched()) {			\
+			_raw_spin_unlock(lock);		\
+			preempt_enable_no_resched();	\
+			__cond_resched();		\
+			spin_lock(lock);		\
+		}					\
+	} while (0)
+#endif
 
 /* Reevaluate whether the task has signals pending delivery.
    This is required every time the blocked sigset_t changes.
diff -puN fs/reiserfs/journal.c~cond_resched_lock-smp-fix fs/reiserfs/journal.c
--- 25/fs/reiserfs/journal.c~cond_resched_lock-smp-fix	2004-07-13 16:58:42.462152896 -0700
+++ 25-akpm/fs/reiserfs/journal.c	2004-07-13 17:03:43.131444216 -0700
@@ -796,6 +796,7 @@ static int write_ordered_buffers(spinloc
     struct buffer_chunk chunk;
     struct list_head tmp;
     INIT_LIST_HEAD(&tmp);
+    int resched_counter = 0;
 
     chunk.nr = 0;
     spin_lock(lock);
@@ -827,7 +828,7 @@ static int write_ordered_buffers(spinloc
 	}
 loop_next:
 	put_bh(bh);
-	cond_resched_lock(lock);
+	cond_resched_lock(lock, resched_counter, 64);
     }
     if (chunk.nr) {
 	spin_unlock(lock);
@@ -848,7 +849,7 @@ loop_next:
 	if (!buffer_uptodate(bh))
 	    ret = -EIO;
 	put_bh(bh);
-	cond_resched_lock(lock);
+	cond_resched_lock(lock, resched_counter, 64);
     }
     spin_unlock(lock);
     return ret;
diff -puN mm/shmem.c~cond_resched_lock-smp-fix mm/shmem.c
--- 25/mm/shmem.c~cond_resched_lock-smp-fix	2004-07-13 16:58:42.487149096 -0700
+++ 25-akpm/mm/shmem.c	2004-07-13 17:04:06.310920400 -0700
@@ -424,6 +424,7 @@ static void shmem_truncate(struct inode 
 	swp_entry_t *ptr;
 	int offset;
 	int freed;
+	int resched_counter = 0;
 
 	inode->i_ctime = inode->i_mtime = CURRENT_TIME;
 	idx = (inode->i_size + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
@@ -496,7 +497,7 @@ static void shmem_truncate(struct inode 
 				shmem_free_block(inode);
 			}
 			empty = subdir;
-			cond_resched_lock(&info->lock);
+			cond_resched_lock(&info->lock, resched_counter, 32);
 			dir = shmem_dir_map(subdir);
 		}
 		subdir = *dir;
diff -puN mm/memory.c~cond_resched_lock-smp-fix mm/memory.c
--- 25/mm/memory.c~cond_resched_lock-smp-fix	2004-07-13 16:58:42.504146512 -0700
+++ 25-akpm/mm/memory.c	2004-07-13 17:05:43.943078056 -0700
@@ -224,6 +224,7 @@ int copy_page_range(struct mm_struct *ds
 	unsigned long address = vma->vm_start;
 	unsigned long end = vma->vm_end;
 	unsigned long cow;
+	int resched_counter = 0;
 
 	if (is_vm_hugetlb_page(vma))
 		return copy_hugetlb_page_range(dst, src, vma);
@@ -341,7 +342,8 @@ cont_copy_pte_range_noset:
 			pte_unmap_nested(src_pte-1);
 			pte_unmap(dst_pte-1);
 			spin_unlock(&src->page_table_lock);
-			cond_resched_lock(&dst->page_table_lock);
+			cond_resched_lock(&dst->page_table_lock,
+						resched_counter, 256);
 cont_copy_pmd_range:
 			src_pmd++;
 			dst_pmd++;
@@ -530,6 +532,7 @@ int unmap_vmas(struct mmu_gather **tlbp,
 	int tlb_start_valid = 0;
 	int ret = 0;
 	int atomic = details && details->atomic;
+	int resched_counter = 0;
 
 	for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) {
 		unsigned long start;
@@ -567,10 +570,11 @@ int unmap_vmas(struct mmu_gather **tlbp,
 			zap_bytes -= block;
 			if ((long)zap_bytes > 0)
 				continue;
-			if (!atomic && need_resched()) {
+			if (!atomic) {
 				int fullmm = tlb_is_full_mm(*tlbp);
 				tlb_finish_mmu(*tlbp, tlb_start, start);
-				cond_resched_lock(&mm->page_table_lock);
+				cond_resched_lock(&mm->page_table_lock,
+						resched_counter, 1);
 				*tlbp = tlb_gather_mmu(mm, fullmm);
 				tlb_start_valid = 0;
 			}
diff -puN mm/rmap.c~cond_resched_lock-smp-fix mm/rmap.c
--- 25/mm/rmap.c~cond_resched_lock-smp-fix	2004-07-13 16:58:42.521143928 -0700
+++ 25-akpm/mm/rmap.c	2004-07-13 17:06:59.866535936 -0700
@@ -661,6 +661,7 @@ static inline int try_to_unmap_file(stru
 	unsigned long max_nl_cursor = 0;
 	unsigned long max_nl_size = 0;
 	unsigned int mapcount;
+	int resched_counter = 0;
 
 	if (!spin_trylock(&mapping->i_mmap_lock))
 		return ret;
@@ -699,7 +700,7 @@ static inline int try_to_unmap_file(stru
 	 */
 	mapcount = page->mapcount;
 	page_map_unlock(page);
-	cond_resched_lock(&mapping->i_mmap_lock);
+	cond_resched_lock(&mapping->i_mmap_lock, resched_counter, 1);
 
 	max_nl_size = (max_nl_size + CLUSTER_SIZE - 1) & CLUSTER_MASK;
 	if (max_nl_cursor == 0)
@@ -728,7 +729,7 @@ static inline int try_to_unmap_file(stru
 					(void *) max_nl_cursor;
 			ret = SWAP_AGAIN;
 		}
-		cond_resched_lock(&mapping->i_mmap_lock);
+		cond_resched_lock(&mapping->i_mmap_lock, resched_counter, 8);
 		max_nl_cursor += CLUSTER_SIZE;
 	} while (max_nl_cursor <= max_nl_size);
 
_

Re: [patch] voluntary-preempt-2.6.8-rc2-J3

[Lee Revell]

On Mon, 2004-07-26 at 16:36, Ingo Molnar wrote:
> * Andrew Morton <akpm@osdl.org> wrote:
> 
> > The bigger this thing gets, the more worried I get.  Sometime this is
> > going to need to be split up into individual fixes, and they need to
> > be based upon an overall approach which we haven't yet settled on.
> 
> i will do that splitup. Right now i'm simply mapping how widespread the
> problem is and what type of fixes we need. The situation isnt all that
> bad but we might need (an optional) mechanism to make softirqs
> synchronous. All of this stuff is nicely modular and i'll do a splitup
> post 2.6.8 (i dont think we want to disturb 2.6.8 with any of this).
> 

>From a user's perspective, and based on my own testing, I do not see the
patch having to get much bigger, the vast majority of the hot spots have
been fixed, and the patch remains quite comprehensible even to someone
with a sketchy knowledge of the kernel.  There is only one issue I can
think of that has not been addressed at all (the PS/2 Caps Lock issue). 
All that seems to remain are tweaks to individual fixes that are already
in the patch.

Lee

Re: no luck with max_sectors_kb (Re: voluntary-preempt-2.6.8-rc2-J4)

[Rudo Thomas]

> > After setting it to 32 (the hw max is 128), userland programs fail
> > with I/O errors. Setting it back to 128 gets it back to working, sort
> > of. The errors probably get bufferred somewhere.
> > 
> > During the "bad" setting (32), messages like this one show up in kernel log.
> > 
> > bio too big device hda3 (104 > 64)
> 
> does the patch below, ontop of -J7, help?

I tried it, but did not test tuning the value, as the patch has problems of its
own: with device-mapper.

bio too big device dm-0 (256 > 255)

Again, some files cannot be read on the device.

Bye for now.

Rudo.

Re: [patch] voluntary-preempt-2.6.8-rc2-J4

[Jens Axboe]

On Mon, Jul 26 2004, Ingo Molnar wrote:
> 
> i've uploaded the -J4 patch:
> 
>    http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-J4
> 
> Changes since -J3:
> 
>  - make block device max_sectors sysfs tunable. There's a new entry
>    /sys/block/*/queue/max_sectors_kb which stores the current max 
>    request size in KB. You can write it to change the size.
> 
>    Jens: i've attached a standalone patch against 2.6.8-rc2 below. 
>    Please apply if it looks good to you. (I've added extra locking to 
>    make sure max_sectors and readahead_pages gets updated at once, for 
>    the unlikely event of two CPUs updating max_sectors at once.) I've 
>    tested this on IDE using the UP kernel.
> 
>  - include the unlock-tty changes suggested by akpm and tested by Lee 
>    Revell. I've also unlocked the tty_read path - only the release/open 
>    paths remain under the big kernel lock. Please report any tty related 
>    regressions.
> 
> 	Ingo
> 
> --
> introduce max_sectors_kb tunable.
> 
> Signed-off-by: Ingo Molnar <mingo@elte.hu>
> 
> --- linux/drivers/block/ll_rw_blk.c.orig	
> +++ linux/drivers/block/ll_rw_blk.c	
> @@ -3062,13 +3063,50 @@ queue_ra_store(struct request_queue *q, 
>  	unsigned long ra_kb;
>  	ssize_t ret = queue_var_store(&ra_kb, page, count);
>  
> +	spin_lock_irq(q->queue_lock);
>  	if (ra_kb > (q->max_sectors >> 1))
>  		ra_kb = (q->max_sectors >> 1);
>  
>  	q->backing_dev_info.ra_pages = ra_kb >> (PAGE_CACHE_SHIFT - 10);
> +	spin_unlock_irq(q->queue_lock);
> +
> +	return ret;
> +}
> +
> +static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
> +{
> +	int max_sectors_kb = q->max_sectors >> 1;
> +
> +	return queue_var_show(max_sectors_kb, (page));
> +}
> +
> +static ssize_t
> +queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
> +{
> +	unsigned long max_sectors_kb;
> +	ssize_t ret = queue_var_store(&max_sectors_kb, page, count);
> +	int ra_kb;
> +
> +	/*
> +	 * Take the queue lock to update the readahead and max_sectors
> +	 * values synchronously:
> +	 */
> +	spin_lock_irq(q->queue_lock);
> +	/*
> +	 * Trim readahead window as well, if necessary:
> +	 */
> +	ra_kb = q->backing_dev_info.ra_pages << (PAGE_CACHE_SHIFT - 10);
> +	if (ra_kb > max_sectors_kb)
> +		q->backing_dev_info.ra_pages =
> +				max_sectors_kb >> (PAGE_CACHE_SHIFT - 10);
> +
> +	q->max_sectors = max_sectors_kb << 1;
> +	spin_unlock_irq(q->queue_lock);
> +
>  	return ret;
>  }
>  
> +
>  static struct queue_sysfs_entry queue_requests_entry = {
>  	.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
>  	.show = queue_requests_show,
> @@ -3081,9 +3119,16 @@ static struct queue_sysfs_entry queue_ra
>  	.store = queue_ra_store,
>  };
>  
> +static struct queue_sysfs_entry queue_max_sectors_entry = {
> +	.attr = {.name = "max_sectors_kb", .mode = S_IRUGO | S_IWUSR },
> +	.show = queue_max_sectors_show,
> +	.store = queue_max_sectors_store,
> +};
> +
>  static struct attribute *default_attrs[] = {
>  	&queue_requests_entry.attr,
>  	&queue_ra_entry.attr,
> +	&queue_max_sectors_entry.attr,
>  	NULL,
>  };

I don't like it. First of all, the implementation really should drain
the queue first, then set max value before allowing people to queue more
io. The queue lock doesn't help here, readers don't even attempt to
serialize access to max_sectors. 

Secondly, I don't like the concept of exposing this value. If you want
to do something like this, we must split the value into two like
proposed (and patched) some months ago into a hardware and user value.

I don't see why we can't just drop ata48 default value to 256kb instead.
There's very little command over head on ide, I bet the majority of the
change in performance when playing with 256kb vs 1024kb is not the
command overhead itself, rather things like read-ahead that could be
more intelligent.

-- 
Jens Axboe

Re: [patch] voluntary-preempt-2.6.8-rc2-J4

[Jens Axboe]

On Tue, Jul 27 2004, Ingo Molnar wrote:
> 
> [i've sent a second patch too since the first version.]
> 
> * Jens Axboe <axboe@suse.de> wrote:
> 
> > I don't like it. First of all, the implementation really should drain
> > the queue first, then set max value before allowing people to queue
> > more io. The queue lock doesn't help here, readers don't even attempt
> > to serialize access to max_sectors. 
> 
> why should the queue be drained? There might be a few leftover big
> requests, but these are not a problem.

The patch I commented on did not split sectors value into two, in that
case it's a big problem. When split and expose only the one, it's not an
issue.

> > Secondly, I don't like the concept of exposing this value. If you want
> > to do something like this, we must split the value into two like
> > proposed (and patched) some months ago into a hardware and user value.
> 
> yes, agreed - that's what the second patch does.

Great.

> > I don't see why we can't just drop ata48 default value to 256kb
> > instead. There's very little command over head on ide, I bet the
> > majority of the change in performance when playing with 256kb vs
> > 1024kb is not the command overhead itself, rather things like
> > read-ahead that could be more intelligent.
> 
> 256kb isnt enough from a latency POV either - and if a user wants some
> extreme setting like 16KB per request why not allow it? Especially since
> these tunables cause zero runtime overhead.

Note that I've been travelling and didn't really track the thread
closely before that, but it seems to me that most of the latency
incurred is an artifact of long code paths hanging off bi_end_io. Maybe
it would be better to try and fix some of that and get both good irq
completion latencies, while still maintaining decent drive throughput.

That said, this second version looks better from an inclusion point of
view. Better fix those manual q->max_sectors settings first.

-- 
Jens Axboe

Re: no luck with max_sectors_kb (Re: voluntary-preempt-2.6.8-rc2-J4)

[Ingo Molnar]

* Rudo Thomas <rudo@matfyz.cz> wrote:

> > does the patch below, ontop of -J7, help?
> 
> I tried it, but did not test tuning the value, as the patch has
> problems of its own: with device-mapper.
> 
> bio too big device dm-0 (256 > 255)
> 
> Again, some files cannot be read on the device.

ok, dm (and some other layered block drivers) set q->max_sectors
directly instead of using blk_queue_max_sectors().

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-J4

[Ingo Molnar]

[i've sent a second patch too since the first version.]

* Jens Axboe <axboe@suse.de> wrote:

> I don't like it. First of all, the implementation really should drain
> the queue first, then set max value before allowing people to queue
> more io. The queue lock doesn't help here, readers don't even attempt
> to serialize access to max_sectors. 

why should the queue be drained? There might be a few leftover big
requests, but these are not a problem.

> Secondly, I don't like the concept of exposing this value. If you want
> to do something like this, we must split the value into two like
> proposed (and patched) some months ago into a hardware and user value.

yes, agreed - that's what the second patch does.

> I don't see why we can't just drop ata48 default value to 256kb
> instead. There's very little command over head on ide, I bet the
> majority of the change in performance when playing with 256kb vs
> 1024kb is not the command overhead itself, rather things like
> read-ahead that could be more intelligent.

256kb isnt enough from a latency POV either - and if a user wants some
extreme setting like 16KB per request why not allow it? Especially since
these tunables cause zero runtime overhead.

	Ingo

Re: no luck with max_sectors_kb (Re: voluntary-preempt-2.6.8-rc2-J4)

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> ok, dm (and some other layered block drivers) set q->max_sectors
> directly instead of using blk_queue_max_sectors().

does the patch below fix your DM problems?

	Ingo

--- linux/drivers/md/dm-table.c.orig	
+++ linux/drivers/md/dm-table.c	
@@ -825,7 +825,7 @@ void dm_table_set_restrictions(struct dm
 	 * Make sure we obey the optimistic sub devices
 	 * restrictions.
 	 */
-	q->max_sectors = t->limits.max_sectors;
+	blk_queue_max_sectors(q, t->limits.max_sectors);
 	q->max_phys_segments = t->limits.max_phys_segments;
 	q->max_hw_segments = t->limits.max_hw_segments;
 	q->hardsect_size = t->limits.hardsect_size;

Re: [patch] voluntary-preempt-2.6.8-rc2-J4

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> [i've sent a second patch too since the first version.]

and here's a third one:

  http://redhat.com/~mingo/voluntary-preempt/tune-max-sectors-2.6.8-rc2-A0

this includes the dm-table.c fix and also fixes other stacked drivers 
and updates blk_queue_stack_limits() to change max_hw_sectors too.

this patch should be pretty complete i believe.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-J3

[Junio C Hamano]

>>>>> "AM" == Andrew Morton <akpm@osdl.org> writes:

AM> +#ifdef CONFIG_SMP
AM> +#define cond_resched_lock(lock, counter, limit)		\
AM> +	do {						\
AM> +		if (++(counter) >= limit) {		\
AM> +			spin_unlock(lock);		\
AM> +			cpu_relax();			\
AM> +			spin_lock(lock);		\
AM> +		}					\
AM> +		(counter) = 0;				\
AM> +	} while (0)
AM> +#else

I am wondering if you meant to reset the counter to zero inside
of the if(){}, probably after reaquiring the lock...

 

Re: [patch] voluntary-preempt-2.6.8-rc2-J4

[Ingo Molnar]

* Jens Axboe <axboe@suse.de> wrote:

> That said, this second version looks better from an inclusion point of
> view. Better fix those manual q->max_sectors settings first.

ok - the third patch i sent is supposed to deal with all those cases as
well.

	Ingo

Re: no luck with max_sectors_kb (Re: voluntary-preempt-2.6.8-rc2-J4)

[Rudo Thomas]

> > ok, dm (and some other layered block drivers) set q->max_sectors
> > directly instead of using blk_queue_max_sectors().
> 
> does the patch below fix your DM problems?

Indeed it does.

Rudo.

[patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Ingo Molnar]

i've uploaded -L2:
 
  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-L2

the big change in the -L2 release is a new latency-reduction feature:
the redirection of hardirqs to irqd. This enables lock-break methods
that make hardirqs preemptable.

I've done the lock-break of the IDE driver's completion path and i now
cannot trigger larger than 100 usecs hardirq latencies via the IDE
driver anymore, on a 2GHz x86 box - even if max_sectors is kept at the
default 1024K value. (!)

the hardirq-redirection feature is activated via voluntary-preemption=3
(default). All irqs except the timer irq are redirected. (the timer irq
needs to run from irq context - but it has constant latency and constant
frequency so it's not an issue. Soft timers are executed from the timer
softirq which is redirected and hence preemptable.)

this means that with this patch applied the 2.6 UP kernel is quite close
to being hard-RT capable (using controlled drivers and workloads). All
unbound-latency asynchronous workloads have been unloaded into
synchronous, schedulable process contexts - so nothing can delay a
high-prio RT task. (assuming we caught all the latencies. Any remaining
latency can be fixed with the existing methods.)

other changes in -L2:

i've done a softirq lock-break in the atkbd and ps2mouse drivers - this
should fix the big latencies triggered by NumLock/CapsLock, reported by
Lee Revell.

BIG WARNING: the hardirq-redirection feature is quite intrusive to
drivers so it's possible that some drivers will break under it.  The
changes done to the IDE driver might also endanger stability. Be careful
when applying this patch to production systems. If your system doesnt
boot then please try the voluntary-preempt=2 boot-option to turn off
hardirq redirection.

I've done some stresstesting on a desktop-style IDE-based system and
everything seems to work fine. YMMV.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Felipe Alfaro Solana]

[-- Attachment #1: Type: text/plain, Size: 637 bytes --]

On Tue, 2004-07-27 at 18:27 +0200, Ingo Molnar wrote:

> BIG WARNING: the hardirq-redirection feature is quite intrusive to
> drivers so it's possible that some drivers will break under it.  The
> changes done to the IDE driver might also endanger stability. Be careful
> when applying this patch to production systems. If your system doesnt
> boot then please try the voluntary-preempt=2 boot-option to turn off
> hardirq redirection.

I've seen an oops on my P4 machine when booting with voluntary-
preempt=3, but not with voluntary-preempt<3. I think it's related to the
serial controller (IRQ3 and IRQ4). Please, see attached dmesg.

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #2: dmesg --]
[-- Type: text/plain; name=dmesg; charset=utf-8, Size: 12417 bytes --]

Linux version 2.6.8-rc2-bk6-ck6 (root@glass.felipe-alfaro.com) (gcc version 3.4.1 20040714 (Red Hat 3.4.1-5)) #1 Wed Jul 28 00:00:29 CEST 2004
BIOS-provided physical RAM map:
 BIOS-e820: 0000000000000000 - 00000000000a0000 (usable)
 BIOS-e820: 00000000000f0000 - 0000000000100000 (reserved)
 BIOS-e820: 0000000000100000 - 000000001fff0000 (usable)
 BIOS-e820: 000000001fff0000 - 000000001fff3000 (ACPI NVS)
 BIOS-e820: 000000001fff3000 - 0000000020000000 (ACPI data)
 BIOS-e820: 00000000fec00000 - 0000000100000000 (reserved)
511MB LOWMEM available.
found SMP MP-table at 000f5610
On node 0 totalpages: 131056
  DMA zone: 4096 pages, LIFO batch:1
  Normal zone: 126960 pages, LIFO batch:16
  HighMem zone: 0 pages, LIFO batch:1
DMI 2.3 present.
ACPI: RSDP (v000 IntelR                                    ) @ 0x000f6fe0
ACPI: RSDT (v001 IntelR AWRDACPI 0x42302e31 AWRD 0x00000000) @ 0x1fff3000
ACPI: FADT (v001 IntelR AWRDACPI 0x42302e31 AWRD 0x00000000) @ 0x1fff3040
ACPI: MADT (v001 IntelR AWRDACPI 0x42302e31 AWRD 0x00000000) @ 0x1fff6dc0
ACPI: DSDT (v001 INTELR AWRDACPI 0x00001000 MSFT 0x0100000c) @ 0x00000000
ACPI: Local APIC address 0xfee00000
ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled)
Processor #0 15:2 APIC version 20
ACPI: LAPIC (acpi_id[0x01] lapic_id[0x01] disabled)
ACPI: IOAPIC (id[0x02] address[0xfec00000] gsi_base[0])
IOAPIC[0]: Assigned apic_id 2
IOAPIC[0]: apic_id 2, version 32, address 0xfec00000, GSI 0-23
ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
ACPI: IRQ0 used by override.
ACPI: IRQ2 used by override.
ACPI: IRQ9 used by override.
Enabling APIC mode:  Flat.  Using 1 I/O APICs
Using ACPI (MADT) for SMP configuration information
Built 1 zonelists
Kernel command line: ro root=/dev/hda2
Initializing CPU#0
CPU 0 irqstacks, hard=c039f000 soft=c039e000
PID hash table entries: 2048 (order 11: 16384 bytes)
Detected 2004.916 MHz processor.
Using tsc for high-res timesource
Console: colour VGA+ 80x25
Dentry cache hash table entries: 131072 (order: 7, 524288 bytes)
Inode-cache hash table entries: 65536 (order: 6, 262144 bytes)
Memory: 515648k/524224k available (1779k kernel code, 7816k reserved, 759k data, 128k init, 0k highmem)
Checking if this processor honours the WP bit even in supervisor mode... Ok.
Calibrating delay loop... 3956.73 BogoMIPS
Mount-cache hash table entries: 512 (order: 0, 4096 bytes)
CPU: After generic identify, caps: 3febfbff 00000000 00000000 00000000
CPU: After vendor identify, caps:  3febfbff 00000000 00000000 00000000
CPU: Trace cache: 12K uops, L1 D cache: 8K
CPU: L2 cache: 512K
CPU: After all inits, caps:        3febfbff 00000000 00000000 00000080
CPU: Intel(R) Pentium(R) 4 CPU 2.00GHz stepping 04
Enabling fast FPU save and restore... done.
Enabling unmasked SIMD FPU exception support... done.
Checking 'hlt' instruction... OK.
enabled ExtINT on CPU#0
ESR value before enabling vector: 00000000
ESR value after enabling vector: 00000000
ENABLING IO-APIC IRQs
init IO_APIC IRQs
 IO-APIC (apicid-pin) 2-0, 2-16, 2-17, 2-18, 2-19, 2-20, 2-21, 2-22, 2-23 not connected.
..TIMER: vector=0x31 pin1=2 pin2=-1
Using local APIC timer interrupts.
calibrating APIC timer ...
..... CPU clock speed is 2004.0312 MHz.
..... host bus clock speed is 100.0215 MHz.
NET: Registered protocol family 16
PCI: PCI BIOS revision 2.10 entry at 0xfb0a0, last bus=2
PCI: Using configuration type 1
mtrr: v2.0 (20020519)
ACPI: Subsystem revision 20040326
ACPI: Interpreter enabled
ACPI: Using IOAPIC for interrupt routing
ACPI: PCI Root Bridge [PCI0] (00:00)
PCI: Probing PCI hardware (bus 00)
PCI: Transparent bridge - 0000:00:1e.0
ACPI: PCI Interrupt Routing Table [\_SB_.PCI0._PRT]
ACPI: PCI Interrupt Routing Table [\_SB_.PCI0.HUB0._PRT]
ACPI: PCI Interrupt Link [LNKA] (IRQs 3 4 5 6 7 9 10 11 12 14 15) *0, disabled.
ACPI: PCI Interrupt Link [LNKB] (IRQs 3 4 *5 6 7 9 10 11 12 14 15)
ACPI: PCI Interrupt Link [LNKC] (IRQs 3 4 5 6 7 9 10 11 12 14 15) *0, disabled.
ACPI: PCI Interrupt Link [LNKD] (IRQs 3 4 5 6 7 9 10 *11 12 14 15)
ACPI: PCI Interrupt Link [LNKE] (IRQs 3 4 5 6 7 9 10 *11 12 14 15)
ACPI: PCI Interrupt Link [LNKF] (IRQs 3 4 5 6 7 9 10 11 *12 14 15)
ACPI: PCI Interrupt Link [LNK0] (IRQs 3 4 5 6 7 9 10 11 12 14 15) *0, disabled.
ACPI: PCI Interrupt Link [LNK1] (IRQs 3 4 5 6 7 *9 10 11 12 14 15)
PCI: Using ACPI for IRQ routing
ACPI: PCI interrupt 0000:00:1f.2[D] -> GSI 19 (level, low) -> IRQ 19
ACPI: PCI interrupt 0000:00:1f.4[C] -> GSI 23 (level, low) -> IRQ 23
ACPI: PCI interrupt 0000:00:1f.5[B] -> GSI 17 (level, low) -> IRQ 17
ACPI: PCI interrupt 0000:02:0c.0[A] -> GSI 20 (level, low) -> IRQ 20
ACPI: PCI interrupt 0000:02:0d.0[A] -> GSI 21 (level, low) -> IRQ 21
number of MP IRQ sources: 15.
number of IO-APIC #2 registers: 24.
testing the IO APIC.......................
IO APIC #2......
.... register #00: 02000000
.......    : physical APIC id: 02
.......    : Delivery Type: 0
.......    : LTS          : 0
.... register #01: 00178020
.......     : max redirection entries: 0017
.......     : PRQ implemented: 1
.......     : IO APIC version: 0020
.... register #02: 00000000
.......     : arbitration: 00
.... register #03: 00000001
.......     : Boot DT    : 1
.... IRQ redirection table:
 NR Log Phy Mask Trig IRR Pol Stat Dest Deli Vect:   
 00 000 00  1    0    0   0   0    0    0    00
 01 001 01  0    0    0   0   0    1    1    39
 02 001 01  0    0    0   0   0    1    1    31
 03 001 01  0    0    0   0   0    1    1    41
 04 001 01  0    0    0   0   0    1    1    49
 05 001 01  0    0    0   0   0    1    1    51
 06 001 01  0    0    0   0   0    1    1    59
 07 001 01  0    0    0   0   0    1    1    61
 08 001 01  0    0    0   0   0    1    1    69
 09 001 01  0    1    0   0   0    1    1    71
 0a 001 01  0    0    0   0   0    1    1    79
 0b 001 01  0    0    0   0   0    1    1    81
 0c 001 01  0    0    0   0   0    1    1    89
 0d 001 01  0    0    0   0   0    1    1    91
 0e 001 01  0    0    0   0   0    1    1    99
 0f 001 01  0    0    0   0   0    1    1    A1
 10 000 00  1    0    0   0   0    0    0    00
 11 001 01  1    1    0   1   0    1    1    B9
 12 000 00  1    0    0   0   0    0    0    00
 13 001 01  1    1    0   1   0    1    1    A9
 14 001 01  1    1    0   1   0    1    1    C1
 15 001 01  1    1    0   1   0    1    1    C9
 16 000 00  1    0    0   0   0    0    0    00
 17 001 01  1    1    0   1   0    1    1    B1
IRQ to pin mappings:
IRQ0 -> 0:2
IRQ1 -> 0:1
IRQ3 -> 0:3
IRQ4 -> 0:4
IRQ5 -> 0:5
IRQ6 -> 0:6
IRQ7 -> 0:7
IRQ8 -> 0:8
IRQ9 -> 0:9
IRQ10 -> 0:10
IRQ11 -> 0:11
IRQ12 -> 0:12
IRQ13 -> 0:13
IRQ14 -> 0:14
IRQ15 -> 0:15
IRQ17 -> 0:17
IRQ19 -> 0:19
IRQ20 -> 0:20
IRQ21 -> 0:21
IRQ23 -> 0:23
.................................... done.
TC classifier action (bugs to netdev@oss.sgi.com cc hadi@cyberus.ca)
audit: initializing netlink socket (disabled)
audit(1090974113.678:0): initialized
Total HugeTLB memory allocated, 0
Initializing Cryptographic API
ACPI: Power Button (FF) [PWRF]
ACPI: Sleep Button (CM) [SLPB]
ACPI: Fan [FAN] (on)
ACPI: Processor [CPU0] (supports C1)
ACPI: Thermal Zone [THRM] (55 C)
Real Time Clock Driver v1.12
Linux agpgart interface v0.100 (c) Dave Jones
agpgart: Detected an Intel i845 Chipset.
agpgart: Maximum main memory to use for agp memory: 439M
agpgart: AGP aperture is 64M @ 0xe8000000
[drm] Initialized radeon 1.11.0 20020828 on minor 0: ATI Technologies Inc Radeon RV100 QY [Radeon 7000/VE]
Serial: 8250/16550 driver $Revision: 1.90 $ 8 ports, IRQ sharing disabled
ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A
ttyS1 at I/O 0x2f8 (irq = 3) is a 16550A
Uniform Multi-Platform E-IDE driver Revision: 7.00alpha2
ide: Assuming 33MHz system bus speed for PIO modes; override with idebus=xx
ICH2: IDE controller at PCI slot 0000:00:1f.1
ICH2: chipset revision 5
ICH2: not 100% native mode: will probe irqs later
    ide0: BM-DMA at 0xf000-0xf007, BIOS settings: hda:DMA, hdb:DMA
    ide1: BM-DMA at 0xf008-0xf00f, BIOS settings: hdc:DMA, hdd:DMA
hda: ST3120022A, ATA DISK drive
hdb: ST380021A, ATA DISK drive
Using cfq io scheduler
ide0 at 0x1f0-0x1f7,0x3f6 on irq 14
hdc: HL-DT-STDVD-ROM GDR8161B, ATAPI CD/DVD-ROM drive
hdd: PLEXTOR CD-R PX-W4824A, ATAPI CD/DVD-ROM drive
ide1 at 0x170-0x177,0x376 on irq 15
hda: max request size: 1024KiB
hda: 234441648 sectors (120034 MB) w/2048KiB Cache, CHS=16383/255/63, UDMA(100)
 hda: hda1 hda2
hdb: max request size: 128KiB
hdb: 156301488 sectors (80026 MB) w/2048KiB Cache, CHS=65535/16/63, UDMA(100)
 hdb: hdb1
mice: PS/2 mouse device common for all mice
input: PC Speaker
serio: i8042 AUX port at 0x60,0x64 irq 12
serio: i8042 KBD port at 0x60,0x64 irq 1
input: AT Translated Set 2 keyboard on isa0060/serio0
NET: Registered protocol family 2
IP: routing cache hash table of 4096 buckets, 32Kbytes
TCP: Hash tables configured (established 32768 bind 65536)
Initializing IPsec netlink socket
NET: Registered protocol family 1
NET: Registered protocol family 10
IPv6 over IPv4 tunneling driver
NET: Registered protocol family 17
NET: Registered protocol family 15
p4-clockmod: P4/Xeon(TM) CPU On-Demand Clock Modulation available
ACPI: (supports S0 S1 S3 S4bios S5)
BIOS EDD facility v0.16 2004-Jun-25, 2 devices found
kjournald starting.  Commit interval 5 seconds
EXT3-fs: mounted filesystem with ordered data mode.
VFS: Mounted root (ext3 filesystem) readonly.
Freeing unused kernel memory: 128k freed
usbcore: registered new driver usbfs
usbcore: registered new driver hub
USB Universal Host Controller Interface driver v2.2
ACPI: PCI interrupt 0000:00:1f.2[D] -> GSI 19 (level, low) -> IRQ 19
uhci_hcd 0000:00:1f.2: Intel Corp. 82801BA/BAM USB (Hub #1)
PCI: Setting latency timer of device 0000:00:1f.2 to 64
uhci_hcd 0000:00:1f.2: irq 19, io base 0000b000
uhci_hcd 0000:00:1f.2: new USB bus registered, assigned bus number 1
hub 1-0:1.0: USB hub found
hub 1-0:1.0: 2 ports detected
ACPI: PCI interrupt 0000:00:1f.4[C] -> GSI 23 (level, low) -> IRQ 23
uhci_hcd 0000:00:1f.4: Intel Corp. 82801BA/BAM USB (Hub #2)
PCI: Setting latency timer of device 0000:00:1f.4 to 64
uhci_hcd 0000:00:1f.4: irq 23, io base 0000b800
uhci_hcd 0000:00:1f.4: new USB bus registered, assigned bus number 2
hub 2-0:1.0: USB hub found
hub 2-0:1.0: 2 ports detected
EXT3 FS on hda2, internal journal
usb 1-2: new low speed USB device using address 2
usbcore: registered new driver hiddev
input: USB HID v1.00 Mouse [Microsoft Microsoft IntelliMouse® Explorer] on usb-0000:00:1f.2-2
usbcore: registered new driver usbhid
drivers/usb/input/hid-core.c: v2.0:USB HID core driver
Adding 262136k swap on /swap.  Priority:-1 extents:67
kjournald starting.  Commit interval 5 seconds
EXT3 FS on hda1, internal journal
EXT3-fs: mounted filesystem with ordered data mode.
kjournald starting.  Commit interval 5 seconds
EXT3 FS on hdb1, internal journal
EXT3-fs: mounted filesystem with ordered data mode.
hdc: ATAPI 48X DVD-ROM drive, 256kB Cache, UDMA(33)
Uniform CD-ROM driver Revision: 3.20
hdd: ATAPI 40X CD-ROM CD-R/RW drive, 4096kB Cache, UDMA(33)
ip_tables: (C) 2000-2002 Netfilter core team
ip_tables: (C) 2000-2002 Netfilter core team
ACPI: PCI interrupt 0000:02:0c.0[A] -> GSI 20 (level, low) -> IRQ 20
3c59x: Donald Becker and others. www.scyld.com/network/vortex.html
0000:02:0c.0: 3Com PCI 3c905C Tornado at 0xa000. Vers LK1.1.19
ACPI: PCI interrupt 0000:02:0d.0[A] -> GSI 21 (level, low) -> IRQ 21
0000:02:0d.0: 3Com PCI 3c905C Tornado at 0xa400. Vers LK1.1.19
ip_tables: (C) 2000-2002 Netfilter core team
irq event 4: bogus return value ffffffff
 [<c0105b7e>] __report_bad_irq+0x24/0x7d
 [<c0105c3e>] note_interrupt+0x49/0x88
 [<c02bb3c2>] schedule+0x222/0x48d
 [<c0105f05>] do_hardirq+0x10f/0x191
 [<c0123cb5>] irqd+0x0/0xad
 [<c0123d55>] irqd+0xa0/0xad
 [<c0130010>] kthread+0x7c/0xa4
 [<c012ff94>] kthread+0x0/0xa4
 [<c0102249>] kernel_thread_helper+0x5/0xb
handlers:
irq event 3: bogus return value ffffffff
 [<c0105b7e>] __report_bad_irq+0x24/0x7d
 [<c0105c3e>] note_interrupt+0x49/0x88
 [<c02bb3c2>] schedule+0x222/0x48d
 [<c0105f05>] do_hardirq+0x10f/0x191
 [<c0123cb5>] irqd+0x0/0xad
 [<c0123d55>] irqd+0xa0/0xad
 [<c0130010>] kthread+0x7c/0xa4
 [<c012ff94>] kthread+0x0/0xa4
 [<c0102249>] kernel_thread_helper+0x5/0xb
handlers:
agpgart: Found an AGP 2.0 compliant device at 0000:00:00.0.
agpgart: Putting AGP V2 device at 0000:00:00.0 into 1x mode
agpgart: Putting AGP V2 device at 0000:01:01.0 into 1x mode

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Lee Revell]

On Tue, 2004-07-27 at 12:27, Ingo Molnar wrote:

> i've done a softirq lock-break in the atkbd and ps2mouse drivers - this
> should fix the big latencies triggered by NumLock/CapsLock, reported by
> Lee Revell.
> 

L2 does not fix this problem.  Previously, toggling CapsLock would
trigger a single large XRUN.  Now it seems to actually prevent the
soundcard interrupt handler from executing in time, as well as
triggering smaller XRUNs.  This is with preempt=1, voluntary-preempt=3.

ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:211: Unexpected hw_pointer value [1] (stream = 0, delta: -12, max jitter = 32): wrong interrupt acknowledge?
 [<c01066a7>] dump_stack+0x17/0x20
 [<de936497>] snd_pcm_period_elapsed+0x1c7/0x3e0 [snd_pcm]
 [<de956477>] snd_emu10k1_interrupt+0x337/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107c2f>] do_hardirq+0xaf/0x180
 [<c0113cbf>] cond_resched_softirq+0x4f/0x90
 [<c0211245>] atkbd_sendbyte+0x45/0x90
 [<c021144f>] atkbd_command+0x1bf/0x200
 [<c02115fb>] atkbd_event+0x16b/0x200
 [<c020e775>] input_event+0x115/0x3d0
 [<c01e7d9b>] kbd_bh+0xbb/0x160
 [<c011a7d4>] tasklet_action+0x44/0x70
 [<c011a593>] ___do_softirq+0x73/0x80
 [<c011a5e9>] _do_softirq+0x9/0x10
 [<c011a975>] irqd+0x75/0xc0
 [<c01284f4>] kthread+0xa4/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107c2f>] do_hardirq+0xaf/0x180
 [<c0113cbf>] cond_resched_softirq+0x4f/0x90
 [<c0211245>] atkbd_sendbyte+0x45/0x90
 [<c021144f>] atkbd_command+0x1bf/0x200
 [<c02115fb>] atkbd_event+0x16b/0x200
 [<c020e775>] input_event+0x115/0x3d0
 [<c01e7d9b>] kbd_bh+0xbb/0x160
 [<c011a7d4>] tasklet_action+0x44/0x70
 [<c011a593>] ___do_softirq+0x73/0x80
 [<c011a5e9>] _do_softirq+0x9/0x10
 [<c011a975>] irqd+0x75/0xc0
 [<c01284f4>] kthread+0xa4/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:211: Unexpected hw_pointer value [1] (stream = 0, delta: -7, max jitter = 32): wrong interrupt acknowledge?
 [<c01066a7>] dump_stack+0x17/0x20
 [<de936497>] snd_pcm_period_elapsed+0x1c7/0x3e0 [snd_pcm]
 [<de956477>] snd_emu10k1_interrupt+0x337/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107c2f>] do_hardirq+0xaf/0x180
 [<c011a99b>] irqd+0x9b/0xc0
 [<c01284f4>] kthread+0xa4/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Lee Revell]

On Tue, 2004-07-27 at 12:27, Ingo Molnar wrote:
> i've uploaded -L2:
>  
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-L2
> 
> the hardirq-redirection feature is activated via voluntary-preemption=3
> (default). All irqs except the timer irq are redirected. (the timer irq
> needs to run from irq context - but it has constant latency and constant
> frequency so it's not an issue. Soft timers are executed from the timer
> softirq which is redirected and hence preemptable.)
> 
> this means that with this patch applied the 2.6 UP kernel is quite close
> to being hard-RT capable (using controlled drivers and workloads). All
> unbound-latency asynchronous workloads have been unloaded into
> synchronous, schedulable process contexts - so nothing can delay a
> high-prio RT task. (assuming we caught all the latencies. Any remaining
> latency can be fixed with the existing methods.)
> 
> i've done a softirq lock-break in the atkbd and ps2mouse drivers - this
> should fix the big latencies triggered by NumLock/CapsLock, reported by
> Lee Revell.
> 

The Caps Lock problem goes away with voluntary-preempt=2.  It makes
sense that 3 would be problematic, as for audio work you don't want the
soundcard interrupt redirected.

The obvious next feature to add would be to make certain IRQs
non-schedulable, like you would for an RT system.  For an audio system
this would be just the soundcard interrupt (and timer as stated above). 
Then, while it still might not be hard-RT, it would blow away anything
achievable on the other OS'es people do audio work with.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Lee Revell]

On Tue, 2004-07-27 at 12:27, Ingo Molnar wrote:
> i've uploaded -L2:
>  
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-L2
> 

Much better.  With 2:1 Bonnie produced a single XRUN:

ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c01413ef>] filemap_sync_pte_range+0x9f/0xc0
 [<c01414a0>] filemap_sync+0x90/0x110
 [<c014156b>] msync_interval+0x4b/0xe0
 [<c0141724>] sys_msync+0x124/0x136
 [<c0106047>] syscall_call+0x7/0xb

And the mlockall/mmap2 problem is not completely solved.  Starting a
jackd while another is running still produces XRUNs:

ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D0p
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956477>] snd_emu10k1_interrupt+0x337/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c013d89e>] do_no_page+0x4e/0x310
 [<c013dd21>] handle_mm_fault+0xc1/0x170
 [<c013c712>] get_user_pages+0x102/0x370
 [<c013de68>] make_pages_present+0x68/0x90
 [<c013f5d6>] do_mmap_pgoff+0x3e6/0x620
 [<c010be66>] sys_mmap2+0x76/0xb0
 [<c0106047>] syscall_call+0x7/0xb
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D0p
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956477>] snd_emu10k1_interrupt+0x337/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c013de68>] make_pages_present+0x68/0x90
 [<c013e2bd>] mlock_fixup+0x8d/0xb0
 [<c013e5a0>] do_mlockall+0x70/0x90
 [<c013e659>] sys_mlockall+0x99/0xa0
 [<c0106047>] syscall_call+0x7/0xb

Works great otherwise.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Dmitry Torokhov]

On Tuesday 27 July 2004 11:27 am, Ingo Molnar wrote:
> other changes in -L2:
> 
> i've done a softirq lock-break in the atkbd and ps2mouse drivers - this
> should fix the big latencies triggered by NumLock/CapsLock, reported by
> Lee Revell.
> 

Actually the following seems to be working on my laptop and I was thinking
about pushing it to Vojtech. Any reason why this should not be done?

(The patch is on top of Vojtech's tee + my other changes so it won't apply
to anything.)

===== drivers/input/keyboard/atkbd.c 1.66 vs edited =====
--- 1.66/drivers/input/keyboard/atkbd.c	2004-07-26 01:35:39 -05:00
+++ edited/drivers/input/keyboard/atkbd.c	2004-07-26 22:54:41 -05:00
@@ -209,10 +209,25 @@
 	unsigned int last;
 	unsigned long time;
 
+	/* Ensures that only one command is executing at a time */
+	struct semaphore cmd_sem;
+
+	/* Used to signal completion from interrupt handler */
+	wait_queue_head_t wait;
+
 	/* Flags */
 	unsigned long flags;
 };
 
+/* Work structure to schedule execution of a command */
+struct atkbd_work {
+	struct work_struct work;
+	struct atkbd *atkbd;
+	int command;
+	unsigned char param[0];
+};
+
+
 static void atkbd_report_key(struct input_dev *dev, struct pt_regs *regs, int code, int value)
 {
 	input_regs(dev, regs);
@@ -262,10 +277,12 @@
 					set_bit(ATKBD_FLAG_CMD1, &atkbd->flags);
 				}
 				clear_bit(ATKBD_FLAG_ACK, &atkbd->flags);
+				wake_up_interruptible(&atkbd->wait);
 				break;
 			case ATKBD_RET_NAK:
 				atkbd->nak = 1;
 				clear_bit(ATKBD_FLAG_ACK, &atkbd->flags);
+				wake_up_interruptible(&atkbd->wait);
 				break;
 		}
 		goto out;
@@ -276,10 +293,13 @@
 		if (atkbd->cmdcnt)
 			atkbd->cmdbuf[--atkbd->cmdcnt] = code;
 
-		clear_bit(ATKBD_FLAG_CMD1, &atkbd->flags);
-		if (!atkbd->cmdcnt)
-			clear_bit(ATKBD_FLAG_CMD, &atkbd->flags);
+		if (test_and_clear_bit(ATKBD_FLAG_CMD1, &atkbd->flags) && atkbd->cmdcnt)
+			wake_up_interruptible(&atkbd->wait);
 
+		if (!atkbd->cmdcnt) {
+			clear_bit(ATKBD_FLAG_CMD, &atkbd->flags);
+			wake_up_interruptible(&atkbd->wait);
+		}
 		goto out;
 	}
 
@@ -412,12 +432,12 @@
  * acknowledge. It doesn't handle resends according to the keyboard
  * protocol specs, because if these are needed, the keyboard needs
  * replacement anyway, and they only make a mess in the protocol.
+ *
+ * atkbd_sendbyte() can only be called from a process context
  */
 
 static int atkbd_sendbyte(struct atkbd *atkbd, unsigned char byte)
 {
-	int timeout = 200000; /* 200 msec */
-
 #ifdef ATKBD_DEBUG
 	printk(KERN_DEBUG "atkbd.c: Sent: %02x\n", byte);
 #endif
@@ -425,8 +445,9 @@
 	set_bit(ATKBD_FLAG_ACK, &atkbd->flags);
 
 	if (serio_write(atkbd->serio, byte) == 0)
-		while (test_bit(ATKBD_FLAG_ACK, &atkbd->flags) && timeout--)
-			udelay(1);
+		wait_event_interruptible_timeout(atkbd->wait,
+				!test_bit(ATKBD_FLAG_ACK, &atkbd->flags),
+				msecs_to_jiffies(200));
 
 	clear_bit(ATKBD_FLAG_ACK, &atkbd->flags);
 	return -atkbd->nak;
@@ -435,19 +456,21 @@
 /*
  * atkbd_command() sends a command, and its parameters to the keyboard,
  * then waits for the response and puts it in the param array.
+ *
+ * atkbd_command() can only be called from a process context
  */
 
 static int atkbd_command(struct atkbd *atkbd, unsigned char *param, int command)
 {
-	int timeout = 500000; /* 500 msec */
+	int timeout;
 	int send = (command >> 12) & 0xf;
 	int receive = (command >> 8) & 0xf;
 	int rc = -1;
 	int i;
 
-	if (command == ATKBD_CMD_RESET_BAT)
-		timeout = 4000000; /* 4 sec */
+	timeout = msecs_to_jiffies(command == ATKBD_CMD_RESET_BAT ? 4000 : 500);
 
+	down(&atkbd->cmd_sem);
 	clear_bit(ATKBD_FLAG_CMD, &atkbd->flags);
 
 	if (receive && param)
@@ -464,27 +487,26 @@
 		if (atkbd_sendbyte(atkbd, param[i]))
 			goto out;
 
-	while (test_bit(ATKBD_FLAG_CMD, &atkbd->flags) && timeout--) {
-
-		if (!test_bit(ATKBD_FLAG_CMD1, &atkbd->flags)) {
-
-			if (command == ATKBD_CMD_RESET_BAT && timeout > 100000)
-				timeout = 100000;
+	timeout = wait_event_interruptible_timeout(atkbd->wait,
+				!test_bit(ATKBD_FLAG_CMD1, &atkbd->flags), timeout);
 
-			if (command == ATKBD_CMD_GETID &&
-			    atkbd->cmdbuf[receive - 1] != 0xab && atkbd->cmdbuf[receive - 1] != 0xac) {
-				/*
-				 * Device behind the port is not a keyboard
-				 * so we don't need to wait for the 2nd byte
-				 * of ID response.
-				 */
-				clear_bit(ATKBD_FLAG_CMD, &atkbd->flags);
-				atkbd->cmdcnt = 0;
-				break;
-			}
+	if (atkbd->cmdcnt && timeout > 0) {
+		if (command == ATKBD_CMD_RESET_BAT && jiffies_to_msecs(timeout) > 100)
+			timeout = msecs_to_jiffies(100);
+
+		if (command == ATKBD_CMD_GETID &&
+		    atkbd->cmdbuf[receive - 1] != 0xab && atkbd->cmdbuf[receive - 1] != 0xac) {
+			/*
+			 * Device behind the port is not a keyboard
+			 * so we don't need to wait for the 2nd byte
+			 * of ID response.
+			 */
+			clear_bit(ATKBD_FLAG_CMD, &atkbd->flags);
+			atkbd->cmdcnt = 0;
 		}
 
-		udelay(1);
+		wait_event_interruptible_timeout(atkbd->wait,
+				!test_bit(ATKBD_FLAG_CMD, &atkbd->flags), timeout);
 	}
 
 	if (param)
@@ -499,11 +521,59 @@
 out:
 	clear_bit(ATKBD_FLAG_CMD, &atkbd->flags);
 	clear_bit(ATKBD_FLAG_CMD1, &atkbd->flags);
+	up(&atkbd->cmd_sem);
 
 	return rc;
 }
 
 /*
+ * atkbd_execute_scheduled_command() sends a command, previously scheduled by
+ * atkbd_schedule_command(), to the keyboard.
+ */
+
+static void atkbd_execute_scheduled_command(void *data)
+{
+	struct atkbd_work *atkbd_work = data;
+
+	atkbd_command(atkbd_work->atkbd, atkbd_work->param, atkbd_work->command);
+
+	kfree(atkbd_work);
+}
+
+/*
+ * atkbd_schedule_command() allows to schedule delayed execution of a keyboard
+ * command and can be used to issue a command from an interrupt or softirq
+ * context.
+ */
+
+static int atkbd_schedule_command(struct atkbd *atkbd, unsigned char *param, int command)
+{
+	struct atkbd_work *atkbd_work;
+	int send = (command >> 12) & 0xf;
+	int receive = (command >> 8) & 0xf;
+
+	if (!test_bit(ATKBD_FLAG_ENABLED, &atkbd->flags))
+		return -1;
+
+	if (!(atkbd_work = kmalloc(sizeof(struct atkbd_work) + max(send, receive), GFP_ATOMIC)))
+		return -1;
+
+	memset(atkbd_work, 0, sizeof(struct atkbd_work));
+	atkbd_work->atkbd = atkbd;
+	atkbd_work->command = command;
+	memcpy(atkbd_work->param, param, send);
+	INIT_WORK(&atkbd_work->work, atkbd_execute_scheduled_command, atkbd_work);
+
+	if (!schedule_work(&atkbd_work->work)) {
+		kfree(atkbd_work);
+		return -1;
+	}
+
+	return 0;
+}
+
+
+/*
  * Event callback from the input module. Events that change the state of
  * the hardware are processed here.
  */
@@ -529,7 +599,7 @@
 			param[0] = (test_bit(LED_SCROLLL, dev->led) ? 1 : 0)
 			         | (test_bit(LED_NUML,    dev->led) ? 2 : 0)
 			         | (test_bit(LED_CAPSL,   dev->led) ? 4 : 0);
-		        atkbd_command(atkbd, param, ATKBD_CMD_SETLEDS);
+		        atkbd_schedule_command(atkbd, param, ATKBD_CMD_SETLEDS);
 
 			if (atkbd->extra) {
 				param[0] = 0;
@@ -538,7 +608,7 @@
 					 | (test_bit(LED_SUSPEND, dev->led) ? 0x04 : 0)
 				         | (test_bit(LED_MISC,    dev->led) ? 0x10 : 0)
 				         | (test_bit(LED_MUTE,    dev->led) ? 0x20 : 0);
-				atkbd_command(atkbd, param, ATKBD_CMD_EX_SETLEDS);
+				atkbd_schedule_command(atkbd, param, ATKBD_CMD_EX_SETLEDS);
 			}
 
 			return 0;
@@ -554,7 +624,7 @@
 			dev->rep[REP_PERIOD] = period[i];
 			dev->rep[REP_DELAY] = delay[j];
 			param[0] = i | (j << 5);
-			atkbd_command(atkbd, param, ATKBD_CMD_SETREP);
+			atkbd_schedule_command(atkbd, param, ATKBD_CMD_SETREP);
 
 			return 0;
 	}
@@ -726,7 +796,11 @@
 static void atkbd_disconnect(struct serio *serio)
 {
 	struct atkbd *atkbd = serio->private;
+
 	clear_bit(ATKBD_FLAG_ENABLED, &atkbd->flags);
+	synchronize_kernel();
+	flush_scheduled_work();
+
 	input_unregister_device(&atkbd->dev);
 	serio_close(serio);
 	kfree(atkbd);
@@ -747,6 +821,9 @@
 	if (!(atkbd = kmalloc(sizeof(struct atkbd), GFP_KERNEL)))
 		return;
 	memset(atkbd, 0, sizeof(struct atkbd));
+
+	init_MUTEX(&atkbd->cmd_sem);
+	init_waitqueue_head(&atkbd->wait);
 
 	switch (serio->type & SERIO_TYPE) {
 

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Lee Revell]

On Tue, 2004-07-27 at 12:27, Ingo Molnar wrote:
> i've uploaded -L2:
>  
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-L2
> 
Some more results:


I can no longer trigger XRUNs with bonnie, so I switched to iozone.  Here is 
what I got:

 ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a4ce>] balance_pgdat+0x1be/0x220
 [<c013a5d9>] kswapd+0xa9/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c015deff>] do_poll+0x5f/0xc0
 [<c015e091>] sys_poll+0x131/0x220
 [<c0106047>] syscall_call+0x7/0xb
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a197>] shrink_caches+0x57/0x60
 [<c013a238>] try_to_free_pages+0x98/0x170
 [<c0132707>] __alloc_pages+0x137/0x340
 [<c01302c5>] generic_file_aio_write_nolock+0x355/0xb80
 [<c0130bd2>] generic_file_aio_write+0x62/0x90
 [<c01a9458>] ext3_file_write+0x28/0xc0
 [<c014b01a>] do_sync_write+0x6a/0xa0
 [<c014b10c>] vfs_write+0xbc/0x110
 [<c014b1de>] sys_write+0x2e/0x50
 [<c0106047>] syscall_call+0x7/0xb
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a4ce>] balance_pgdat+0x1be/0x220
 [<c013a5d9>] kswapd+0xa9/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a4ce>] balance_pgdat+0x1be/0x220
 [<c013a5d9>] kswapd+0xa9/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a4ce>] balance_pgdat+0x1be/0x220
 [<c013a5d9>] kswapd+0xa9/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a197>] shrink_caches+0x57/0x60
 [<c013a238>] try_to_free_pages+0x98/0x170
 [<c0132707>] __alloc_pages+0x137/0x340
 [<c01302c5>] generic_file_aio_write_nolock+0x355/0xb80
 [<c0130bd2>] generic_file_aio_write+0x62/0x90
 [<c01a9458>] ext3_file_write+0x28/0xc0
 [<c014b01a>] do_sync_write+0x6a/0xa0
 [<c014b10c>] vfs_write+0xbc/0x110
 [<c014b1de>] sys_write+0x2e/0x50
 [<c0106047>] syscall_call+0x7/0xb
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a4ce>] balance_pgdat+0x1be/0x220
 [<c013a5d9>] kswapd+0xa9/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a197>] shrink_caches+0x57/0x60
 [<c013a238>] try_to_free_pages+0x98/0x170
 [<c0132707>] __alloc_pages+0x137/0x340
 [<c01302c5>] generic_file_aio_write_nolock+0x355/0xb80
 [<c0130bd2>] generic_file_aio_write+0x62/0x90
 [<c01a9458>] ext3_file_write+0x28/0xc0
 [<c014b01a>] do_sync_write+0x6a/0xa0
 [<c014b10c>] vfs_write+0xbc/0x110
 [<c014b1de>] sys_write+0x2e/0x50
 [<c0106047>] syscall_call+0x7/0xb
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a4ce>] balance_pgdat+0x1be/0x220
 [<c013a5d9>] kswapd+0xa9/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a4ce>] balance_pgdat+0x1be/0x220
 [<c013a5d9>] kswapd+0xa9/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93654b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de956211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078c8>] handle_IRQ_event+0x38/0x80
 [<c0107e5d>] do_IRQ+0xbd/0x1a0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146751>] add_to_swap+0x21/0xc0
 [<c0139426>] shrink_list+0x156/0x4b0
 [<c01398cd>] shrink_cache+0x14d/0x370
 [<c013a0f8>] shrink_zone+0xa8/0xf0
 [<c013a4ce>] balance_pgdat+0x1be/0x220
 [<c013a5d9>] kswapd+0xa9/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Ingo Molnar]

* Dmitry Torokhov <dtor_core@ameritech.net> wrote:

> On Tuesday 27 July 2004 11:27 am, Ingo Molnar wrote:
> > other changes in -L2:
> > 
> > i've done a softirq lock-break in the atkbd and ps2mouse drivers - this
> > should fix the big latencies triggered by NumLock/CapsLock, reported by
> > Lee Revell.
> > 
> 
> Actually the following seems to be working on my laptop and I was
> thinking about pushing it to Vojtech. Any reason why this should not
> be done?
> 
> (The patch is on top of Vojtech's tee + my other changes so it won't
> apply to anything.)

your solution of pushing completion work into a workqueue is of course
the right approach! My change was just papering over this artificial
softirq latency. Please push it to Vojtech ASAP.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Ingo Molnar]

* Felipe Alfaro Solana <felipe_alfaro@linuxmail.org> wrote:

> I've seen an oops on my P4 machine when booting with voluntary-
> preempt=3, but not with voluntary-preempt<3. I think it's related to
> the serial controller (IRQ3 and IRQ4). Please, see attached dmesg.

do you mean this one:

> irq event 4: bogus return value ffffffff
>  [<c0105b7e>] __report_bad_irq+0x24/0x7d
>  [<c0105c3e>] note_interrupt+0x49/0x88
>  [<c02bb3c2>] schedule+0x222/0x48d
>  [<c0105f05>] do_hardirq+0x10f/0x191
>  [<c0123cb5>] irqd+0x0/0xad
>  [<c0123d55>] irqd+0xa0/0xad
>  [<c0130010>] kthread+0x7c/0xa4
>  [<c012ff94>] kthread+0x0/0xa4
>  [<c0102249>] kernel_thread_helper+0x5/0xb
> handlers:

this isnt an oops, it is essentially just a warning. I am too getting
this a couple of times during bootup when using the serial console.

irqd changes timings and apparently in this case the serial IRQ line
gets deregistered for some amount of time while the hw still produces an
interrupt. (it might also be an irqd bug, but all seems to be functional
and i have no problems using the serial console.)

so unless you see some instability later on (or see a large flood of
such messages), you can disregard this warning.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> The obvious next feature to add would be to make certain IRQs
> non-schedulable, like you would for an RT system.  For an audio system
> this would be just the soundcard interrupt (and timer as stated
> above).  Then, while it still might not be hard-RT, it would blow away
> anything achievable on the other OS'es people do audio work with.

yes, this is the next step. Does v=3 work on your system? (even if the
delaying of the soundcard irq causes latencies.)

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > i've done a softirq lock-break in the atkbd and ps2mouse drivers - this
> > should fix the big latencies triggered by NumLock/CapsLock, reported by
> > Lee Revell.
> 
> L2 does not fix this problem.  Previously, toggling CapsLock would
> trigger a single large XRUN.  Now it seems to actually prevent the
> soundcard interrupt handler from executing in time, as well as
> triggering smaller XRUNs.  This is with preempt=1,
> voluntary-preempt=3.

yeah, this is because right now irqd lacks any configurability: it is
executing all hardirqs (and all softirqs) in one context without any
particular prioritization.

if your soundcard doesnt share the irq line with any other 'heavy' 
interrupt then you can make the irq 'direct' via a simple change to 
arch/i386/kernel/irq.c, change this line from:

 #define redirectable_irq(irq) ((irq) != 0)

to:

 #define redirectable_irq(irq) (((irq) != 0) && ((irq) != 10))

(if the soundcard is on IRQ 10).

does such a change combined with v=3 fix the latencies you are seeing?

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Lee Revell]

On Wed, 2004-07-28 at 00:59, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > The obvious next feature to add would be to make certain IRQs
> > non-schedulable, like you would for an RT system.  For an audio system
> > this would be just the soundcard interrupt (and timer as stated
> > above).  Then, while it still might not be hard-RT, it would blow away
> > anything achievable on the other OS'es people do audio work with.
> 
> yes, this is the next step. Does v=3 work on your system? (even if the
> delaying of the soundcard irq causes latencies.)
> 

Yes, it works well, except for this behavior which is as expected.  I
will test making this interrupt 'direct' tomorrow, it is not shared with
anything too heavy.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Thomas Charbonnel]

> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > i've done a softirq lock-break in the atkbd and ps2mouse drivers - this
> > > should fix the big latencies triggered by NumLock/CapsLock, reported by
> > > Lee Revell.
> > 
> > L2 does not fix this problem.  Previously, toggling CapsLock would
> > trigger a single large XRUN.  Now it seems to actually prevent the
> > soundcard interrupt handler from executing in time, as well as
> > triggering smaller XRUNs.  This is with preempt=1,
> > voluntary-preempt=3.
> 
> yeah, this is because right now irqd lacks any configurability: it is
> executing all hardirqs (and all softirqs) in one context without any
> particular prioritization.
> 
> if your soundcard doesnt share the irq line with any other 'heavy' 
> interrupt then you can make the irq 'direct' via a simple change to 
> arch/i386/kernel/irq.c, change this line from:
> 
>  #define redirectable_irq(irq) ((irq) != 0)
> 
> to:
> 
>  #define redirectable_irq(irq) (((irq) != 0) && ((irq) != 10))
> 
> (if the soundcard is on IRQ 10).
> 
> does such a change combined with v=3 fix the latencies you are seeing?
> 
> 	Ingo

Testing with -L2 and the redirectable_irq change, the xruns triggered by
the keyboard on 8.079 secs boundaries are still here (but the CAPS_LOCK
one is gone since the the redirectable change). I ran some latency tests
with vp:3 kp:0, progressively rising the rtc freqency (2048, 4096), and
I realized that those latency spikes every 8.079 seconds are here even
if I don't touch the keyboard. Here's the typical showtrace output for
those spikes :

T=1.14231 diff=0.978288
 rtc_interrupt (+bd)
 handle_IRQ_event (+50)
 do_hardirq (+bc)
 irqd (+ac)
 kthread (+aa)
 irqd (+0)
 kthread (+0)
 kernel_thread_helper (+5)

The pattern is very similar to the previous one I reported on those
regular xruns :

XRUN: pcmC2D0c
 [<c0105f6e>] dump_stack+0x1e/0x30
 [<c03673b1>] snd_pcm_period_elapsed+0x2e1/0x420
 [<c039d3d4>] snd_hdsp_interrupt+0x174/0x180
 [<c01073bb>] handle_IRQ_event+0x3b/0x70
 [<c0107746>] do_IRQ+0x96/0x150
 [<c0105b14>] common_interrupt+0x18/0x20
 [<c0107746>] do_IRQ+0x96/0x150
 [<c0105b14>] common_interrupt+0x18/0x20
 [<c01030f4>] cpu_idle+0x34/0x40
 [<c054880d>] start_kernel+0x16d/0x190
 [<c010019f>] 0xc010019f

The other source of xrun was seen during the disk write tests, once the
memory runs out (spikes up to 14 ms, but more generally 6 ms)

Thomas

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Ingo Molnar]

* Thomas Charbonnel <thomas@undata.org> wrote:

> The other source of xrun was seen during the disk write tests, once
> the memory runs out (spikes up to 14 ms, but more generally 6 ms)

which particular filesystem and disk driver are you using? (ext3/IDE?) 
The 'disk write tests' are those of latencytest-0.5.4?

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Thomas Charbonnel]

> * Thomas Charbonnel <thomas@undata.org> wrote:
> 
> > The other source of xrun was seen during the disk write tests, once
> > the memory runs out (spikes up to 14 ms, but more generally 6 ms)
> 
> which particular filesystem and disk driver are you using? (ext3/IDE?) 
> The 'disk write tests' are those of latencytest-0.5.4?
> 
> 	Ingo

It's an IDE reiserfs disk.
Those are indeed the disk write tests from latencytest-0.5.4
My mistake about the spikes, they are more 3 ms than 6.
You can see the results here :
http://www.undata.org/~thomas/latencytest-0.5.4/rtc_freq_2048/
http://www.undata.org/~thomas/latencytest-0.5.4/rtc_freq_4096/
http://www.undata.org/~thomas/latencytest-0.5.4/rtc_freq_8192/

The traces always look the same : 

T=29.6014 diff=14.4758
 rtc_interrupt (+bd)
 handle_IRQ_event (+50)
 do_hardirq (+bc)
 irqd (+ac)
 kthread (+aa)
 irqd (+0)
 kthread (+0)
 kernel_thread_helper (+5)


Thomas

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Lee Revell]

On Wed, 2004-07-28 at 01:05, Ingo Molnar wrote:
> if your soundcard doesnt share the irq line with any other 'heavy' 
> interrupt then you can make the irq 'direct' via a simple change to 
> arch/i386/kernel/irq.c, change this line from:
> 
>  #define redirectable_irq(irq) ((irq) != 0)
> 
> to:
> 
>  #define redirectable_irq(irq) (((irq) != 0) && ((irq) != 10))
> 
> (if the soundcard is on IRQ 10).
> 
> does such a change combined with v=3 fix the latencies you are seeing?

This fixes the PS/2 problem, but running iozone still produces these
(very small, ~0.1ms) XRUNs:

ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93d54b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de979211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078d7>] handle_IRQ_event+0x47/0x90
 [<c0107e93>] do_IRQ+0xe3/0x1b0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c020793c>] ide_dma_intr+0x7c/0xa0
 [<c02014a9>] ide_intr+0xf9/0x1a0
 [<c01078d7>] handle_IRQ_event+0x47/0x90
 [<c0107c3f>] do_hardirq+0xaf/0x180
 [<c011a9bb>] irqd+0x9b/0xc0
 [<c0128514>] kthread+0xa4/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10

This is with the default max_sectors_kb of 1024.  If I turn this down 
to 512, I do not get these, but I did get this one, only once:

ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93d54b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de979211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078d7>] handle_IRQ_event+0x47/0x90
 [<c0107e93>] do_IRQ+0xe3/0x1b0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0262957>] schedule+0x2c7/0x580
 [<c02630d7>] schedule_timeout+0x57/0xa0
 [<c015df61>] do_poll+0xa1/0xc0
 [<c015e0b1>] sys_poll+0x131/0x220
 [<c0106047>] syscall_call+0x7/0xb

If I turn max_sectors_kb down to 256, I do not get any XRUNs at all, 
and the maximum delay reported by running JACK for 5 minutes (several 
million soundcard IRQs) is only 42 usecs!  Previously, with 1:2, I could 
only get this down to ~120 usecs, and this required setting max_sectors_kb 
to 32-64.  Going any lower than this did not reduce latency any more, it 
seems like some other code path is responsible.

I am not seeing the latency spikes at ~8 second intervals that Thomas reported.
Could this be a bug in latencytest? 

Lee 

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Lee Revell]

On Wed, 2004-07-28 at 01:05, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> if your soundcard doesnt share the irq line with any other 'heavy' 
> interrupt then you can make the irq 'direct' via a simple change to 
> arch/i386/kernel/irq.c, change this line from:
> 
>  #define redirectable_irq(irq) ((irq) != 0)
> 
> to:
> 
>  #define redirectable_irq(irq) (((irq) != 0) && ((irq) != 10))
> 
> (if the soundcard is on IRQ 10).
> 
> does such a change combined with v=3 fix the latencies you are seeing?

With L2, 1:3, max_sectors_kb=256, and the above change, the performance
is truly amazing.  Over 20 million interrupts, on a 600Mhz machine, the
worst latency I was able to trigger was 46 usecs.  There does not seem
to be any adverse affect on any aspect of the system.

This looks like the perfect setup for a DAW.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Bill Huey]

On Wed, Jul 28, 2004 at 05:50:52PM -0400, Lee Revell wrote:
> With L2, 1:3, max_sectors_kb=256, and the above change, the performance
> is truly amazing.  Over 20 million interrupts, on a 600Mhz machine, the
> worst latency I was able to trigger was 46 usecs.  There does not seem
> to be any adverse affect on any aspect of the system.
> 
> This looks like the perfect setup for a DAW.

And as a musician into the notion of owning tons of pro-audio gear (I
don't at this time), I encourage you and the ALSA folks to go with it. :)

bill

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Lee Revell]

On Wed, 2004-07-28 at 17:50, Lee Revell wrote:
> On Wed, 2004-07-28 at 01:05, Ingo Molnar wrote:
> > * Lee Revell <rlrevell@joe-job.com> wrote:
> > if your soundcard doesnt share the irq line with any other 'heavy' 
> > interrupt then you can make the irq 'direct' via a simple change to 
> > arch/i386/kernel/irq.c, change this line from:
> > 
> >  #define redirectable_irq(irq) ((irq) != 0)
> > 
> > to:
> > 
> >  #define redirectable_irq(irq) (((irq) != 0) && ((irq) != 10))
> > 
> > (if the soundcard is on IRQ 10).
> > 
> > does such a change combined with v=3 fix the latencies you are seeing?
> 
> With L2, 1:3, max_sectors_kb=256, and the above change, the performance
> is truly amazing.  Over 20 million interrupts, on a 600Mhz machine, the
> worst latency I was able to trigger was 46 usecs.  There does not seem
> to be any adverse affect on any aspect of the system.
> 

Here are some more results.  I am up to 56 million interrupts and I have
yet to trigger a latency higher than 46 usecs.  It looks like this is a
hard upper limit.

I did get a couple XRUNs, some of which I have never seen before.  One
seemed to be caused by apt-get update/upgrade or exiting dselect.

The 'rtc: lost some interrupts' message is also new.  I believe mplayer
is using the RTC for timing.

Jul 28 20:23:17 mindpipe kernel: rtc: lost some interrupts at 1024Hz.
Jul 28 20:23:40 mindpipe kernel: rtc: lost some interrupts at 1024Hz.
Jul 28 20:23:54 mindpipe kernel: ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
Jul 28 20:23:54 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Jul 28 20:23:54 mindpipe kernel:  [__crc_totalram_pages+29841/2369476] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
Jul 28 20:23:54 mindpipe kernel:  [__crc_totalram_pages+274775/2369476] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
Jul 28 20:23:54 mindpipe kernel:  [handle_IRQ_event+71/144] handle_IRQ_event+0x47/0x90
Jul 28 20:23:54 mindpipe kernel:  [do_IRQ+227/432] do_IRQ+0xe3/0x1b0
Jul 28 20:23:54 mindpipe kernel:  [common_interrupt+24/32] common_interrupt+0x18/0x20
Jul 28 20:23:54 mindpipe kernel:  [set_page_dirty+70/96] set_page_dirty+0x46/0x60
Jul 28 20:23:54 mindpipe kernel:  [filemap_sync_pte+92/112] filemap_sync_pte+0x5c/0x70
Jul 28 20:23:54 mindpipe kernel:  [filemap_sync_pte_range+159/192] filemap_sync_pte_range+0x9f/0xc0
Jul 28 20:23:54 mindpipe kernel:  [filemap_sync+144/272] filemap_sync+0x90/0x110
Jul 28 20:23:54 mindpipe kernel:  [msync_interval+75/224] msync_interval+0x4b/0xe0
Jul 28 20:23:54 mindpipe kernel:  [sys_msync+292/310] sys_msync+0x124/0x136
Jul 28 20:23:54 mindpipe kernel:  [syscall_call+7/11] syscall_call+0x7/0xb
Jul 28 20:30:24 mindpipe kernel: ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
Jul 28 20:30:24 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Jul 28 20:30:24 mindpipe kernel:  [__crc_totalram_pages+29841/2369476] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
Jul 28 20:30:24 mindpipe kernel:  [__crc_totalram_pages+274775/2369476] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
Jul 28 20:30:24 mindpipe kernel:  [handle_IRQ_event+71/144] handle_IRQ_event+0x47/0x90
Jul 28 20:30:24 mindpipe kernel:  [do_IRQ+227/432] do_IRQ+0xe3/0x1b0
Jul 28 20:30:24 mindpipe kernel:  [common_interrupt+24/32] common_interrupt+0x18/0x20
Jul 28 20:30:26 mindpipe kernel: ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
Jul 28 20:30:26 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Jul 28 20:30:26 mindpipe kernel:  [__crc_totalram_pages+29841/2369476] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
Jul 28 20:30:26 mindpipe kernel:  [__crc_totalram_pages+274775/2369476] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
Jul 28 20:30:26 mindpipe kernel:  [handle_IRQ_event+71/144] handle_IRQ_event+0x47/0x90
Jul 28 20:30:26 mindpipe kernel:  [do_IRQ+227/432] do_IRQ+0xe3/0x1b0
Jul 28 20:30:26 mindpipe kernel:  [common_interrupt+24/32] common_interrupt+0x18/0x20
Jul 28 20:30:26 mindpipe kernel:  [do_anonymous_page+124/384] do_anonymous_page+0x7c/0x180
Jul 28 20:30:26 mindpipe kernel:  [do_no_page+78/784] do_no_page+0x4e/0x310
Jul 28 20:30:26 mindpipe kernel:  [handle_mm_fault+193/368] handle_mm_fault+0xc1/0x170
Jul 28 20:30:26 mindpipe kernel:  [get_user_pages+258/880] get_user_pages+0x102/0x370
Jul 28 20:30:26 mindpipe kernel:  [make_pages_present+104/144] make_pages_present+0x68/0x90
Jul 28 20:30:26 mindpipe kernel:  [do_mmap_pgoff+998/1568] do_mmap_pgoff+0x3e6/0x620
Jul 28 20:30:26 mindpipe kernel:  [sys_mmap2+118/176] sys_mmap2+0x76/0xb0
Jul 28 20:30:26 mindpipe kernel:  [syscall_call+7/11] syscall_call+0x7/0xb
Jul 28 20:32:18 mindpipe kernel: ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
Jul 28 20:32:18 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Jul 28 20:32:18 mindpipe kernel:  [__crc_totalram_pages+29841/2369476] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
Jul 28 20:32:18 mindpipe kernel:  [__crc_totalram_pages+274775/2369476] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
Jul 28 20:32:18 mindpipe kernel:  [handle_IRQ_event+71/144] handle_IRQ_event+0x47/0x90
Jul 28 20:32:18 mindpipe kernel:  [do_IRQ+227/432] do_IRQ+0xe3/0x1b0
Jul 28 20:32:18 mindpipe kernel:  [common_interrupt+24/32] common_interrupt+0x18/0x20
Jul 28 20:32:18 mindpipe kernel:  [set_page_dirty+70/96] set_page_dirty+0x46/0x60
Jul 28 20:32:18 mindpipe kernel:  [filemap_sync_pte+92/112] filemap_sync_pte+0x5c/0x70
Jul 28 20:32:18 mindpipe kernel:  [filemap_sync_pte_range+159/192] filemap_sync_pte_range+0x9f/0xc0
Jul 28 20:32:18 mindpipe kernel:  [filemap_sync+144/272] filemap_sync+0x90/0x110
Jul 28 20:32:18 mindpipe kernel:  [msync_interval+75/224] msync_interval+0x4b/0xe0
Jul 28 20:32:18 mindpipe kernel:  [sys_msync+292/310] sys_msync+0x124/0x136
Jul 28 20:32:18 mindpipe kernel:  [syscall_call+7/11] syscall_call+0x7/0xb

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Lee Revell]

On Wed, 2004-07-28 at 21:08, Lee Revell wrote:
> On Wed, 2004-07-28 at 17:50, Lee Revell wrote:
> > On Wed, 2004-07-28 at 01:05, Ingo Molnar wrote:
> > > * Lee Revell <rlrevell@joe-job.com> wrote:
> > > if your soundcard doesnt share the irq line with any other 'heavy' 
> > > interrupt then you can make the irq 'direct' via a simple change to 
> > > arch/i386/kernel/irq.c, change this line from:
> > > 
> > >  #define redirectable_irq(irq) ((irq) != 0)
> > > 
> > > to:
> > > 
> > >  #define redirectable_irq(irq) (((irq) != 0) && ((irq) != 10))
> > > 
> > > (if the soundcard is on IRQ 10).
> > > 
> > > does such a change combined with v=3 fix the latencies you are seeing?
> > 
> > With L2, 1:3, max_sectors_kb=256, and the above change, the performance
> > is truly amazing.  Over 20 million interrupts, on a 600Mhz machine, the
> > worst latency I was able to trigger was 46 usecs.  There does not seem
> > to be any adverse affect on any aspect of the system.
> > 
> 
> Here are some more results.  I am up to 56 million interrupts and I have
> yet to trigger a latency higher than 46 usecs.  It looks like this is a
> hard upper limit.

I have also found that if I stress the VM subsystem severely using
sysbench --threads=128 --test=memory, jackd will not start, eventually
its watchdog thread will kill it before it opens the audio ports.  It
seems likely that under pressure the mlockall() would never return.  I
can add some debugging code to jackd if you need to see which system
call is timing out.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Here are some more results.  I am up to 56 million interrupts and I
> have yet to trigger a latency higher than 46 usecs.  It looks like
> this is a hard upper limit.

nice - what is the average (and minimum?) latency reported by jackd? 
I'd say 46 usecs on a 600 MHz box is quite close to what it takes to
handle an interrupt and schedule to the cache-cold jackd task. It should
definitely be well below the latency required for jackd to do its job
reliably.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-L2, preemptable hardirqs

[Lee Revell]

On Thu, 2004-07-29 at 14:04, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Here are some more results.  I am up to 56 million interrupts and I
> > have yet to trigger a latency higher than 46 usecs.  It looks like
> > this is a hard upper limit.
> 
> nice - what is the average (and minimum?) latency reported by jackd? 
> I'd say 46 usecs on a 600 MHz box is quite close to what it takes to
> handle an interrupt and schedule to the cache-cold jackd task. It should
> definitely be well below the latency required for jackd to do its job
> reliably.

Yes, I am quite impressed with these results.  The jitter is about 5-10%
of the total available time at the very lowest latency settings.  This
is an order of magnitude better than 2.4 + ll.  It went as high as 50
usecs, but this took about 100 million interrupts.  I have not hacked
jackd yet to keep track of the average and minimum value, but eyeballing
it I would say 7-8 usecs is the minimum and 25-30 the average.  The
distribution seems very normal.

I am going to try making the RTC irq non-redirectable as well, as I got
a few 'rtc: lost some interrupts at 1024 Hz' messages, and the RTC is
important for MIDI timing.

Other than the few outliers I reported earlier, this is very close to
being ready for general pro audio use.  I did notice that the msync()
XRUN can be reliably reproduced by 'apt-get update && apt-get upgrade'.

I think I can speak on behalf of all Linux audio users when I say:
thanks a million, to all kernel developers.

Lee

[patch] voluntary-preempt-2.6.8-rc2-M5

[Ingo Molnar]

i've uploaded the latest version of the voluntary-preempt patch:
 
   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-M5

the biggest change is that i've integrated the irq threads code from
Scott Wood. I've done a number of usability enhancements:

added a new mechanism for finegrained configuration of threadedness /
nonthreadedness at the handler level: there are new:

   /proc/irq/<N>/<handler>/threaded

entries that control this behavior. Writing 0 to such an entry makes
that particular handler 'directly executed', writing 1 to it turns it
back to be handled by its own IRQ kernel thread.

E.g. the following command changes the serial line interrupt back to
non-threaded:

   echo 0 > /proc/irq/*/serial/threaded

the IRQ threads show up at low PID numbers (typically between 100 and
200) and their RT priority can be set via the 'chrt' utility (part of
schedutils). E.g. setting IRQ 10's irq thread priority back to the
non-RT SCHED_OTHER class can be done via:

  chrt --other --pid 0 `pidof 'IRQ 10'`

and to change IRQ 4's thread to SCHED_FIFO and the highest RT priority:

  chrt --fifo --pid 99 `pidof 'IRQ 4'`

to get good audio latencies i'd suggest to set the the audio driver's
and the RT-clock driver's IRQ handler to be non-threaded, and to set
jackd's RT priority to higher than 50 (which is the default of the IRQ
threads).

But it would also be interesting to see how the maximum latencies look
like if both the RT-clock and the audio handlers are threaded, and the
two affected IRQ threads are set to SCHED_FIFO 99 priority - they should
preempt everything. The latency increase compared to the 'direct' setup
should show us the real-life overhead of hardirq redirection.

the patch also changes the way the IDE latencies are avoided: based on
suggestions from Jens the latency-critical portion of the driver is now
done without holding ide_lock - and this makes the driver preemptable if
the handler is running in a thread.

if booting with voluntary-preempt=2/0/1 (the default is 3) then all
interrupts default to being non-threaded. Note that the /proc entries
can be used to turn threadedness back on even in this case - this can be
used to debug problematic drivers.

i made the irq-threads code work on SMP too: the IRQ threads now bind
themselves according to the value of /proc/irq/<N>/smp_affinity and
change the binding of that value is modified. IRQ threads only migrate
when it's safe - there will be no migration to another CPU while
executing a hardirq.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Thu, 2004-07-29 at 18:26, Ingo Molnar wrote:
> i've uploaded the latest version of the voluntary-preempt patch:
>  
>    http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-M5
> 

After running jackd with L2 all night, the only repeated XRUN was this one:

ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c01066a7>] dump_stack+0x17/0x20
 [<de93d54b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de979211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c01078d7>] handle_IRQ_event+0x47/0x90
 [<c0107e93>] do_IRQ+0xe3/0x1b0
 [<c0106268>] common_interrupt+0x18/0x20
 [<c0146771>] add_to_swap+0x21/0xc0
 [<c0139446>] shrink_list+0x156/0x4b0
 [<c01398ed>] shrink_cache+0x14d/0x370
 [<c013a118>] shrink_zone+0xa8/0xf0
 [<c013a4ee>] balance_pgdat+0x1be/0x220
 [<c013a5f9>] kswapd+0xa9/0xb0
 [<c0104395>] kernel_thread_helper+0x5/0x10

This produced a few ~2ms XRUNs.  The shrink_zone -> shrink_cache -> shrink_list 
is a recurring motif.

Is this addressed in M2?

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-J3

[Eric St-Laurent]

On Mon, 2004-07-26 at 16:36, Ingo Molnar wrote:
> one alternative technique to yours would be to notify _all_ CPUs that a
> high-prio RT task is ready to run (via a broadcast need-resched). That
> way the UP latency-break techniques map to SMP in a 1:1 way.
> 
> non-RT tasks dont get this benefit, which is a difference to the UP
> situation, but i dont think it would be appropriate to use the UP
> behavior, due to the overhead of broadcasting.
> 
> a combination of the two techniques could be used too: a global 'break
> locks from now on' flag which gets set if a (RT?) task wants to
> reschedule. Normally this flag would be zero and the cacheline would be
> clean and shared between all CPUs, causing no overhead. Once a task

It might be possible to eliminate the need_resched flag.  Here is an
article that talk about a event-driven preemption model.

Quoting :

Over the long term, MontaVista is investigating whether preemption locks
can be eliminated (or at least greatly reduced in number) by protecting
all the short-duration critical regions with spinlocks that also disable
interrupts on the local CPU, and the long-duration critical regions with
mutex locks. ("Long duration" means much greater than the time taken by
two context switches.) This will reduce the overhead of the preemptable
kernel, since there will no longer be any need to test for preemption
("polling for preemption") at the end of a preemption-locked region
(which could happen tens of thousands of times per second on an average
system). Instead, preemption would happen automatically as part of the
interrupt servicing that causes a higher-priority process to become
runnable ("event-driven preemption"). Typically, this only happens a few
times to a few tens of times per second with an average system workload,
making the "event-driven preemption" model much more efficient than the
"polling for preemption" model. This method also has an added efficiency
in that the system will take advantage of the cache disruption caused by
the interrupt (which is unavoidable) to continue with the preemption.

Reference:

http://www.linuxdevices.com/cgi-bin/printerfriendly.cgi?id=AT4185744181


Best regards,

Eric St-Laurent

Re: [patch] voluntary-preempt-2.6.8-rc2-J3

[Lee Revell]

On Thu, 2004-07-29 at 22:31, Eric St-Laurent wrote:
> It might be possible to eliminate the need_resched flag.  Here is an
> article that talk about a event-driven preemption model.
> 
> Quoting :
> 
> Over the long term, MontaVista is investigating whether preemption locks
> can be eliminated (or at least greatly reduced in number) by protecting
> all the short-duration critical regions with spinlocks that also disable
> interrupts on the local CPU, and the long-duration critical regions with
> mutex locks.

This is a pretty old article, from 2000, describing the preemption model
they implemented on 2.4.  I believe the above paragraph describes the
advantages of their model vs. the approach taken by the old 2.4 low
latency patches.

My understanding is that the preemptible 2.6 kernel already works this
way.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> After running jackd with L2 all night, the only repeated XRUN was this
> one:
> 
> ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
>  [<c01066a7>] dump_stack+0x17/0x20
>  [<de93d54b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
>  [<de979211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
>  [<c01078d7>] handle_IRQ_event+0x47/0x90
>  [<c0107e93>] do_IRQ+0xe3/0x1b0
>  [<c0106268>] common_interrupt+0x18/0x20
>  [<c0146771>] add_to_swap+0x21/0xc0
>  [<c0139446>] shrink_list+0x156/0x4b0
>  [<c01398ed>] shrink_cache+0x14d/0x370
>  [<c013a118>] shrink_zone+0xa8/0xf0
>  [<c013a4ee>] balance_pgdat+0x1be/0x220
>  [<c013a5f9>] kswapd+0xa9/0xb0
>  [<c0104395>] kernel_thread_helper+0x5/0x10
> 
> This produced a few ~2ms XRUNs.  The shrink_zone -> shrink_cache ->
> shrink_list is a recurring motif.
> 
> Is this addressed in M2?

not yet. I havent seen this latency yet, nor are there any immediately
clear clues in the xrun logs you sent. (it would still be nice to check
out -M2, to see whether with all those configurability changes it
matches the latencies of L2+your-irq.c-hack.)

shrink_list() itself is preemptable once every iteration so that
function alone shouldnt be able to generate a 2msec latency.

a strong suspect is add_to_swap(), it could be looping. Could you do a
'echo m > /proc/sysrq-trigger' and send me the results? In particular
are there any significant 'race' counts in the 'Swap cache:' stats?

the other possible suspect is get_swap_page() - it's called from
add_to_swap() and has the potential to introduce latencies (it does
bitmap scans, etc.) - but it never shows up in your xrun logs, which is
a bit weird.

(could you perhaps also try wli's latency printing patch? That prints
out latencies closer to where they really happen.)

plus it seems the latency is related to certain VM activities - you
still cannot name any particular reproducer workload - it just happens
occasionally, right?

	Ingo

[patch] voluntary-preempt-2.6.8-rc2-mm1-M5

[Ingo Molnar]

Upon popular request i've merged the latest voluntary-preempt patch to
the latest -mm kernel:

  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-mm1-M5

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Fri, 2004-07-30 at 02:44, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:

> shrink_list() itself is preemptable once every iteration so that
> function alone shouldnt be able to generate a 2msec latency.
> 
> a strong suspect is add_to_swap(), it could be looping. Could you do a
> 'echo m > /proc/sysrq-trigger' and send me the results? In particular
> are there any significant 'race' counts in the 'Swap cache:' stats?
> 
> the other possible suspect is get_swap_page() - it's called from
> add_to_swap() and has the potential to introduce latencies (it does
> bitmap scans, etc.) - but it never shows up in your xrun logs, which is
> a bit weird.
> 
> (could you perhaps also try wli's latency printing patch? That prints
> out latencies closer to where they really happen.)
> 
> plus it seems the latency is related to certain VM activities - you
> still cannot name any particular reproducer workload - it just happens
> occasionally, right?
> 

I think iozone -a or one of the sysbench tests triggers this one.  But
it also happened a few times overnight.  I will try to reproduce it
today.  You are referring to wli's 'preempt timing' patch, correct?

Also, I sent another report of XRUNS related to msync().  I have
narrowed this down a bit.  This one happens whenever 'apt-get update'
replaces its package lists.  Presumably it mmaps the files in
/var/lib/apt/lists and then calls msync at the end to write them out,
which triggers the XRUN.

Here are the results of 'echo m > /proc/sysrq-trigger':

SysRq : Show Memory
Mem-info:
DMA per-cpu:
cpu 0 hot: low 2, high 6, batch 1
cpu 0 cold: low 0, high 2, batch 1
Normal per-cpu:
cpu 0 hot: low 32, high 96, batch 16
cpu 0 cold: low 0, high 32, batch 16
HighMem per-cpu: empty

Free pages:       12064kB (0kB HighMem)
Active:70285 inactive:41081 dirty:24 writeback:0 unstable:0 free:3016 slab:5405 mapped:34481 pagetables:233
DMA free:3352kB min:20kB low:40kB high:60kB active:5288kB inactive:2128kB present:16384kB
protections[]: 10 348 348
Normal free:8712kB min:676kB low:1352kB high:2028kB active:275852kB inactive:162196kB present:475072kB
protections[]: 0 338 338
HighMem free:0kB min:128kB low:256kB high:384kB active:0kB inactive:0kB present:0kB
protections[]: 0 0 0
DMA: 382*4kB 52*8kB 2*16kB 1*32kB 1*64kB 0*128kB 1*256kB 0*512kB 1*1024kB 0*2048kB 0*4096kB = 3352kB
Normal: 188*4kB 159*8kB 82*16kB 100*32kB 8*64kB 3*128kB 1*256kB 0*512kB 1*1024kB 0*2048kB 0*4096kB = 8712kB
HighMem: empty
Swap cache: add 3, delete 1, find 0/0, race 0+0
Free swap:       499916kB
122864 pages of RAM
0 pages of HIGHMEM
2054 reserved pages
96028 pages shared
2 pages swap cached

Looks like no significant races.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-mm1-M5

[Muli Ben-Yehuda]

On Fri, Jul 30, 2004 at 10:13:26AM +0200, Ingo Molnar wrote:
> 
> Upon popular request i've merged the latest voluntary-preempt patch to
> the latest -mm kernel:
> 
>
http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-mm1-M5

Works fairly nicely for me (nice job!), except the mouse does not work
in both console (gpm) and X. This is on a thinkpad T21, 2.6.8-rc2-mm2
with the M5 patch. Doing 'echo 0 > /proc/irq/{1|12}/i8042/threaded'
does not work. This trivial workaround fixes it for me:

--- 2.6.8-rc2-mm1/drivers/input/serio/i8042.c	2004-07-30 14:49:14.018816320 +0000
+++ 2.6.8-rc2-mm1-mx/drivers/input/serio/i8042.c	2004-07-30 14:48:35.274706320 +0000
@@ -302,8 +302,9 @@
 		if (i8042_mux_open++)
 			return 0;
 
+	printk(KERN_INFO "i8042 requesting IRQ %d\n", values->irq);
 	if (request_irq(values->irq, i8042_interrupt,
-			SA_SHIRQ, "i8042", i8042_request_irq_cookie)) {
+			SA_SHIRQ | SA_NOTHREAD, "i8042", i8042_request_irq_cookie)) {
 		printk(KERN_ERR "i8042.c: Can't get irq %d for %s, unregistering the port.\n", values->irq, values->name);
 		goto irq_fail;
 	}

Other things that might be relevant

- both with and without the workaround, I get this:
"Jul 30 08:42:40 hydra kernel: atkbd.c: Spurious ACK on
isa0060/serio0. Some program, like XFree86, might be trying access
hardware directly."

- both with and without the workaround, I get this:
Jul 30 08:42:27 hydra kernel: Badness in free_irq at arch/i386/kernel/irq.c:721
Jul 30 08:42:27 hydra kernel:  [<c010895d>] free_irq+0x190/0x1aa
Jul 30 08:42:27 hydra kernel:  [<c02632d6>] floppy_release_irq_and_dma+0x259/0x272
Jul 30 08:42:27 hydra kernel:  [<c025cb48>] set_dor+0xe0/0x16f
Jul 30 08:42:27 hydra kernel:  [<c025ceac>] motor_off_callback+0x0/0x36
Jul 30 08:42:27 hydra kernel:  [<c025cede>] motor_off_callback+0x32/0x36
Jul 30 08:42:27 hydra kernel:  [<c0128a42>] run_timer_softirq+0xee/0x1e4
Jul 30 08:42:27 hydra kernel:  [<c0124884>] ksoftirqd+0x0/0xd7
Jul 30 08:42:27 hydra kernel:  [<c0124481>] ___do_softirq+0x83/0x8a
Jul 30 08:42:27 hydra kernel:  [<c01244b9>] _do_softirq+0x6/0x8
Jul 30 08:42:27 hydra kernel:  [<c012490c>] ksoftirqd+0x88/0xd7
Jul 30 08:42:27 hydra kernel:  [<c01341cd>] kthread+0xb7/0xbd
Jul 30 08:42:27 hydra kernel:  [<c0134116>] kthread+0x0/0xbd
Jul 30 08:42:27 hydra kernel:  [<c0103e41>] kernel_thread_helper+0x5/0xb

- with the workaround, I see this
"Jul 30 08:42:33 hydra kernel: psmouse.c: Mouse at
isa0060/serio1/input0 lost synchronization, throwing 1 bytes away."

Cheers, 
Muli
-- 
Muli Ben-Yehuda
http://www.mulix.org | http://mulix.livejournal.com/

Re: [patch] voluntary-preempt-2.6.8-rc2-mm1-M5

[Alan Cox]

On Gwe, 2004-07-30 at 23:00, Muli Ben-Yehuda wrote:
> - both with and without the workaround, I get this:
> "Jul 30 08:42:40 hydra kernel: atkbd.c: Spurious ACK on
> isa0060/serio0. Some program, like XFree86, might be trying access
> hardware directly."

If you have an old Xorg or run XFree86 this may well be a real report.
Old X tries to bang the controller directly for some things. It also
turns up in some normal situations unfortunately and is very confusing
for end users. Really this printk should die ASAP.

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Fri, 2004-07-30 at 02:44, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > After running jackd with L2 all night, the only repeated XRUN was this
> > one:
> > 
> > ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
> >  [<c01066a7>] dump_stack+0x17/0x20
> >  [<de93d54b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
> >  [<de979211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
> >  [<c01078d7>] handle_IRQ_event+0x47/0x90
> >  [<c0107e93>] do_IRQ+0xe3/0x1b0
> >  [<c0106268>] common_interrupt+0x18/0x20
> >  [<c0146771>] add_to_swap+0x21/0xc0
> >  [<c0139446>] shrink_list+0x156/0x4b0
> >  [<c01398ed>] shrink_cache+0x14d/0x370
> >  [<c013a118>] shrink_zone+0xa8/0xf0
> >  [<c013a4ee>] balance_pgdat+0x1be/0x220
> >  [<c013a5f9>] kswapd+0xa9/0xb0
> >  [<c0104395>] kernel_thread_helper+0x5/0x10
> > 
> > This produced a few ~2ms XRUNs.  The shrink_zone -> shrink_cache ->
> > shrink_list is a recurring motif.
> > 
> > Is this addressed in M2?
> 
> not yet. I havent seen this latency yet, nor are there any immediately
> clear clues in the xrun logs you sent. (it would still be nice to check
> out -M2, to see whether with all those configurability changes it
> matches the latencies of L2+your-irq.c-hack.)

I discovered that a few of the XRUN traces were spurious - jackd
apparently does something while stopping and starting that produces an
XRUN trace but that jackd does not consider an error.  I will fix this
in jackd.  The msync() related XRUN triggered by apt-get is definitely
real.

Here are my results with 2.6.8-rc2-mm1-M5.

							max usecs
All IRQs threaded					349
Soundcard IRQ not threaded				227
Soundcard IRQ not threaded + max_sectors_kb -> 64	119

I will test next with 2.6.8-rc2-M5.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Fri, 2004-07-30 at 18:54, Lee Revell wrote:

> Here are my results with 2.6.8-rc2-mm1-M5.
> 
> 							max usecs
> All IRQs threaded					349
> Soundcard IRQ not threaded				227
> Soundcard IRQ not threaded + max_sectors_kb -> 64	119
> 
> I will test next with 2.6.8-rc2-M5.
> 

Results with 2.6.8-rc2-M5:

Configuration						max usecs
-----------------------------------------------------------------
All IRQs threaded					370 
Soundcard IRQ not threaded				335
Soundcard IRQ not threaded + max_sectors_kb -> 64	161

So, it looks like the added configurability does add some overhead - 161
usecs vs. 50.  mm1 is significantly better than the stock kernel.  The
above results imply that 2.6.8-rc2-mm1 with the irq.c hack would be
frighteningly good.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Ingo Molnar]

On Fri, 30 Jul 2004, Lee Revell wrote:

> Results with 2.6.8-rc2-M5:
> 
> Configuration						max usecs
> -----------------------------------------------------------------
> All IRQs threaded					370 
> Soundcard IRQ not threaded				335
> Soundcard IRQ not threaded + max_sectors_kb -> 64	161
> 
> So, it looks like the added configurability does add some overhead - 161
> usecs vs. 50. [...]

+110 usecs is too much to be explained by redirection and configurability
overhead. The configurability overhead is near zero.

could you try to repeat the '50 usecs' test with -L2 [that was the one you
used?] to make sure it's repeatable? The latencies of -L2 and -M5 should
be near identical. The configurability should at most cause a 1-2 usecs
overhead - definitely not two orders of magnitude higher. So if there's a
difference then i must have degraded one of the latency reduction changes
between L2 and M5.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Thomas Charbonnel]

Ingo Molnar wrote :
> i've uploaded the latest version of the voluntary-preempt patch:
>  
>    http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-M5
> 

This patch doesn't solve the latency spikes I see every ~8 seconds on my
system. They still appear in latencytest when the rtc interrupt is the
only non threaded interrupt on the system, and they still produce xruns
in jack in conjunction with the keyboard even if the the sound card's
interrupt is the only non threaded irq and jack's SCHED_FIFO priority is
higher than the irq threads. I suspected a clock issue and switched from
tsc to pmtmr without success. I now suspect a hardware issue (the
machine is a laptop with a PIII M 1GHz cpu) or a scheduler corner case.
Running jack at 96000Hz with 2 periods of 64 samples, I can see regular
bursts of cpu usage every 2 or 3 seconds. Oprofile reveals that schedule
is then the second most CPU intensive function on the system.

Thomas

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Sun, 2004-08-01 at 07:28, Ingo Molnar wrote:
> On Fri, 30 Jul 2004, Lee Revell wrote:
> 
> > Results with 2.6.8-rc2-M5:
> > 
> > Configuration						max usecs
> > -----------------------------------------------------------------
> > All IRQs threaded					370 
> > Soundcard IRQ not threaded				335
> > Soundcard IRQ not threaded + max_sectors_kb -> 64	161
> > 
> > So, it looks like the added configurability does add some overhead - 161
> > usecs vs. 50. [...]
> 
> +110 usecs is too much to be explained by redirection and configurability
> overhead. The configurability overhead is near zero.
> 
> could you try to repeat the '50 usecs' test with -L2 [that was the one you
> used?] to make sure it's repeatable? The latencies of -L2 and -M5 should
> be near identical. The configurability should at most cause a 1-2 usecs
> overhead - definitely not two orders of magnitude higher. So if there's a
> difference then i must have degraded one of the latency reduction changes
> between L2 and M5.

The above numbers really do not tell you much, I have some better ones. 
Here is the histogram for a 5,000,000 sample run with M5:

 Delay   Count
-----   -----
6       98189
7       2013275
8       51025
9       120465
10      93339
11      72087
12      116165
13      106433
14      85930
15      57178
16      59835
17      51096
18      71671
19      106738
20      109491
21      84906
22      2215
23      946
24      1755
25      1977
26      1589
27      1117
28      1889
29      1616
30      1858
31      2926
32      3111
33      2558
34      400
35      1366
36      2172
37      2840
38      1088
39      1042
40      172
41      26
42      32
43      43
44      41
45      10
46      8
47      36
48      109
49      125
50      83
51      45
52      36
53      7
55      1
56      1
58      1
61      1
62      1
68      1
75      1
77      1
79      1
89      1
97      1

So if you discard the highest ~10 of 5,000,000, it's the same as L2.  Maybe a 
corner case, or a new race condition?  I will run the above test with L2.

I wonder if the multiple spikes represent different code paths, or the same 
but with a hot/warm/cold cache?

Lee

[patch] voluntary-preempt-2.6.8-rc2-O2

[Ingo Molnar]

here's the latest version of the voluntary-preempt patch:
  
  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-O2

this patch is mainly a stabilization effort. I dropped the irq-threads
code added in -M5 and rewrote it from scratch based on -L2 - it is
simpler and should be more robust.

The same /proc/irq/* configuration switches are still present, but i
added the following additional rule: if _any_ handler of a given IRQ is
marked as non-threaded then all handlers will be executed non-threaded
as well.

E.g. if you have the following handlers on IRQ 10:

 10:      11584   IO-APIC-level  eth0, eth1, eth2

and you change /proc/irq/16/eth1/threaded from 1 to 0 then the eth0 and
eth2 handlers will be executed non-threaded as well. (This rule only
enforces what the hardware enforces anyway, none of the previous patches
allowed true separation of these handlers.)

i also changed the IO-APIC level-triggered code to be robust when
redirection is done. The noapic workaround should not be necessary
anymore.

the keyboard lockups are now hopefully all gone too - i've tested
IO-APIC and non-IO-APIC setups as well and NumLock/ScrollLock works fine
in all sorts of workloads.

Let me know if you still have any problems.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-O2

[Felipe Alfaro Solana]

On Sun, 2004-08-01 at 21:30 +0200, Ingo Molnar wrote:
> here's the latest version of the voluntary-preempt patch:
>   
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-O2
> 
> this patch is mainly a stabilization effort. I dropped the irq-threads
> code added in -M5 and rewrote it from scratch based on -L2 - it is
> simpler and should be more robust.
> 
> The same /proc/irq/* configuration switches are still present, but i
> added the following additional rule: if _any_ handler of a given IRQ is
> marked as non-threaded then all handlers will be executed non-threaded
> as well.
> 
> E.g. if you have the following handlers on IRQ 10:
> 
>  10:      11584   IO-APIC-level  eth0, eth1, eth2
> 
> and you change /proc/irq/16/eth1/threaded from 1 to 0 then the eth0 and
> eth2 handlers will be executed non-threaded as well. (This rule only
> enforces what the hardware enforces anyway, none of the previous patches
> allowed true separation of these handlers.)
> 
> i also changed the IO-APIC level-triggered code to be robust when
> redirection is done. The noapic workaround should not be necessary
> anymore.
> 
> the keyboard lockups are now hopefully all gone too - i've tested
> IO-APIC and non-IO-APIC setups as well and NumLock/ScrollLock works fine
> in all sorts of workloads.
> 
> Let me know if you still have any problems.

I'll be away on vacations for about two weeks, but before leaving, I
wanted to test run 2.6.8-rc2-bk11 plus voluntary-preempt-O2 on my P4 box
with both ACPI and IO/APIC enabled, voluntary-preempt=3 and preempt=1
and it seems to be working fine (I still haven't seen any hard locks).

# cat /proc/interrupts
           CPU0
  0:     114044    IO-APIC-edge  timer
  1:        497    IO-APIC-edge  i8042
  8:          1    IO-APIC-edge  rtc
  9:          0   IO-APIC-level  acpi
 14:       9289    IO-APIC-edge  ide0
 15:         54    IO-APIC-edge  ide1
 17:       1299   IO-APIC-level  Intel 82801BA-ICH2
 19:      13408   IO-APIC-level  uhci_hcd
 20:        279   IO-APIC-level  eth0
 23:          0   IO-APIC-level  uhci_hcd
NMI:          0
LOC:     113967
ERR:          0
MIS:          0

# grep . /proc/irq/*/*/threaded
/proc/irq/14/ide0/threaded:1
/proc/irq/15/ide1/threaded:1
/proc/irq/17/Intel 82801BA-ICH2/threaded:1
/proc/irq/19/uhci_hcd/threaded:1
/proc/irq/1/i8042/threaded:1
/proc/irq/20/eth0/threaded:1
/proc/irq/23/uhci_hcd/threaded:1
/proc/irq/8/rtc/threaded:1
/proc/irq/9/acpi/threaded:1

Re: [patch] voluntary-preempt-2.6.8-rc2-O2

[Daniel Schmitt]

On Sunday 01 August 2004 21:30, Ingo Molnar wrote:
> here's the latest version of the voluntary-preempt patch:
>
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-O2

One minor problem: this crashes hard (in interrupt context, stack pointer is 
bogus) during early boot iff CONFIG_REGPARM is set. With the earlier patches, 
this didn't happen. No ill effects so far with the default ABI; performance 
(apart from the usual reiserfs problems) is flawless.

Context: plain 2.6.8-rc2 + O2, gcc (GCC) 3.3.4 20040623 (Gentoo Linux 
3.3.4-r1, ssp-3.3.2-2, pie-8.7.6), EPoX 8RDA3+ (nForce2) motherboard. If you 
need more information, let me know.

Ciao,

Daniel.

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Sun, 2004-08-01 at 07:28, Ingo Molnar wrote:
> could you try to repeat the '50 usecs' test with -L2 [that was the one you
> used?] to make sure it's repeatable?

Results with L2, soundcard + RTC interrupts 'direct', N=1,000,000:

Delay   Count
-----   -----
6       24330
7       429668
8       34474
9       26184
10      12210
11      9060
12      9104
13      8460
14      11011
15      13615
16      14584
17      13371
18      12169
19      10864
20      11936
21      7974
22      6256
23      4888
24      2385
25      640
26      164
27      113
28      86
29      105
30      90
31      86
32      103
33      140
34      149
35      183
36      160
37      141
38      147
39      146
40      172
41      140
42      131
43      89
44      54
45      10
46      3
47      2
49      3

It's as I expected - same as M5, minus the outliers.  Any idea what
caused these?  I will try O2 next, I would expect it to be similar.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-O2

[Matt Heler]

Ingo,

I get the following error below on with your patch applied on stock 
2.6.8-rc2 ...

  CC      kernel/softirq.o
  CC      kernel/hardirq.o
kernel/hardirq.c:51: error: conflicting types for 'generic_handle_IRQ_event'
include/linux/irq.h:78: error: previous declaration of 
'generic_handle_IRQ_event' was here
kernel/hardirq.c:51: error: conflicting types for 'generic_handle_IRQ_event'
include/linux/irq.h:78: error: previous declaration of 
'generic_handle_IRQ_event' was here
make[1]: *** [kernel/hardirq.o] Error 1
make: *** [kernel] Error 2


Matt H.


On Sunday 01 August 2004 12:30 pm, Ingo Molnar wrote:
> here's the latest version of the voluntary-preempt patch:
>
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-O2
>
> this patch is mainly a stabilization effort. I dropped the irq-threads
> code added in -M5 and rewrote it from scratch based on -L2 - it is
> simpler and should be more robust.
>
> The same /proc/irq/* configuration switches are still present, but i
> added the following additional rule: if _any_ handler of a given IRQ is
> marked as non-threaded then all handlers will be executed non-threaded
> as well.
>
> E.g. if you have the following handlers on IRQ 10:
>
>  10:      11584   IO-APIC-level  eth0, eth1, eth2
>
> and you change /proc/irq/16/eth1/threaded from 1 to 0 then the eth0 and
> eth2 handlers will be executed non-threaded as well. (This rule only
> enforces what the hardware enforces anyway, none of the previous patches
> allowed true separation of these handlers.)
>
> i also changed the IO-APIC level-triggered code to be robust when
> redirection is done. The noapic workaround should not be necessary
> anymore.
>
> the keyboard lockups are now hopefully all gone too - i've tested
> IO-APIC and non-IO-APIC setups as well and NumLock/ScrollLock works fine
> in all sorts of workloads.
>
> Let me know if you still have any problems.
>
> 	Ingo
> -
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Sun, 2004-08-01 at 19:37, Lee Revell wrote:
> On Sun, 2004-08-01 at 07:28, Ingo Molnar wrote:
> > could you try to repeat the '50 usecs' test with -L2 [that was the one you
> > used?] to make sure it's repeatable?
> 
> Results with L2, soundcard + RTC interrupts 'direct', N=1,000,000:
> 
> Delay   Count
> -----   -----
> 6       24330
> 7       429668
> 8       34474
> 9       26184
> 10      12210
> 11      9060
> 12      9104
> 13      8460
> 14      11011
> 15      13615
> 16      14584
> 17      13371
> 18      12169
> 19      10864
> 20      11936
> 21      7974
> 22      6256
> 23      4888
> 24      2385
> 25      640
> 26      164
> 27      113
> 28      86
> 29      105
> 30      90
> 31      86
> 32      103
> 33      140
> 34      149
> 35      183
> 36      160
> 37      141
> 38      147
> 39      146
> 40      172
> 41      140
> 42      131
> 43      89
> 44      54
> 45      10
> 46      3
> 47      2
> 49      3
> 

The above histogram was generated with normal desktop activity going
on.  Another run under the same conditions produced the same three
"humps".  Here are the results with an idle system:

Delay   Count
-----   -----
6       8460
7       481093
8       66311
9       43167
10      23021
11      5536
12      3883
13      3370
14      3635
15      3923
16      3419
17      2892
18      2615
19      2659
20      5239
21      3607
22      1289
23      662
24      237
25      59
26      28
27      15
28      17
29      24
30      29
31      50
32      65
33      34
34      34
35      23
36      29
37      19
38      25
39      36
40      42
41      22
42      18
43      8
44      5

The second hump centered at 20-21 usecs is still present, but smaller. 
The third hump centered at 31-32 is barely detectable, but present, 
repeating the test showed the same thing.

So the first hump is the fast path, interrupt -> scheduler -> jackd with
the fewest possible context switches, and a hot or warm cache.  The
third would have to be the worst case scenario (relatively speaking),
where we preempt another process in one of the longer common critical
sections.

The second hump seems like either the same situation as the fast path,
except with a cold cache, or where we have to preempt another process
that is not in a critical section at all, or a short one.

If we have any suspects for the code paths involved, couldn't this be
verified by adding a udelay(10) to the path, and verifying that the hump
moves by 10?  This technique could also be used to distinguish different
code paths with similar execution times.  It looks like they are finite
and few in number.

At this point there are no latency problems I can see, all that remains
is to understand the causes of the observed latencies.  Then assuming
any "direct-irq" drivers and anything you run SCHED_FIFO is realtime
safe, what else remains to be done in order to make hard realtime
guarantees?

Lee

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-O2

[Lee Revell]

On Sun, 2004-08-01 at 15:30, Ingo Molnar wrote:
> here's the latest version of the voluntary-preempt patch:
>   
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-O2

This works as well as L2, possibly better, because the 42 usec result
was with the rtc irq threaded, it is now direct.  I will post numbers
soon.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-O2

[Lee Revell]

On Sun, 2004-08-01 at 21:45, Lee Revell wrote:
> I will post numbers soon.

I will be out of town for a few days, so this is the last batch.  On my
hardware at least, all the latency problems have been resolved.

Delay   Count
-----   -----
6       2880
7       173347
8       311550
9       51870
10      16382
11      12687
12      11757
13      16355
14      14338
15      10980
16      8152
17      6837
18      5331
19      6593
20      5175
21      4181
22      2189
23      1280
24      622
25      461
26      327
27      230
28      195
29      183
30      277
31      271
32      263
33      224
34      143
35      78
36      62
37      78
38      66
39      60
40      55
41      39
42      25
43      14
44      7
45      6
46      5
47      4
48      1
49      5
50      6
51      3
52      1
54      1
55      2
56      2

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-O2

[Felipe Alfaro Solana]

On Mon, 2004-08-02 at 01:20 +0200, Daniel Schmitt wrote:
> On Sunday 01 August 2004 21:30, Ingo Molnar wrote:
> > here's the latest version of the voluntary-preempt patch:
> >
> >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-O2
> 
> One minor problem: this crashes hard (in interrupt context, stack pointer is 
> bogus) during early boot iff CONFIG_REGPARM is set. With the earlier patches, 
> this didn't happen. No ill effects so far with the default ABI; performance 
> (apart from the usual reiserfs problems) is flawless.

Hmmm... I have CONFIG_REGPARM enabled and it didn't crash for me at all.

Re: [patch] voluntary-preempt-2.6.8-rc2-O2

[Felipe Alfaro Solana]

On Sun, 2004-08-01 at 16:44 -0700, Matt Heler wrote:
> Ingo,
> 
> I get the following error below on with your patch applied on stock 
> 2.6.8-rc2 ...
> 
>   CC      kernel/softirq.o
>   CC      kernel/hardirq.o
> kernel/hardirq.c:51: error: conflicting types for 'generic_handle_IRQ_event'
> include/linux/irq.h:78: error: previous declaration of 
> 'generic_handle_IRQ_event' was here
> kernel/hardirq.c:51: error: conflicting types for 'generic_handle_IRQ_event'
> include/linux/irq.h:78: error: previous declaration of 
> 'generic_handle_IRQ_event' was here
> make[1]: *** [kernel/hardirq.o] Error 1
> make: *** [kernel] Error 2

Try removing the "asmlinkage" from the definition of
"generic_handle_IRQ_event" in file "kernel/hardirq.c".

Re: [patch] voluntary-preempt-2.6.8-rc2-O2 didn't link

[Helge Hafting]

Ingo Molnar wrote:

>here's the latest version of the voluntary-preempt patch:
>  
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-O2
>  
>
This didn't link:

  CC      arch/i386/lib/usercopy.o
  AR      arch/i386/lib/lib.a
  GEN     .version
  CHK     include/linux/compile.h
  UPD     include/linux/compile.h
  CC      init/version.o
  LD      init/built-in.o
  LD      .tmp_vmlinux1
init/built-in.o(.text+0x1be): In function `init':
init/main.c:708: undefined reference to `spawn_irq_threads'
make: *** [.tmp_vmlinux1] Error 1

Helge Hafting

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> If we have any suspects for the code paths involved, couldn't this be
> verified by adding a udelay(10) to the path, and verifying that the
> hump moves by 10?  This technique could also be used to distinguish
> different code paths with similar execution times.  It looks like they
> are finite and few in number.

i believe wli's latency-timing patch gives a pretty good indication of
the code path(s) involved - i'm using something quite similar to that. 
The printout triggers immediately at the end of a high latency, so the
stack contains a number of clues about what went on before.

> At this point there are no latency problems I can see, all that
> remains is to understand the causes of the observed latencies.  Then
> assuming any "direct-irq" drivers and anything you run SCHED_FIFO is
> realtime safe, what else remains to be done in order to make hard
> realtime guarantees?

what remains i believe is to improve wli's patch and to observe (and
fix) the maximum latencies. Also, your workload is a subset of what
other people might be running so wider testing is obviously needed as
well.

btw., with what max_sectors setting were you testing during these tests? 
In -O2 the hardirqs are not preemptable (yet) so with 1024 sectors you
should see quite high latencies due to the completion activity.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-O2

[Ingo Molnar]

* Felipe Alfaro Solana <felipe_alfaro@linuxmail.org> wrote:

> > kernel/hardirq.c:51: error: conflicting types for 'generic_handle_IRQ_event'
> > include/linux/irq.h:78: error: previous declaration of 
> > 'generic_handle_IRQ_event' was here

> Try removing the "asmlinkage" from the definition of
> "generic_handle_IRQ_event" in file "kernel/hardirq.c".

i solved it by adding asmlinkage to irq.h - i've updated the -O2 patch
to include this fix. It needs to stay asmlinkage because e.g. the 4K
stacks code calls it from within an assembly section which assumes the
function follows the normal calling convention (and not e.g. regparm).

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-O2 didn't link

[Ingo Molnar]

* Helge Hafting <helge.hafting@hist.no> wrote:

> >here's the latest version of the voluntary-preempt patch:
> > 
> > http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-O2
> > 
> >
> This didn't link:

>  LD      .tmp_vmlinux1
> init/built-in.o(.text+0x1be): In function `init':
> init/main.c:708: undefined reference to `spawn_irq_threads'
> make: *** [.tmp_vmlinux1] Error 1

hm, the -O2 patch doesnt have any spawn_irq_threads() function. Are you
sure your build is correct?

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-O2

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> On Sun, 2004-08-01 at 21:45, Lee Revell wrote:
> > I will post numbers soon.
> 
> I will be out of town for a few days, so this is the last batch.  On my
> hardware at least, all the latency problems have been resolved.

> 55      2
> 56      2

nice. I'd not worry about the identity of the -M5 latencies too much if
they are gone in -O2.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Mon, 2004-08-02 at 03:39, Ingo Molnar wrote:

> btw., with what max_sectors setting were you testing during these tests? 
> In -O2 the hardirqs are not preemptable (yet) so with 1024 sectors you
> should see quite high latencies due to the completion activity.

256.  It will be a few days before I can do any more testing.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Mon, 2004-08-02 at 03:39, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > If we have any suspects for the code paths involved, couldn't this be
> > verified by adding a udelay(10) to the path, and verifying that the
> > hump moves by 10?  This technique could also be used to distinguish
> > different code paths with similar execution times.  It looks like they
> > are finite and few in number.
> 
> i believe wli's latency-timing patch gives a pretty good indication of
> the code path(s) involved - i'm using something quite similar to that. 
> The printout triggers immediately at the end of a high latency, so the
> stack contains a number of clues about what went on before.
> 

Can you post this patch, or add it to the voluntary preempt series?  The
one posted several weeks ago worked for me but had to be applied
manually and has probably been improved since.

Just to clarify the last numbers I posted were for O2.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Can you post this patch, or add it to the voluntary preempt series? 
> The one posted several weeks ago worked for me but had to be applied
> manually and has probably been improved since.
> 
> Just to clarify the last numbers I posted were for O2.

i've uploaded the preempt-timing patch (relative to -O2):

   http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc2-O2

QuickStart for those who havent used it yet: enable PREEMPT_TIMING in
.config and add preempt_thresh=1000 [== 1000 usec threshold] to the
kernel's boot options. 

i changed the original patch to make it a bit more usable - the
threshold can be changed runtime via /proc/sys/kernel/preempt_thresh,
and the units are microseconds not milliseconds.

the timings should be pretty accurate in vp=0,1,2,3 modes as well.

Note that this patch only measures latencies of critical sections
(preempt-off sections), not other latencies that occur when running
!CONFIG_PREEMPT or with kernel_preemption=0.

	Ingo

[patch] preempt-timing-on-2.6.8-rc2-O2

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> i've uploaded the preempt-timing patch (relative to -O2):
> 
>    http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc2-O2
> 
> QuickStart for those who havent used it yet: enable PREEMPT_TIMING in
> .config and add preempt_thresh=1000 [== 1000 usec threshold] to the
> kernel's boot options. 
> 
> i changed the original patch to make it a bit more usable - the
> threshold can be changed runtime via /proc/sys/kernel/preempt_thresh,
> and the units are microseconds not milliseconds.

i've uploaded a new version of the patch, this solves a problem when
using a smaller than 1000 usecs threshold: idle time got accounted as
delay too which flooded the log. The new patch makes idle=poll the
default and that function calls touch_preempt_timing().

	Ingo

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[William Lee Irwin III]

On Mon, Aug 02, 2004 at 12:08:15PM +0200, Ingo Molnar wrote:
> i've uploaded a new version of the patch, this solves a problem when
> using a smaller than 1000 usecs threshold: idle time got accounted as
> delay too which flooded the log. The new patch makes idle=poll the
> default and that function calls touch_preempt_timing().

Sorry about not updating things, but there's also another bug, which is
that touch_preempt_timing() forgets to check if the threshold has been
violated in the interim.


-- wli

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[Ingo Molnar]

* William Lee Irwin III <wli@holomorphy.com> wrote:

> On Mon, Aug 02, 2004 at 12:08:15PM +0200, Ingo Molnar wrote:
> > i've uploaded a new version of the patch, this solves a problem when
> > using a smaller than 1000 usecs threshold: idle time got accounted as
> > delay too which flooded the log. The new patch makes idle=poll the
> > default and that function calls touch_preempt_timing().
> 
> Sorry about not updating things, but there's also another bug, which
> is that touch_preempt_timing() forgets to check if the threshold has
> been violated in the interim.

i've re-uploaded the -O2 patch with this fixed:

 http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc2-O2

but i'm still seeing some latencies:

 (kjournald/189): 997us non-preemptible critical section violated 100 us
 preempt threshold starting at journal_commit_transaction+0x642/0x2b10
 and ending at journal_commit_transaction+0x24ce/0x2b10

  [<c0105d7e>] dump_stack+0x1e/0x30
  [<c011ad0f>] dec_preempt_count+0x3f/0x50
  [<c01dfd3e>] journal_commit_transaction+0x24ce/0x2b10
  [<c01e3bf4>] kjournald+0x1a4/0x710
  [<c0102765>] kernel_thread_helper+0x5/0x10

these should not happen with vp=3 and max_sectors=16. Could you
double-check the patch above, does it have any other timing thinko?

	Ingo

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

>  (kjournald/189): 997us non-preemptible critical section violated 100 us
>  preempt threshold starting at journal_commit_transaction+0x642/0x2b10
>  and ending at journal_commit_transaction+0x24ce/0x2b10
> 
>   [<c0105d7e>] dump_stack+0x1e/0x30
>   [<c011ad0f>] dec_preempt_count+0x3f/0x50
>   [<c01dfd3e>] journal_commit_transaction+0x24ce/0x2b10
>   [<c01e3bf4>] kjournald+0x1a4/0x710
>   [<c0102765>] kernel_thread_helper+0x5/0x10

ok, found it - it's a false positive in commit.c due to need_resched()
not doing a touch_preempt_timing(). Newest patch at:

 http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc2-O2

i changed need_resched() to do a touch_preempt_timing() - this also got
rid of some other changes. All code i checked really takes
need_resched() seriously if it looks at it - any reason why you didnt
add this to need_resched() before?

	Ingo

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[William Lee Irwin III]

On Mon, Aug 02, 2004 at 12:35:16PM +0200, Ingo Molnar wrote:
> i've re-uploaded the -O2 patch with this fixed:
>  http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc2-O2
> but i'm still seeing some latencies:
>  (kjournald/189): 997us non-preemptible critical section violated 100 us
>  preempt threshold starting at journal_commit_transaction+0x642/0x2b10
>  and ending at journal_commit_transaction+0x24ce/0x2b10
>   [<c0105d7e>] dump_stack+0x1e/0x30
>   [<c011ad0f>] dec_preempt_count+0x3f/0x50
>   [<c01dfd3e>] journal_commit_transaction+0x24ce/0x2b10
>   [<c01e3bf4>] kjournald+0x1a4/0x710
>   [<c0102765>] kernel_thread_helper+0x5/0x10
> these should not happen with vp=3 and max_sectors=16. Could you
> double-check the patch above, does it have any other timing thinko?

The false negative I mentioned was the last unresolved issue I knew of.
A brief examination of your updated patch didn't help me to spot any
false positives that haven't been reported yet.

It may help to find the line numbers corresponding to the above
(actually, the way the macros are hooked, the files, functions, line
numbers could be reported directly by the implementation) so we can get
a better idea if this is a false positive or a real 997us long non-
preemptible critical section.


-- wli

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[William Lee Irwin III]

* Ingo Molnar <mingo@elte.hu> wrote:
>>  (kjournald/189): 997us non-preemptible critical section violated 100 us
>>  preempt threshold starting at journal_commit_transaction+0x642/0x2b10
>>  and ending at journal_commit_transaction+0x24ce/0x2b10
>>   [<c0105d7e>] dump_stack+0x1e/0x30
>>   [<c011ad0f>] dec_preempt_count+0x3f/0x50
>>   [<c01dfd3e>] journal_commit_transaction+0x24ce/0x2b10
>>   [<c01e3bf4>] kjournald+0x1a4/0x710
>>   [<c0102765>] kernel_thread_helper+0x5/0x10

On Mon, Aug 02, 2004 at 12:51:00PM +0200, Ingo Molnar wrote:
> ok, found it - it's a false positive in commit.c due to need_resched()
> not doing a touch_preempt_timing(). Newest patch at:
>  http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc2-O2

Great, thanks for fixing this up.


On Mon, Aug 02, 2004 at 12:51:00PM +0200, Ingo Molnar wrote:
> i changed need_resched() to do a touch_preempt_timing() - this also got
> rid of some other changes. All code i checked really takes
> need_resched() seriously if it looks at it - any reason why you didnt
> add this to need_resched() before?

I was less sure of this as I hadn't audited need_resched() callers to
be sure they did cond_resched() or similar. As you've carried out the
audit of need_resched() callers, you have the certainty I didn't.


-- wli

Re: [patch] voluntary-preempt-2.6.8-rc2-O2

[Lenar Lõhmus]

Ingo Molnar wrote:

>here's the latest version of the voluntary-preempt patch:
>  
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc2-O2
>  
>
Rediff against 2.6.8-rc2-mm2 would be nice if possible (there are some 
rejects right now).

Lenar

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Takashi Iwai]

At Fri, 30 Jul 2004 18:54:39 -0400,
Lee Revell wrote:
> 
> On Fri, 2004-07-30 at 02:44, Ingo Molnar wrote:
> > * Lee Revell <rlrevell@joe-job.com> wrote:
> > 
> > > After running jackd with L2 all night, the only repeated XRUN was this
> > > one:
> > > 
> > > ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
> > >  [<c01066a7>] dump_stack+0x17/0x20
> > >  [<de93d54b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
> > >  [<de979211>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
> > >  [<c01078d7>] handle_IRQ_event+0x47/0x90
> > >  [<c0107e93>] do_IRQ+0xe3/0x1b0
> > >  [<c0106268>] common_interrupt+0x18/0x20
> > >  [<c0146771>] add_to_swap+0x21/0xc0
> > >  [<c0139446>] shrink_list+0x156/0x4b0
> > >  [<c01398ed>] shrink_cache+0x14d/0x370
> > >  [<c013a118>] shrink_zone+0xa8/0xf0
> > >  [<c013a4ee>] balance_pgdat+0x1be/0x220
> > >  [<c013a5f9>] kswapd+0xa9/0xb0
> > >  [<c0104395>] kernel_thread_helper+0x5/0x10
> > > 
> > > This produced a few ~2ms XRUNs.  The shrink_zone -> shrink_cache ->
> > > shrink_list is a recurring motif.
> > > 
> > > Is this addressed in M2?
> > 
> > not yet. I havent seen this latency yet, nor are there any immediately
> > clear clues in the xrun logs you sent. (it would still be nice to check
> > out -M2, to see whether with all those configurability changes it
> > matches the latencies of L2+your-irq.c-hack.)
> 
> I discovered that a few of the XRUN traces were spurious - jackd
> apparently does something while stopping and starting that produces an
> XRUN trace but that jackd does not consider an error.  I will fix this
> in jackd.  The msync() related XRUN triggered by apt-get is definitely
> real.

Yes.  There is a bogus report at stopping (snd_pcm_drain is called).
It was fixed in the recent ALSA cvs tree, but seems not propagated to
bk yet...


Takashi

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Mon, 2004-08-02 at 11:19, Takashi Iwai wrote:
> At Fri, 30 Jul 2004 18:54:39 -0400,
> Lee Revell wrote:
> > 
> > I discovered that a few of the XRUN traces were spurious - jackd
> > apparently does something while stopping and starting that produces an
> > XRUN trace but that jackd does not consider an error.  I will fix this
> > in jackd.  The msync() related XRUN triggered by apt-get is definitely
> > real.
> 
> Yes.  There is a bogus report at stopping (snd_pcm_drain is called).
> It was fixed in the recent ALSA cvs tree, but seems not propagated to
> bk yet...
> 

It also seems to produce an xrun at startup.  This is with the latest
ALSA CVS.  Is this behavior by design?

Lee

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[Lee Revell]

On Mon, 2004-08-02 at 06:51, Ingo Molnar wrote:
> Newest patch at:
> 
>  http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc2-O2
> 

I get a *lot* of these:

(swapper/0): 996us non-preemptible critical section violated 100 us
preempt threshold starting at schedule+0x55/0x5a0 and ending at
do_IRQ+0x110/0x180
 [<c0106717>] dump_stack+0x17/0x20
 [<c0113eec>] dec_preempt_count+0x3c/0x50
 [<c0107a20>] do_IRQ+0x110/0x180
 [<c01062d8>] common_interrupt+0x18/0x20
 [<c0104222>] cpu_idle+0x32/0x50
 [<c030a782>] start_kernel+0x1a2/0x1f0
 [<c010019f>] 0xc010019f

This has to be a false positive, because if there were really ~1ms
non-preemptible critical sections all over the place I would be getting
xruns.

Lee 

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[William Lee Irwin III]

On Mon, 2004-08-02 at 06:51, Ingo Molnar wrote:
>> Newest patch at:
>>  http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc2-O2

On Sat, Aug 07, 2004 at 10:31:06PM -0400, Lee Revell wrote:
> I get a *lot* of these:
> (swapper/0): 996us non-preemptible critical section violated 100 us
> preempt threshold starting at schedule+0x55/0x5a0 and ending at
> do_IRQ+0x110/0x180
>  [<c0106717>] dump_stack+0x17/0x20
>  [<c0113eec>] dec_preempt_count+0x3c/0x50
>  [<c0107a20>] do_IRQ+0x110/0x180
>  [<c01062d8>] common_interrupt+0x18/0x20
>  [<c0104222>] cpu_idle+0x32/0x50
>  [<c030a782>] start_kernel+0x1a2/0x1f0
>  [<c010019f>] 0xc010019f
> This has to be a false positive, because if there were really ~1ms
> non-preemptible critical sections all over the place I would be getting
> xruns.

This seems to be 996us spent in do_IRQ() when it interrupts idle tasks.

-- wli

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[Lee Revell]

On Sat, 2004-08-07 at 22:33, William Lee Irwin III wrote:
> On Sat, Aug 07, 2004 at 10:31:06PM -0400, Lee Revell wrote:
> >  [<c010019f>] 0xc010019f
> > This has to be a false positive, because if there were really ~1ms
> > non-preemptible critical sections all over the place I would be getting
> > xruns.
> 
> This seems to be 996us spent in do_IRQ() when it interrupts idle tasks.
> 

I am not sure I understand how preemption can be disabled for 996us and
not cause an xrun.

Lee

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[William Lee Irwin III]

On Sat, 2004-08-07 at 22:33, William Lee Irwin III wrote:
>> This seems to be 996us spent in do_IRQ() when it interrupts idle tasks.

On Sat, Aug 07, 2004 at 10:36:55PM -0400, Lee Revell wrote:
> I am not sure I understand how preemption can be disabled for 996us and
> not cause an xrun.

You were idling at the time, so the latency affected nothing, except
perhaps task migration on SMP.


-- wli

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[Lee Revell]

On Sat, 2004-08-07 at 22:39, William Lee Irwin III wrote:
> On Sat, 2004-08-07 at 22:33, William Lee Irwin III wrote:
> >> This seems to be 996us spent in do_IRQ() when it interrupts idle tasks.
> 
> On Sat, Aug 07, 2004 at 10:36:55PM -0400, Lee Revell wrote:
> > I am not sure I understand how preemption can be disabled for 996us and
> > not cause an xrun.
> 
> You were idling at the time, so the latency affected nothing, except
> perhaps task migration on SMP.
> 

Would it be possible to detect this situation and treat it like a
preemptible section, just so the sheer volume of printks does not cause
a problem?

Lee

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[William Lee Irwin III]

On Sat, 2004-08-07 at 22:39, William Lee Irwin III wrote:
>> You were idling at the time, so the latency affected nothing, except
>> perhaps task migration on SMP.

On Sat, Aug 07, 2004 at 10:47:09PM -0400, Lee Revell wrote:
> Would it be possible to detect this situation and treat it like a
> preemptible section, just so the sheer volume of printks does not cause
> a problem?

It's probably actually a false positive. What I was explaining was why,
if it were in fact a non-preemptible critical section, it wouldn't cause
xruns.


-- wli

Re: [patch] preempt-timing-on-2.6.8-rc2-O2

[Lee Revell]

On Sat, 2004-08-07 at 22:48, William Lee Irwin III wrote:

> It's probably actually a false positive. What I was explaining was why,
> if it were in fact a non-preemptible critical section, it wouldn't cause
> xruns.
> 

These two seem to be real.  It looks like get_user_pages is still
problematic in 2.6.8-rc2.

(jackd/6219): 10943us non-preemptible critical section violated 1000 us preempt threshold starting at kmap_atomic+0x10/0x60 and ending at kunmap_atomic+0x8/0x20
 [<c0106717>] dump_stack+0x17/0x20
 [<c0113eec>] dec_preempt_count+0x3c/0x50
 [<c0111ce8>] kunmap_atomic+0x8/0x20
 [<c013e2cb>] do_anonymous_page+0x8b/0x190
 [<c013e41e>] do_no_page+0x4e/0x310
 [<c013e8a1>] handle_mm_fault+0xc1/0x170
 [<c013d280>] get_user_pages+0x110/0x380
 [<c013e9f8>] make_pages_present+0x68/0x90
 [<c0140196>] do_mmap_pgoff+0x3e6/0x620
 [<c010b656>] sys_mmap2+0x76/0xb0
 [<c01060b7>] syscall_call+0x7/0xb
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c0106717>] dump_stack+0x17/0x20
 [<de93d64b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de9791d1>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c011a7d3>] generic_handle_IRQ_event+0x33/0x60
 [<c01079c2>] do_IRQ+0xb2/0x180
 [<c01062d8>] common_interrupt+0x18/0x20
 [<c0113e18>] __touch_preempt_timing+0x8/0x20
 [<c0113ea2>] inc_preempt_count+0x32/0x40
 [<c014cb8e>] fget+0x1e/0x60
 [<c015ecdd>] do_pollfd+0x2d/0xa0
 [<c015edaf>] do_poll+0x5f/0xc0
 [<c015ef41>] sys_poll+0x131/0x220
 [<c01060b7>] syscall_call+0x7/0xb
(gnome-terminal/914): 10738us non-preemptible critical section violated 1000 us preempt threshold starting at fget+0x1e/0x60 and ending at fget+0x3d/0x60
 [<c0106717>] dump_stack+0x17/0x20
 [<c0113eec>] dec_preempt_count+0x3c/0x50
 [<c014cbad>] fget+0x3d/0x60
 [<c015ecdd>] do_pollfd+0x2d/0xa0
 [<c015edaf>] do_poll+0x5f/0xc0
 [<c015ef41>] sys_poll+0x131/0x220
 [<c01060b7>] syscall_call+0x7/0xb

Lee

[patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

here's the latest version of the voluntary-preempt patch:
   
  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc3-O4

(it should apply to -rc2 cleanly too.)

-O4 fixes a two more latency sources:

 - mlockall() triggering latencies (reported by Thomas Charbonnel, Lee
   Revell and Florian Schmidt)

 - CDROM umount triggering latencies (reported by Florian Schmidt)

there's also a fix for the CONFIG_PCI_MSI compilation bug reported by
Edouard Gomez.

(the APIC bug has not been found yet so either turn off all APIC options
in .config or use noapic if you intend to use voluntary_preempt=3.)

the preempt-timing-on-2.6.8-rc2-O2 patch applies cleanly to -O4.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> (the APIC bug has not been found yet so either turn off all APIC
> options in .config or use noapic if you intend to use
> voluntary_preempt=3.)

but it would be nice if those experiencing APIC lockups could test the
following: does the lockup still occur with -O4 if kernel_preemption=0?
(while keeping voluntary_preemption=3)

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Florian Schmidt]

On Mon, 9 Aug 2004 15:05:58 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> 
> * Ingo Molnar <mingo@elte.hu> wrote:
> 
> > (the APIC bug has not been found yet so either turn off all APIC
> > options in .config or use noapic if you intend to use
> > voluntary_preempt=3.)
> 
> but it would be nice if those experiencing APIC lockups could test the
> following: does the lockup still occur with -O4 if
> kernel_preemption=0?(while keeping voluntary_preemption=3)

I don't use APIC, since it never worked good for me.. But i wanted to
report that the mlockall latency still seems to be there.. I can easily
trigger it with mlockall'ing > ~10000kb. Need to recompile with the
preempt-timing patch, but here's an xrun trace that happened when
mlockall'ing 20000kb:

Aug  9 18:53:51 mango kernel: ALSA sound/core/pcm_lib.c:169: XRUN:
pcmC0D0p
Aug  9 18:53:51 mango kernel:  [<c010636e>] dump_stack+0x1e/0x20
Aug  9 18:53:51 mango kernel:  [<f08befae>]
snd_pcm_period_elapsed+0x2ee/0x460 [snd_pcm]
Aug  9 18:53:51 mango kernel:  [<f08e1b84>]
snd_cs46xx_interrupt+0x1b4/0x1f0 [snd_cs46xx]
Aug  9 18:53:51 mango kernel:  [<c011a5db>]
generic_handle_IRQ_event+0x3b/0x70
Aug  9 18:53:51 mango kernel:  [<c0107725>] do_IRQ+0xa5/0x150
Aug  9 18:53:51 mango kernel:  [<c0105f88>] common_interrupt+0x18/0x20
Aug  9 18:53:51 mango kernel:  [<c013cc4a>] get_user_pages+0xca/0x380
Aug  9 18:53:51 mango kernel:  [<c013e33d>] make_pages_present+0x8d/0xb0
Aug  9 18:53:51 mango kernel:  [<c013e7e6>] mlock_fixup+0xa6/0xc0
Aug  9 18:53:51 mango kernel:  [<c013eadb>] do_mlockall+0x7b/0x90
Aug  9 18:53:51 mango kernel:  [<c013eb8e>] sys_mlockall+0x9e/0xb0
Aug  9 18:53:51 mango kernel:  [<c0105e1b>] syscall_call+0x7/0xb

I can also see some of these [sparse they are though]:

Aug  9 18:58:08 mango kernel: ALSA sound/core/pcm_lib.c:199: Unexpected
hw_pointer value [1] (stream = 0, delta: -32, max jitter = 64): wrong
interrupt acknowledge?
Aug  9 18:58:08 mango kernel:  [<c010636e>] dump_stack+0x1e/0x20
Aug  9 18:58:08 mango kernel:  [<f08bee8d>]
snd_pcm_period_elapsed+0x1cd/0x460 [snd_pcm]
Aug  9 18:58:08 mango kernel:  [<f08e1b84>]
snd_cs46xx_interrupt+0x1b4/0x1f0 [snd_cs46xx]
Aug  9 18:58:08 mango kernel:  [<c011a5db>]
generic_handle_IRQ_event+0x3b/0x70
Aug  9 18:58:08 mango kernel:  [<c0107725>] do_IRQ+0xa5/0x150
Aug  9 18:58:08 mango kernel:  [<c0105f88>] common_interrupt+0x18/0x20

Flo


-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Mon, 2004-08-09 at 13:02, Florian Schmidt wrote:

> I don't use APIC, since it never worked good for me.. But i wanted to
> report that the mlockall latency still seems to be there.. I can easily
> trigger it with mlockall'ing > ~10000kb. Need to recompile with the
> preempt-timing patch, but here's an xrun trace that happened when
> mlockall'ing 20000kb:
> 

Ingo, do you plan to maintain the voluntary preempt patch against the
-mm series?  From looking at Andrew's announcement yesterday it looks
like many latency issues fixed in the voluntary preemption patches are
also fixed in -mm, so it seems like the patch would be much smaller. 
One thing that might be useful is breaking out the irq threading code as
a patch against -mm.  Judging from all the -mm latency fixes it seems
like this would work as well as the vanilla kernel+voluntary preempt.

This would also make it easier to identify which are the important
latency fixes from -mm enabling them to be pushed into mainline sooner. 
On some of my tests I got 10-20% better results using vol-preempt+mm vs
vol-preempt+vanilla, it would be nice to identify what changes are
responsible.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Mon, 2004-08-09 at 13:02, Florian Schmidt wrote:
> On Mon, 9 Aug 2004 15:05:58 +0200
> Ingo Molnar <mingo@elte.hu> wrote:
> 
> > 
> > * Ingo Molnar <mingo@elte.hu> wrote:
> > 
> > > (the APIC bug has not been found yet so either turn off all APIC
> > > options in .config or use noapic if you intend to use
> > > voluntary_preempt=3.)
> > 
> > but it would be nice if those experiencing APIC lockups could test the
> > following: does the lockup still occur with -O4 if
> > kernel_preemption=0?(while keeping voluntary_preemption=3)
> 
> I don't use APIC, since it never worked good for me.. But i wanted to
> report that the mlockall latency still seems to be there.. I can easily
> trigger it with mlockall'ing > ~10000kb. Need to recompile with the
> preempt-timing patch, but here's an xrun trace that happened when
> mlockall'ing 20000kb:
> 

Same results here, the mlockall problem is not fixed by -O4:

ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139:
XRUN: pcmC0D2c
 [<c0106717>] dump_stack+0x17/0x20
 [<de93664b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de9571d1>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c011a7d3>] generic_handle_IRQ_event+0x33/0x60
 [<c01079c2>] do_IRQ+0xb2/0x180
 [<c01062d8>] common_interrupt+0x18/0x20
 [<c013e2ee>] do_anonymous_page+0x7e/0x190
 [<c013e44e>] do_no_page+0x4e/0x310
 [<c013e8d1>] handle_mm_fault+0xc1/0x170
 [<c013d2a0>] get_user_pages+0x130/0x3b0
 [<c013ea28>] make_pages_present+0x68/0x90
 [<c01401d8>] do_mmap_pgoff+0x3f8/0x640
 [<c010b656>] sys_mmap2+0x76/0xb0
 [<c01060b7>] syscall_call+0x7/0xb
(jackd/778): 10984us non-preemptible critical section violated 1100 us
preempt threshold starting at kmap_atomic+0x10/0x60 and ending at
kunmap_atomic+0x8/0x20
 [<c0106717>] dump_stack+0x17/0x20
 [<c0113eec>] dec_preempt_count+0x3c/0x50
 [<c0111ce8>] kunmap_atomic+0x8/0x20
 [<c013e2fb>] do_anonymous_page+0x8b/0x190
 [<c013e44e>] do_no_page+0x4e/0x310
 [<c013e8d1>] handle_mm_fault+0xc1/0x170
 [<c013d2a0>] get_user_pages+0x130/0x3b0
 [<c013ea28>] make_pages_present+0x68/0x90
 [<c01401d8>] do_mmap_pgoff+0x3f8/0x640
 [<c010b656>] sys_mmap2+0x76/0xb0
 [<c01060b7>] syscall_call+0x7/0xb

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Mon, 2004-08-09 at 22:06, Lee Revell wrote:

> Same results here, the mlockall problem is not fixed by -O4:

Correction, those traces did not involve mlockall at all, but
kmap_atomic and get_user_pages.

Here is another one I got starting jackd.  Never seen it before today.

(jackd/778): 14583us non-preemptible critical section violated 1100 us preempt threshold starting at schedule+0x55/0x5a0 and ending at schedule+0x2ed/0x5a0
 [<c0106717>] dump_stack+0x17/0x20
 [<c0113eec>] dec_preempt_count+0x3c/0x50
 [<c0264d4d>] schedule+0x2ed/0x5a0
 [<c026554e>] schedule_timeout+0x9e/0xa0
 [<c0121dcf>] sys_rt_sigtimedwait+0x1df/0x2e0
 [<c01060b7>] syscall_call+0x7/0xb
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c0106717>] dump_stack+0x17/0x20
 [<de93664b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de9571d1>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c011a7d3>] generic_handle_IRQ_event+0x33/0x60
 [<c01079c2>] do_IRQ+0xb2/0x180
 [<c01062d8>] common_interrupt+0x18/0x20

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Ingo, do you plan to maintain the voluntary preempt patch against the
> -mm series?  From looking at Andrew's announcement yesterday it looks
> like many latency issues fixed in the voluntary preemption patches are
> also fixed in -mm, so it seems like the patch would be much smaller. 

yeah, and in addition we've already pushed 99% of our might_sleep()
additions to -mm too so that reduces the patch size too, quite
significantly.

time is the only limiting factor. Due to these partial merges (we are
trying to get all uncontroversial bits into -mm, hence into upstream)
the merge to -mm is hard. Especially for lock-breaks that i've done
differently than Andrew. I sent a consolidation patch yesterday but this
is still work in progress. So i'll do an -mm merge very time i get to do
it, but the primary testing still remains on the vanilla kernel (which
most people use).

> One thing that might be useful is breaking out the irq threading code
> as a patch against -mm.  Judging from all the -mm latency fixes it
> seems like this would work as well as the vanilla kernel+voluntary
> preempt.

both softirq threading and hardirq threading will be done separately,
yes. Also, most of the switches and source-level distinctions between
cond_resched and voluntary_resched variants can go away too. The
'-clean' patch in the voluntary-preempt directory shows how it would
look like in the end.

> This would also make it easier to identify which are the important
> latency fixes from -mm enabling them to be pushed into mainline
> sooner.  On some of my tests I got 10-20% better results using
> vol-preempt+mm vs vol-preempt+vanilla, it would be nice to identify
> what changes are responsible.

the plan right now is to push all the known-good stuff into 2.6.9 once
2.6.8 is out. That will unify all the improvements in a natural way. The
latest update kernel of FC2 also includes an earlier version of the
voluntary-preempt patch (sans any irq threading bits) - and it worked
out fine so far.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

[-- Attachment #1: Type: text/plain, Size: 617 bytes --]


* Lee Revell <rlrevell@joe-job.com> wrote:

> On Mon, 2004-08-09 at 22:06, Lee Revell wrote:
> 
> > Same results here, the mlockall problem is not fixed by -O4:
> 
> Correction, those traces did not involve mlockall at all, but
> kmap_atomic and get_user_pages.
> 
> Here is another one I got starting jackd.  Never seen it before today.
> 
> (jackd/778): 14583us non-preemptible critical section violated 1100 us
> preempt threshold starting at schedule+0x55/0x5a0 and ending at
> schedule+0x2ed/0x5a0

can you trigger similar latencies via the attached mlock testcode?
(written by Florian. Run it as root.)

	Ingo

[-- Attachment #2: mlockall-test.cc --]
[-- Type: text/plain, Size: 866 bytes --]

// here is the code i used to test the mlockall caused xruns
#include <sys/mman.h>
#include <iostream>
#include <sstream>
#include <unistd.h>

int main (int argc, char *argv[]) {
        if (argc < 2) {
                std::cout << "how many kbytes you want allocated and mlockall'ed?" << std::endl;
        }

        std::stringstream stream(argv[1]);
        int kbytes;
        stream >> kbytes;
        char *mem = new char[kbytes*1024];
        std::cout << "filling with 0's" << std::endl;
        for (int i = 0; i < kbytes*1024; ++i) {
                mem[i] = 0;
        }

        std::cout << "ok, you want " << kbytes << "kb of memory mlocked.  going for it.." << std::endl;
        int error = mlockall(MCL_CURRENT);
        if (error != 0) { std::cout << "mlock error" << std::endl; }
        else { std::cout << "mlock successfull" << std::endl;}
}

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Here is another one I got starting jackd.  Never seen it before today.
> 
> (jackd/778): 14583us non-preemptible critical section violated 1100 us
> preempt threshold starting at schedule+0x55/0x5a0 and ending at
> schedule+0x2ed/0x5a0

just to make sure this is not a false positive - is this accompanied by
ALSA-detected xruns as well? (i suspect it is.)

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 04:09, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Here is another one I got starting jackd.  Never seen it before today.
> > 
> > (jackd/778): 14583us non-preemptible critical section violated 1100 us
> > preempt threshold starting at schedule+0x55/0x5a0 and ending at
> > schedule+0x2ed/0x5a0
> 
> just to make sure this is not a false positive - is this accompanied by
> ALSA-detected xruns as well? (i suspect it is.)

Yes, here it is:

Aug  9 22:12:48 mindpipe kernel: ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
Aug  9 22:12:48 mindpipe kernel:  [dump_stack+23/32] dump_stack+0x17/0x20
Aug  9 22:12:48 mindpipe kernel:  [__crc_totalram_pages+1425/2369476] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
Aug  9 22:12:48 mindpipe kernel:  [__crc_totalram_pages+135447/2369476] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
Aug  9 22:12:48 mindpipe kernel:  [generic_handle_IRQ_event+51/96] generic_handle_IRQ_event+0x33/0x60
Aug  9 22:12:48 mindpipe kernel:  [do_IRQ+178/384] do_IRQ+0xb2/0x180
Aug  9 22:12:48 mindpipe kernel:  [common_interrupt+24/32] common_interrupt+0x18/0x20
Aug  9 22:12:48 mindpipe kernel:  [schedule+727/1440] schedule+0x2d7/0x5a0
Aug  9 22:12:48 mindpipe kernel:  [schedule_timeout+158/160] schedule_timeout+0x9e/0xa0
Aug  9 22:12:48 mindpipe kernel:  [sys_rt_sigtimedwait+479/736] sys_rt_sigtimedwait+0x1df/0x2e0
Aug  9 22:12:48 mindpipe kernel:  [syscall_call+7/11] syscall_call+0x7/0xb

I do not seem to get false positives anymore, IOW, all the traces I send 
you are accompanied by an ALSA xrun.  I suspect that with a lower 
threshold (100us), the overhead from all the printks and stack dumps was 
causing one violation to lead to a domino effect.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

>  [<c013e2ee>] do_anonymous_page+0x7e/0x190
>  [<c013e44e>] do_no_page+0x4e/0x310
>  [<c013e8d1>] handle_mm_fault+0xc1/0x170
>  [<c013d2a0>] get_user_pages+0x130/0x3b0
>  [<c013ea28>] make_pages_present+0x68/0x90
>  [<c01401d8>] do_mmap_pgoff+0x3f8/0x640
>  [<c010b656>] sys_mmap2+0x76/0xb0

could you turn off CONFIG_HIGHMEM altogether? Can you still reproduce
this mlockall() latency?

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

>  [<c01062d8>] common_interrupt+0x18/0x20
>  [<c013e2ee>] do_anonymous_page+0x7e/0x190
>  [<c013e44e>] do_no_page+0x4e/0x310
>  [<c013e8d1>] handle_mm_fault+0xc1/0x170
>  [<c013d2a0>] get_user_pages+0x130/0x3b0
>  [<c013ea28>] make_pages_present+0x68/0x90
>  [<c01401d8>] do_mmap_pgoff+0x3f8/0x640
>  [<c010b656>] sys_mmap2+0x76/0xb0
>  [<c01060b7>] syscall_call+0x7/0xb

another idea: you are running this on a C3, using CONFIG_MCYRIXIII,
correct? That is one of the rare configs that triggers X86_USE_3DNOW and
MMX ops. If 3dnow is in any way handicapped in that CPU then that could
cause trouble. Could you compile for e.g. CONFIG_M586TSC? [that option
should be fully compatible with a C3.] - this will exclude the MMX page
clearing ops.

what makes me suspicious is this entry in your call-trace:

>  [<c013e2ee>] do_anonymous_page+0x7e/0x190

this is the next instruction after:

   call   c01d1910 <mmx_clear_page>

but it's all a bit weird - the ALSA interrupt should have hit the
fast_clear_page() function but it didnt. Maybe it doesnt show up because
the call to fast_clear_page() within mmx_clear_page() is tail-optimized
and possibly the stack entry is lost. But still ...

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> another idea: you are running this on a C3, using CONFIG_MCYRIXIII,
> correct? That is one of the rare configs that triggers X86_USE_3DNOW
> and MMX ops. If 3dnow is in any way handicapped in that CPU then that
> could cause trouble. Could you compile for e.g. CONFIG_M586TSC? [that
> option should be fully compatible with a C3.] - this will exclude the
> MMX page clearing ops.

another (more remote) possibility is that the timestamp counter gets
somehow messed up during MMX ops. Does the ALSA detector use the
timestamp counter, or does it only use jiffies? (if it only used jiffies
that would give us some robustness since it's an independent
time-source.) I suspect 'music indeed skips' isnt a good enough test for
this case, given that jackd starts up ...

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

i've uploaded a new version of the preempt-timing patch:

  http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc3-O4.patch

this patch fixes a number of false positives and false negatives. In
particular it fixes the idle-task false positives, and it now correctly
measures preemption delays in softirq and hardirq contexts and in
bh-disabled process contexts. Maybe this sheds a light on some of the
more mysterious delays that we've seen. (and which were never directly
measured before.)

(the patch also got alot simpler, which should help portability.)

	Ingo

Re: [patch] preempt-timing-on-2.6.8-rc3-O4.patch

[William Lee Irwin III]

On Tue, Aug 10, 2004 at 12:13:03PM +0200, Ingo Molnar wrote:
> i've uploaded a new version of the preempt-timing patch:
> http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc3-O4.patch
> this patch fixes a number of false positives and false negatives. In
> particular it fixes the idle-task false positives, and it now correctly
> measures preemption delays in softirq and hardirq contexts and in
> bh-disabled process contexts. Maybe this sheds a light on some of the
> more mysterious delays that we've seen. (and which were never directly
> measured before.)
> (the patch also got alot simpler, which should help portability.)


Looks really good, this thing is really starting to look slick from all
the time you've put in on it.

The adding in of the FOOIRQ_OFFSET bits were a rather major oversisght
on my part!  Very good catch.


-- wli

Re: [patch] preempt-timing-on-2.6.8-rc3-O4.patch

[William Lee Irwin III]

On Tue, Aug 10, 2004 at 12:13:03PM +0200, Ingo Molnar wrote:
>> i've uploaded a new version of the preempt-timing patch:
>> http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc3-O4.patch
>> this patch fixes a number of false positives and false negatives. In
>> particular it fixes the idle-task false positives, and it now correctly
>> measures preemption delays in softirq and hardirq contexts and in
>> bh-disabled process contexts. Maybe this sheds a light on some of the
>> more mysterious delays that we've seen. (and which were never directly
>> measured before.)
>> (the patch also got alot simpler, which should help portability.)

On Tue, Aug 10, 2004 at 03:20:19AM -0700, William Lee Irwin III wrote:
> Looks really good, this thing is really starting to look slick from all
> the time you've put in on it.
> The adding in of the FOOIRQ_OFFSET bits were a rather major oversisght
> on my part!  Very good catch.

Feh, there has to be a way to say "this is a good patch" without sounding
like I'm reviewing freshman code. This is a good patch.


-- wli

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Florian Schmidt]

On Mon, 09 Aug 2004 22:06:20 -0400
Lee Revell <rlrevell@joe-job.com> wrote:

> > I don't use APIC, since it never worked good for me.. But i wanted
> > to report that the mlockall latency still seems to be there.. I can
> > easily trigger it with mlockall'ing > ~10000kb. Need to recompile
> > with the preempt-timing patch, but here's an xrun trace that
> > happened when mlockall'ing 20000kb:
> > 
> 
> Same results here, the mlockall problem is not fixed by -O4:
> 
> ALSA
> /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139:
> XRUN: pcmC0D2c
>  [<c0106717>] dump_stack+0x17/0x20
[snip]
> (jackd/778): 10984us non-preemptible critical section violated 1100 us
> preempt threshold starting at kmap_atomic+0x10/0x60 and ending at
> kunmap_atomic+0x8/0x20
[snip]


Hi,

i just applied the new preempt-timing patch and i made the observation
that if jackd is not running then mlocking 500megs of ram doesn't cause
the preempt-timing [thresh = 1000] to report anything.

But when jackd is running i can easily cause an xrun by mlocking even
only 50megs. In this case also a critical section report is printed by
the preempt-timing:

Aug 10 13:13:05 mango kernel: ALSA sound/core/pcm_lib.c:169: XRUN:
pcmC0D0p
Aug 10 13:13:05 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 10 13:13:05 mango kernel:  [<f08c2431>]
snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
Aug 10 13:13:05 mango kernel:  [<f089ebce>]
snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
Aug 10 13:13:05 mango kernel:  [<c011b55b>]
generic_handle_IRQ_event+0x3b/0x70
Aug 10 13:13:05 mango kernel:  [<c01078c6>] do_IRQ+0xb6/0x170
Aug 10 13:13:05 mango kernel:  [<c0106098>] common_interrupt+0x18/0x20
Aug 10 13:13:05 mango kernel:  [<c013f073>] get_user_pages+0xd3/0x3d0
Aug 10 13:13:05 mango kernel:  [<c01408fd>] make_pages_present+0x8d/0xb0
Aug 10 13:13:05 mango kernel:  [<c0140da6>] mlock_fixup+0xa6/0xc0
Aug 10 13:13:05 mango kernel:  [<c014109b>] do_mlockall+0x7b/0x90
Aug 10 13:13:05 mango kernel:  [<c014114e>] sys_mlockall+0x9e/0xb0
Aug 10 13:13:05 mango kernel:  [<c0105f2b>] syscall_call+0x7/0xb
Aug 10 13:13:05 mango kernel: (mlockall_test/877): 2706us
non-preemptible critical section violated 1000 us preempt threshold
starting at get_user_pages+0x2f3/0x3d0 and ending at
get_user_pages+0xa5/0x3d0
Aug 10 13:13:05 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 10 13:13:05 mango kernel:  [<c01146b3>]
touch_preempt_timing+0x33/0x40
Aug 10 13:13:05 mango kernel:  [<c013f045>] get_user_pages+0xa5/0x3d0
Aug 10 13:13:05 mango kernel:  [<c01408fd>] make_pages_present+0x8d/0xb0
Aug 10 13:13:05 mango kernel:  [<c0140da6>] mlock_fixup+0xa6/0xc0
Aug 10 13:13:05 mango kernel:  [<c014109b>] do_mlockall+0x7b/0x90
Aug 10 13:13:05 mango kernel:  [<c014114e>] sys_mlockall+0x9e/0xb0
Aug 10 13:13:05 mango kernel:  [<c0105f2b>] syscall_call+0x7/0xb

So it seems that the combination of the two is somehow triggering this
behaviour:

ALSA sound/core/pcm_lib.c:169: XRUN: pcmC0D0p
 [<c01064ae>] dump_stack+0x1e/0x20
 [<f08c2431>] snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
 [<f089ebce>] snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
 [<c011b55b>] generic_handle_IRQ_event+0x3b/0x70
 [<c01078c6>] do_IRQ+0xb6/0x170
 [<c0106098>] common_interrupt+0x18/0x20
 [<c013f293>] get_user_pages+0x2f3/0x3d0
 [<c01408fd>] make_pages_present+0x8d/0xb0
 [<c0140da6>] mlock_fixup+0xa6/0xc0
 [<c014109b>] do_mlockall+0x7b/0x90
 [<c014114e>] sys_mlockall+0x9e/0xb0
 [<c0105f2b>] syscall_call+0x7/0xb
(mlockall_test/862): 2701us non-preemptible critical section violated
1000 us preempt threshold starting at get_user_pages+0x2f3/0x3d0 and
ending at get_user_pages+0xa5/0x3d0
 [<c01064ae>] dump_stack+0x1e/0x20
 [<c01146b3>] touch_preempt_timing+0x33/0x40
 [<c013f045>] get_user_pages+0xa5/0x3d0
 [<c01408fd>] make_pages_present+0x8d/0xb0
 [<c0140da6>] mlock_fixup+0xa6/0xc0
 [<c014109b>] do_mlockall+0x7b/0x90
 [<c014114e>] sys_mlockall+0x9e/0xb0
 [<c0105f2b>] syscall_call+0x7/0xb
ALSA sound/core/pcm_lib.c:169: XRUN: pcmC0D0p
 [<c01064ae>] dump_stack+0x1e/0x20
 [<f08c2431>] snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
 [<f089ebce>] snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
 [<c011b55b>] generic_handle_IRQ_event+0x3b/0x70
 [<c01078c6>] do_IRQ+0xb6/0x170
 [<c0106098>] common_interrupt+0x18/0x20
 [<c0114576>] check_preempt_timing+0x16/0x100
 [<c0114735>] sub_preempt_count+0x45/0x60
 [<c013f282>] get_user_pages+0x2e2/0x3d0
 [<c01408fd>] make_pages_present+0x8d/0xb0
 [<c0140da6>] mlock_fixup+0xa6/0xc0
 [<c014109b>] do_mlockall+0x7b/0x90
 [<c014114e>] sys_mlockall+0x9e/0xb0
 [<c0105f2b>] syscall_call+0x7/0xb
ALSA sound/core/pcm_lib.c:169: XRUN: pcmC0D0p
 [<c01064ae>] dump_stack+0x1e/0x20
 [<f08c2431>] snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
 [<f089ebce>] snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
 [<c011b55b>] generic_handle_IRQ_event+0x3b/0x70
 [<c01078c6>] do_IRQ+0xb6/0x170
 [<c0106098>] common_interrupt+0x18/0x20
ALSA sound/core/pcm_lib.c:169: XRUN: pcmC0D0p
 [<c01064ae>] dump_stack+0x1e/0x20
 [<f08c2431>] snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
 [<f089ebce>] snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
 [<c011b55b>] generic_handle_IRQ_event+0x3b/0x70
 [<c01078c6>] do_IRQ+0xb6/0x170
 [<c0106098>] common_interrupt+0x18/0x20
ALSA sound/core/pcm_lib.c:169: XRUN: pcmC0D0p
 [<c01064ae>] dump_stack+0x1e/0x20
 [<f08c2431>] snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
 [<f089ebce>] snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
 [<c011b55b>] generic_handle_IRQ_event+0x3b/0x70
 [<c01078c6>] do_IRQ+0xb6/0x170
 [<c0106098>] common_interrupt+0x18/0x20


I also tried to disable syslogd and looked into dmesg to see the xrun
reports. i don't know if they differ in any essential way:

 [<c0105f2b>] syscall_call+0x7/0xb
ALSA sound/core/pcm_lib.c:169: XRUN: pcmC0D0p
 [<c01064ae>] dump_stack+0x1e/0x20
 [<f08c2431>] snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
 [<f089ebce>] snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
 [<c011b55b>] generic_handle_IRQ_event+0x3b/0x70
 [<c01078c6>] do_IRQ+0xb6/0x170
 [<c0106098>] common_interrupt+0x18/0x20
 [<c0114668>] __touch_preempt_timing+0x8/0x20
 [<c01146be>] touch_preempt_timing+0x3e/0x40
 [<c013f045>] get_user_pages+0xa5/0x3d0
 [<c01408fd>] make_pages_present+0x8d/0xb0
 [<c0140da6>] mlock_fixup+0xa6/0xc0
 [<c014109b>] do_mlockall+0x7b/0x90
 [<c014114e>] sys_mlockall+0x9e/0xb0
 [<c0105f2b>] syscall_call+0x7/0xb
(mlockall_test/913): 3053us non-preemptible critical section violated
1000 us preempt threshold starting at get_user_pages+0xa5/0x3d0 and
ending at get_user_pages+0x2e2/0x3d0
 [<c01064ae>] dump_stack+0x1e/0x20
 [<c0114735>] sub_preempt_count+0x45/0x60
 [<c013f282>] get_user_pages+0x2e2/0x3d0
 [<c01408fd>] make_pages_present+0x8d/0xb0
 [<c0140da6>] mlock_fixup+0xa6/0xc0
 [<c014109b>] do_mlockall+0x7b/0x90
 [<c014114e>] sys_mlockall+0x9e/0xb0
 [<c0105f2b>] syscall_call+0x7/0xb
ALSA sound/core/pcm_lib.c:169: XRUN: pcmC0D0p
 [<c01064ae>] dump_stack+0x1e/0x20
 [<f08c2431>] snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
 [<f089ebce>] snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
 [<c011b55b>] generic_handle_IRQ_event+0x3b/0x70
 [<c01078c6>] do_IRQ+0xb6/0x170
 [<c0106098>] common_interrupt+0x18/0x20
 [<c013f293>] get_user_pages+0x2f3/0x3d0
 [<c01408fd>] make_pages_present+0x8d/0xb0
 [<c0140da6>] mlock_fixup+0xa6/0xc0
 [<c014109b>] do_mlockall+0x7b/0x90
 [<c014114e>] sys_mlockall+0x9e/0xb0
 [<c0105f2b>] syscall_call+0x7/0xb
ALSA sound/core/pcm_lib.c:169: XRUN: pcmC0D0p
 [<c01064ae>] dump_stack+0x1e/0x20
 [<f08c2431>] snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
 [<f089ebce>] snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
 [<c011b55b>] generic_handle_IRQ_event+0x3b/0x70
 [<c01078c6>] do_IRQ+0xb6/0x170
 [<c0106098>] common_interrupt+0x18/0x20
ALSA sound/core/pcm_lib.c:169: XRUN: pcmC0D0c
 [<c01064ae>] dump_stack+0x1e/0x20
 [<f08c2431>] snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
 [<f089ebce>] snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
 [<c011b55b>] generic_handle_IRQ_event+0x3b/0x70
 [<c01078c6>] do_IRQ+0xb6/0x170
 [<c0106098>] common_interrupt+0x18/0x20
 [<c013f045>] get_user_pages+0xa5/0x3d0
 [<c01408fd>] make_pages_present+0x8d/0xb0
 [<c0140da6>] mlock_fixup+0xa6/0xc0
 [<c014109b>] do_mlockall+0x7b/0x90
 [<c014114e>] sys_mlockall+0x9e/0xb0
 [<c0105f2b>] syscall_call+0x7/0xb
(mlockall_test/916): 2703us non-preemptible critical section violated
1000 us preempt threshold starting at get_user_pages+0xa5/0x3d0 and
ending at get_user_pages+0x2e2/0x3d0
 [<c01064ae>] dump_stack+0x1e/0x20
 [<c0114735>] sub_preempt_count+0x45/0x60
 [<c013f282>] get_user_pages+0x2e2/0x3d0
 [<c01408fd>] make_pages_present+0x8d/0xb0
 [<c0140da6>] mlock_fixup+0xa6/0xc0
 [<c014109b>] do_mlockall+0x7b/0x90
 [<c014114e>] sys_mlockall+0x9e/0xb0
 [<c0105f2b>] syscall_call+0x7/0xb
ALSA sound/core/pcm_lib.c:169: XRUN: pcmC0D0p
 [<c01064ae>] dump_stack+0x1e/0x20
 [<f08c2431>] snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
 [<f089ebce>] snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
 [<c011b55b>] generic_handle_IRQ_event+0x3b/0x70
 [<c01078c6>] do_IRQ+0xb6/0x170
 [<c0106098>] common_interrupt+0x18/0x20
 [<c0114576>] check_preempt_timing+0x16/0x100
 [<c0114735>] sub_preempt_count+0x45/0x60
 [<c0291bc5>] schedule+0x305/0x5b0
 [<c029241e>] schedule_timeout+0x5e/0xb0
 [<c0162fd9>] do_poll+0x99/0xc0
 [<c016313f>] sys_poll+0x13f/0x240
 [<c0105f2b>] syscall_call+0x7/0xb
(qjackctl/859): 5549us non-preemptible critical section violated 1000 us
preempt threshold starting at schedule+0x5c/0x5b0 and ending at
schedule+0x305/0x5b0
 [<c01064ae>] dump_stack+0x1e/0x20
 [<c0114735>] sub_preempt_count+0x45/0x60
 [<c0291bc5>] schedule+0x305/0x5b0
 [<c029241e>] schedule_timeout+0x5e/0xb0
 [<c0162fd9>] do_poll+0x99/0xc0
 [<c016313f>] sys_poll+0x13f/0x240
 [<c0105f2b>] syscall_call+0x7/0xb


Flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Alan Cox]

On Maw, 2004-08-10 at 10:22, Ingo Molnar wrote:
> another (more remote) possibility is that the timestamp counter gets
> somehow messed up during MMX ops. Does the ALSA detector use the
> timestamp counter, or does it only use jiffies? (if it only used jiffies
> that would give us some robustness since it's an independent
> time-source.) I suspect 'music indeed skips' isnt a good enough test for
> this case, given that jackd starts up ...

The standard VIA EPIA boards are 133Mhz SDRAM or 266Mhz DDR, which
is shared with video and the graphics engine. Could the MMX copier
simply be eating all the remaining memory bandwidth so that its
in fact memory not latency ?

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

* Alan Cox <alan@lxorguk.ukuu.org.uk> wrote:

> > another (more remote) possibility is that the timestamp counter gets
> > somehow messed up during MMX ops. Does the ALSA detector use the
> > timestamp counter, or does it only use jiffies? (if it only used jiffies
> > that would give us some robustness since it's an independent
> > time-source.) I suspect 'music indeed skips' isnt a good enough test for
> > this case, given that jackd starts up ...
> 
> The standard VIA EPIA boards are 133Mhz SDRAM or 266Mhz DDR, which is
> shared with video and the graphics engine. Could the MMX copier simply
> be eating all the remaining memory bandwidth so that its in fact
> memory not latency ?

it's a 10 msec latency that got measured, for a single page copy (!) -
if it's memory latency then it must be something really bad...

	Ingo

[patch] voluntary-preempt-2.6.8-rc3-O5

[Ingo Molnar]

i've uploaded the latest version of the voluntary-preempt patch:
    
  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc3-O5

-O5 fixes the APIC lockup issues. The bug was primarily caused by PCI
POST delays causing IRQ storms of level-triggered IRQ sources that were
hardirq-redirected. Also found some bugs in delayed-IRQ masking and
unmasking. SMP should thus work again too.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 03:53, Ingo Molnar wrote:
> can you trigger similar latencies via the attached mlock testcode?
> (written by Florian. Run it as root.)
> 

Yup, using only 100000 bytes, I get a bunch of these:

(mlockall-test/6561): 10353us non-preemptible critical section violated 1100 us preempt threshold starting at get_user_pages+0xa6/0x3b0 and ending at get_user_pages+0x2c5/0x3b0
 [<c0106717>] dump_stack+0x17/0x20
 [<c0113eec>] dec_preempt_count+0x3c/0x50
 [<c013d435>] get_user_pages+0x2c5/0x3b0
 [<c013ea28>] make_pages_present+0x68/0x90
 [<c013ee8d>] mlock_fixup+0x8d/0xb0
 [<c013f170>] do_mlockall+0x70/0x90
 [<c013f229>] sys_mlockall+0x99/0xa0
 [<c01060b7>] syscall_call+0x7/0xb
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c0106717>] dump_stack+0x17/0x20
 [<de93d64b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de9791d1>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c011a7d3>] generic_handle_IRQ_event+0x33/0x60
 [<c01079c2>] do_IRQ+0xb2/0x180
 [<c01062d8>] common_interrupt+0x18/0x20
 [<c0113d31>] check_preempt_timing+0x11/0xf0
 [<c0113e62>] touch_preempt_timing+0x32/0x40
 [<c013d216>] get_user_pages+0xa6/0x3b0
 [<c013ea28>] make_pages_present+0x68/0x90
 [<c013ee8d>] mlock_fixup+0x8d/0xb0
(mlockall-test/6561): 9168us non-preemptible critical section violated 1100 us preempt threshold starting at get_user_pages+0x2cf/0x3b0 and ending at get_user_pages+0xa6/0x3b0
 [<c0106717>] dump_stack+0x17/0x20
 [<c0113e62>] touch_preempt_timing+0x32/0x40
 [<c013d216>] get_user_pages+0xa6/0x3b0
 [<c013ea28>] make_pages_present+0x68/0x90
 [<c013ee8d>] mlock_fixup+0x8d/0xb0
 [<c013f170>] do_mlockall+0x70/0x90
 [<c013f229>] sys_mlockall+0x99/0xa0
 [<c01060b7>] syscall_call+0x7/0xb

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Florian Schmidt]

On Mon, 9 Aug 2004 12:46:49 +0200
Ingo Molnar <mingo@elte.hu> wrote:

>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc3-O4
> 
> (it should apply to -rc2 cleanly too.)
> 
> -O4 fixes a two more latency sources:

Hi, i also wanted to report that i also still see these mysterious (at
least to me) "(swapper/0)" critical section reports (i use O4 with the
preempt-timing-on-2.6.8-rc3-O4.patch): These are basically the only
xruns/critical section reports i see when using jackd at 2*64 and with
some clients connected. I'm not sure if i see them when jackd is not
running.. Will test later this evening.

Flo

Aug 10 16:05:50 mango kernel: (swapper/0): 2463us non-preemptible
critical section violated 1000 us preempt threshold starting at
do_IRQ+0x19/0x170 and ending at do_IRQ+0x126/0x170
Aug 10 16:05:50 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 10 16:05:50 mango kernel:  [<c0114735>] sub_preempt_count+0x45/0x60
Aug 10 16:05:50 mango kernel:  [<c0107936>] do_IRQ+0x126/0x170
Aug 10 16:05:50 mango kernel:  [<c0106098>] common_interrupt+0x18/0x20
Aug 10 16:05:50 mango kernel:  [<c0103eb9>] cpu_idle+0x39/0x50
Aug 10 16:05:50 mango kernel:  [<c03148c1>] start_kernel+0x1a1/0x1f0
Aug 10 16:05:50 mango kernel:  [<c010019f>] 0xc010019f
Aug 10 16:05:50 mango kernel: ALSA sound/core/pcm_lib.c:169: XRUN:
pcmC0D0p
Aug 10 16:05:50 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 10 16:05:50 mango kernel:  [<f08c2431>]
snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
Aug 10 16:05:50 mango kernel:  [<f089ebce>]
snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
Aug 10 16:05:50 mango kernel:  [<c011b55b>]
generic_handle_IRQ_event+0x3b/0x70
Aug 10 16:05:50 mango kernel:  [<c01078c6>] do_IRQ+0xb6/0x170
Aug 10 16:05:50 mango kernel:  [<c0106098>] common_interrupt+0x18/0x20
Aug 10 16:07:11 mango kernel: ALSA sound/core/pcm_lib.c:169: XRUN:
pcmC0D0c
Aug 10 16:07:11 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 10 16:07:11 mango kernel:  [<f08c2431>]
snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
Aug 10 16:07:11 mango kernel:  [<f089ebce>]
snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
Aug 10 16:07:11 mango kernel:  [<c011b55b>]
generic_handle_IRQ_event+0x3b/0x70
Aug 10 16:07:11 mango kernel:  [<c01078c6>] do_IRQ+0xb6/0x170
Aug 10 16:07:11 mango kernel:  [<c0106098>] common_interrupt+0x18/0x20
Aug 10 16:07:11 mango kernel:  [<c0103eb9>] cpu_idle+0x39/0x50
Aug 10 16:07:11 mango kernel:  [<c03148c1>] start_kernel+0x1a1/0x1f0
Aug 10 16:07:11 mango kernel:  [<c010019f>] 0xc010019f
Aug 10 16:07:11 mango kernel: (swapper/0): 2462us non-preemptible
critical section violated 1000 us preempt threshold starting at
do_IRQ+0x19/0x170 and ending at do_IRQ+0x126/0x170
Aug 10 16:07:11 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 10 16:07:11 mango kernel:  [<c0114735>] sub_preempt_count+0x45/0x60
Aug 10 16:07:11 mango kernel:  [<c0107936>] do_IRQ+0x126/0x170
Aug 10 16:07:11 mango kernel:  [<c0106098>] common_interrupt+0x18/0x20
Aug 10 16:07:11 mango kernel:  [<c0103eb9>] cpu_idle+0x39/0x50
Aug 10 16:07:11 mango kernel:  [<c03148c1>] start_kernel+0x1a1/0x1f0
Aug 10 16:07:11 mango kernel:  [<c010019f>] 0xc010019f
Aug 10 16:07:11 mango kernel: ALSA sound/core/pcm_lib.c:169: XRUN:
pcmC0D0p
Aug 10 16:07:11 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 10 16:07:11 mango kernel:  [<f08c2431>]
snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
Aug 10 16:07:11 mango kernel:  [<f089ebce>]
snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
Aug 10 16:07:11 mango kernel:  [<c011b55b>]
generic_handle_IRQ_event+0x3b/0x70
Aug 10 16:07:11 mango kernel:  [<c01078c6>] do_IRQ+0xb6/0x170
Aug 10 16:07:11 mango kernel:  [<c0106098>] common_interrupt+0x18/0x20
Aug 10 16:09:52 mango kernel: ALSA sound/core/pcm_lib.c:169: XRUN:
pcmC0D0p
Aug 10 16:09:52 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 10 16:09:52 mango kernel:  [<f08c2431>]
snd_pcm_period_elapsed+0x311/0x480 [snd_pcm]
Aug 10 16:09:52 mango kernel:  [<f089ebce>]
snd_cs46xx_interrupt+0x1be/0x1f0 [snd_cs46xx]
Aug 10 16:09:52 mango kernel:  [<c011b55b>]
generic_handle_IRQ_event+0x3b/0x70
Aug 10 16:09:52 mango kernel:  [<c01078c6>] do_IRQ+0xb6/0x170
Aug 10 16:09:52 mango kernel:  [<c0106098>] common_interrupt+0x18/0x20
Aug 10 16:09:52 mango kernel:  [<c0103eb9>] cpu_idle+0x39/0x50
Aug 10 16:09:52 mango kernel:  [<c03148c1>] start_kernel+0x1a1/0x1f0
Aug 10 16:09:52 mango kernel:  [<c010019f>] 0xc010019f
Aug 10 16:09:52 mango kernel: (swapper/0): 2459us non-preemptible
critical section violated 1000 us preempt threshold starting at
do_IRQ+0x19/0x170 and ending at do_IRQ+0x126/0x170
Aug 10 16:09:52 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 10 16:09:52 mango kernel:  [<c0114735>] sub_preempt_count+0x45/0x60
Aug 10 16:09:52 mango kernel:  [<c0107936>] do_IRQ+0x126/0x170
Aug 10 16:09:52 mango kernel:  [<c0106098>] common_interrupt+0x18/0x20
Aug 10 16:09:52 mango kernel:  [<c0103eb9>] cpu_idle+0x39/0x50
Aug 10 16:09:52 mango kernel:  [<c03148c1>] start_kernel+0x1a1/0x1f0
Aug 10 16:09:52 mango kernel:  [<c010019f>] 0xc010019f


System info:

Linux mango.fruits.de 2.6.8-rc3 #2 Tue Aug 10 12:46:08 CEST 2004 i686
GNU/Linux
-----------
processor	: 0
vendor_id	: AuthenticAMD
cpu family	: 6
model		: 7
model name	: AMD Duron(tm) Processor
stepping	: 1
cpu MHz		: 1194.993
cache size	: 64 KB
fdiv_bug	: no
hlt_bug		: no
f00f_bug	: no
coma_bug	: no
fpu		: yes
fpu_exception	: yes
cpuid level	: 1
wp		: yes
flags		: fpu vme de pse tsc msr pae mce cx8 sep mtrr pge mca cmov pat
pse36 mmx fxsr sse syscall mp mmxext 3dnowext 3dnow
bogomips	: 2359.29

-----------
/proc/interrupts:
           CPU0       
  0:    8341883          XT-PIC  timer
  1:      11031          XT-PIC  i8042
  2:          0          XT-PIC  cascade
  5:   10546999          XT-PIC  CS46XX
  8:          4          XT-PIC  rtc
 10:      17097          XT-PIC  eth0
 12:     292978          XT-PIC  i8042
 14:       2675          XT-PIC  ide0
 15:      43327          XT-PIC  ide1
NMI:          0 
ERR:          0
-----------
/proc/irq/10/eth0/threaded:1
/proc/irq/12/i8042/threaded:1
/proc/irq/14/ide0/threaded:1
/proc/irq/15/ide1/threaded:1
/proc/irq/1/i8042/threaded:1
/proc/irq/5/CS46XX/threaded:0
/proc/irq/8/rtc/threaded:0
-----------
/sys/block/hda/queue/max_sectors_kb:16
/sys/block/hdd/queue/max_sectors_kb:16
-----------
voluntary_preemption
3
kernel_preemption
1
-----------
preempt_thresh
1000


-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Florian Schmidt]

On Tue, 10 Aug 2004 10:16:55 -0400
Lee Revell <rlrevell@joe-job.com> wrote:

> On Tue, 2004-08-10 at 03:53, Ingo Molnar wrote:
> > can you trigger similar latencies via the attached mlock testcode?
> > (written by Florian. Run it as root.)
> > 
> 
> Yup, using only 100000 bytes, I get a bunch of these:

Hi, just to make sure: Those weren't 100000 kbytes actually?

Flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 11:04, Florian Schmidt wrote:
> On Tue, 10 Aug 2004 10:16:55 -0400
> Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > On Tue, 2004-08-10 at 03:53, Ingo Molnar wrote:
> > > can you trigger similar latencies via the attached mlock testcode?
> > > (written by Florian. Run it as root.)
> > > 
> > 
> > Yup, using only 100000 bytes, I get a bunch of these:
> 
> Hi, just to make sure: Those weren't 100000 kbytes actually?
> 

Oops, 100000KB.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 08:40, Ingo Molnar wrote:

> it's a 10 msec latency that got measured, for a single page copy (!) -
> if it's memory latency then it must be something really bad...
> 

Here's another one that I have seen a few times.

(gnome-terminal/850): 15487us non-preemptible critical section violated 1100 us preempt threshold starting at add_wait_queue+0x15/0x50 and ending at add_wait_queue+0x2c/0x50
 [<c0106717>] dump_stack+0x17/0x20
 [<c0113eec>] dec_preempt_count+0x3c/0x50
 [<c011406c>] add_wait_queue+0x2c/0x50
 [<c01e334d>] normal_poll+0x3d/0x177
 [<c01df281>] tty_poll+0x61/0x80
 [<c015f0a1>] do_pollfd+0x91/0xa0
 [<c015f10f>] do_poll+0x5f/0xc0
 [<c015f2a1>] sys_poll+0x131/0x220
 [<c01060b7>] syscall_call+0x7/0xb
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c0106717>] dump_stack+0x17/0x20
 [<de93d64b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de9791d1>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c011a7d3>] generic_handle_IRQ_event+0x33/0x60
 [<c01079c2>] do_IRQ+0xb2/0x180
 [<c01062d8>] common_interrupt+0x18/0x20
 [<c015f0a1>] do_pollfd+0x91/0xa0
 [<c015f10f>] do_poll+0x5f/0xc0
 [<c015f2a1>] sys_poll+0x131/0x220
 [<c01060b7>] syscall_call+0x7/0xb

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Alan Cox]

On Maw, 2004-08-10 at 18:10, Lee Revell wrote:
> OK, with CONFIG_M586TSC, I am getting a lot of lockups.  A few happened
> during normal desktop use, and it locks up hard when starting jackd. 
> Could this have anything to do with the ALSA drivers (which I am
> compiling seperately from ALSA cvs) detecting my build system as i686? 
> I have read that the C3 is more like a 486 (with MMX & 3DNow) than a
> 686.

The C3 is a full 686 instruction set. The kernel is different because
the GNU tool people couldn't read manuals and once the error was made 
it was a bit too late to fix it.

Thus ALSA deciding its 686 is fine.

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 04:58, Ingo Molnar wrote:

> another idea: you are running this on a C3, using CONFIG_MCYRIXIII,
> correct? That is one of the rare configs that triggers X86_USE_3DNOW and
> MMX ops. If 3dnow is in any way handicapped in that CPU then that could
> cause trouble. Could you compile for e.g. CONFIG_M586TSC? [that option
> should be fully compatible with a C3.] - this will exclude the MMX page
> clearing ops.
> 

OK, with CONFIG_M586TSC, I am getting a lot of lockups.  A few happened
during normal desktop use, and it locks up hard when starting jackd. 
Could this have anything to do with the ALSA drivers (which I am
compiling seperately from ALSA cvs) detecting my build system as i686? 
I have read that the C3 is more like a 486 (with MMX & 3DNow) than a
686.

Lee

 

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> OK, with CONFIG_M586TSC, I am getting a lot of lockups.  A few
> happened during normal desktop use, and it locks up hard when starting
> jackd.  Could this have anything to do with the ALSA drivers (which I
> am compiling seperately from ALSA cvs) detecting my build system as
> i686?  I have read that the C3 is more like a 486 (with MMX & 3DNow)
> than a 686.

well, could you try M486 then?

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 13:12, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > OK, with CONFIG_M586TSC, I am getting a lot of lockups.  A few
> > happened during normal desktop use, and it locks up hard when starting
> > jackd.  Could this have anything to do with the ALSA drivers (which I
> > am compiling seperately from ALSA cvs) detecting my build system as
> > i686?  I have read that the C3 is more like a 486 (with MMX & 3DNow)
> > than a 686.
> 
> well, could you try M486 then?
> 

Sure, I want to try recompiling alsa-lib as well, there was a version
mismatch that may be the problem.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Dave Jones]

On Tue, Aug 10, 2004 at 01:10:42PM -0400, Lee Revell wrote:
 > On Tue, 2004-08-10 at 04:58, Ingo Molnar wrote:
 > 
 > > another idea: you are running this on a C3, using CONFIG_MCYRIXIII,
 > > correct? That is one of the rare configs that triggers X86_USE_3DNOW and
 > > MMX ops. If 3dnow is in any way handicapped in that CPU then that could
 > > cause trouble. Could you compile for e.g. CONFIG_M586TSC? [that option
 > > should be fully compatible with a C3.] - this will exclude the MMX page
 > > clearing ops.
 > > 
 > 
 > OK, with CONFIG_M586TSC, I am getting a lot of lockups.  A few happened
 > during normal desktop use, and it locks up hard when starting jackd. 
 > Could this have anything to do with the ALSA drivers (which I am
 > compiling seperately from ALSA cvs) detecting my build system as i686? 

If you have an early C3  (ie, pre Nehemiah model), then you will lack
the cmov instruction that gcc assumes is in 686's.  If the ALSA scripts
aren't aware of this, they will generate code which your CPU cannot run.

 > I have read that the C3 is more like a 486 (with MMX & 3DNow) than a
 > 686.

It's 686, just lacking various extensions. The current models have
all the same optional features you could find on a Pentium Pro,
plus a bunch of extra exclusive bits.

		Dave

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Dave Jones]

On Tue, Aug 10, 2004 at 05:25:48PM +0100, Alan Cox wrote:
 > On Maw, 2004-08-10 at 18:10, Lee Revell wrote:
 > > OK, with CONFIG_M586TSC, I am getting a lot of lockups.  A few happened
 > > during normal desktop use, and it locks up hard when starting jackd. 
 > > Could this have anything to do with the ALSA drivers (which I am
 > > compiling seperately from ALSA cvs) detecting my build system as i686? 
 > > I have read that the C3 is more like a 486 (with MMX & 3DNow) than a
 > > 686.
 > 
 > The C3 is a full 686 instruction set. The kernel is different because
 > the GNU tool people couldn't read manuals and once the error was made 
 > it was a bit too late to fix it.
 > 
 > Thus ALSA deciding its 686 is fine.

Depends which C3.  If OP's C3 lacks cmov, it definitly is not ok.
Any cmov ending up in that module will blow up.

		Dave

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 13:33, Dave Jones wrote:
> On Tue, Aug 10, 2004 at 01:10:42PM -0400, Lee Revell wrote:
> If you have an early C3  (ie, pre Nehemiah model), then you will lack
> the cmov instruction that gcc assumes is in 686's.  If the ALSA scripts
> aren't aware of this, they will generate code which your CPU cannot run.
> 

This is what I have, the first of their fanless models I believe, 600Mhz
C3.

However ALSA always detected the CPU as a 686 and I never had problems,
it's definitely the change from CONFIG_MCYRIXIII to CONFIG_M586TSC that
started the lockups.

I recompiled all the ALSA libs and jackd, same problem - starting jackd
causes immediate lockup.  Now building with M486.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 03:51, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Ingo, do you plan to maintain the voluntary preempt patch against the
> > -mm series?  From looking at Andrew's announcement yesterday it looks
> > like many latency issues fixed in the voluntary preemption patches are
> > also fixed in -mm, so it seems like the patch would be much smaller. 
> 
> yeah, and in addition we've already pushed 99% of our might_sleep()
> additions to -mm too so that reduces the patch size too, quite
> significantly.
> 
> time is the only limiting factor. Due to these partial merges (we are
> trying to get all uncontroversial bits into -mm, hence into upstream)
> the merge to -mm is hard. Especially for lock-breaks that i've done
> differently than Andrew. I sent a consolidation patch yesterday but this
> is still work in progress. So i'll do an -mm merge very time i get to do
> it, but the primary testing still remains on the vanilla kernel (which
> most people use).
> 

Rather than having to maintain the voluntary preempt patch for the -mm
series, after the next -mm merge, maybe you could just post or send me
an incremental diff against the last voluntary preempt patch for the
vanilla kernel when you update it, and I or someone else from the Linux
audio community could maintain the patch against the -mm series.  It
seems that lately the changes from one version of the patch to the next
are small and easy enough to comprehend.  We could also filter the bug
reports a bit.  Maybe a voluntary preemption mailing list is in order,
if this gets to be a constant source of huge log postings to LKML.

This would also improve the ability of the Linux audio community to
influence the direction of the kernel, by separately analyzing the
impact of different changes, especially if the -mm series is going to
function as 2.7 for the time being.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Peter Zijlstra]

[-- Attachment #1: Type: text/plain, Size: 870 bytes --]

On Tue, 2004-08-10 at 15:26 +0200, Ingo Molnar wrote:
> i've uploaded the latest version of the voluntary-preempt patch:
>     
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc3-O5
> 
> -O5 fixes the APIC lockup issues. The bug was primarily caused by PCI
> POST delays causing IRQ storms of level-triggered IRQ sources that were
> hardirq-redirected. Also found some bugs in delayed-IRQ masking and
> unmasking. SMP should thus work again too.
> 
> 	Ingo
Hi Ingo,

I've rediffed O5 agains 2.6.8-rc4-mm1-staircase. It compiles and boots
for me when using voluntary-preempt=1; 2 and 3 lock up the system right
after the line: 'Probing IDE interface ide1...' (dmesg & .config
included).

staircase patch used: 
  http://ck.kolivas.org/patches/2.6/2.6.8/2.6.8-rc4-mm1/from_2.6.8-rc4-
mm1_to_staircase7.I

Thanks for all the effort,

Peter Zijlstra

[-- Attachment #2: voluntary-preempt-2.6.8-rc4-mm1-O5.patch.gz --]
[-- Type: application/x-gzip, Size: 19704 bytes --]

[-- Attachment #3: dmesg-2.6.8-rc4-mm1-staircase-O5.gz --]
[-- Type: application/x-gzip, Size: 5366 bytes --]

[-- Attachment #4: config-2.6.8-rc4-mm1-staircase-O5.gz --]
[-- Type: application/x-gzip, Size: 7692 bytes --]

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 13:12, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > OK, with CONFIG_M586TSC, I am getting a lot of lockups.  A few
> > happened during normal desktop use, and it locks up hard when starting
> > jackd.  Could this have anything to do with the ALSA drivers (which I
> > am compiling seperately from ALSA cvs) detecting my build system as
> > i686?  I have read that the C3 is more like a 486 (with MMX & 3DNow)
> > than a 686.
> 
> well, could you try M486 then?
> 

Still does not work.  Starting jackd hangs the system.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 13:37, Dave Jones wrote:

> Depends which C3.  If OP's C3 lacks cmov, it definitly is not ok.
> Any cmov ending up in that module will blow up.
> 

Argh, never mind, this was an ALSA bug, reverting to CVS from two days
ago fixed it.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Tue, 2004-08-10 at 09:26, Ingo Molnar wrote:
> i've uploaded the latest version of the voluntary-preempt patch:
>     
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc3-O5
> 
> -O5 fixes the APIC lockup issues. The bug was primarily caused by PCI
> POST delays causing IRQ storms of level-triggered IRQ sources that were
> hardirq-redirected. Also found some bugs in delayed-IRQ masking and
> unmasking. SMP should thus work again too.
> 

The mlockall() issue seems to be fixed.  Now I get this one when
starting jackd:

(jackd/12427): 10882us non-preemptible critical section violated 400 us preempt threshold starting at kernel_fpu_begin+0x10/0x60 and ending at fast_clear_page+0x75/0xa0
 [<c0106777>] dump_stack+0x17/0x20
 [<c01140eb>] sub_preempt_count+0x4b/0x60
 [<c01d1585>] fast_clear_page+0x75/0xa0
 [<c013ea86>] do_anonymous_page+0x86/0x180
 [<c013ebd0>] do_no_page+0x50/0x300
 [<c013f041>] handle_mm_fault+0xc1/0x170
 [<c013da33>] get_user_pages+0x133/0x3d0
 [<c013f198>] make_pages_present+0x68/0x90
 [<c0140948>] do_mmap_pgoff+0x3f8/0x640
 [<c010b7f6>] sys_mmap2+0x76/0xb0
 [<c0106117>] syscall_call+0x7/0xb

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Tue, 2004-08-10 at 09:26, Ingo Molnar wrote:
> i've uploaded the latest version of the voluntary-preempt patch:
>     
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc3-O5
> 
> -O5 fixes the APIC lockup issues. The bug was primarily caused by PCI
> POST delays causing IRQ storms of level-triggered IRQ sources that were
> hardirq-redirected. Also found some bugs in delayed-IRQ masking and
> unmasking. SMP should thus work again too.
> 

Correction, mlockall-test *does* cause xruns.  Here is the trace:

(mlockall-test/12466): 10303us non-preemptible critical section violated 400 us preempt threshold starting at get_user_pages+0xa5/0x3d0 and ending at get_user_pages+0x2e7/0x3d0
 [<c0106777>] dump_stack+0x17/0x20
 [<c01140eb>] sub_preempt_count+0x4b/0x60
 [<c013dbe7>] get_user_pages+0x2e7/0x3d0
 [<c013f198>] make_pages_present+0x68/0x90
 [<c013f5ed>] mlock_fixup+0x8d/0xb0
 [<c013f8d0>] do_mlockall+0x70/0x90
 [<c013f989>] sys_mlockall+0x99/0xa0
 [<c0106117>] syscall_call+0x7/0xb
ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D2c
 [<c0106777>] dump_stack+0x17/0x20
 [<de92f79b>] snd_pcm_period_elapsed+0x28b/0x3f0 [snd_pcm]
 [<de96c1d1>] snd_emu10k1_interrupt+0xd1/0x3c0 [snd_emu10k1]
 [<c011ac63>] generic_handle_IRQ_event+0x33/0x60
 [<c0107a27>] do_IRQ+0xb7/0x190
 [<c0106338>] common_interrupt+0x18/0x20
 [<c0114008>] __touch_preempt_timing+0x8/0x20
 [<c013dbf4>] get_user_pages+0x2f4/0x3d0
 [<c013f198>] make_pages_present+0x68/0x90
 [<c013f5ed>] mlock_fixup+0x8d/0xb0
 [<c013f8d0>] do_mlockall+0x70/0x90
 [<c013f989>] sys_mlockall+0x99/0xa0
 [<c0106117>] syscall_call+0x7/0xb

Here is a different one:

ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139: XRUN: pcmC0D0p
 [<c0106777>] dump_stack+0x17/0x20
 [<de92f79b>] snd_pcm_period_elapsed+0x28b/0x3f0 [snd_pcm]
 [<de96c437>] snd_emu10k1_interrupt+0x337/0x3c0 [snd_emu10k1]
 [<c011ac63>] generic_handle_IRQ_event+0x33/0x60
 [<c0107a27>] do_IRQ+0xb7/0x190
 [<c0106338>] common_interrupt+0x18/0x20
 [<c0266777>] schedule+0x2d7/0x5b0
 [<c0266fa7>] schedule_timeout+0x57/0xa0
 [<c015fef1>] do_poll+0xa1/0xc0
 [<c0160041>] sys_poll+0x131/0x220
 [<c0106117>] syscall_call+0x7/0xb
(jackd/12430): 17135us non-preemptible critical section violated 400 us preempt threshold starting at schedule+0x57/0x5b0 and ending at schedule+0x2ef/0x5b0
 [<c0106777>] dump_stack+0x17/0x20
 [<c01140eb>] sub_preempt_count+0x4b/0x60
 [<c026678f>] schedule+0x2ef/0x5b0
 [<c0266fa7>] schedule_timeout+0x57/0xa0
 [<c015fef1>] do_poll+0xa1/0xc0
 [<c0160041>] sys_poll+0x131/0x220
 [<c0106117>] syscall_call+0x7/0xb

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Lee Revell]

On Tue, 2004-08-10 at 13:12, Ingo Molnar wrote:

> well, could you try M486 then?
> 

Results with CONFIG_M586TSC are basically the same as CONFIG_MCYRIXIII. 
I have posted my latest results here:

http://www.members.dca.net/rlrevell/testresults/testresults.html

There are graphs (linear and logarithmic scale) for each of 5 trials,
followed by all the XRUN traces and preempt violations I got around the
time of the test.

iozone -a was running also.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Florian Schmidt]

On Tue, 10 Aug 2004 17:56:00 -0400
Lee Revell <rlrevell@joe-job.com> wrote:

> The mlockall() issue seems to be fixed.  Now I get this one when
> starting jackd:
> 
> (jackd/12427): 10882us non-preemptible critical section violated 400
> us preempt threshold starting at kernel_fpu_begin+0x10/0x60 and ending
> at fast_clear_page+0x75/0xa0
>  [<c0106777>] dump_stack+0x17/0x20
>  [<c01140eb>] sub_preempt_count+0x4b/0x60
>  [<c01d1585>] fast_clear_page+0x75/0xa0
>  [<c013ea86>] do_anonymous_page+0x86/0x180
>  [<c013ebd0>] do_no_page+0x50/0x300
>  [<c013f041>] handle_mm_fault+0xc1/0x170
>  [<c013da33>] get_user_pages+0x133/0x3d0
>  [<c013f198>] make_pages_present+0x68/0x90
>  [<c0140948>] do_mmap_pgoff+0x3f8/0x640
>  [<c010b7f6>] sys_mmap2+0x76/0xb0
>  [<c0106117>] syscall_call+0x7/0xb

I see these, too [at jackd startup and at client startup]:

Aug 11 01:15:21 mango kernel: (jackd/843): 3215us non-preemptible
critical secti
on violated 1000 us preempt threshold starting at
kernel_fpu_begin+0x1a/0x60 and
 ending at fast_clear_page+0x54/0x80
Aug 11 01:15:21 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 11 01:15:21 mango kernel:  [<c0114735>] sub_preempt_count+0x45/0x60
Aug 11 01:15:21 mango kernel:  [<c01daa94>] fast_clear_page+0x54/0x80
Aug 11 01:15:21 mango kernel:  [<c0140150>] do_anonymous_page+0x90/0x190
Aug 11 01:15:21 mango kernel:  [<c01402b1>] do_no_page+0x61/0x320
Aug 11 01:15:21 mango kernel:  [<c0140763>] handle_mm_fault+0xe3/0x1a0
Aug 11 01:15:21 mango kernel:  [<c013f0e7>] get_user_pages+0x147/0x3d0
Aug 11 01:15:21 mango kernel:  [<c01408fd>] make_pages_present+0x8d/0xb0
Aug 11 01:15:21 mango kernel:  [<c0142255>] do_mmap_pgoff+0x445/0x6b0
Aug 11 01:15:21 mango kernel:  [<c010b800>] old_mmap+0xe0/0x120
Aug 11 01:15:21 mango kernel:  [<c0105f2b>] syscall_call+0x7/0xb
Aug 11 01:15:21 mango kernel: ALSA sound/core/pcm_lib.c:169: XRUN:
pcmC0D0p
Aug 11 01:15:21 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 11 01:15:21 mango kernel:  [<f08c2431>]
snd_pcm_period_elapsed+0x311/0x480 [
snd_pcm]
Aug 11 01:15:21 mango kernel:  [<f089ebce>]
snd_cs46xx_interrupt+0x1be/0x1f0 [sn
d_cs46xx]
Aug 11 01:15:21 mango kernel:  [<c011b55b>]
generic_handle_IRQ_event+0x3b/0x70
Aug 11 01:15:21 mango kernel:  [<c01078c6>] do_IRQ+0xb6/0x170
Aug 11 01:15:21 mango kernel:  [<c0106098>] common_interrupt+0x18/0x20
Aug 11 01:15:40 mango gconfd (tapas-803): GConf server is not in use,
shutting d
own.
Aug 11 01:15:40 mango gconfd (tapas-803): Exiting
Aug 11 01:16:01 mango kernel: ALSA sound/core/pcm_lib.c:169: XRUN:
pcmC0D0p
Aug 11 01:16:01 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 11 01:16:01 mango kernel:  [<f08c2431>]
snd_pcm_period_elapsed+0x311/0x480 [
snd_pcm]
Aug 11 01:16:01 mango kernel:  [<f089ebce>]
snd_cs46xx_interrupt+0x1be/0x1f0 [sn
d_cs46xx]
Aug 11 01:16:01 mango kernel:  [<c011b55b>]
generic_handle_IRQ_event+0x3b/0x70
Aug 11 01:16:01 mango kernel:  [<c01078c6>] do_IRQ+0xb6/0x170
Aug 11 01:16:01 mango kernel:  [<c0106098>] common_interrupt+0x18/0x20
Aug 11 01:16:01 mango kernel:  [<c0140150>] do_anonymous_page+0x90/0x190
Aug 11 01:16:01 mango kernel:  [<c01402b1>] do_no_page+0x61/0x320
Aug 11 01:16:01 mango kernel:  [<c0140763>] handle_mm_fault+0xe3/0x1a0
Aug 11 01:16:01 mango kernel:  [<c013f0e7>] get_user_pages+0x147/0x3d0
Aug 11 01:16:01 mango kernel:  [<c01408fd>] make_pages_present+0x8d/0xb0
Aug 11 01:16:01 mango kernel:  [<c0142255>] do_mmap_pgoff+0x445/0x6b0
Aug 11 01:16:01 mango kernel:  [<c010b800>] old_mmap+0xe0/0x120
Aug 11 01:16:01 mango kernel:  [<c0105f2b>] syscall_call+0x7/0xb
Aug 11 01:16:01 mango kernel: (scsynth/870): 3230us non-preemptible
critical sec
tion violated 1000 us preempt threshold starting at
kernel_fpu_begin+0x1a/0x60 a
nd ending at fast_clear_page+0x54/0x80
Aug 11 01:16:01 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 11 01:16:01 mango kernel:  [<c0114735>] sub_preempt_count+0x45/0x60
Aug 11 01:16:01 mango kernel:  [<c01daa94>] fast_clear_page+0x54/0x80
Aug 11 01:16:01 mango kernel:  [<c0140150>] do_anonymous_page+0x90/0x190
Aug 11 01:16:01 mango kernel:  [<c01402b1>] do_no_page+0x61/0x320
Aug 11 01:16:01 mango kernel:  [<c0140763>] handle_mm_fault+0xe3/0x1a0
Aug 11 01:16:01 mango kernel:  [<c013f0e7>] get_user_pages+0x147/0x3d0
Aug 11 01:16:01 mango kernel:  [<c01408fd>] make_pages_present+0x8d/0xb0
Aug 11 01:16:01 mango kernel:  [<c0142255>] do_mmap_pgoff+0x445/0x6b0
Aug 11 01:16:01 mango kernel:  [<c010b800>] old_mmap+0xe0/0x120
Aug 11 01:16:01 mango kernel:  [<c0105f2b>] syscall_call+0x7/0xb

Flo


-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Tim Wright]

On Tue, 2004-08-10 at 10:37, Dave Jones wrote:
> On Tue, Aug 10, 2004 at 05:25:48PM +0100, Alan Cox wrote:
>  > On Maw, 2004-08-10 at 18:10, Lee Revell wrote:
>  > > OK, with CONFIG_M586TSC, I am getting a lot of lockups.  A few happened
>  > > during normal desktop use, and it locks up hard when starting jackd. 
>  > > Could this have anything to do with the ALSA drivers (which I am
>  > > compiling seperately from ALSA cvs) detecting my build system as i686? 
>  > > I have read that the C3 is more like a 486 (with MMX & 3DNow) than a
>  > > 686.
>  > 
>  > The C3 is a full 686 instruction set. The kernel is different because
>  > the GNU tool people couldn't read manuals and once the error was made 
>  > it was a bit too late to fix it.
>  > 
>  > Thus ALSA deciding its 686 is fine.
> 
> Depends which C3.  If OP's C3 lacks cmov, it definitly is not ok.
> Any cmov ending up in that module will blow up.
> 

Not that it helps at all, but the conditional move instruction was
marked as optional by Intel, hence the 'cmov' capability flag (in the
flags part of /proc/cpuinfo). As you say, the earlier C3 chips (like the
one in my firewall) lack support for this instruction:

model name      : VIA Samuel 2
cpu MHz         : 533.365
flags           : fpu de tsc msr cx8 mtrr pge mmx 3dnow

So, strictly speaking, you shouldn't use cmov for something that's
simply "i686" compatible. Of course, gcc etc. uses it because it's very
nice from the point of view of avoiding having to do branches and the
associated malarky and someone failed to spot that it was optional. So,
yes, it's not OK to use -march=i686 on these chips. Modern gccs know
about -march=c3, which ought to be a good bet for these chips (one would
hope).

Tim

-- 
Tim Wright <timw@splhi.com>
Splhi

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> (jackd/12427): 10882us non-preemptible critical section violated 400
> us preempt threshold starting at kernel_fpu_begin+0x10/0x60 and ending
> at fast_clear_page+0x75/0xa0

to make sure this is a real latency and not some rdtsc weirdness, could
you try the latest version of preempt-timing:

 http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc3-O5-A2

this adds jiffies-based latency values to the printout, e.g.:

  (ksoftirqd/0/2): 3860us [3 jiffy] non-preemptible critical section
  violated 100 us preempt threshold starting at ___do_softirq+0x1b/0x90
  and ending at cond_resched_softirq+0x57/0x70

shows that a 10 jiffy (10 msec) latency happened - which matches the
rdtsc-based 3860 usecs value.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> > (jackd/12427): 10882us non-preemptible critical section violated 400
> > us preempt threshold starting at kernel_fpu_begin+0x10/0x60 and ending
> > at fast_clear_page+0x75/0xa0
> 
> to make sure this is a real latency and not some rdtsc weirdness, could
> you try the latest version of preempt-timing:
> 
>  http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc3-O5-A2
> 
> this adds jiffies-based latency values to the printout, e.g.:

could you also apply the patch below? This splits up fast_clear_page()
overhead into 3 pieces to find out where the overhead comes from. Note
that this will likely cause 'reduced' latency printouts, but those are
not real because the touch_preempt_timing() lines dont show real
preemptible points. You should still get a larger latency in one of
those regions. The kernel-FPU code (which is triggered by the MMX ops)
could be another source of overhead - but 10 msecs is so large ...

(another thing: could you turn on CONFIG_DEBUG_HIGHMEM,
CONFIG_DEBUG_SPINLOCK and CONFIG_DEBUG_SPINLOCK_SLEEP? Lets make sure
that what we are seeing here is not somehow caused by atomicity
violations.)

	Ingo

--- linux/arch/i386/lib/mmx.c.orig
+++ linux/arch/i386/lib/mmx.c
@@ -132,7 +132,9 @@ static void fast_clear_page(void *page)
 {
 	int i;
 
+	touch_preempt_timing();	
 	kernel_fpu_begin();
+	touch_preempt_timing();	
 	
 	__asm__ __volatile__ (
 		"  pxor %%mm0, %%mm0\n" : :
@@ -158,7 +160,9 @@ static void fast_clear_page(void *page)
 	__asm__ __volatile__ (
 		"  sfence \n" : :
 	);
+	touch_preempt_timing();	
 	kernel_fpu_end();
+	touch_preempt_timing();	
 }
 
 static void fast_copy_page(void *to, void *from)
@@ -260,8 +264,10 @@ static void fast_copy_page(void *to, voi
 static void fast_clear_page(void *page)
 {
 	int i;
-	
+
+	touch_preempt_timing();	
 	kernel_fpu_begin();
+	touch_preempt_timing();	
 	
 	__asm__ __volatile__ (
 		"  pxor %%mm0, %%mm0\n" : :
@@ -290,7 +296,9 @@ static void fast_clear_page(void *page)
 		page+=128;
 	}
 
+	touch_preempt_timing();	
 	kernel_fpu_end();
+	touch_preempt_timing();	
 }
 
 static void fast_copy_page(void *to, void *from)

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Wed, 2004-08-11 at 03:42, Ingo Molnar wrote:

> (another thing: could you turn on CONFIG_DEBUG_HIGHMEM,
> CONFIG_DEBUG_SPINLOCK and CONFIG_DEBUG_SPINLOCK_SLEEP? Lets make sure
> that what we are seeing here is not somehow caused by atomicity
> violations.)

I have highmem disabled per your previous request.  Should I turn it
back on?

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > (another thing: could you turn on CONFIG_DEBUG_HIGHMEM,
> > CONFIG_DEBUG_SPINLOCK and CONFIG_DEBUG_SPINLOCK_SLEEP? Lets make sure
> > that what we are seeing here is not somehow caused by atomicity
> > violations.)
> 
> I have highmem disabled per your previous request.  Should I turn it
> back on?

nope, please keep it disabled still. (You can turn the C3 CPU option
back on though, it doesnt seem to have any impact on latency.)

then it's just the spinlock debugging options that should be enabled. 
(they make sense on UP kernels too, if CONFIG_PREEMPT is enabled.)

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Ingo Molnar]

another thing: do you see the 10 msec latency every time you run
mlockall-test, or does it only happen sporadically?

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Ingo Molnar]

[-- Attachment #1: Type: text/plain, Size: 1882 bytes --]


most of the remaining latencies look quite suspect. E.g. the 
select()/tty_poll() ones:

  (gnome-terminal/826): 15491us non-preemptible critical section 
   violated 1100 us preempt threshold starting at
   add_wait_queue+0x15/0x50 and ending at add_wait_queue+0x2c/0x50

  [dump_stack+23/32] dump_stack+0x17/0x20
  [dec_preempt_count+60/80] dec_preempt_count+0x3c/0x50
  [add_wait_queue+44/80] add_wait_queue+0x2c/0x50
  [normal_poll+61/375] normal_poll+0x3d/0x177
  [tty_poll+97/128] tty_poll+0x61/0x80
  [do_pollfd+145/160] do_pollfd+0x91/0xa0
  [do_poll+95/192] do_poll+0x5f/0xc0
  [sys_poll+305/544] sys_poll+0x131/0x220
  [syscall_call+7/11] syscall_call+0x7/0xb

according to the trace this latency happened in a point where it's near
impossible to happen. add_wait_queue() is just a couple of straight
instructions on UP.

do you have any powersaving mode enabled in the BIOS? SMM handlers can
introduce such latencies (low probability).

the only other possibility is either a measurement error, or some mystic
IRQ overhead. But almost all IRQs are redirected so the IRQ overhead can
be eliminated almost completely. Plus direct-IRQ overhead should also
show up via the latest preempt-timing patch. Wrt. measurement error, the
jiffies based printout ought to help somewhat.

i'm currently running a loop of mlockall-test 100MB on a 466 MHz
Celeron, and not a single blip on the radar with a 1000 usecs threshold,
after 1 hour of runtime ...

i've previously seen RDTSC (cycle-counter) weirdnesses on another box,
in userspace. To exclude this possibility i've attached yet another
patch, it tries to make all the kernel rdtsc variants more robust. Does
this patch make any difference to the latency printouts? [this patch
doesnt handle cases where the rdtsc value jumps forward in time
permanently, but it handles the occasional blips i've seen on the other
box.]

	Ingo

[-- Attachment #2: rdtsc-robust-2.6.8-rc3-A0 --]
[-- Type: text/plain, Size: 1427 bytes --]

--- linux/include/asm-i386/msr.h.orig3	
+++ linux/include/asm-i386/msr.h	
@@ -32,15 +32,50 @@ static inline void wrmsrl (unsigned long
 	wrmsr (msr, lo, hi);
 }
 
-#define rdtsc(low,high) \
+#define __rdtsc(low,high) \
      __asm__ __volatile__("rdtsc" : "=a" (low), "=d" (high))
 
-#define rdtscl(low) \
+#define rdtsc(low,high) do {						\
+	unsigned int __low1, __high1, __low2, __high2;			\
+	for (;;) {							\
+		__rdtsc(__low1,__high1);				\
+		__rdtsc(__low2,__high2);				\
+		if (__high1 == __high2 && __low2 - __low1 < 1000)	\
+			break;						\
+	}								\
+	low = __low2;							\
+	high = __high2;							\
+} while (0)
+
+#define __rdtscl(low) \
      __asm__ __volatile__("rdtsc" : "=a" (low) : : "edx")
 
-#define rdtscll(val) \
+#define rdtscl(low) do {						\
+	unsigned int __low1, __low2;					\
+	for (;;) {							\
+		__rdtscl(__low1);					\
+		__rdtscl(__low2);					\
+		if (__low2 - __low1 < 1000)				\
+			break;						\
+	}								\
+	low = __low2;							\
+} while (0)
+
+#define __rdtscll(val) \
      __asm__ __volatile__("rdtsc" : "=A" (val))
 
+#define rdtscll(val) do {						\
+	unsigned long long __val1, __val2;				\
+	for (;;) {							\
+		__rdtscll(__val1);					\
+		__rdtscll(__val2);					\
+		if (__val2 - __val1 < 1000ULL)				\
+			break;						\
+	}								\
+	val = __val2;							\
+} while (0)
+
+
 #define write_tsc(val1,val2) wrmsr(0x10, val1, val2)
 
 #define rdpmc(counter,low,high) \

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Linh Dang]

Hi,

I'm not running the voluntary-preempt-* patches. but I do see some
long latencies with:

        vanilla 2.6.7+preempt-timing+defer-softirq

which were NOT reported here. Is it useful the report them?

Regards
-- 
Linh Dang

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Florian Schmidt]

On Wed, 11 Aug 2004 11:06:39 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> i'm currently running a loop of mlockall-test 100MB on a 466 MHz
> Celeron, and not a single blip on the radar with a 1000 usecs
> threshold, after 1 hour of runtime ...

I suppose you're not using jackd. As i have noticed that these critical
sections only get reported when jackd is running. It seems jackd is
producing a certain kind of load which exposes them..

Flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Ingo Molnar]

* Florian Schmidt <mista.tapas@gmx.net> wrote:

> > i'm currently running a loop of mlockall-test 100MB on a 466 MHz
> > Celeron, and not a single blip on the radar with a 1000 usecs
> > threshold, after 1 hour of runtime ...
> 
> I suppose you're not using jackd. As i have noticed that these
> critical sections only get reported when jackd is running. It seems
> jackd is producing a certain kind of load which exposes them..

so you can only trigger the latencies via mlockall-test if jackd is also 
running? Or do the latencies only trigger in jackd (and related 
programs)?

if the later, then i'm wondering whether any of the audio code turns off
caching for specific pages or does DMA to user pages, or mmap()s device
(PCI) memory?

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Florian Schmidt]

On Wed, 11 Aug 2004 14:43:42 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> 
> * Florian Schmidt <mista.tapas@gmx.net> wrote:
> 
> > > i'm currently running a loop of mlockall-test 100MB on a 466 MHz
> > > Celeron, and not a single blip on the radar with a 1000 usecs
> > > threshold, after 1 hour of runtime ...
> > 
> > I suppose you're not using jackd. As i have noticed that these
> > critical sections only get reported when jackd is running. It seems
> > jackd is producing a certain kind of load which exposes them..
> 
> so you can only trigger the latencies via mlockall-test if jackd is
> also running? 

Yes. I can happily mlockall 500 megs of ram when jackd is not running
and i do not get any preempt-timing reports.. As soon as jackd is
running even mlockall'ing only 20megs triggers a preempt-timing report
and also a alsa xrun report. Hmm, let me try turning off xrun_debug
traces while leaving preempt timing reports enabled... 

Ok, when xrun_debug traces are disabled [but preempt timing is on].
mlocking 20 megs of memory triggers an xrun [now only reported by jackd
in it's stdout, not in syslog] but not a preempt timing report. Lee, can
you verify? 

So it seems that the xrun_debug reports from the alsa interface jackd is
using triggers the preempt timing reports. So these are really
suspicious...

So i'm asking myself: How can jackd experience an xrun, when there's no
preempt-timing report showing a kernel latency. Jackd is running
SCHED_FIFO, so the mlockall_test program should not take away the cpu
from jackd. Is mlockall() special in another way? Sorry, i know too
little about the kernel internals to ask the right question. Maybe the
alsa driver is reacting to the mlockall_test..

Another piece of info which might be valuable (i cannot judge this): I
use jackd and mlockall not as root, but as normal user utilizing the
realtime lsm which allows non root users to mlock, mlockall  and change
the scheduling class (via sched_setsched()?). I'll try runnign jackd and
the mlockall_test as root to see if the results differ..

> Or do the latencies only trigger in jackd (and related 
> programs)?

Sorry, i don't understand that question..

> 
> if the later, then i'm wondering whether any of the audio code turns
> off caching for specific pages or does DMA to user pages, or mmap()s
> device(PCI) memory?

Flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Florian Schmidt]

On Wed, 11 Aug 2004 14:43:42 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> so you can only trigger the latencies via mlockall-test if jackd is
> also running? Or do the latencies only trigger in jackd (and related 
> programs)?

Hi,

i also want to say that some of the sprocadic xruns i see do also not
seem to be cause by latencies. The preempt timing reports i saw where
rather results of the xrun_debug reports i think..

How do i come to this conclusion? Well i have jack running w/o ALSA
xrun_debug reports but with preempt timing running for quite a while and
i see about the same number of xruns as before, but i don't see any
preempt timing reports anymore...

So i think jackd itself is doing some weird stuff. Or the driver of my
soundcard. 

Flo


-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Wed, 2004-08-11 at 08:43, Ingo Molnar wrote:
> * Florian Schmidt <mista.tapas@gmx.net> wrote:
> 
> > > i'm currently running a loop of mlockall-test 100MB on a 466 MHz
> > > Celeron, and not a single blip on the radar with a 1000 usecs
> > > threshold, after 1 hour of runtime ...
> > 
> > I suppose you're not using jackd. As i have noticed that these
> > critical sections only get reported when jackd is running. It seems
> > jackd is producing a certain kind of load which exposes them..
> 
> so you can only trigger the latencies via mlockall-test if jackd is also 
> running? Or do the latencies only trigger in jackd (and related 
> programs)?
> 
> if the later, then i'm wondering whether any of the audio code turns off
> caching for specific pages or does DMA to user pages, or mmap()s device
> (PCI) memory?
> 

I believe that jackd may do all of these.  I am adding Paul Davis to the
cc: list as he would know better.

Whatever is going on, it only happens at jackd startup.  Jackd does not
report an xrun because the developers added code not to report an xrun
within the first 64 frames, so this message would not confuse users.

There is definitely some subtle bug in the preempt-timing patch, because
I am getting reports of long non-preemptible sections, which do not
correspond to an xrun in jackd - if these were real then even a 400usec
non-preemptible section would cause an xrun.  I do not seem to get many
xruns during normal jackd operation.

If, during initialization, jackd called some function that could sleep
*after* starting the PCM, for example if it tried to allocate memory in
the same thread as the audio, this would cause an xrun, because the
soundcard interrupt would occur, but jackd would not be woken up because
it is sleeping on some other resource, correct?  Then, when jackd
eventually woke up, it would see that a lot of time had passed, and
report an xrun.  This would look the same as if jackd had been ready to
run and had not been scheduled in time.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Wed, 2004-08-11 at 07:48, Linh Dang wrote:
> Hi,
> 
> I'm not running the voluntary-preempt-* patches. but I do see some
> long latencies with:
> 
>         vanilla 2.6.7+preempt-timing+defer-softirq
> 
> which were NOT reported here. Is it useful the report them?
> 

Probably not.  Many latency issues as well as bugs in the preempt timing
patch have been fixed since then.  You should try the latest version.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Linh Dang]

Lee Revell <rlrevell@joe-job.com> wrote:

> On Wed, 2004-08-11 at 07:48, Linh Dang wrote:
>> Hi,
>>
>> I'm not running the voluntary-preempt-* patches. but I do see some
>> long latencies with:
>>
>> vanilla 2.6.7+preempt-timing+defer-softirq
>>
>> which were NOT reported here. Is it useful the report them?
>>
>
> Probably not.  Many latency issues as well as bugs in the preempt
> timing patch have been fixed since then.  You should try the latest
> version.

which "latest"? Linus's 2.6.8-rcX or Andrew's -mm's or Ingo's patches.

I've looked at Ingo patches but didn't see fixes for the followings:

1. In sys_mount(): do_mount() is called with the BKL held. on jffs2
   system, jffs2 might for a big media-scan and lock preemption for
   several msecs. even if jffs2_scan_eraseblock() calls cond_resched()
   for every flash sector, scanning one sector is still very long.

2. netif_receive_skb(): the rcu_read_lock() is too broad. IMHO, it's
   only needed around the 2 list_for_each_entry_rcu()s. There's no
   reason why rcu_read_lock() is needed when calling ip_recv on the
   skb.

3. with Ingo's patches, if all softirqs are done by the daemon then
   there's should be NO need to call local_bh_disable()/enable()
   around the processing loop in ___do_softirq().

4. in cfi_cmdset_0002.c, using spin_lock_bh() to guard struct flchip
   seems to be an overkill. why a semaphore is NOT sufficient?


I'm a total newbie so it's possible that I'm totally wrong about the
aboves.


Regards

-- 
Linh Dang

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> There is definitely some subtle bug in the preempt-timing patch,
> because I am getting reports of long non-preemptible sections, which
> do not correspond to an xrun in jackd - if these were real then even a
> 400usec non-preemptible section would cause an xrun.  I do not seem to
> get many xruns during normal jackd operation.

could you send me these latest preempt-timing warnings?

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Thu, 2004-08-12 at 03:21, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > There is definitely some subtle bug in the preempt-timing patch,
> > because I am getting reports of long non-preemptible sections, which
> > do not correspond to an xrun in jackd - if these were real then even a
> > 400usec non-preemptible section would cause an xrun.  I do not seem to
> > get many xruns during normal jackd operation.
> 
> could you send me these latest preempt-timing warnings?
> 

Here are all the XRUN traces and preempt-timing warnings for the last
~24 hours.  I still have not had a chance to test with the latest
patches & config changes.

http://krustophenia.net/2.6.8-rc3-O5/kern.log.txt

Here are some graphs from other test runs:

http://krustophenia.net/testresults.php?dataset=2.6.8-rc3-O5
http://krustophenia.net/testresults.php?dataset=2.6.8-rc2-mm2-O3

For reference here's an -mm kernel without the voluntary-preempt
patches:

http://krustophenia.net/testresults.php?dataset=2.6.8-rc3-mm2

Source code for the php script:

http://krustophenia.net/testresults.phps

I based the php script on this perl script: 

http://www.oddmuse.org/cgi-bin/oddmuse/GnuPlot_Extension

The web server is my desktop machine, so please don't complain if it
becomes unavailable without warning.  I will try to keep the links
working.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Wed, 2004-08-11 at 03:31, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > (jackd/12427): 10882us non-preemptible critical section violated 400
> > us preempt threshold starting at kernel_fpu_begin+0x10/0x60 and ending
> > at fast_clear_page+0x75/0xa0
> 
> to make sure this is a real latency and not some rdtsc weirdness, could
> you try the latest version of preempt-timing:
> 
>  http://redhat.com/~mingo/voluntary-preempt/preempt-timing-on-2.6.8-rc3-O5-A2
> 
> this adds jiffies-based latency values to the printout, e.g.:
> 
>   (ksoftirqd/0/2): 3860us [3 jiffy] non-preemptible critical section
>   violated 100 us preempt threshold starting at ___do_softirq+0x1b/0x90
>   and ending at cond_resched_softirq+0x57/0x70
> 
> shows that a 10 jiffy (10 msec) latency happened - which matches the
> rdtsc-based 3860 usecs value.
> 

Here is the preempt-timing violation and accompanying xrun I got when
starting jackd.  It looks like the jiffies value matches the rdtsc:

ALSA /home/rlrevell/cvs/alsa/alsa-driver/alsa-kernel/core/pcm_lib.c:139:
XRUN: pcmC0D2c
 [<c0106497>] dump_stack+0x17/0x20
 [<de92714b>] snd_pcm_period_elapsed+0x27b/0x3e0 [snd_pcm]
 [<de963171>] snd_emu10k1_interrupt+0xd1/0x380 [snd_emu10k1]
 [<c011a693>] generic_handle_IRQ_event+0x33/0x60
 [<c010765e>] do_IRQ+0xbe/0x180
 [<c0106078>] common_interrupt+0x18/0x20
 [<c013d28e>] do_no_page+0x4e/0x300
 [<c013d6e1>] handle_mm_fault+0xb1/0x150
 [<c013c118>] get_user_pages+0x138/0x3e0
 [<c013d828>] make_pages_present+0x68/0x90
 [<c013ef5f>] do_mmap_pgoff+0x40f/0x670
 [<c010b245>] sys_mmap2+0x75/0xb0
 [<c0105e57>] syscall_call+0x7/0xb
(jackd/1359): 9174us [10 jiffy] non-preemptible critical section
violated 400 us preempt threshold starting at do_IRQ+0x19/0x180 and
ending at do_IRQ+0x121/0x180 [<c0106497>] dump_stack+0x17/0x20
 [<c0113e5b>] sub_preempt_count+0x4b/0x60
 [<c01076c1>] do_IRQ+0x121/0x180
 [<c0106078>] common_interrupt+0x18/0x20
 [<c013d28e>] do_no_page+0x4e/0x300
 [<c013d6e1>] handle_mm_fault+0xb1/0x150
 [<c013c118>] get_user_pages+0x138/0x3e0
 [<c013d828>] make_pages_present+0x68/0x90
 [<c013ef5f>] do_mmap_pgoff+0x40f/0x670
 [<c010b245>] sys_mmap2+0x75/0xb0
 [<c0105e57>] syscall_call+0x7/0xb

Lee

[patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

i've uploaded the latest version of the voluntary-preempt patch:
     
  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6

during the past couple of weeks there has been a steady trend towards
rarer and harder to analyze latencies.

The preempt-timing patch was a nice starting point but it only prints
limited info about the beginning and the end of a critical section -
often leaving us in the dark about what happened within the critical
section. Often the trace only contains generic entry/exit points like
e.g. do_poll() which are not very helpful in determining the precise
reason for the latency.

so in -O6 i've implemented a 'latency tracer', which automatically
records all kernel functions called during a maximum-latency incident.
This typically means thousands of functions per critical section. I've
combined this tracer with the preempt-timing approach to produce a
pretty powerful tool to find & squash latencies.

there's a new /proc/latency_trace file that holds the current latency
trace (belonging to the previous high-latency event). It has a format
that is intended to make it as easy as possible for kernel developers to
fix any particular latency source. Audio developers and users can
generate such traces and send them along to kernel developers as text
files.

Sample use of the latency tracer:

E.g. the following incident:

 (default.hotplug/1470): 121 us critical section violates 100 us threshold.
  => started at: <kmap_high+0x2b/0x2d0>
  => ended at:   <kmap_high+0x1a9/0x2d0>
  [<c0105a23>] dump_stack+0x23/0x30
  [<c0140d14>] check_preempt_timing+0x184/0x1e0
  [<c0140e84>] sub_preempt_count+0x54/0x5d
  [<c0152959>] kmap_high+0x1a9/0x2d0
  [<c017655a>] copy_strings+0xea/0x230
  [<c01766db>] copy_strings_kernel+0x3b/0x50
  [<c017840d>] do_execve+0x12d/0x1f0
  [<c0103284>] sys_execve+0x44/0x80
  [<c0104b95>] sysenter_past_esp+0x52/0x71

this doesnt tell us too much about why it took 121 usecs to get from one
end of kmap_high() to the other end of kmap_high(). Looking at
/proc/latency_trace tells us the full story:

  preemption latency trace v1.0
  -----------------------------
   latency: 121 us, entries: 1032 (1032)
   process: default.hotplug/1470, uid: 0
   nice: -10, policy: 0, rt_priority: 0
  =======>
   0.000ms (+0.000ms): page_address (kmap_high)
   0.000ms (+0.000ms): page_slot (page_address)
   0.000ms (+0.000ms): flush_all_zero_pkmaps (kmap_high)
   0.000ms (+0.000ms): set_page_address (flush_all_zero_pkmaps)
  [...]
   0.118ms (+0.000ms): page_slot (set_page_address)
   0.118ms (+0.000ms): check_preempt_timing (sub_preempt_count)

it's the rare but possible call to flush_all_zero_pkmaps() that
generates this particular latency.

as can be seen in the above the example, the trace contains a header
portion and a trace line for every kernel function called. Only function
entries are recorded (not function returns) so i've added the parent
function to the trace too, for easier identification of the call
sequence.

there's a MAX_TRACE define in kernel/latency.c - set to 4000 currently -
this is the maximum number of function calls traced per critical
section. Feel free to increase/decrease this. The header portion shows
the true number of functions called in a critical section, e.g.:

   latency: 1531 us, entries: 4000 (16098)

tells us that there were 16098 trace entries but only the first 4000
were recorded.

-O6 also adds another timing option besides preempt_thresh: if
preempt_thresh is set to 0 then the tracer will automatically track the
largest-previous latency. (i.e. the system does a search for the
absolute maximum latency.) The /proc/sys/kernel/preempt_max_latency 
control can be used to reset this value to conduct a new search for a 
new workload, without having to reboot the system.

-O6 also does some SMP improvements: the IRQ threads now listen to the
/proc/irq/*/smp_affinity mask and bind themselves to the configured CPU. 
This means that e.g. the irqbalance daemon will work as expected.

-O6 also fixes and cleans up a number of other aspects of the
preempt-timing mechanism.

the latency tracer can be turned on/off via CONFIG_LATENCY_TRACE at
compile time. An active tracer means considerable runtime overhead. 
Especially code that does alot of small function calls will see a
performance hit. I'm seeing a ~10% overhead on a 2GHz system, but YMMV. 

reports, suggestions welcome,

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Thu, 2004-08-12 at 17:54, Lee Revell wrote:
> I still have not had a chance to test with the latest
> patches & config changes.

Here are all the log entries produced during this test:

rlrevell@mindpipe:~$ for test in 1 2 3 4 5; do echo; echo "Test $test";
jackd -n 100000 --realtime -d alsa --rate 48000 -D -P hw:0,0 -C hw:0,2
-p 32 -S >> mm2; done

http://krustophenia.net/2.6.8-rc3-O5-A2/kern.log.txt

When one jackd process is running, starting another jackd process, 
using a different device produces an xrun in the first.  If mlockall-test 
is run while a jackd process is running, this also produces an xrun in 
the jackd process.

So, it seems that if a SCHED_FIFO process opens a PCM device using mmap, 
then mlockall's the memory, then another process mlockall's memory, the 
result is an xrun 100% of the time.

Lee
 

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Thu, 2004-08-12 at 20:04, Lee Revell wrote:

> So, it seems that if a SCHED_FIFO process opens a PCM device using mmap, 
> then mlockall's the memory, then another process mlockall's memory, the 
> result is an xrun 100% of the time.
> 

I have found that around 1400 KB is a magic number on my system, this
triggers the preempt violation/xrun about 50% of the time.  1300 never
triggers it, 1500 always triggers it.

Also the amount of memory being mlockall'ed does not affect the length
of the preemption violation - if we hit it at all, there's a 10ms
latency, whether we lock 1400KB or 100MB.

Hopefully O6 will give enough info to track this down.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Thu, 2004-08-12 at 20:55, Florian Schmidt wrote:
> i think that the mlockall and client/jackd startup xruns often do not
> seem to correspond to a critical timing report.. Try the following: turn
> off xrun_debug but leave the preempt-timing stuff on. On my system, the
> mlockall_test provokes an xrun in jackd's output but i do not get a
> preempt-timing report (thresh = 500). 
> 
> OTOH when the xrun_debug is on, the xrun_debug report actually seems to
> trigger the preempt-timing report.
> 

Even if it is not a long non-preemptible section, mlockall-test is still
doing *something* that causes an xrun in jackd.  Maybe interrupts are
being disabled for a long time.  Whatever it is, it cannot possibly be a
jackd bug, because mlockall by an unrelated, normal-priority process
causes an xrun in the SCHED_FIFO jackd process.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Florian Schmidt]

On Thu, 12 Aug 2004 20:27:57 -0400
Lee Revell <rlrevell@joe-job.com> wrote:

> On Thu, 2004-08-12 at 20:04, Lee Revell wrote:
> 
> > So, it seems that if a SCHED_FIFO process opens a PCM device using
> > mmap, then mlockall's the memory, then another process mlockall's
> > memory, the result is an xrun 100% of the time.
> > 
> 
> I have found that around 1400 KB is a magic number on my system, this
> triggers the preempt violation/xrun about 50% of the time.  1300 never
> triggers it, 1500 always triggers it.
> 
> Also the amount of memory being mlockall'ed does not affect the length
> of the preemption violation - if we hit it at all, there's a 10ms
> latency, whether we lock 1400KB or 100MB.
> 
> Hopefully O6 will give enough info to track this down.

Hi, 

i think that the mlockall and client/jackd startup xruns often do not
seem to correspond to a critical timing report.. Try the following: turn
off xrun_debug but leave the preempt-timing stuff on. On my system, the
mlockall_test provokes an xrun in jackd's output but i do not get a
preempt-timing report (thresh = 500). 

OTOH when the xrun_debug is on, the xrun_debug report actually seems to
trigger the preempt-timing report.

I think many of the jackd xruns are really jacks business. But maybe i
misinterpret the symptom.

BTW: on my system with 2*64 frames the magic mlockall number seems to be
around 15megs.. mlockall'ing only 10megs is very unlikely to trigger an
xrun in jackd  And even mlockall'ing 100 megs, while always producing an
xrun in jackd, doesn't show a corresponding preempt timing report either
[with preempt_thresh = 500] as long as the xrun_debug is off. 

Flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Thu, 2004-08-12 at 21:18, Florian Schmidt wrote:
> On Thu, 12 Aug 2004 20:55:30 -0400
> Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Even if it is not a long non-preemptible section, mlockall-test is
> > still doing *something* that causes an xrun in jackd.  Maybe
> > interrupts are being disabled for a long time.  Whatever it is, it
> > cannot possibly be a jackd bug, because mlockall by an unrelated,
> > normal-priority process causes an xrun in the SCHED_FIFO jackd
> > process.
> 
> Hmm, yes, that makes sense. I wonder: does mlockall have any influence
> on IPC?
> 

Good question.  I did repeat your test (disabling xrun_debug) and I can
confirm that I no longer get a preempt timing violation, but
mlockall-test still triggers xruns in jackd every time.  So there is
something else going on here.

Lee

Re: [Jackit-devel] Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Lee Revell]

On Thu, 2004-08-12 at 20:55, Florian Schmidt wrote:
> I think many of the jackd xruns are really jacks business. But maybe i
> misinterpret the symptom.
> 

I also get this one with apt-get dist-upgrade:

(dpkg/11902): 444us [0 jiffy] non-preemptible critical section violated
300 us preempt threshold starting at copy_mm+0x31b/0x3f0 and ending at
copy_page_range+0x144/0x290
 [<c0106497>] dump_stack+0x17/0x20
 [<c0113dc2>] touch_preempt_timing+0x32/0x40
 [<c013b7b4>] copy_page_range+0x144/0x290
 [<c0114922>] copy_mm+0x362/0x3f0
 [<c0115243>] copy_process+0x3d3/0xb60
 [<c0115a11>] do_fork+0x41/0x1b3
 [<c01049b9>] sys_clone+0x29/0x30
 [<c0105e57>] syscall_call+0x7/0xb

Shouldn't fork()'ing a dpkg process be a lot faster than that, with
copy-on-write etc?

Lee

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Florian Schmidt]

On Thu, 12 Aug 2004 20:55:30 -0400
Lee Revell <rlrevell@joe-job.com> wrote:

> Even if it is not a long non-preemptible section, mlockall-test is
> still doing *something* that causes an xrun in jackd.  Maybe
> interrupts are being disabled for a long time.  Whatever it is, it
> cannot possibly be a jackd bug, because mlockall by an unrelated,
> normal-priority process causes an xrun in the SCHED_FIFO jackd
> process.

Hmm, yes, that makes sense. I wonder: does mlockall have any influence
on IPC?

Flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-12 at 19:51, Ingo Molnar wrote:
> i've uploaded the latest version of the voluntary-preempt patch:
>      
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6
> 

Does not compile.  For each module I get:

WARNING: /lib/modules/2.6.8-rc4-O6/kernel/drivers/ieee1394/ohci1394.ko
needs unknown symbol mcount

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-12 at 21:25, Lee Revell wrote:
> Does not compile.  For each module I get:
> 

Never mind, stupid mistake on my part.

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-12 at 21:31, Lee Revell wrote:
> On Thu, 2004-08-12 at 21:25, Lee Revell wrote:
> > Does not compile.  For each module I get:
> > 
> 
> Never mind, stupid mistake on my part.
> 

Argh, this is actually a fatal bug, and not a mistake on my part. 
mcount is an unknown symbol, and make modules_install does not like
that.

I checked Module.symvers and it is not in there, but this seems to be a
generated file, and I have no idea why mcount does not appear.

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-12 at 22:39, Lee Revell wrote:
> On Thu, 2004-08-12 at 21:31, Lee Revell wrote:
> > On Thu, 2004-08-12 at 21:25, Lee Revell wrote:
> > > Does not compile.  For each module I get:
> > > 
> > 
> > Never mind, stupid mistake on my part.
> > 
> 
> Argh, this is actually a fatal bug, and not a mistake on my part. 
> mcount is an unknown symbol, and make modules_install does not like
> that.
> 
> I checked Module.symvers and it is not in there, but this seems to be a
> generated file, and I have no idea why mcount does not appear.
> 

I think appending this to i386_ksyms.c fixes the problem:

#ifdef CONFIG_PREEMPT_TIMING
EXPORT_SYMBOL(mcount);
#endif

Possibly that should be CONFIG_LATENCY_TRACE.

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-12 at 23:54, Lee Revell wrote:
> On Thu, 2004-08-12 at 22:39, Lee Revell wrote:
> > On Thu, 2004-08-12 at 21:31, Lee Revell wrote:
> > > On Thu, 2004-08-12 at 21:25, Lee Revell wrote:
> > > > Does not compile.  For each module I get:
> > > > 
> > > 
> > > Never mind, stupid mistake on my part.
> > > 
> > 
> > Argh, this is actually a fatal bug, and not a mistake on my part. 
> > mcount is an unknown symbol, and make modules_install does not like
> > that.
> > 
> > I checked Module.symvers and it is not in there, but this seems to be a
> > generated file, and I have no idea why mcount does not appear.
> > 
> 
> I think appending this to i386_ksyms.c fixes the problem:
> 
> #ifdef CONFIG_PREEMPT_TIMING
> EXPORT_SYMBOL(mcount);
> #endif
> 
> Possibly that should be CONFIG_LATENCY_TRACE.

I believe this is the correct patch, based on
arch/sparc64/kernel/sparc64_ksyms.c.  Ingo, are you using a sparc64 for
your testing?

--- arch/i386/kernel/i386_ksyms.c.orig	2004-08-13 00:18:51.000000000 -0400
+++ arch/i386/kernel/i386_ksyms.c	2004-08-13 00:19:46.000000000 -0400
@@ -59,6 +59,11 @@
 extern unsigned long cpu_khz;
 extern unsigned long get_cmos_time(void);
 
+#if defined(CONFIG_MCOUNT)
+extern void mcount(void);
+EXPORT_SYMBOL_NOVERS(mcount);
+#endif
+
 /* platform dependent support */
 EXPORT_SYMBOL(boot_cpu_data);
 EXPORT_SYMBOL(MCA_bus);

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Roland Dreier]

    Lee> I believe this is the correct patch, based on
    Lee> arch/sparc64/kernel/sparc64_ksyms.c.  Ingo, are you using a
    Lee> sparc64 for your testing?

He's probably just not using modules.  There's no way LATENCY_TRACE
can work on anything except i386, since that's the only definition of
mcount that's provided (and if one were being anal, it would probably
make more sense to add the config stuff to arch/i386/Kconfig rather
than init/Kconfig).

 - R.

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Fri, 2004-08-13 at 00:35, Roland Dreier wrote:
>     Lee> I believe this is the correct patch, based on
>     Lee> arch/sparc64/kernel/sparc64_ksyms.c.  Ingo, are you using a
>     Lee> sparc64 for your testing?
> 
> He's probably just not using modules.  There's no way LATENCY_TRACE
> can work on anything except i386, since that's the only definition of
> mcount that's provided (and if one were being anal, it would probably
> make more sense to add the config stuff to arch/i386/Kconfig rather
> than init/Kconfig).
> 

It also exists on sparc64, it's just called _mcount.

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Roland Dreier]

    Lee> It also exists on sparc64, it's just called _mcount.

That's something completely different (used for stack overflow
debugging, I think).  (and ".globl mcount, _mcount" means it's also
called mcount)

Look at Ingo's patch; it only adds mcount() to arch/i386 (although
__mcount is defined in kernel/latency.c, there's no way for any other
arch to call it).

 - Roland

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Matt Heler]

Ingo,

I get the following error when I have CONFIG_PREEMPT_TIMING=n

  AS      arch/i386/kernel/entry.o
  CC      arch/i386/kernel/traps.o
arch/i386/kernel/traps.c: In function `do_nmi':
arch/i386/kernel/traps.c:539: error: syntax error before "do"
arch/i386/kernel/traps.c:539: error: syntax error before ')' token
arch/i386/kernel/traps.c:537: warning: unused variable `cpu'
arch/i386/kernel/traps.c: At top level:
arch/i386/kernel/traps.c:541: warning: type defaults to `int' in declaration 
of `cpu'
arch/i386/kernel/traps.c:541: warning: data definition has no type or storage 
class
arch/i386/kernel/traps.c:542: error: syntax error before '++' token
arch/i386/kernel/traps.c:500: warning: `default_do_nmi' defined but not used
make[1]: *** [arch/i386/kernel/traps.o] Error 1
make: *** [arch/i386/kernel] Error 2


Matt 

On Thursday 12 August 2004 4:51 pm, Ingo Molnar wrote:
> i've uploaded the latest version of the voluntary-preempt patch:
>
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6
>
> during the past couple of weeks there has been a steady trend towards
> rarer and harder to analyze latencies.
>
> The preempt-timing patch was a nice starting point but it only prints
> limited info about the beginning and the end of a critical section -
> often leaving us in the dark about what happened within the critical
> section. Often the trace only contains generic entry/exit points like
> e.g. do_poll() which are not very helpful in determining the precise
> reason for the latency.
>
> so in -O6 i've implemented a 'latency tracer', which automatically
> records all kernel functions called during a maximum-latency incident.
> This typically means thousands of functions per critical section. I've
> combined this tracer with the preempt-timing approach to produce a
> pretty powerful tool to find & squash latencies.
>
> there's a new /proc/latency_trace file that holds the current latency
> trace (belonging to the previous high-latency event). It has a format
> that is intended to make it as easy as possible for kernel developers to
> fix any particular latency source. Audio developers and users can
> generate such traces and send them along to kernel developers as text
> files.
>
> Sample use of the latency tracer:
>
> E.g. the following incident:
>
>  (default.hotplug/1470): 121 us critical section violates 100 us threshold.
>   => started at: <kmap_high+0x2b/0x2d0>
>   => ended at:   <kmap_high+0x1a9/0x2d0>
>   [<c0105a23>] dump_stack+0x23/0x30
>   [<c0140d14>] check_preempt_timing+0x184/0x1e0
>   [<c0140e84>] sub_preempt_count+0x54/0x5d
>   [<c0152959>] kmap_high+0x1a9/0x2d0
>   [<c017655a>] copy_strings+0xea/0x230
>   [<c01766db>] copy_strings_kernel+0x3b/0x50
>   [<c017840d>] do_execve+0x12d/0x1f0
>   [<c0103284>] sys_execve+0x44/0x80
>   [<c0104b95>] sysenter_past_esp+0x52/0x71
>
> this doesnt tell us too much about why it took 121 usecs to get from one
> end of kmap_high() to the other end of kmap_high(). Looking at
> /proc/latency_trace tells us the full story:
>
>   preemption latency trace v1.0
>   -----------------------------
>    latency: 121 us, entries: 1032 (1032)
>    process: default.hotplug/1470, uid: 0
>    nice: -10, policy: 0, rt_priority: 0
>   =======>
>    0.000ms (+0.000ms): page_address (kmap_high)
>    0.000ms (+0.000ms): page_slot (page_address)
>    0.000ms (+0.000ms): flush_all_zero_pkmaps (kmap_high)
>    0.000ms (+0.000ms): set_page_address (flush_all_zero_pkmaps)
>   [...]
>    0.118ms (+0.000ms): page_slot (set_page_address)
>    0.118ms (+0.000ms): check_preempt_timing (sub_preempt_count)
>
> it's the rare but possible call to flush_all_zero_pkmaps() that
> generates this particular latency.
>
> as can be seen in the above the example, the trace contains a header
> portion and a trace line for every kernel function called. Only function
> entries are recorded (not function returns) so i've added the parent
> function to the trace too, for easier identification of the call
> sequence.
>
> there's a MAX_TRACE define in kernel/latency.c - set to 4000 currently -
> this is the maximum number of function calls traced per critical
> section. Feel free to increase/decrease this. The header portion shows
> the true number of functions called in a critical section, e.g.:
>
>    latency: 1531 us, entries: 4000 (16098)
>
> tells us that there were 16098 trace entries but only the first 4000
> were recorded.
>
> -O6 also adds another timing option besides preempt_thresh: if
> preempt_thresh is set to 0 then the tracer will automatically track the
> largest-previous latency. (i.e. the system does a search for the
> absolute maximum latency.) The /proc/sys/kernel/preempt_max_latency
> control can be used to reset this value to conduct a new search for a
> new workload, without having to reboot the system.
>
> -O6 also does some SMP improvements: the IRQ threads now listen to the
> /proc/irq/*/smp_affinity mask and bind themselves to the configured CPU.
> This means that e.g. the irqbalance daemon will work as expected.
>
> -O6 also fixes and cleans up a number of other aspects of the
> preempt-timing mechanism.
>
> the latency tracer can be turned on/off via CONFIG_LATENCY_TRACE at
> compile time. An active tracer means considerable runtime overhead.
> Especially code that does alot of small function calls will see a
> performance hit. I'm seeing a ~10% overhead on a 2GHz system, but YMMV.
>
> reports, suggestions welcome,
>
> 	Ingo
> -
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-12 at 19:51, Ingo Molnar wrote:
> i've uploaded the latest version of the voluntary-preempt patch:
>      
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6
> 

Interesting results.  One of the problems is kallsyms_lookup, triggered
by the printks:

 0.100ms (+0.000ms): emit_log_char (printk)
 0.100ms (+0.000ms): emit_log_char (printk)
 0.101ms (+0.000ms): emit_log_char (printk)
 0.101ms (+0.000ms): preempt_schedule (printk)
 0.101ms (+0.000ms): release_console_sem (printk)
 0.102ms (+0.000ms): preempt_schedule (release_console_sem)
 0.102ms (+0.000ms): call_console_drivers (release_console_sem)
 0.103ms (+0.000ms): _call_console_drivers (call_console_drivers)
 0.103ms (+0.000ms): __call_console_drivers (call_console_drivers)
 0.105ms (+0.001ms): vt_console_print (__call_console_drivers)
 0.106ms (+0.001ms): vc_cons_allocated (vt_console_print)
 0.108ms (+0.001ms): preempt_schedule (release_console_sem)
 0.108ms (+0.000ms): __print_symbol (print_context_stack)
 0.109ms (+0.001ms): kallsyms_lookup (__print_symbol)
 0.491ms (+0.381ms): sprintf (__print_symbol)
 0.492ms (+0.000ms): vsprintf (sprintf)
 0.492ms (+0.000ms): vsnprintf (vsprintf)
 0.494ms (+0.001ms): number (vsnprintf)
 0.496ms (+0.001ms): number (vsnprintf)
 0.497ms (+0.001ms): printk (__print_symbol)
 0.497ms (+0.000ms): vscnprintf (printk)
 0.497ms (+0.000ms): vsnprintf (vscnprintf)
 0.499ms (+0.001ms): emit_log_char (printk)
 0.499ms (+0.000ms): emit_log_char (printk)

 0.778ms (+0.000ms): emit_log_char (printk)
 0.779ms (+0.000ms): emit_log_char (printk)
 0.779ms (+0.000ms): preempt_schedule (printk)
 0.779ms (+0.000ms): release_console_sem (printk)
 0.780ms (+0.000ms): preempt_schedule (release_console_sem)
 0.780ms (+0.000ms): call_console_drivers (release_console_sem)
 0.781ms (+0.000ms): _call_console_drivers (call_console_drivers)
 0.781ms (+0.000ms): __call_console_drivers (call_console_drivers)
 0.781ms (+0.000ms): vt_console_print (__call_console_drivers)
 0.782ms (+0.000ms): vc_cons_allocated (vt_console_print)
 0.782ms (+0.000ms): preempt_schedule (release_console_sem)
 0.782ms (+0.000ms): __print_symbol (print_context_stack)
 0.783ms (+0.000ms): kallsyms_lookup (__print_symbol)
 1.448ms (+0.665ms): sprintf (__print_symbol)
 1.448ms (+0.000ms): vsprintf (sprintf)
 1.448ms (+0.000ms): vsnprintf (vsprintf)
 1.450ms (+0.001ms): number (vsnprintf)
 1.452ms (+0.001ms): number (vsnprintf)
 1.453ms (+0.000ms): printk (__print_symbol)
 1.453ms (+0.000ms): vscnprintf (printk)
 1.453ms (+0.000ms): vsnprintf (vscnprintf)
 1.455ms (+0.001ms): emit_log_char (printk)
 1.456ms (+0.000ms): emit_log_char (printk)

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-12 at 19:51, Ingo Molnar wrote:
> i've uploaded the latest version of the voluntary-preempt patch:
>      
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6

My results with -O6 confirm Florian's results that when xrun_debug is
off, mlockall() does not produce a long non-preemptible section at all,
but does cause xruns in jackd.  I have no idea how this is possible.

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Fri, 2004-08-13 at 01:27, Lee Revell wrote:
> On Thu, 2004-08-12 at 19:51, Ingo Molnar wrote:
> > i've uploaded the latest version of the voluntary-preempt patch:
> >      
> >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6
> 

Ugh, this is a bad one:

preemption latency trace v1.0
-----------------------------
 latency: 506 us, entries: 157 (157)
 process: evolution/3461, uid: 1000
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): get_random_bytes (__check_and_rekey)
 0.000ms (+0.000ms): extract_entropy (get_random_bytes)
 0.002ms (+0.001ms): extract_entropy (extract_entropy)
 0.003ms (+0.001ms): SHATransform (extract_entropy)
 0.004ms (+0.000ms): memcpy (SHATransform)
 0.014ms (+0.010ms): add_entropy_words (extract_entropy)
 0.016ms (+0.002ms): SHATransform (extract_entropy)
 0.016ms (+0.000ms): memcpy (SHATransform)
 0.025ms (+0.008ms): add_entropy_words (extract_entropy)
 0.026ms (+0.000ms): SHATransform (extract_entropy)
 0.026ms (+0.000ms): memcpy (SHATransform)
 0.035ms (+0.008ms): add_entropy_words (extract_entropy)
 0.035ms (+0.000ms): SHATransform (extract_entropy)
 0.036ms (+0.000ms): memcpy (SHATransform)
 0.044ms (+0.008ms): add_entropy_words (extract_entropy)
 0.045ms (+0.000ms): SHATransform (extract_entropy)
 0.045ms (+0.000ms): memcpy (SHATransform)
 0.053ms (+0.008ms): add_entropy_words (extract_entropy)
 0.054ms (+0.000ms): SHATransform (extract_entropy)
 0.055ms (+0.000ms): memcpy (SHATransform)
 0.063ms (+0.008ms): add_entropy_words (extract_entropy)
 0.064ms (+0.000ms): SHATransform (extract_entropy)
 0.064ms (+0.000ms): memcpy (SHATransform)
 0.072ms (+0.008ms): add_entropy_words (extract_entropy)
 0.073ms (+0.000ms): SHATransform (extract_entropy)
 0.073ms (+0.000ms): memcpy (SHATransform)
 0.082ms (+0.008ms): add_entropy_words (extract_entropy)
 0.083ms (+0.001ms): SHATransform (extract_entropy)
 0.083ms (+0.000ms): memcpy (SHATransform)
 0.091ms (+0.008ms): add_entropy_words (extract_entropy)
 0.092ms (+0.000ms): SHATransform (extract_entropy)
 0.092ms (+0.000ms): memcpy (SHATransform)
 0.101ms (+0.008ms): add_entropy_words (extract_entropy)
 0.101ms (+0.000ms): SHATransform (extract_entropy)
 0.102ms (+0.000ms): memcpy (SHATransform)
 0.110ms (+0.008ms): add_entropy_words (extract_entropy)
 0.111ms (+0.000ms): SHATransform (extract_entropy)
 0.111ms (+0.000ms): memcpy (SHATransform)
 0.119ms (+0.008ms): add_entropy_words (extract_entropy)
 0.120ms (+0.000ms): SHATransform (extract_entropy)
 0.120ms (+0.000ms): memcpy (SHATransform)
 0.129ms (+0.008ms): add_entropy_words (extract_entropy)
 0.129ms (+0.000ms): SHATransform (extract_entropy)
 0.129ms (+0.000ms): memcpy (SHATransform)
 0.138ms (+0.008ms): add_entropy_words (extract_entropy)
 0.139ms (+0.000ms): SHATransform (extract_entropy)
 0.139ms (+0.000ms): memcpy (SHATransform)
 0.147ms (+0.008ms): add_entropy_words (extract_entropy)
 0.148ms (+0.000ms): SHATransform (extract_entropy)
 0.148ms (+0.000ms): memcpy (SHATransform)
 0.157ms (+0.008ms): add_entropy_words (extract_entropy)
 0.158ms (+0.000ms): SHATransform (extract_entropy)
 0.158ms (+0.000ms): memcpy (SHATransform)
 0.166ms (+0.008ms): add_entropy_words (extract_entropy)
 0.167ms (+0.000ms): SHATransform (extract_entropy)
 0.167ms (+0.000ms): memcpy (SHATransform)
 0.176ms (+0.008ms): add_entropy_words (extract_entropy)
 0.177ms (+0.000ms): SHATransform (extract_entropy)
 0.177ms (+0.000ms): memcpy (SHATransform)
 0.185ms (+0.008ms): add_entropy_words (extract_entropy)
 0.186ms (+0.000ms): SHATransform (extract_entropy)
 0.186ms (+0.000ms): memcpy (SHATransform)
 0.195ms (+0.008ms): add_entropy_words (extract_entropy)
 0.196ms (+0.000ms): SHATransform (extract_entropy)
 0.196ms (+0.000ms): memcpy (SHATransform)
 0.204ms (+0.008ms): add_entropy_words (extract_entropy)
 0.205ms (+0.000ms): SHATransform (extract_entropy)
 0.205ms (+0.000ms): memcpy (SHATransform)
 0.214ms (+0.008ms): add_entropy_words (extract_entropy)
 0.214ms (+0.000ms): SHATransform (extract_entropy)
 0.215ms (+0.000ms): memcpy (SHATransform)
 0.223ms (+0.008ms): add_entropy_words (extract_entropy)
 0.224ms (+0.000ms): SHATransform (extract_entropy)
 0.224ms (+0.000ms): memcpy (SHATransform)
 0.233ms (+0.008ms): add_entropy_words (extract_entropy)
 0.234ms (+0.001ms): SHATransform (extract_entropy)
 0.234ms (+0.000ms): memcpy (SHATransform)
 0.242ms (+0.008ms): add_entropy_words (extract_entropy)
 0.243ms (+0.000ms): SHATransform (extract_entropy)
 0.243ms (+0.000ms): memcpy (SHATransform)
 0.252ms (+0.008ms): add_entropy_words (extract_entropy)
 0.252ms (+0.000ms): SHATransform (extract_entropy)
 0.253ms (+0.000ms): memcpy (SHATransform)
 0.261ms (+0.008ms): add_entropy_words (extract_entropy)
 0.262ms (+0.000ms): SHATransform (extract_entropy)
 0.262ms (+0.000ms): memcpy (SHATransform)
 0.271ms (+0.008ms): add_entropy_words (extract_entropy)
 0.271ms (+0.000ms): SHATransform (extract_entropy)
 0.272ms (+0.000ms): memcpy (SHATransform)
 0.280ms (+0.008ms): add_entropy_words (extract_entropy)
 0.281ms (+0.000ms): SHATransform (extract_entropy)
 0.281ms (+0.000ms): memcpy (SHATransform)
 0.289ms (+0.008ms): add_entropy_words (extract_entropy)
 0.290ms (+0.000ms): SHATransform (extract_entropy)
 0.290ms (+0.000ms): memcpy (SHATransform)
 0.299ms (+0.008ms): add_entropy_words (extract_entropy)
 0.300ms (+0.000ms): SHATransform (extract_entropy)
 0.300ms (+0.000ms): memcpy (SHATransform)
 0.308ms (+0.008ms): add_entropy_words (extract_entropy)
 0.309ms (+0.000ms): SHATransform (extract_entropy)
 0.309ms (+0.000ms): memcpy (SHATransform)
 0.318ms (+0.008ms): add_entropy_words (extract_entropy)
 0.319ms (+0.000ms): SHATransform (extract_entropy)
 0.319ms (+0.000ms): memcpy (SHATransform)
 0.327ms (+0.008ms): add_entropy_words (extract_entropy)
 0.328ms (+0.000ms): SHATransform (extract_entropy)
 0.328ms (+0.000ms): memcpy (SHATransform)
 0.337ms (+0.008ms): add_entropy_words (extract_entropy)
 0.338ms (+0.000ms): SHATransform (extract_entropy)
 0.338ms (+0.000ms): memcpy (SHATransform)
 0.346ms (+0.008ms): add_entropy_words (extract_entropy)
 0.347ms (+0.000ms): SHATransform (extract_entropy)
 0.347ms (+0.000ms): memcpy (SHATransform)
 0.356ms (+0.008ms): add_entropy_words (extract_entropy)
 0.356ms (+0.000ms): SHATransform (extract_entropy)
 0.357ms (+0.000ms): memcpy (SHATransform)
 0.365ms (+0.008ms): add_entropy_words (extract_entropy)
 0.366ms (+0.000ms): SHATransform (extract_entropy)
 0.366ms (+0.000ms): memcpy (SHATransform)
 0.375ms (+0.008ms): add_entropy_words (extract_entropy)
 0.375ms (+0.000ms): SHATransform (extract_entropy)
 0.376ms (+0.000ms): memcpy (SHATransform)
 0.384ms (+0.008ms): add_entropy_words (extract_entropy)
 0.385ms (+0.001ms): add_entropy_words (extract_entropy)
 0.396ms (+0.010ms): credit_entropy_store (extract_entropy)
 0.397ms (+0.001ms): SHATransform (extract_entropy)
 0.397ms (+0.000ms): memcpy (SHATransform)
 0.405ms (+0.008ms): add_entropy_words (extract_entropy)
 0.406ms (+0.000ms): SHATransform (extract_entropy)
 0.407ms (+0.000ms): memcpy (SHATransform)
 0.415ms (+0.008ms): add_entropy_words (extract_entropy)
 0.416ms (+0.001ms): SHATransform (extract_entropy)
 0.416ms (+0.000ms): memcpy (SHATransform)
 0.425ms (+0.008ms): add_entropy_words (extract_entropy)
 0.425ms (+0.000ms): SHATransform (extract_entropy)
 0.426ms (+0.000ms): memcpy (SHATransform)
 0.434ms (+0.008ms): add_entropy_words (extract_entropy)
 0.435ms (+0.001ms): SHATransform (extract_entropy)
 0.436ms (+0.000ms): memcpy (SHATransform)
 0.444ms (+0.008ms): add_entropy_words (extract_entropy)
 0.445ms (+0.000ms): SHATransform (extract_entropy)
 0.445ms (+0.000ms): memcpy (SHATransform)
 0.453ms (+0.008ms): add_entropy_words (extract_entropy)
 0.455ms (+0.001ms): SHATransform (extract_entropy)
 0.455ms (+0.000ms): memcpy (SHATransform)
 0.463ms (+0.008ms): add_entropy_words (extract_entropy)
 0.464ms (+0.000ms): SHATransform (extract_entropy)
 0.464ms (+0.000ms): memcpy (SHATransform)
 0.473ms (+0.008ms): add_entropy_words (extract_entropy)
 0.474ms (+0.001ms): SHATransform (extract_entropy)
 0.474ms (+0.000ms): memcpy (SHATransform)
 0.482ms (+0.008ms): add_entropy_words (extract_entropy)
 0.483ms (+0.000ms): SHATransform (extract_entropy)
 0.483ms (+0.000ms): memcpy (SHATransform)
 0.492ms (+0.008ms): add_entropy_words (extract_entropy)
 0.493ms (+0.001ms): local_bh_enable (__check_and_rekey)
 0.494ms (+0.000ms): check_preempt_timing (sub_preempt_count)

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-12 at 19:51, Ingo Molnar wrote:
> i've uploaded the latest version of the voluntary-preempt patch:
>      
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6

Here's another weird one.  This happens when you create a new tab in
gnome-terminal, which causes a fork().

preemption latency trace v1.0
-----------------------------
 latency: 613 us, entries: 697 (697)
 process: gnome-terminal/3467, uid: 1000
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): do_IRQ (common_interrupt)
 0.000ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.003ms (+0.003ms): generic_redirect_hardirq (do_IRQ)
 0.004ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.004ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 0.005ms (+0.000ms): mark_offset_tsc (timer_interrupt)
 0.011ms (+0.006ms): do_timer (timer_interrupt)
 0.011ms (+0.000ms): update_process_times (do_timer)
 0.012ms (+0.000ms): update_one_process (update_process_times)
 0.012ms (+0.000ms): run_local_timers (update_process_times)
 0.012ms (+0.000ms): raise_softirq (update_process_times)
 0.013ms (+0.000ms): scheduler_tick (update_process_times)
 0.013ms (+0.000ms): sched_clock (scheduler_tick)
 0.014ms (+0.001ms): task_timeslice (scheduler_tick)
 0.015ms (+0.000ms): update_wall_time (do_timer)
 0.015ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 0.017ms (+0.001ms): generic_note_interrupt (do_IRQ)
 0.017ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.017ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.019ms (+0.001ms): do_softirq (do_IRQ)
 0.019ms (+0.000ms): __do_softirq (do_softirq)
 0.019ms (+0.000ms): wake_up_process (do_softirq)
 0.020ms (+0.000ms): try_to_wake_up (wake_up_process)
 0.020ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.020ms (+0.000ms): activate_task (try_to_wake_up)
 0.021ms (+0.000ms): sched_clock (activate_task)
 0.021ms (+0.000ms): recalc_task_prio (activate_task)
 0.022ms (+0.000ms): effective_prio (recalc_task_prio)
 0.022ms (+0.000ms): enqueue_task (activate_task)
 0.023ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.025ms (+0.000ms): preempt_schedule (copy_page_range)
 0.026ms (+0.000ms): preempt_schedule (copy_page_range)

...about 400+ of the same deleted...

 0.459ms (+0.000ms): preempt_schedule (copy_page_range)
 0.459ms (+0.000ms): preempt_schedule (copy_page_range)
 0.460ms (+0.000ms): preempt_schedule (copy_page_range)
 0.461ms (+0.000ms): preempt_schedule (copy_page_range)
 0.461ms (+0.000ms): preempt_schedule (copy_page_range)
 0.461ms (+0.000ms): check_preempt_timing (touch_preempt_timing)

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Peter Zijlstra]

On Thu, 2004-08-12 at 21:49 -0700, Matt Heler wrote:
> Ingo,
> 
> I get the following error when I have CONFIG_PREEMPT_TIMING=n
> 
>   AS      arch/i386/kernel/entry.o
>   CC      arch/i386/kernel/traps.o
> arch/i386/kernel/traps.c: In function `do_nmi':
> arch/i386/kernel/traps.c:539: error: syntax error before "do"
> arch/i386/kernel/traps.c:539: error: syntax error before ')' token
> arch/i386/kernel/traps.c:537: warning: unused variable `cpu'
> arch/i386/kernel/traps.c: At top level:
> arch/i386/kernel/traps.c:541: warning: type defaults to `int' in declaration 
> of `cpu'
> arch/i386/kernel/traps.c:541: warning: data definition has no type or storage 
> class
> arch/i386/kernel/traps.c:542: error: syntax error before '++' token
> arch/i386/kernel/traps.c:500: warning: `default_do_nmi' defined but not used
> make[1]: *** [arch/i386/kernel/traps.o] Error 1
> make: *** [arch/i386/kernel] Error 2
> 
> 
> Matt 
> 

This fixes it.

--- ./include/asm-i386/hardirq.h~       2004-08-13 11:17:38.668333125 +0200
+++ ./include/asm-i386/hardirq.h        2004-08-13 11:51:40.835968747 +0200
@@ -73,7 +73,7 @@
 #define hardirq_endlock()      do { } while (0)
 
 #define irq_enter()            add_preempt_count(HARDIRQ_OFFSET)
-#define nmi_enter()            (irq_enter())
+#define nmi_enter()            irq_enter()
 #define nmi_exit()             sub_preempt_count(HARDIRQ_OFFSET)
 
 #ifdef CONFIG_PREEMPT



-- 
Peter Zijlstra <a.p.zijlstra@chello.nl>

Re: [patch] voluntary-preempt-2.6.8-rc3-O5

[Ingo Molnar]

* Florian Schmidt <mista.tapas@gmx.net> wrote:

> i think that the mlockall and client/jackd startup xruns often do not
> seem to correspond to a critical timing report.. Try the following:
> turn off xrun_debug but leave the preempt-timing stuff on. On my
> system, the mlockall_test provokes an xrun in jackd's output but i do
> not get a preempt-timing report (thresh = 500). 
> 
> OTOH when the xrun_debug is on, the xrun_debug report actually seems
> to trigger the preempt-timing report.

this later phenomenon is expected and unrelated: printk-ing to the
console (as the ALSA xrun kernel message does) is quite expensive,
especially with a full stack dump included. So this fact alone doesnt
tell us much about why the xrun itself happened.

if there is no preempt-timing report when the ALSA xrun debugging is
disabled it strongly suggests that whatever causes the xrun, it's not
due to the length of a non-preemptible critical section, but some other
phenomenon (either in userspace or in kernelspace) that causes an xrun.

irqs being left disabled by accident is one such possibility - the
preempt-timing patch does not (yet) track irqs-off sections.

> I think many of the jackd xruns are really jacks business. But maybe i
> misinterpret the symptom.

either jack's or the kernel's - but it's less likely to be the classical
non-preemptible sections we were focused on so far. (it could still be a
bug in the preempt-timing patch producing a false negative - but the
likelyhood is low i'd say.)

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> +#if defined(CONFIG_MCOUNT)
> +extern void mcount(void);
> +EXPORT_SYMBOL_NOVERS(mcount);
> +#endif

yeah, we need this export. (i've put it into kernel/latency.c and it's
fine to do it as an EXPORT_SYMBOL()). Will show up in -O7.

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:

> > arch/i386/kernel/traps.c: In function `do_nmi':
> > arch/i386/kernel/traps.c:539: error: syntax error before "do"

> This fixes it.
> 
> --- ./include/asm-i386/hardirq.h~       2004-08-13 11:17:38.668333125 +0200
> +++ ./include/asm-i386/hardirq.h        2004-08-13 11:51:40.835968747 +0200
> @@ -73,7 +73,7 @@
>  #define hardirq_endlock()      do { } while (0)
>  
>  #define irq_enter()            add_preempt_count(HARDIRQ_OFFSET)
> -#define nmi_enter()            (irq_enter())
> +#define nmi_enter()            irq_enter()

yep - thx, this fix too will be in -O7. It seems this compilation error
only happens with older gcc and i'm using 3.3.

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* Roland Dreier <roland@topspin.com> wrote:

> Look at Ingo's patch; it only adds mcount() to arch/i386 (although
> __mcount is defined in kernel/latency.c, there's no way for any other
> arch to call it).

should be trivial to add it to other arches. But the patch indeed is
x86-only for the time being - 100% of the testers so far were x86 users
so i dont see any point in bloating the patch with untested arch
changes. Tested contributions are welcome of course!

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Interesting results.  One of the problems is kallsyms_lookup,
> triggered by the printks:

yeah - kallsyms_lookup does a linear search over thousands of symbols. 
Especially since /proc/latency_trace uses it too it would be worthwile
to implement some sort of binary searching.

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Peter Zijlstra]

On Fri, 2004-08-13 at 12:19 +0200, Ingo Molnar wrote:
> * Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
> 
> > > arch/i386/kernel/traps.c: In function `do_nmi':
> > > arch/i386/kernel/traps.c:539: error: syntax error before "do"
> 
> > This fixes it.
> > 
> > --- ./include/asm-i386/hardirq.h~       2004-08-13 11:17:38.668333125 +0200
> > +++ ./include/asm-i386/hardirq.h        2004-08-13 11:51:40.835968747 +0200
> > @@ -73,7 +73,7 @@
> >  #define hardirq_endlock()      do { } while (0)
> >  
> >  #define irq_enter()            add_preempt_count(HARDIRQ_OFFSET)
> > -#define nmi_enter()            (irq_enter())
> > +#define nmi_enter()            irq_enter()
> 
> yep - thx, this fix too will be in -O7. It seems this compilation error
> only happens with older gcc and i'm using 3.3.
> 
> 	Ingo
FYI,

peter@twins ~ $ gcc --version
gcc (GCC) 3.3.4 20040623 (Gentoo Linux 3.3.4-r1, ssp-3.3.2-2, pie-8.7.6)
Copyright (C) 2003 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

and it hits here too.

-- 
Peter Zijlstra <a.p.zijlstra@chello.nl>

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6
> 
> Ugh, this is a bad one:
> 
> preemption latency trace v1.0
> -----------------------------
>  latency: 506 us, entries: 157 (157)
>  process: evolution/3461, uid: 1000
>  nice: 0, policy: 0, rt_priority: 0
> =======>
>  0.000ms (+0.000ms): get_random_bytes (__check_and_rekey)
[...]
>  0.493ms (+0.001ms): local_bh_enable (__check_and_rekey)

indeed this is a new one. Entropy rekeying every 300 seconds. Most of
the overhead comes from the memcpy's - 10 usecs a pop!

this could possibly explain some earlier reports of RTC problems every
couple of minutes - on slower boxes it could easily be more than the 0.5
msec you got. (and with a 8192 Hz RTC the interrupt period is 122
usecs.)

such bhs-off spinlocked sections exclude all softirq traffic - and in
the redirected hardirqs case the hardirqs are excluded too.

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Florian Schmidt]

On Fri, 13 Aug 2004 01:51:16 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> -O6 also adds another timing option besides preempt_thresh: if
> preempt_thresh is set to 0 then the tracer will automatically track
> the largest-previous latency. (i.e. the system does a search for the
> absolute maximum latency.) The /proc/sys/kernel/preempt_max_latency 
> control can be used to reset this value to conduct a new search for a 
> new workload, without having to reboot the system.

Hmm, should a new report only be generated when the newly measured
latency is really > than all other ones? I get the feeling that >= seems
to be enough. Here a dmesg extract:

(swapper/1): new 6 us maximum-latency critical section.
 => started at: <schedule+0x5c/0x5b0>
 => ended at:   <finish_task_switch+0x32/0x90>
 [<c01064ae>] dump_stack+0x1e/0x20
 [<c0130c64>] check_preempt_timing+0x184/0x1e0
 [<c0130db4>] sub_preempt_count+0x44/0x50
 [<c0112c22>] finish_task_switch+0x32/0x90
 [<c0112c9e>] schedule_tail+0x1e/0x60
 [<c0105df6>] ret_from_fork+0x6/0x14
(swapper/1): new 8 us maximum-latency critical section.
 => started at: <voluntary_resched+0x1e/0x50>
 => ended at:   <voluntary_resched+0x1e/0x50>
 [<c01064ae>] dump_stack+0x1e/0x20
 [<c0130c64>] check_preempt_timing+0x184/0x1e0
 [<c0130cf8>] touch_preempt_timing+0x38/0x60
 [<c0292a9e>] voluntary_resched+0x1e/0x50
 [<c013a1b6>] kmem_cache_alloc+0x56/0x60
 [<c01160c0>] copy_process+0x9e0/0xbd0
 [<c0116302>] do_fork+0x52/0x1b3
 [<c0104135>] kernel_thread+0x85/0x90
 [<c012a907>] keventd_create_kthread+0x27/0x50
 [<c012a9a7>] kthread_create+0x77/0xd0
 [<c031dc9b>] cpu_callback+0x5b/0xc0
 [<c031dd22>] spawn_ksoftirqd+0x22/0x50
 [<c0100334>] init+0x34/0x170
 [<c01040a5>] kernel_thread_helper+0x5/0x10
(swapper/1): new 9 us maximum-latency critical section.
 => started at: <voluntary_resched+0x1e/0x50>
 => ended at:   <voluntary_resched+0x1e/0x50>
 [<c01064ae>] dump_stack+0x1e/0x20
 [<c0130c64>] check_preempt_timing+0x184/0x1e0
 [<c0130cf8>] touch_preempt_timing+0x38/0x60
 [<c0292a9e>] voluntary_resched+0x1e/0x50
 [<c013a1b6>] kmem_cache_alloc+0x56/0x60
 [<c0116009>] copy_process+0x929/0xbd0
 [<c0116302>] do_fork+0x52/0x1b3
 [<c0104135>] kernel_thread+0x85/0x90
 [<c012a907>] keventd_create_kthread+0x27/0x50
 [<c012a9a7>] kthread_create+0x77/0xd0
 [<c031dc9b>] cpu_callback+0x5b/0xc0
 [<c031dd22>] spawn_ksoftirqd+0x22/0x50
 [<c0100334>] init+0x34/0x170
 [<c01040a5>] kernel_thread_helper+0x5/0x10


Here are two reports with the same maximum-latency (31 us):

(swapper/1): new 31 us maximum-latency critical section.
 => started at: <voluntary_resched+0x1e/0x50>
 => ended at:   <voluntary_resched+0x1e/0x50>
 [<c01064ae>] dump_stack+0x1e/0x20
 [<c0130c64>] check_preempt_timing+0x184/0x1e0
 [<c0130cf8>] touch_preempt_timing+0x38/0x60
 [<c0292a9e>] voluntary_resched+0x1e/0x50
 [<c013a3f0>] __kmalloc+0x80/0x90
 [<c0316279>] malloc+0x19/0x20
 [<c01024d4>] huft_build+0x2d4/0x590
 [<c0102ea1>] inflate_fixed+0xc1/0x1a0
 [<c010379a>] inflate+0x4a/0xb0
 [<c0103b89>] gunzip+0x2f9/0x530
 [<c0316e41>] unpack_to_rootfs+0x191/0x220
 [<c0316efb>] populate_rootfs+0x2b/0x130
 [<c010033e>] init+0x3e/0x170
 [<c01040a5>] kernel_thread_helper+0x5/0x10
(swapper/1): new 31 us maximum-latency critical section.
 => started at: <voluntary_resched+0x1e/0x50>
 => ended at:   <voluntary_resched+0x1e/0x50>
 [<c01064ae>] dump_stack+0x1e/0x20
 [<c0130c64>] check_preempt_timing+0x184/0x1e0
 [<c0130cf8>] touch_preempt_timing+0x38/0x60
 [<c0292a9e>] voluntary_resched+0x1e/0x50
 [<c013a1b6>] kmem_cache_alloc+0x56/0x60
 [<c015c8e9>] getname+0x29/0xc0
 [<c015f3fd>] sys_mkdir+0x1d/0xf0
 [<c03169af>] do_name+0x10f/0x1b0
 [<c0316bb1>] write_buffer+0x21/0x30
 [<c0316be4>] flush_buffer+0x24/0x60
 [<c0316c64>] flush_window+0x24/0x70
 [<c01037db>] inflate+0x8b/0xb0
 [<c0103b89>] gunzip+0x2f9/0x530
 [<c0316e41>] unpack_to_rootfs+0x191/0x220
 [<c0316efb>] populate_rootfs+0x2b/0x130
 [<c010033e>] init+0x3e/0x170
 [<c01040a5>] kernel_thread_helper+0x5/0x10
NET: Registered protocol family 16

Btw: i do have some regular ca. 300 us latencies.. Here are some traces
(these happen with an average frequency of ca. 0.3hz):


Aug 13 12:40:49 mango kernel: (ksoftirqd/0/2): 307 us critical section
violates 250 us threshold.
Aug 13 12:40:49 mango kernel:  => started at: <___do_softirq+0x20/0x90>
Aug 13 12:40:49 mango kernel:  => ended at:  
<cond_resched_softirq+0x59/0x70>
Aug 13 12:40:49 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 13 12:40:49 mango kernel:  [<c0130c64>]
check_preempt_timing+0x184/0x1e0
Aug 13 12:40:49 mango kernel:  [<c0130cf8>]
touch_preempt_timing+0x38/0x60
Aug 13 12:40:49 mango kernel:  [<c0113df9>]
cond_resched_softirq+0x59/0x70
Aug 13 12:40:49 mango kernel:  [<c011faef>] run_timer_softirq+0xdf/0x1d0
Aug 13 12:40:49 mango kernel:  [<c011adbc>] ___do_softirq+0x7c/0x90
Aug 13 12:40:49 mango kernel:  [<c011ae19>] _do_softirq+0x9/0x10
Aug 13 12:40:49 mango kernel:  [<c011b1d4>] ksoftirqd+0x84/0xd0
Aug 13 12:40:49 mango kernel:  [<c012a8da>] kthread+0xaa/0xb0
Aug 13 12:40:49 mango kernel:  [<c01040a5>]
kernel_thread_helper+0x5/0x10
Aug 13 12:40:54 mango kernel: (ksoftirqd/0/2): 307 us critical section
violates 250 us threshold.
Aug 13 12:40:54 mango kernel:  => started at: <___do_softirq+0x20/0x90>
Aug 13 12:40:54 mango kernel:  => ended at:  
<cond_resched_softirq+0x59/0x70>
Aug 13 12:40:54 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 13 12:40:54 mango kernel:  [<c0130c64>]
check_preempt_timing+0x184/0x1e0
Aug 13 12:40:54 mango kernel:  [<c0130cf8>]
touch_preempt_timing+0x38/0x60
Aug 13 12:40:54 mango kernel:  [<c0113df9>]
cond_resched_softirq+0x59/0x70
Aug 13 12:40:54 mango kernel:  [<c011faef>] run_timer_softirq+0xdf/0x1d0
Aug 13 12:40:54 mango kernel:  [<c011adbc>] ___do_softirq+0x7c/0x90
Aug 13 12:40:54 mango kernel:  [<c011ae19>] _do_softirq+0x9/0x10
Aug 13 12:40:54 mango kernel:  [<c011b1d4>] ksoftirqd+0x84/0xd0
Aug 13 12:40:54 mango kernel:  [<c012a8da>] kthread+0xaa/0xb0
Aug 13 12:40:54 mango kernel:  [<c01040a5>]
kernel_thread_helper+0x5/0x10
Aug 13 12:40:59 mango kernel: (ksoftirqd/0/2): 308 us critical section
violates 250 us threshold.
Aug 13 12:40:59 mango kernel:  => started at: <___do_softirq+0x20/0x90>
Aug 13 12:40:59 mango kernel:  => ended at:  
<cond_resched_softirq+0x59/0x70>
Aug 13 12:40:59 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 13 12:40:59 mango kernel:  [<c0130c64>]
check_preempt_timing+0x184/0x1e0
Aug 13 12:40:59 mango kernel:  [<c0130cf8>]
touch_preempt_timing+0x38/0x60
Aug 13 12:40:59 mango kernel:  [<c0113df9>]
cond_resched_softirq+0x59/0x70
Aug 13 12:40:59 mango kernel:  [<c011faef>] run_timer_softirq+0xdf/0x1d0
Aug 13 12:40:59 mango kernel:  [<c011adbc>] ___do_softirq+0x7c/0x90
Aug 13 12:40:59 mango kernel:  [<c011ae19>] _do_softirq+0x9/0x10
Aug 13 12:40:59 mango kernel:  [<c011b1d4>] ksoftirqd+0x84/0xd0
Aug 13 12:40:59 mango kernel:  [<c012a8da>] kthread+0xaa/0xb0
Aug 13 12:40:59 mango kernel:  [<c01040a5>]
kernel_thread_helper+0x5/0x10
Aug 13 12:41:04 mango kernel: (ksoftirqd/0/2): 307 us critical section
violates 250 us threshold.
Aug 13 12:41:04 mango kernel:  => started at: <___do_softirq+0x20/0x90>
Aug 13 12:41:04 mango kernel:  => ended at:  
<cond_resched_softirq+0x59/0x70>
Aug 13 12:41:04 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 13 12:41:04 mango kernel:  [<c0130c64>]
check_preempt_timing+0x184/0x1e0
Aug 13 12:41:04 mango kernel:  [<c0130cf8>]
touch_preempt_timing+0x38/0x60
Aug 13 12:41:04 mango kernel:  [<c0113df9>]
cond_resched_softirq+0x59/0x70
Aug 13 12:41:04 mango kernel:  [<c011faef>] run_timer_softirq+0xdf/0x1d0
Aug 13 12:41:04 mango kernel:  [<c011adbc>] ___do_softirq+0x7c/0x90
Aug 13 12:41:04 mango kernel:  [<c011ae19>] _do_softirq+0x9/0x10
Aug 13 12:41:04 mango kernel:  [<c011b1d4>] ksoftirqd+0x84/0xd0
Aug 13 12:41:04 mango kernel:  [<c012a8da>] kthread+0xaa/0xb0
Aug 13 12:41:04 mango kernel:  [<c01040a5>]
kernel_thread_helper+0x5/0x10
Aug 13 12:41:09 mango kernel: (ksoftirqd/0/2): 307 us critical section
violates 250 us threshold.
Aug 13 12:41:09 mango kernel:  => started at: <___do_softirq+0x20/0x90>
Aug 13 12:41:09 mango kernel:  => ended at:  
<cond_resched_softirq+0x59/0x70>
Aug 13 12:41:09 mango kernel:  [<c01064ae>] dump_stack+0x1e/0x20
Aug 13 12:41:09 mango kernel:  [<c0130c64>]
check_preempt_timing+0x184/0x1e0
Aug 13 12:41:09 mango kernel:  [<c0130cf8>]
touch_preempt_timing+0x38/0x60
Aug 13 12:41:09 mango kernel:  [<c0113df9>]
cond_resched_softirq+0x59/0x70
Aug 13 12:41:09 mango kernel:  [<c011faef>] run_timer_softirq+0xdf/0x1d0
Aug 13 12:41:09 mango kernel:  [<c011adbc>] ___do_softirq+0x7c/0x90
Aug 13 12:41:09 mango kernel:  [<c011ae19>] _do_softirq+0x9/0x10
Aug 13 12:41:09 mango kernel:  [<c011b1d4>] ksoftirqd+0x84/0xd0
Aug 13 12:41:09 mango kernel:  [<c012a8da>] kthread+0xaa/0xb0
Aug 13 12:41:09 mango kernel:  [<c01040a5>]
kernel_thread_helper+0x5/0x10

Some system info:

I do not habe preempt-tracing enabled, only voluntary preempt and
preempt timing..

Linux mango.fruits.de 2.6.8-rc4 #1 Fri Aug 13 03:05:04 CEST 2004 i686
GNU/Linux
-----------
processor       : 0
vendor_id       : AuthenticAMD
cpu family      : 6
model           : 7
model name      : AMD Duron(tm) Processor
stepping        : 1
cpu MHz         : 1195.497
cache size      : 64 KB
fdiv_bug        : no
hlt_bug         : no
f00f_bug        : no
coma_bug        : no
fpu             : yes
fpu_exception   : yes
cpuid level     : 1
wp              : yes
flags           : fpu vme de pse tsc msr pae mce cx8 sep mtrr pge mca
cmov pat pse36 mmx fxsr sse syscall mp mmxext 3dnowext 3dnow
bogomips        : 2359.29

-----------
/proc/interrupts:
           CPU0       
  0:    1832325          XT-PIC  timer
  1:       8892          XT-PIC  i8042
  2:          0          XT-PIC  cascade
  5:          0          XT-PIC  CS46XX
  8:          4          XT-PIC  rtc
 10:       2066          XT-PIC  eth0
 12:      39786          XT-PIC  i8042
 14:        900          XT-PIC  ide0
 15:      12719          XT-PIC  ide1
NMI:          0 
ERR:          0
-----------
/proc/irq/10/eth0/threaded:1
/proc/irq/12/i8042/threaded:1
/proc/irq/14/ide0/threaded:1
/proc/irq/15/ide1/threaded:1
/proc/irq/1/i8042/threaded:1
/proc/irq/5/CS46XX/threaded:0
/proc/irq/8/rtc/threaded:0
-----------
/sys/block/hda/queue/max_sectors_kb:16
/sys/block/hdd/queue/max_sectors_kb:16
-----------
voluntary_preemption
3
kernel_preemption
1
-----------
preempt_thresh
250


-- 
Palimm Palimm!
http://affenbande.org/~tapas/

[patch] voluntary-preempt-2.6.8-rc4-O7

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Here's another weird one.  This happens when you create a new tab in
> gnome-terminal, which causes a fork().
> 
> preemption latency trace v1.0
> -----------------------------
>  latency: 613 us, entries: 697 (697)
>  process: gnome-terminal/3467, uid: 1000
>  nice: 0, policy: 0, rt_priority: 0
> =======>

>  0.459ms (+0.000ms): preempt_schedule (copy_page_range)
>  0.459ms (+0.000ms): preempt_schedule (copy_page_range)
>  0.460ms (+0.000ms): preempt_schedule (copy_page_range)
>  0.461ms (+0.000ms): preempt_schedule (copy_page_range)

ok, it seems the lock-break of the outer loop was not enough - the up to
1024 iterations in the inner loop can generate quite high latencies too.

in -O7 i've added a runtime flag to disable/enable tracing without
having to recompile the kernel:

 http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O7

/proc/sys/kernel/tracing_enabled controls whether full tracing is done
or not. (some overhead still occurs due to the instrumentation but most
of the tracing overhead should vanish.)

It would be nice to check whether the same overhead occurs with tracing
disabled too - the copy_page_range() loop could be one piece of code
that could get roughly twice as slow with tracing enabled as with
tracing disabled. But 300 usecs is too much too.

-O7 also adds your EXPORT_SYMBOL fix and Peter Zijlstra's compilation
fix.

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* Florian Schmidt <mista.tapas@gmx.net> wrote:

> Hmm, should a new report only be generated when the newly measured
> latency is really > than all other ones? I get the feeling that >=
> seems to be enough. Here a dmesg extract:
> 
> (swapper/1): new 6 us maximum-latency critical section.
> (swapper/1): new 8 us maximum-latency critical section.
> (swapper/1): new 9 us maximum-latency critical section.

> Here are two reports with the same maximum-latency (31 us):
> 
> (swapper/1): new 31 us maximum-latency critical section.
> (swapper/1): new 31 us maximum-latency critical section.

the latency tracer tracks latencies in cycle units - but they are
displayed at microsecond accuracy - hence these 'equal' latencies.

to jump-start all those smaller latencies you can do this:

	echo 100 > /proc/sys/kernel/preempt_max_latency
	echo 0 > /proc/sys/kernel/preempt_thresh

this way the maximum-searching only starts at 100 usecs.

> Btw: i do have some regular ca. 300 us latencies.. Here are some traces
> (these happen with an average frequency of ca. 0.3hz):

> (ksoftirqd/0/2): 307 us critical section violates 250 us threshold.
>  => started at: <___do_softirq+0x20/0x90>
>  => ended at: <cond_resched_softirq+0x59/0x70>

this is too opaque - could you try -O7, enable tracing and save a
/proc/latency_trace instance of such a latency? It looks like some sort
of softirq latency - perhaps one particular driver's timer fn causes it
- we'll be able to tell more from the trace.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc4-O7

[Ingo Molnar]

* Florian Schmidt <mista.tapas@gmx.net> wrote:

> > in -O7 i've added a runtime flag to disable/enable tracing without
> > having to recompile the kernel:
> > 
> >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O7
> 
> I have one question: Are kernel frame pointers required for the
> latency traces? I had frame pointers on in my kernel config but wonder
> if they are nessecary or if i can save the cycles and kb [i'm not too
> big on kernel internals, so forgive the possibly stupid question]?..

-pg requires frame pointers, yes. The Kconfig change i did forces
framepointers on so it's automatic. (gcc bails out with an error on -pg
-fomit-frame-pointer)

> The other thing i wanted to say is: way to go man!! These patches rock
> my linux audio world. A big thank you from me and [probably] the whole
> linux audio community for the work you put into this!

you are welcome!

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc4-O7

[Florian Schmidt]

On Fri, 13 Aug 2004 12:48:17 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> in -O7 i've added a runtime flag to disable/enable tracing without
> having to recompile the kernel:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O7

I have one question: Are kernel frame pointers required for the latency
traces? I had frame pointers on in my kernel config but wonder if they
are nessecary or if i can save the cycles and kb [i'm not too big on
kernel internals, so forgive the possibly stupid question]?..

The other thing i wanted to say is: way to go man!! These patches rock
my linux audio world. A big thank you from me and [probably] the whole
linux audio community for the work you put into this!

Compiling O7 right now with traces enabled, so i'll send a trace soon
about the ca 300 us latencies i see.

Flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* Florian Schmidt <mista.tapas@gmx.net> wrote:

> On Fri, 13 Aug 2004 12:54:06 +0200
> Ingo Molnar <mingo@elte.hu> wrote:
> 
> > > (ksoftirqd/0/2): 307 us critical section violates 250 us threshold.
> > >  => started at: <___do_softirq+0x20/0x90>
> > >  => ended at: <cond_resched_softirq+0x59/0x70>
> > 
> > this is too opaque - could you try -O7, enable tracing and save a
> > /proc/latency_trace instance of such a latency? It looks like some
> > sort of softirq latency - perhaps one particular driver's timer fn
> > causes it- we'll be able to tell more from the trace.
> 
> Hi, this looks like one of them:
> 
> mango:~# cat /proc/latency_trace 
> preemption latency trace v1.0
> -----------------------------
>  latency: 308 us, entries: 12 (12)
>  process: ksoftirqd/0/2, uid: 0
>  nice: -10, policy: 0, rt_priority: 0
> =======>
>  0.000ms (+0.000ms): run_timer_softirq (___do_softirq)
>  0.000ms (+0.000ms): sis900_timer (run_timer_softirq)
>  0.001ms (+0.000ms): mdio_read (sis900_timer)
>  0.002ms (+0.000ms): mdio_reset (mdio_read)
>  0.071ms (+0.069ms): mdio_idle (mdio_read)
>  0.151ms (+0.079ms): mdio_read (sis900_timer)
>  0.151ms (+0.000ms): mdio_reset (mdio_read)
>  0.220ms (+0.069ms): mdio_idle (mdio_read)
>  0.300ms (+0.079ms): __mod_timer (sis900_timer)
>  0.300ms (+0.000ms): internal_add_timer (__mod_timer)
>  0.300ms (+0.000ms): cond_resched_softirq (run_timer_softirq)
>  0.301ms (+0.000ms): check_preempt_timing (touch_preempt_timing)

indeed this seems to be a driver related timer: driver/net/sis900.c's
sis900_timer() function. This timer polls your network card for link
status once every second. The mdio_read() function does alot of in/out
instructions which are quite slow.

could you try the following and disable mdio_delay():

	#define mdio_delay() do { } while (0)

normally it should work just fine. Worst-case you'd get a non-working
network driver. Does this change reduce the latency?

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Florian Schmidt]

On Fri, 13 Aug 2004 12:54:06 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> > (ksoftirqd/0/2): 307 us critical section violates 250 us threshold.
> >  => started at: <___do_softirq+0x20/0x90>
> >  => ended at: <cond_resched_softirq+0x59/0x70>
> 
> this is too opaque - could you try -O7, enable tracing and save a
> /proc/latency_trace instance of such a latency? It looks like some
> sort of softirq latency - perhaps one particular driver's timer fn
> causes it- we'll be able to tell more from the trace.

Hi, this looks like one of them:

mango:~# cat /proc/latency_trace 
preemption latency trace v1.0
-----------------------------
 latency: 308 us, entries: 12 (12)
 process: ksoftirqd/0/2, uid: 0
 nice: -10, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): run_timer_softirq (___do_softirq)
 0.000ms (+0.000ms): sis900_timer (run_timer_softirq)
 0.001ms (+0.000ms): mdio_read (sis900_timer)
 0.002ms (+0.000ms): mdio_reset (mdio_read)
 0.071ms (+0.069ms): mdio_idle (mdio_read)
 0.151ms (+0.079ms): mdio_read (sis900_timer)
 0.151ms (+0.000ms): mdio_reset (mdio_read)
 0.220ms (+0.069ms): mdio_idle (mdio_read)
 0.300ms (+0.079ms): __mod_timer (sis900_timer)
 0.300ms (+0.000ms): internal_add_timer (__mod_timer)
 0.300ms (+0.000ms): cond_resched_softirq (run_timer_softirq)
 0.301ms (+0.000ms): check_preempt_timing (touch_preempt_timing)

Flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Fri, 2004-08-13 at 06:22, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Interesting results.  One of the problems is kallsyms_lookup,
> > triggered by the printks:
> 
> yeah - kallsyms_lookup does a linear search over thousands of symbols. 
> Especially since /proc/latency_trace uses it too it would be worthwile
> to implement some sort of binary searching.
> 

Would it be easier to have a mode where the symbols are not resolved,
and would just require the traces to be postprocessed? 

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Fri, 2004-08-13 at 06:31, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6
> > 
> > Ugh, this is a bad one:
> > 
> > preemption latency trace v1.0
> > -----------------------------
> >  latency: 506 us, entries: 157 (157)
> >  process: evolution/3461, uid: 1000
> >  nice: 0, policy: 0, rt_priority: 0
> > =======>
> >  0.000ms (+0.000ms): get_random_bytes (__check_and_rekey)
> [...]
> >  0.493ms (+0.001ms): local_bh_enable (__check_and_rekey)
> 
> indeed this is a new one. Entropy rekeying every 300 seconds. Most of
> the overhead comes from the memcpy's - 10 usecs a pop!

Here's another case where those memcpy's hurt.  This was the biggest
latency reported last night:

preemption latency trace v1.0
-----------------------------
 latency: 683 us, entries: 329 (329)
 process: ksoftirqd/0/2, uid: 0
 nice: -10, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): process_backlog (net_rx_action)
 0.000ms (+0.000ms): netif_receive_skb (process_backlog)
 0.002ms (+0.002ms): packet_rcv_spkt (netif_receive_skb)
 0.003ms (+0.000ms): skb_clone (packet_rcv_spkt)
 0.003ms (+0.000ms): kmem_cache_alloc (skb_clone)
 0.004ms (+0.000ms): memcpy (skb_clone)
 0.006ms (+0.001ms): strlcpy (packet_rcv_spkt)
 0.007ms (+0.001ms): sk_run_filter (packet_rcv_spkt)
 0.010ms (+0.002ms): __kfree_skb (packet_rcv_spkt)
 0.010ms (+0.000ms): kfree_skbmem (__kfree_skb)
 0.010ms (+0.000ms): skb_release_data (kfree_skbmem)
 0.011ms (+0.000ms): kmem_cache_free (kfree_skbmem)
 0.011ms (+0.000ms): ip_rcv (netif_receive_skb)
 0.013ms (+0.001ms): ip_route_input (ip_rcv)
 0.014ms (+0.000ms): rt_hash_code (ip_route_input)
 0.015ms (+0.001ms): ip_route_input_slow (ip_rcv)
 0.017ms (+0.001ms): rt_hash_code (ip_route_input_slow)
 0.018ms (+0.001ms): fn_hash_lookup (ip_route_input_slow)
 0.020ms (+0.001ms): fib_semantic_match (fn_hash_lookup)
 0.022ms (+0.002ms): fib_validate_source (ip_route_input_slow)
 0.023ms (+0.001ms): fn_hash_lookup (fib_validate_source)
 0.024ms (+0.001ms): fn_hash_lookup (fib_validate_source)
 0.026ms (+0.001ms): fib_semantic_match (fn_hash_lookup)
 0.027ms (+0.000ms): __fib_res_prefsrc (fib_validate_source)
 0.027ms (+0.000ms): inet_select_addr (__fib_res_prefsrc)
 0.030ms (+0.002ms): do_IRQ (common_interrupt)
 0.030ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.034ms (+0.003ms): generic_redirect_hardirq (do_IRQ)
 0.034ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.034ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 0.035ms (+0.000ms): mark_offset_tsc (timer_interrupt)
 0.041ms (+0.005ms): do_timer (timer_interrupt)
 0.041ms (+0.000ms): update_process_times (do_timer)
 0.041ms (+0.000ms): update_one_process (update_process_times)
 0.042ms (+0.000ms): run_local_timers (update_process_times)
 0.042ms (+0.000ms): raise_softirq (update_process_times)
 0.043ms (+0.000ms): scheduler_tick (update_process_times)
 0.043ms (+0.000ms): sched_clock (scheduler_tick)
 0.044ms (+0.001ms): task_timeslice (scheduler_tick)
 0.045ms (+0.000ms): update_wall_time (do_timer)
 0.045ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 0.046ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.046ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.046ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.048ms (+0.001ms): dst_alloc (ip_route_input_slow)
 0.049ms (+0.000ms): kmem_cache_alloc (dst_alloc)
 0.050ms (+0.000ms): cache_alloc_refill (kmem_cache_alloc)
 0.055ms (+0.005ms): rt_intern_hash (ip_route_input_slow)
 0.056ms (+0.001ms): local_bh_enable (rt_intern_hash)
 0.058ms (+0.001ms): ip_local_deliver (ip_rcv)
 0.059ms (+0.001ms): tcp_v4_rcv (ip_local_deliver)
 0.060ms (+0.000ms): tcp_v4_checksum_init (tcp_v4_rcv)
 0.060ms (+0.000ms): skb_checksum (tcp_v4_checksum_init)
 0.065ms (+0.004ms): tcp_v4_do_rcv (tcp_v4_rcv)
 0.066ms (+0.000ms): tcp_v4_hnd_req (tcp_v4_do_rcv)
 0.066ms (+0.000ms): tcp_v4_search_req (tcp_v4_hnd_req)
 0.068ms (+0.002ms): tcp_rcv_state_process (tcp_v4_do_rcv)
 0.069ms (+0.001ms): tcp_v4_conn_request (tcp_rcv_state_process)
 0.070ms (+0.000ms): kmem_cache_alloc (tcp_v4_conn_request)
 0.071ms (+0.000ms): cache_alloc_refill (kmem_cache_alloc)
 0.072ms (+0.001ms): cache_grow (cache_alloc_refill)
 0.073ms (+0.000ms): kmem_flagcheck (cache_grow)
 0.073ms (+0.000ms): kmem_getpages (cache_grow)
 0.073ms (+0.000ms): __get_free_pages (kmem_getpages)
 0.074ms (+0.000ms): __alloc_pages (__get_free_pages)
 0.074ms (+0.000ms): buffered_rmqueue (__alloc_pages)
 0.075ms (+0.000ms): bad_range (buffered_rmqueue)
 0.076ms (+0.000ms): prep_new_page (buffered_rmqueue)
 0.076ms (+0.000ms): zone_statistics (__alloc_pages)
 0.077ms (+0.000ms): alloc_slabmgmt (cache_grow)
 0.077ms (+0.000ms): set_slab_attr (cache_grow)
 0.078ms (+0.000ms): cache_init_objs (cache_grow)
 0.084ms (+0.005ms): tcp_parse_options (tcp_v4_conn_request)
 0.087ms (+0.003ms): secure_tcp_sequence_number (tcp_v4_conn_request)
 0.087ms (+0.000ms): do_gettimeofday (secure_tcp_sequence_number)
 0.088ms (+0.000ms): get_offset_tsc (do_gettimeofday)
 0.089ms (+0.000ms): __check_and_rekey (secure_tcp_sequence_number)
 0.089ms (+0.000ms): get_random_bytes (__check_and_rekey)
 0.090ms (+0.000ms): extract_entropy (get_random_bytes)
 0.091ms (+0.001ms): extract_entropy (extract_entropy)
 0.093ms (+0.001ms): SHATransform (extract_entropy)
 0.093ms (+0.000ms): memcpy (SHATransform)
 0.103ms (+0.009ms): add_entropy_words (extract_entropy)

[...]

 0.572ms (+0.000ms): SHATransform (extract_entropy)
 0.572ms (+0.000ms): memcpy (SHATransform)
 0.581ms (+0.008ms): add_entropy_words (extract_entropy)
 0.582ms (+0.001ms): local_bh_enable (__check_and_rekey)
 0.583ms (+0.000ms): halfMD4Transform (secure_tcp_sequence_number)
 0.584ms (+0.001ms): tcp_v4_send_synack (tcp_v4_conn_request)
 0.585ms (+0.000ms): tcp_v4_route_req (tcp_v4_send_synack)
 0.586ms (+0.001ms): do_IRQ (common_interrupt)
 0.587ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.591ms (+0.004ms): generic_redirect_hardirq (do_IRQ)
 0.592ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.592ms (+0.000ms): rtc_interrupt (generic_handle_IRQ_event)
 0.592ms (+0.000ms): is_hpet_enabled (rtc_interrupt)
 0.596ms (+0.004ms): mod_timer (rtc_interrupt)
 0.597ms (+0.000ms): __mod_timer (rtc_interrupt)
 0.597ms (+0.000ms): internal_add_timer (__mod_timer)
 0.598ms (+0.000ms): __wake_up (rtc_interrupt)
 0.598ms (+0.000ms): __wake_up_common (__wake_up)
 0.599ms (+0.000ms): kill_fasync (rtc_interrupt)
 0.599ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.599ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.600ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.601ms (+0.001ms): ip_route_output_flow (tcp_v4_route_req)
 0.602ms (+0.000ms): __ip_route_output_key (ip_route_output_flow)
 0.602ms (+0.000ms): rt_hash_code (__ip_route_output_key)
 0.603ms (+0.000ms): ip_route_output_slow (ip_route_output_flow)
 0.604ms (+0.001ms): ip_dev_find (ip_route_output_slow)
 0.604ms (+0.000ms): fn_hash_lookup (ip_dev_find)
 0.605ms (+0.000ms): fib_semantic_match (fn_hash_lookup)
 0.606ms (+0.000ms): fn_hash_lookup (ip_route_output_slow)
 0.607ms (+0.000ms): fn_hash_lookup (ip_route_output_slow)
 0.607ms (+0.000ms): fib_semantic_match (fn_hash_lookup)
 0.608ms (+0.000ms): fn_hash_select_default (ip_route_output_slow)
 0.610ms (+0.001ms): dst_alloc (ip_route_output_slow)
 0.610ms (+0.000ms): kmem_cache_alloc (dst_alloc)
 0.613ms (+0.002ms): rt_set_nexthop (ip_route_output_slow)
 0.613ms (+0.000ms): memcpy (rt_set_nexthop)
 0.614ms (+0.000ms): rt_hash_code (ip_route_output_slow)
 0.614ms (+0.000ms): rt_intern_hash (ip_route_output_slow)
 0.615ms (+0.000ms): arp_bind_neighbour (rt_intern_hash)
 0.615ms (+0.000ms): neigh_lookup (arp_bind_neighbour)
 0.616ms (+0.000ms): arp_hash (neigh_lookup)
 0.618ms (+0.001ms): local_bh_enable (neigh_lookup)
 0.618ms (+0.000ms): local_bh_enable (rt_intern_hash)
 0.619ms (+0.001ms): tcp_make_synack (tcp_v4_send_synack)
 0.619ms (+0.000ms): sock_wmalloc (tcp_make_synack)
 0.620ms (+0.000ms): alloc_skb (sock_wmalloc)
 0.620ms (+0.000ms): kmem_cache_alloc (alloc_skb)
 0.621ms (+0.000ms): __kmalloc (alloc_skb)
 0.626ms (+0.005ms): ip_build_and_send_pkt (tcp_v4_send_synack)
 0.628ms (+0.001ms): ip_output (ip_build_and_send_pkt)
 0.628ms (+0.000ms): ip_finish_output (ip_build_and_send_pkt)
 0.629ms (+0.000ms): neigh_resolve_output (ip_finish_output)
 0.630ms (+0.000ms): neigh_hh_init (neigh_resolve_output)
 0.631ms (+0.001ms): eth_header (neigh_resolve_output)
 0.632ms (+0.001ms): local_bh_enable (neigh_resolve_output)
 0.633ms (+0.000ms): dev_queue_xmit (neigh_resolve_output)
 0.634ms (+0.001ms): pfifo_fast_enqueue (dev_queue_xmit)
 0.635ms (+0.000ms): qdisc_restart (dev_queue_xmit)
 0.635ms (+0.000ms): pfifo_fast_dequeue (qdisc_restart)
 0.636ms (+0.000ms): dev_queue_xmit_nit (qdisc_restart)
 0.637ms (+0.000ms): skb_clone (dev_queue_xmit_nit)
 0.637ms (+0.000ms): kmem_cache_alloc (skb_clone)
 0.638ms (+0.000ms): memcpy (skb_clone)
 0.639ms (+0.001ms): packet_rcv_spkt (dev_queue_xmit_nit)
 0.639ms (+0.000ms): strlcpy (packet_rcv_spkt)
 0.640ms (+0.000ms): sk_run_filter (packet_rcv_spkt)
 0.641ms (+0.001ms): __kfree_skb (packet_rcv_spkt)
 0.642ms (+0.000ms): kfree_skbmem (__kfree_skb)
 0.642ms (+0.000ms): skb_release_data (kfree_skbmem)
 0.642ms (+0.000ms): kmem_cache_free (kfree_skbmem)
 0.643ms (+0.000ms): rhine_start_tx (qdisc_restart)
 0.646ms (+0.003ms): qdisc_restart (dev_queue_xmit)
 0.646ms (+0.000ms): pfifo_fast_dequeue (qdisc_restart)
 0.647ms (+0.000ms): local_bh_enable (dev_queue_xmit)
 0.648ms (+0.001ms): tcp_v4_synq_add (tcp_v4_conn_request)
 0.649ms (+0.000ms): tcp_reset_keepalive_timer (tcp_v4_synq_add)
 0.649ms (+0.000ms): sk_reset_timer (tcp_reset_keepalive_timer)
 0.650ms (+0.000ms): mod_timer (sk_reset_timer)
 0.650ms (+0.000ms): __mod_timer (sk_reset_timer)
 0.651ms (+0.000ms): internal_add_timer (__mod_timer)
 0.652ms (+0.001ms): init_westwood (tcp_rcv_state_process)
 0.652ms (+0.000ms): init_bictcp (tcp_rcv_state_process)
 0.653ms (+0.000ms): __kfree_skb (tcp_rcv_state_process)
 0.654ms (+0.000ms): kfree_skbmem (__kfree_skb)
 0.654ms (+0.000ms): skb_release_data (kfree_skbmem)
 0.654ms (+0.000ms): kfree (kfree_skbmem)
 0.656ms (+0.001ms): do_IRQ (common_interrupt)
 0.656ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.660ms (+0.004ms): generic_redirect_hardirq (do_IRQ)
 0.661ms (+0.000ms): wake_up_process (generic_redirect_hardirq)
 0.661ms (+0.000ms): try_to_wake_up (wake_up_process)
 0.661ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.662ms (+0.000ms): activate_task (try_to_wake_up)
 0.662ms (+0.000ms): sched_clock (activate_task)
 0.662ms (+0.000ms): recalc_task_prio (activate_task)
 0.663ms (+0.000ms): effective_prio (recalc_task_prio)
 0.663ms (+0.000ms): enqueue_task (activate_task)
 0.664ms (+0.001ms): kmem_cache_free (kfree_skbmem)
 0.666ms (+0.001ms): check_preempt_timing (touch_preempt_timing)

Lee

Re: [patch] voluntary-preempt-2.6.8-rc4-O7

[Lee Revell]

On Fri, 2004-08-13 at 06:48, Ingo Molnar wrote:

>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O7
> 

Here's another very common one.  I am not sure this one is a problem, 
but possibly points to areas where the VM (or XFree86) could be
improved.

preemption latency trace v1.0
-----------------------------
 latency: 166 us, entries: 305 (305)
 process: XFree86/478, uid: 0
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): unmap_page_range (unmap_vmas)
 0.000ms (+0.000ms): zap_pmd_range (unmap_page_range)
 0.000ms (+0.000ms): zap_pte_range (zap_pmd_range)
 0.002ms (+0.001ms): set_page_dirty (zap_pte_range)
 0.003ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.003ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.004ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.005ms (+0.001ms): free_hot_page (zap_pte_range)
 0.005ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.006ms (+0.001ms): set_page_dirty (zap_pte_range)
 0.007ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.007ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.007ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.008ms (+0.000ms): free_hot_page (zap_pte_range)
 0.008ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.009ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.009ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.010ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.010ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.011ms (+0.000ms): free_hot_page (zap_pte_range)
 0.011ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.012ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.012ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.013ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.013ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.014ms (+0.000ms): free_hot_page (zap_pte_range)
 0.014ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.015ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.015ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.016ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.016ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.016ms (+0.000ms): free_hot_page (zap_pte_range)
 0.017ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.017ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.018ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.018ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.019ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.019ms (+0.000ms): free_hot_page (zap_pte_range)
 0.019ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.020ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.021ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.021ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.021ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.022ms (+0.000ms): free_hot_page (zap_pte_range)
 0.022ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.023ms (+0.000ms): free_pages_bulk (free_hot_cold_page)
 0.037ms (+0.014ms): do_IRQ (common_interrupt)
 0.038ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.041ms (+0.003ms): generic_redirect_hardirq (do_IRQ)
 0.042ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.042ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 0.043ms (+0.000ms): mark_offset_tsc (timer_interrupt)
 0.048ms (+0.005ms): do_timer (timer_interrupt)
 0.049ms (+0.000ms): update_process_times (do_timer)
 0.049ms (+0.000ms): update_one_process (update_process_times)
 0.050ms (+0.000ms): run_local_timers (update_process_times)
 0.050ms (+0.000ms): raise_softirq (update_process_times)
 0.050ms (+0.000ms): scheduler_tick (update_process_times)
 0.051ms (+0.000ms): sched_clock (scheduler_tick)
 0.052ms (+0.001ms): task_timeslice (scheduler_tick)
 0.052ms (+0.000ms): update_wall_time (do_timer)
 0.052ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 0.053ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.053ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.054ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.055ms (+0.001ms): do_softirq (do_IRQ)
 0.055ms (+0.000ms): __do_softirq (do_softirq)
 0.056ms (+0.000ms): wake_up_process (do_softirq)
 0.056ms (+0.000ms): try_to_wake_up (wake_up_process)
 0.056ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.057ms (+0.000ms): activate_task (try_to_wake_up)
 0.057ms (+0.000ms): sched_clock (activate_task)
 0.058ms (+0.000ms): recalc_task_prio (activate_task)
 0.058ms (+0.000ms): effective_prio (recalc_task_prio)
 0.058ms (+0.000ms): enqueue_task (activate_task)
 0.059ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.060ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.060ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.060ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.061ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.061ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.061ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.062ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.062ms (+0.000ms): free_hot_page (zap_pte_range)
 0.063ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.063ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.064ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.064ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.064ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.065ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.065ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.066ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.066ms (+0.000ms): free_hot_page (zap_pte_range)
 0.066ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.067ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.067ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.068ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.068ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.068ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.069ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.069ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.070ms (+0.000ms): free_hot_page (zap_pte_range)
 0.070ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.071ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.071ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.071ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.072ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.072ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.072ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.073ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.073ms (+0.000ms): free_hot_page (zap_pte_range)
 0.073ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.074ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.074ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.075ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.075ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.075ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.076ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.076ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.077ms (+0.000ms): free_hot_page (zap_pte_range)
 0.077ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.078ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.078ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.078ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.079ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.079ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.079ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.080ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.080ms (+0.000ms): free_hot_page (zap_pte_range)
 0.081ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.081ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.082ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.082ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.082ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.083ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.083ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.084ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.084ms (+0.000ms): free_hot_page (zap_pte_range)
 0.084ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.085ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.085ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.086ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.086ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.086ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.087ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.087ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.088ms (+0.000ms): free_hot_page (zap_pte_range)
 0.088ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.088ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.089ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.089ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.090ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.090ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.090ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.091ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.091ms (+0.000ms): free_hot_page (zap_pte_range)
 0.091ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.092ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.092ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.093ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.093ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.093ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.094ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.094ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.095ms (+0.000ms): free_hot_page (zap_pte_range)
 0.095ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.096ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.096ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.096ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.097ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.097ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.097ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.098ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.098ms (+0.000ms): free_hot_page (zap_pte_range)
 0.099ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.099ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.100ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.100ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.100ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.101ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.101ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.102ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.102ms (+0.000ms): free_hot_page (zap_pte_range)
 0.102ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.103ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.103ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.104ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.104ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.104ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.105ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.105ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.106ms (+0.000ms): free_hot_page (zap_pte_range)
 0.106ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.106ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.107ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.107ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.108ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.108ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.108ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.109ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.109ms (+0.000ms): free_hot_page (zap_pte_range)
 0.110ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.110ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.110ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.111ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.111ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.111ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.112ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.112ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.113ms (+0.000ms): free_hot_page (zap_pte_range)
 0.113ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.114ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.114ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.114ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.115ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.115ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.115ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.116ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.116ms (+0.000ms): free_hot_page (zap_pte_range)
 0.117ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.117ms (+0.000ms): free_pages_bulk (free_hot_cold_page)
 0.128ms (+0.011ms): preempt_schedule (free_pages_bulk)
 0.129ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.129ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.130ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.130ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.130ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.131ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.131ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.132ms (+0.000ms): free_hot_page (zap_pte_range)
 0.132ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.132ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.133ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.133ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.134ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.134ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.134ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.135ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.135ms (+0.000ms): free_hot_page (zap_pte_range)
 0.135ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.136ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.136ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.137ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.137ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.137ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.138ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.138ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.139ms (+0.000ms): free_hot_page (zap_pte_range)
 0.139ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.139ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.140ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.140ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.141ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.141ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.141ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.142ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.142ms (+0.000ms): free_hot_page (zap_pte_range)
 0.142ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.143ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.144ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.144ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.145ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.145ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.145ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.146ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.146ms (+0.000ms): free_hot_page (zap_pte_range)
 0.146ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.147ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.147ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.148ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.148ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.148ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.149ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.149ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.149ms (+0.000ms): free_hot_page (zap_pte_range)
 0.150ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.150ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.151ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.151ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.152ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.152ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.152ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.153ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.153ms (+0.000ms): free_hot_page (zap_pte_range)
 0.153ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.154ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.154ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.155ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.155ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.155ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.155ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.156ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.156ms (+0.000ms): free_hot_page (zap_pte_range)
 0.157ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.157ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.158ms (+0.000ms): set_page_dirty (zap_pte_range)
 0.158ms (+0.000ms): page_remove_rmap (zap_pte_range)
 0.159ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.159ms (+0.000ms): free_page_and_swap_cache (zap_pte_range)
 0.159ms (+0.000ms): __page_cache_release (zap_pte_range)
 0.160ms (+0.000ms): preempt_schedule (__page_cache_release)
 0.160ms (+0.000ms): free_hot_page (zap_pte_range)
 0.160ms (+0.000ms): free_hot_cold_page (zap_pte_range)
 0.161ms (+0.000ms): preempt_schedule (zap_pte_range)
 0.161ms (+0.000ms): check_preempt_timing (touch_preempt_timing)

Re: [patch] voluntary-preempt-2.6.8-rc4-O7

[Lee Revell]

On Fri, 2004-08-13 at 06:48, Ingo Molnar wrote:

>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O7
> 

Here is the trace resulting from the apt-get filemap_sync() issue:

preemption latency trace v1.0
-----------------------------
 latency: 3656 us, entries: 4000 (8605)
 process: apt-get/21227, uid: 0
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): filemap_sync_pte_range (filemap_sync)
 0.000ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.002ms (+0.001ms): set_page_dirty (filemap_sync_pte)
 0.002ms (+0.000ms): __set_page_dirty_buffers (set_page_dirty)
 0.004ms (+0.001ms): radix_tree_tag_set (__set_page_dirty_buffers)
 0.006ms (+0.001ms): __mark_inode_dirty (__set_page_dirty_buffers)
 0.008ms (+0.001ms): filemap_sync_pte (filemap_sync_pte_range)
 0.009ms (+0.000ms): set_page_dirty (filemap_sync_pte)
 0.009ms (+0.000ms): __set_page_dirty_buffers (set_page_dirty)
 0.010ms (+0.000ms): radix_tree_tag_set (__set_page_dirty_buffers)
 0.011ms (+0.000ms): __mark_inode_dirty (__set_page_dirty_buffers)
 0.011ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.012ms (+0.000ms): set_page_dirty (filemap_sync_pte)
 0.012ms (+0.000ms): __set_page_dirty_buffers (set_page_dirty)
 0.013ms (+0.000ms): radix_tree_tag_set (__set_page_dirty_buffers)
 0.013ms (+0.000ms): __mark_inode_dirty (__set_page_dirty_buffers)
 0.014ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.014ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.015ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.016ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.016ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.017ms (+0.000ms): set_page_dirty (filemap_sync_pte)
 0.017ms (+0.000ms): __set_page_dirty_buffers (set_page_dirty)
 0.018ms (+0.000ms): radix_tree_tag_set (__set_page_dirty_buffers)
 0.019ms (+0.000ms): __mark_inode_dirty (__set_page_dirty_buffers)
 0.019ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.019ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.020ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.020ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.021ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 0.022ms (+0.000ms): set_page_dirty (filemap_sync_pte)
 0.022ms (+0.000ms): __set_page_dirty_buffers (set_page_dirty)
 0.023ms (+0.000ms): radix_tree_tag_set (__set_page_dirty_buffers)
 0.023ms (+0.000ms): __mark_inode_dirty (__set_page_dirty_buffers)
 
[ zillons of these deleted ]

 1.960ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 1.960ms (+0.000ms): set_page_dirty (filemap_sync_pte)
 1.961ms (+0.000ms): __set_page_dirty_buffers (set_page_dirty)
 1.961ms (+0.000ms): preempt_schedule (__set_page_dirty_buffers)
 1.961ms (+0.000ms): radix_tree_tag_set (__set_page_dirty_buffers)
 1.962ms (+0.000ms): preempt_schedule (__set_page_dirty_buffers)
 1.962ms (+0.000ms): __mark_inode_dirty (__set_page_dirty_buffers)
 1.963ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
 1.964ms (+0.000ms): set_page_dirty (filemap_sync_pte)
 1.964ms (+0.000ms): __set_page_dirty_buffers (set_page_dirty)
 1.964ms (+0.000ms): preempt_schedule (__set_page_dirty_buffers)
 1.965ms (+0.000ms): radix_tree_tag_set (__set_page_dirty_buffers)

Lee

Re: [patch] voluntary-preempt-2.6.8-rc4-O7

[Lee Revell]

On Fri, 2004-08-13 at 06:48, Ingo Molnar wrote:

>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O7
> 

Here is the trace for a long ide i/o completion from the hardirq thread
(IRQ 15 in my case).  These are getting very long, so I modified my
latency-graphing PHP script to display any .txt files I put in that
directory.  All of the traces for a given kernel can be found like so:

http://krustophenia.net/testresults.php?dataset=2.6.8-rc4-bk3-O7

Lee

Re: [patch] voluntary-preempt-2.6.8-rc4-O7

[Lee Revell]

On Fri, 2004-08-13 at 06:48, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:

>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O7
> 

Some more interesting results.  I am not seeing much of a difference in
performance, but the shape of the distribution changes quite a bit:

http://krustophenia.net/testresults.php?dataset=2.6.8-rc3-O5
http://krustophenia.net/testresults.php?dataset=2.6.8-rc4-bk3-O7

Lee

Re: [patch] voluntary-preempt-2.6.8-rc4-O7

[Lee Revell]

On Fri, 2004-08-13 at 06:48, Ingo Molnar wrote:
> ok, it seems the lock-break of the outer loop was not enough - the up to
> 1024 iterations in the inner loop can generate quite high latencies too.
> 

In some of the traces, like this one:

http://krustophenia.net/testresults.php?dataset=2.6.8-rc4-bk3-O7#/var/www/2.6.8-rc4-bk3-O7/mount_reiserfs_latency_trace.txt

there are calls to voluntary_resched.  How is this possible?  Does it
mean that we called voluntary_resched while holding a spinlock, where we
needed to call voluntary_preempt_lock(&foo_lock), and thus failed to
reschedule?

Lee

Re: [patch] voluntary-preempt-2.6.8-rc4-O7

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> In some of the traces, like this one:
> 
> http://krustophenia.net/testresults.php?dataset=2.6.8-rc4-bk3-O7#/var/www/2.6.8-rc4-bk3-O7/mount_reiserfs_latency_trace.txt
> 
> there are calls to voluntary_resched.  How is this possible?  Does it
> mean that we called voluntary_resched while holding a spinlock, where
> we needed to call voluntary_preempt_lock(&foo_lock), and thus failed
> to reschedule?

voluntary_resched() was probably called as part of the might_sleep_if()
done in mm/slab.c. If you have CONFIG_DEBUG_SPINLOCK_SLEEP enabled then
the kernel should have complained about 'Debug: sleeping function ...'.

but what i think happened here is that reiserfs still uses
lock_kernel()/unlock_kernel() quite alot (eg. ext3 or xfs doesnt), which
from a preemptability POV is just as much of a critical section as a
spinlock, but processes can sleep (the scheduler auto-releases and
auto-reacquires the big kernel lock).

	Ingo

--- linux/kernel/sched.c.orig
+++ linux/kernel/sched.c
@@ -3210,11 +3210,11 @@ __setup("voluntary-preempt=", voluntary_
 
 int __sched voluntary_resched(void)
 {
-	if (kernel_preemption || !voluntary_preemption)
-		return 0;
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
 	__might_sleep(__FILE__, __LINE__);
 #endif
+	if (kernel_preemption || !voluntary_preemption)
+		return 0;
 	/*
 	 * The system_state check is somewhat ugly but we might be
 	 * called during early boot when we are not yet ready to reschedule.

[patch] voluntary-preempt-2.6.8-rc4-O8

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Here is the trace resulting from the apt-get filemap_sync() issue:
> 
>  0.000ms (+0.000ms): filemap_sync_pte_range (filemap_sync)
>  0.000ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)

ok, this is one that is fixed in -mm already. I've put the fix into -O8:

 http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O8

other changes in -O8: the massive kallsyms-lookup speedup from Paulo
Marque, and tracing is default-on now.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc4-O8

[Sam Ravnborg]

On Sat, Aug 14, 2004 at 09:20:09AM +0200, Ingo Molnar wrote:
> 
> other changes in -O8: the massive kallsyms-lookup speedup from Paulo
> Marque

Do you have this as an independent patch - or an URL?

	Sam

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* Florian Schmidt <mista.tapas@gmx.net> wrote:

> This is something i wanted to ask anyways: In one of your first VP
> announcements you mentioned you wanted to eliminate all latencies > 1
> ms. To me it seems, that that goal is pretty much reached [at least i
> don't see any longer ones except for at boot and shutdown]. So the
> question is: What is the lower limit for laterncies that you want to
> hear reports about?

well ... i'm interested in all latencies that are well above the typical
average latencies in the system. E.g. when the average is around 20-30
usecs then reports of 200-300 usecs would be interesting.

there's no hard limit, really. Also, sometimes latencies that are 0.3
msec in the report could be 3 msec if triggered properly. So a seemingly
lower than 1 msec latency can very well pinpoint a problem area.

> WRT mlockall: i tried mlockall'ing 500 megs. This produced a new max
> latency of 299 us. the trace is rather long. This one is with jackd
> running and the one below this is w/o jackd running:

>  0.010ms (+0.000ms): free_page_and_swap_cache (clear_page_tables)
>  0.010ms (+0.000ms): __page_cache_release (clear_page_tables)
>  0.010ms (+0.000ms): free_hot_page (clear_page_tables)

hm, the reason for this one is that clear_page_tables() does all the
freeing in a single uninterrupted critical section covered by
mm->page_table_lock.

This function needs a lock-break i believe. Especially in the
process-exit case (exit_mmap()) the lock seems unjustified - the current
task is the sole owner of a never-to-be-used-again collection of
pagetables.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc4-O8

[Lee Revell]

On Sat, 2004-08-14 at 03:20, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Here is the trace resulting from the apt-get filemap_sync() issue:
> > 
> >  0.000ms (+0.000ms): filemap_sync_pte_range (filemap_sync)
> >  0.000ms (+0.000ms): filemap_sync_pte (filemap_sync_pte_range)
> 
> ok, this is one that is fixed in -mm already. I've put the fix into -O8:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O8
> 
> other changes in -O8: the massive kallsyms-lookup speedup from Paulo
> Marque, and tracing is default-on now.
> 

The extract_entropy issue seems to be the worst offender on my system. I
have collected some traces here:

http://krustophenia.net/testresults.php?dataset=2.6.8-rc4-bk3-O7

Lee

Re: [patch] voluntary-preempt-2.6.8-rc4-O8

[Lee Revell]

On Sat, 2004-08-14 at 03:51, Sam Ravnborg wrote:
> On Sat, Aug 14, 2004 at 09:20:09AM +0200, Ingo Molnar wrote:
> > 
> > other changes in -O8: the massive kallsyms-lookup speedup from Paulo
> > Marque
> 
> Do you have this as an independent patch - or an URL?
> 

http://lkml.org/lkml/2004/8/14/3

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[James Courtier-Dutton]

Ingo Molnar wrote:
> i've uploaded the latest version of the voluntary-preempt patch:
>      
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6
> 
<snip>
> reports, suggestions welcome,
> 
> 	Ingo

I used O7.

I have tested this for a day now, and I have noticed problems:
1)
kernel syslog gets a record like this:
 >  (default.hotplug/1470): 121 us critical section violates 100 us 
threshold.
 >   => started at: <kmap_high+0x2b/0x2d0>
 >   => ended at:   <kmap_high+0x1a9/0x2d0>
 >   [<c0105a23>] dump_stack+0x23/0x30
 >   [<c0140d14>] check_preempt_timing+0x184/0x1e0
 >   [<c0140e84>] sub_preempt_count+0x54/0x5d
 >   [<c0152959>] kmap_high+0x1a9/0x2d0
 >   [<c017655a>] copy_strings+0xea/0x230
 >   [<c01766db>] copy_strings_kernel+0x3b/0x50
 >   [<c017840d>] do_execve+0x12d/0x1f0
 >   [<c0103284>] sys_execve+0x44/0x80
 >   [<c0104b95>] sysenter_past_esp+0x52/0x71
and the /proc/latency_trace gets:
 >   preemption latency trace v1.0
 >   -----------------------------
 >    latency: 121 us, entries: 1032 (1032)
 >    process: default.hotplug/1470, uid: 0
 >    nice: -10, policy: 0, rt_priority: 0
 >   =======>
 >    0.000ms (+0.000ms): page_address (kmap_high)
 >    0.000ms (+0.000ms): page_slot (page_address)
 >    0.000ms (+0.000ms): flush_all_zero_pkmaps (kmap_high)
 >    0.000ms (+0.000ms): set_page_address (flush_all_zero_pkmaps)
 >   [...]
 >    0.118ms (+0.000ms): page_slot (set_page_address)
 >    0.118ms (+0.000ms): check_preempt_timing (sub_preempt_count)

Could the patch be adjusted to make the syslog and the 
/proc/latency_trace produce the same output?

2)
I suspect that there is a problem with reiserfs, but when I detect a 
momentary hang in the system(mouse stops moving), no latency_trace appears.

Re: [patch] voluntary-preempt-2.6.8-rc4-O8

[Ingo Molnar]

* Sam Ravnborg <sam@ravnborg.org> wrote:

> > other changes in -O8: the massive kallsyms-lookup speedup from Paulo
> > Marque
> 
> Do you have this as an independent patch - or an URL?

yeah, attached below.

	Ingo

--- linux/kernel/kallsyms.c.orig	
+++ linux/kernel/kallsyms.c	
@@ -22,6 +22,13 @@ extern char kallsyms_names[] __attribute
 /* Defined by the linker script. */
 extern char _stext[], _etext[], _sinittext[], _einittext[];
 
+/* auxiliary markers to speed up symbol lookup */
+#define KALLSYMS_STEM_MARKS	8
+
+static int kallsyms_stem_mark_idx[KALLSYMS_STEM_MARKS];
+static char *kallsyms_stem_mark[KALLSYMS_STEM_MARKS];
+
+
 static inline int is_kernel_inittext(unsigned long addr)
 {
 	if (addr >= (unsigned long)_sinittext
@@ -56,13 +63,42 @@ unsigned long kallsyms_lookup_name(const
 	return module_kallsyms_lookup_name(name);
 }
 
+/* build markers into the compressed symbol table, so that lookups can be faster */
+static void build_stem_marks(void)
+{
+	char *name = kallsyms_names;
+	int i, mark_cnt;
+
+	unsigned prefix;
+
+	mark_cnt = 0;
+	for (i = 0 ; i < kallsyms_num_syms; i++) { 
+		prefix=*name;
+		if (prefix == 0) {
+			/* if this is the first 0-prefix stem in the desired interval */
+			if(i > (mark_cnt + 1) * (kallsyms_num_syms / (KALLSYMS_STEM_MARKS + 1)) && 
+			   kallsyms_stem_mark_idx[mark_cnt]==0) {
+				kallsyms_stem_mark[mark_cnt] = name;
+				kallsyms_stem_mark_idx[mark_cnt] = i;
+				mark_cnt++;
+				if(mark_cnt >= KALLSYMS_STEM_MARKS) break;
+			}
+		}
+		do {
+			name++;
+		} while(*name);
+		name ++;
+	}
+}
 /* Lookup an address.  modname is set to NULL if it's in the kernel. */
 const char *kallsyms_lookup(unsigned long addr,
 			    unsigned long *symbolsize,
 			    unsigned long *offset,
 			    char **modname, char *namebuf)
 {
-	unsigned long i, best = 0;
+	unsigned long i, last_0idx = 0;
+	unsigned long mark, low, high, mid;
+	char *last_0prefix = NULL;
 
 	/* This kernel should never had been booted. */
 	BUG_ON(!kallsyms_addresses);
@@ -72,39 +108,67 @@ const char *kallsyms_lookup(unsigned lon
 
 	if (is_kernel_text(addr) || is_kernel_inittext(addr)) {
 		unsigned long symbol_end;
-		char *name = kallsyms_names;
+		char *name;
 
-		/* They're sorted, we could be clever here, but who cares? */
-		for (i = 0; i < kallsyms_num_syms; i++) {
-			if (kallsyms_addresses[i] > kallsyms_addresses[best] &&
-			    kallsyms_addresses[i] <= addr)
-				best = i;
+		/* do a binary search on the sorted kallsyms_addresses array */
+		low = 0;
+		high = kallsyms_num_syms;
+		while( high-low > 1 ) { 
+			mid = (low + high) / 2;
+			if( kallsyms_addresses[mid] <= addr ) low = mid;
+			else high = mid;
 		}
 
 		/* Grab name */
-		for (i = 0; i <= best; i++) { 
-			unsigned prefix = *name++;
-			strncpy(namebuf + prefix, name, KSYM_NAME_LEN - prefix);
-			name += strlen(name) + 1;
-		}
+		i = 0;
+		name = kallsyms_names;
 
-		/* At worst, symbol ends at end of section. */
-		if (is_kernel_inittext(addr))
-			symbol_end = (unsigned long)_einittext;
-		else
-			symbol_end = (unsigned long)_etext;
+		if(kallsyms_stem_mark_idx[0]==0)
+			build_stem_marks();
+
+		for(mark = 0; mark < KALLSYMS_STEM_MARKS; mark++) {
+			if( low >= kallsyms_stem_mark_idx[mark] ) {
+				i = kallsyms_stem_mark_idx[mark];
+				name = kallsyms_stem_mark[mark];
+			}
+			else break;
+		}
 
-		/* Search for next non-aliased symbol */
-		for (i = best+1; i < kallsyms_num_syms; i++) {
-			if (kallsyms_addresses[i] > kallsyms_addresses[best]) {
-				symbol_end = kallsyms_addresses[i];
-				break;
+		/* find the last stem before the actual symbol that as 0 prefix */
+		unsigned prefix;
+		for (; i <= low; i++) { 
+			prefix=*name;
+			if (prefix == 0) {
+				last_0prefix = name;
+				last_0idx = i;
 			}
+			do {
+				name++;
+			} while(*name);
+			name ++;
 		}
 
-		*symbolsize = symbol_end - kallsyms_addresses[best];
+		/* build the name from there */
+		name = last_0prefix;
+		for (i = last_0idx; i <= low; i++) { 
+			prefix = *name++;
+			strncpy(namebuf + prefix, name, KSYM_NAME_LEN - prefix);
+			name += strlen(name) + 1;
+		}
+
+		if(low == kallsyms_num_syms - 1) {
+			/* At worst, symbol ends at end of section. */
+			if (is_kernel_inittext(addr))
+				symbol_end = (unsigned long)_einittext;
+			else
+				symbol_end = (unsigned long)_etext;
+		}
+		else
+			symbol_end = kallsyms_addresses[low + 1];
+		
+		*symbolsize = symbol_end - kallsyms_addresses[low];
 		*modname = NULL;
-		*offset = addr - kallsyms_addresses[best];
+		*offset = addr - kallsyms_addresses[low];
 		return namebuf;
 	}
 

Re: [patch] voluntary-preempt-2.6.8-rc4-O8

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > > other changes in -O8: the massive kallsyms-lookup speedup from Paulo
> > > Marque
> > 
> > Do you have this as an independent patch - or an URL?
> > 
> 
> http://lkml.org/lkml/2004/8/14/3

this has the (gcc's bogus) compiler warning - i sent the fixed up
version to Sam separately.

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* James Courtier-Dutton <James@superbug.demon.co.uk> wrote:

> I have tested this for a day now, and I have noticed problems:
> 1)
> kernel syslog gets a record like this:
> >  (default.hotplug/1470): 121 us critical section violates 100 us 
> threshold.

> and the /proc/latency_trace gets:
> >   preemption latency trace v1.0
> >   -----------------------------
> >    latency: 121 us, entries: 1032 (1032)
> >    process: default.hotplug/1470, uid: 0
> >    nice: -10, policy: 0, rt_priority: 0
> >   =======>
> >    0.000ms (+0.000ms): page_address (kmap_high)
> >    0.000ms (+0.000ms): page_slot (page_address)
> >    0.000ms (+0.000ms): flush_all_zero_pkmaps (kmap_high)
> >    0.000ms (+0.000ms): set_page_address (flush_all_zero_pkmaps)

> Could the patch be adjusted to make the syslog and the
> /proc/latency_trace produce the same output?

We cannot include the full trace in the syslog - it's possibly thousands
of lines long.

> 2)
> I suspect that there is a problem with reiserfs, but when I detect a 
> momentary hang in the system(mouse stops moving), no latency_trace appears.

well, the mouse could stop moving for a number of reasons. It's handled
via the X server and if the X server is preempted (for whatever reason)
then the mouse pointer isnt updated.

you could try to change the mouse IRQ to be non-threaded. Also, do you
have kernel_preemption set to 1? It defaults to 0.

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[James Courtier-Dutton]

This is just for info, now that we have a nice latency testing tool, we 
might as well collect some useful traces that we can later work on.

Here is a trace showing a latency of 39034us. 
http://www.superbug.demon.co.uk/kernel/

The interesting parts of it are:
0.002ms (+0.000ms): __preempt_spin_lock (schedule)
0.523ms (+0.520ms): do_IRQ (common_interrupt)
...
0.532ms (+0.000ms): end_edge_ioapic_irq (do_IRQ)
0.763ms (+0.230ms): smp_apic_timer_interrupt (apic_timer_interrupt)
...
0.768ms (+0.000ms): preempt_schedule (try_to_wake_up)
1.523ms (+0.754ms): do_IRQ (common_interrupt)
...
1.533ms (+0.000ms): __do_softirq (do_softirq)
1.763ms (+0.229ms): smp_apic_timer_interrupt (apic_timer_interrupt)
...
1.765ms (+0.000ms): __do_softirq (do_softirq)
2.523ms (+0.757ms): do_IRQ (common_interrupt)
...
2.533ms (+0.000ms): do_softirq (do_IRQ)
2.533ms (+0.000ms): __do_softirq (do_softirq)
2.763ms (+0.230ms): smp_apic_timer_interrupt (apic_timer_interrupt)
...

This looks to me to be a bug somewhere. Either in the O7 patch, or in 
the kernel. Surely, do_IRQ should happen quickly, and not take 39ms.

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* James Courtier-Dutton <James@superbug.demon.co.uk> wrote:

> This is just for info, now that we have a nice latency testing tool,
> we might as well collect some useful traces that we can later work on.
> 
> Here is a trace showing a latency of 39034us. 
> http://www.superbug.demon.co.uk/kernel/

> This looks to me to be a bug somewhere. Either in the O7 patch, or in
> the kernel. Surely, do_IRQ should happen quickly, and not take 39ms.

something's wrong indeed. Is this an x86 SMP system? If it's SMP then
please apply the patch below (ontop of -O7), it fixes an SMP false
positive bug in the latency timing code.

the process is looping somewhere. Here are the non-IRQ trace entries:

 0.001ms (+0.000ms): __switch_to (schedule)
 0.002ms (+0.000ms): finish_task_switch (schedule)
 0.002ms (+0.000ms): __preempt_spin_lock (schedule)
... [lots of IRQs] ...
 38.126ms (+0.362ms): preempt_schedule (schedule)
 38.126ms (+0.000ms): sched_clock (schedule)
 38.127ms (+0.000ms): find_next_bit (schedule)
 38.127ms (+0.000ms): task_timeslice (schedule)

this shows that we are looping in __preempt_spin_lock() - most likely
via schedule()'s reqacquire_kernel_lock() code.

i.e. another CPU is holding the big kernel lock and this CPU is looping. 
_but_ this CPU is fully preemptible so the trace produces this false
positive.

	Ingo

--- linux/kernel/sched.c.orig2	
+++ linux/kernel/sched.c	
@@ -4210,7 +4210,9 @@ void __sched __preempt_spin_lock(spinloc
 	do {
 		preempt_enable();
 		while (spin_is_locked(lock)) {
+			preempt_disable();
 			touch_preempt_timing();
+			preempt_enable();
 			cpu_relax();
 		}
 		preempt_disable();
@@ -4229,7 +4231,9 @@ void __sched __preempt_write_lock(rwlock
 	do {
 		preempt_enable();
 		while (rwlock_is_locked(lock)) {
+			preempt_disable();
 			touch_preempt_timing();
+			preempt_enable();
 			cpu_relax();
 		}
 		preempt_disable();

Re: [patch] voluntary-preempt-2.6.8-rc4-O7

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O7
> > 
> 
> Some more interesting results.  I am not seeing much of a difference in performance, but
> the shape of the distribution changes quite a bit:
> 
> http://krustophenia.net/testresults.php?dataset=2.6.8-rc3-O5
> http://krustophenia.net/testresults.php?dataset=2.6.8-rc4-bk3-O7

nice graphs - i suspect the shape difference is due to the tracing overhead: it adds visible
(and measurable) overhead to all kernel execution paths, but since the overwhelming majority
of paths have very low latencies it doesnt degrade the max_latency results visibly. This is
good empirical data - it means what we are after a few offenders who are more than a
magnitude slower than the common paths. It also shows that most of the extra overhead is
rather in isolated functions (e.g. the memcpy's), not in some big latency path involving
many small functions.

i also suspect that if you magnified the 'sharp' nontracing graphs near zero they'd show a
similar shape to the tracing graphs.

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[James Courtier-Dutton]

Ingo Molnar wrote:
> * James Courtier-Dutton <James@superbug.demon.co.uk> wrote:
> 
> 
>>This is just for info, now that we have a nice latency testing tool,
>>we might as well collect some useful traces that we can later work on.
>>
>>Here is a trace showing a latency of 39034us. 
>>http://www.superbug.demon.co.uk/kernel/
> 
> 
>>This looks to me to be a bug somewhere. Either in the O7 patch, or in
>>the kernel. Surely, do_IRQ should happen quickly, and not take 39ms.
> 
> 
> something's wrong indeed. Is this an x86 SMP system? If it's SMP then
> please apply the patch below (ontop of -O7), it fixes an SMP false
> positive bug in the latency timing code.
> 
> the process is looping somewhere. Here are the non-IRQ trace entries:
> 
>  0.001ms (+0.000ms): __switch_to (schedule)
>  0.002ms (+0.000ms): finish_task_switch (schedule)
>  0.002ms (+0.000ms): __preempt_spin_lock (schedule)
> ... [lots of IRQs] ...
>  38.126ms (+0.362ms): preempt_schedule (schedule)
>  38.126ms (+0.000ms): sched_clock (schedule)
>  38.127ms (+0.000ms): find_next_bit (schedule)
>  38.127ms (+0.000ms): task_timeslice (schedule)
> 
> this shows that we are looping in __preempt_spin_lock() - most likely
> via schedule()'s reqacquire_kernel_lock() code.
> 
> i.e. another CPU is holding the big kernel lock and this CPU is looping. 
> _but_ this CPU is fully preemptible so the trace produces this false
> positive.
> 
> 	Ingo
> 

I have not seen that particular problem any more, so I think the patch 
helped.

I have found a new problem though:

# cat latency_trace
preemption latency trace v1.0
-----------------------------
  latency: 1883455195 us, entries: 1 (1)
  process: ksoftirqd/1/5, uid: 0
  nice: -10, policy: 0, rt_priority: 0
=======>
  0.000ms (+0.000ms): cond_resched_softirq (___do_softirq)


That looks bad to me. The user did not notice any latency, but 1883 
seconds seems like a high latency to me!

James

[patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

i've uploaded the -P0 patch:

 http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P0

those who had APIC (and USB, under SMP) problems under previous
versions, are the problems still present in -P0?
 
Changes:

 - make redirected softirqs and hardirqs preemptible by default. I
   reviewed a number of softirq users and it appears to be safe. 
   Process-level enable/disable_bh still disables preemption, but the
   handlers themselves run in a preemptible way. This preemptability
   should fix e.g. the IDE latencies. It could also fix some of the 
   /dev/random related latencies.

 - fixed a bug in hardirq redirection - we didnt re-disable interrupts

 - updated the IO-APIC changes - hopefully it's more robust now

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Peter Zijlstra]

[-- Attachment #1: Type: text/plain, Size: 1301 bytes --]

On Sun, 2004-08-15 at 13:56 +0200, Ingo Molnar wrote:
> i've uploaded the -P0 patch:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P0
> 
> those who had APIC (and USB, under SMP) problems under previous
> versions, are the problems still present in -P0?
>  
> Changes:
> 
>  - make redirected softirqs and hardirqs preemptible by default. I
>    reviewed a number of softirq users and it appears to be safe. 
>    Process-level enable/disable_bh still disables preemption, but the
>    handlers themselves run in a preemptible way. This preemptability
>    should fix e.g. the IDE latencies. It could also fix some of the 
>    /dev/random related latencies.
> 
>  - fixed a bug in hardirq redirection - we didnt re-disable interrupts
> 
>  - updated the IO-APIC changes - hopefully it's more robust now
> 
> 	Ingo

Hi Ingo,

It still locks up hard for me when voluntary-preempt=3, however it does
finish the boot; dmesg attached. The lockup occurs several minutes into
use; usually by by the time I've started X, launched evolution and
selected my first imap folder the machine's dead.

If you need more information or want me to try some patches, just let me
know.

BTW, do you intent on keeping the preempt-smp patch alive?

-- 
Peter Zijlstra <a.p.zijlstra@chello.nl>

[-- Attachment #2: dmesg.2.6.8.1-P0.gz --]
[-- Type: application/x-gzip, Size: 5477 bytes --]

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 10:01, Peter Zijlstra wrote:

> It still locks up hard for me when voluntary-preempt=3, however it does
> finish the boot; dmesg attached. The lockup occurs several minutes into
> use; usually by by the time I've started X, launched evolution and
> selected my first imap folder the machine's dead.
> 
> If you need more information or want me to try some patches, just let me
> know.

These look a bit worrisome:

Aug 15 15:24:11 twins kernel: Total of 2 processors activated (5537.79 BogoMIPS).
Aug 15 15:24:11 twins kernel: WARNING: This combination of AMD processors is not suitable for SMP.
Aug 15 15:24:11 twins kernel: ENABLING IO-APIC IRQs

Aug 15 15:24:11 twins kernel: I/O APIC: AMD Errata #22 may be present. In the event of instability try
Aug 15 15:24:11 twins kernel:         : booting with the "noapic" option.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Peter Zijlstra]

On Sun, 2004-08-15 at 13:33 -0400, Lee Revell wrote:
> On Sun, 2004-08-15 at 10:01, Peter Zijlstra wrote:
> 
> > It still locks up hard for me when voluntary-preempt=3, however it does
> > finish the boot; dmesg attached. The lockup occurs several minutes into
> > use; usually by by the time I've started X, launched evolution and
> > selected my first imap folder the machine's dead.
> > 
> > If you need more information or want me to try some patches, just let me
> > know.
> 
> These look a bit worrisome:
> 
> Aug 15 15:24:11 twins kernel: Total of 2 processors activated (5537.79 BogoMIPS).
> Aug 15 15:24:11 twins kernel: WARNING: This combination of AMD processors is not suitable for SMP.
> Aug 15 15:24:11 twins kernel: ENABLING IO-APIC IRQs
> 
> Aug 15 15:24:11 twins kernel: I/O APIC: AMD Errata #22 may be present. In the event of instability try
> Aug 15 15:24:11 twins kernel:         : booting with the "noapic" option.
> 

Yeah, true. I run with 2 Thunderbirds, not Athlon MPs; AMD does not like
that, the CPUs don't share that inclination.

As for the AMD Errata #22, all other kernels run rock solid without
noapic, so I don't see why these should differ. I did try noapic though,
not on P0 but on one of the later Os.


-- 
Peter Zijlstra <a.p.zijlstra@chello.nl>

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:

> It still locks up hard for me when voluntary-preempt=3, however it
> does finish the boot; dmesg attached. The lockup occurs several
> minutes into use; usually by by the time I've started X, launched
> evolution and selected my first imap folder the machine's dead.

can you set up a serial logging via a null modem cable from that box to
another box? Then enable the NMI watchdog via nmi_watchdog=1 and
reproduce the lockup - the NMI watchdog should trigger and should
produce a traceback to the serial console.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 07:56, Ingo Molnar wrote:
> i've uploaded the -P0 patch:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P0
> 
> those who had APIC (and USB, under SMP) problems under previous
> versions, are the problems still present in -P0?
>  
> Changes:
> 

The ide and /dev/random related latencies are indeed gone.  Here is an
ugly one that I got:

preemption latency trace v1.0
-----------------------------
 latency: 142 us, entries: 4 (4)
 process: ksoftirqd/0/2, uid: 0
 nice: -10, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): rhine_check_duplex (rhine_timer)
 0.000ms (+0.000ms): mdio_read (rhine_check_duplex)
 0.067ms (+0.067ms): mdio_read (rhine_timer)
 0.139ms (+0.071ms): check_preempt_timing (sub_preempt_count)

This looks like the exact same problem Florian had, in his case it was 
the sis900 driver.  Your recommendation was:

        #define mdio_delay() do { } while (0)

Should I try this?

Also, isn't there a better solution than for network drivers to actively 
poll for changes in link status?  Can't they just register a callback that 
will get run on a link state change event?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 07:56, Ingo Molnar wrote:
> i've uploaded the -P0 patch:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P0
> 
> those who had APIC (and USB, under SMP) problems under previous
> versions, are the problems still present in -P0?
>  

The mlockall issue is still not resolved; however, I did manage to get a
trace, which was probably not possible before because some higher
latency but lower frequency event was overwriting /proc/latency_trace. 
So, maybe mlockall does cause xruns by having many shorter, but long
enough to be problematic, non-preemptible sections.

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P0

Also it seems that extract_entropy still causes high latencies, even
though a call to preempt_schedule was added.  I looked at the code in
random.c a bit and this strikes me as an area where the algorithm could
be improved, rather than adding a scheduling point.  Do we really need
*that* much entropy, right then?  And if so, isn't there a zero-copy
solution?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 20:25, Florian Schmidt wrote:
> On Sun, 15 Aug 2004 13:56:49 +0200
> Ingo Molnar <mingo@elte.hu> wrote:
> 
> > 
> > i've uploaded the -P0 patch:
> > 
> >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P0
> 
> I haven't tried this patch yet, but i have a question regarding the
> mlockall issue:
> 
> Jackd also uses IPC mechnisms for remote procedure calls [i think,
> please correct me] and makes heavy use of shared memory. Might
> mlock(all) have influence of this? is jackd maybe producing xruns
> because some IPC stuff blocks when mlockall is used?
> 
> I'm just guessing wildly and uneducatedly..

FWIW, mlockall by an unrelated normal-priority process still causes
xruns in the SCHED_FIFO jackd process even when jackd is run with no
clients.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Florian Schmidt]

On Sun, 15 Aug 2004 13:56:49 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> 
> i've uploaded the -P0 patch:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P0

I haven't tried this patch yet, but i have a question regarding the
mlockall issue:

Jackd also uses IPC mechnisms for remote procedure calls [i think,
please correct me] and makes heavy use of shared memory. Might
mlock(all) have influence of this? is jackd maybe producing xruns
because some IPC stuff blocks when mlockall is used?

I'm just guessing wildly and uneducatedly..

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Florian Schmidt]

On Sun, 15 Aug 2004 20:20:22 -0400
Lee Revell <rlrevell@joe-job.com> wrote:

> > Jackd also uses IPC mechnisms for remote procedure calls [i think,
> > please correct me] and makes heavy use of shared memory. Might
> > mlock(all) have influence of this? is jackd maybe producing xruns
> > because some IPC stuff blocks when mlockall is used?
> > 
> > I'm just guessing wildly and uneducatedly..
> 
> FWIW, mlockall by an unrelated normal-priority process still causes
> xruns in the SCHED_FIFO jackd process even when jackd is run with no
> clients.

Hmm, yes, my thought was that jackd maybe expects some IPC stuff to
return immeaditly or have bounded execution times while this maybe is
not really always garanteed.. Need to look at the source.

Flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 20:25, Florian Schmidt wrote:
> On Sun, 15 Aug 2004 13:56:49 +0200
> Ingo Molnar <mingo@elte.hu> wrote:
> 
> > 
> > i've uploaded the -P0 patch:
> > 
> >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P0
> 
> I haven't tried this patch yet, but i have a question regarding the
> mlockall issue:
> 
> Jackd also uses IPC mechnisms for remote procedure calls [i think,
> please correct me] and makes heavy use of shared memory. Might
> mlock(all) have influence of this? is jackd maybe producing xruns
> because some IPC stuff blocks when mlockall is used?

I reworked the jackd latency histogram to report the time elapsed
between entering and exiting the poll() of PCM fd's in alsa-driver.c. 
This is a different metric than the previous max_usecs histogram, which
I believe indirectly measured latency, this one measures it directly.

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P0

The peaks on this graph should correspond directly to the length of the
non-preemptible critical section reported by Ingo's latency tracer.  I
think the large peak around 580-600usecs is caused by the
extract_entropy issue (which can be hit by regular processes and
ksoftirqd), and the large peak around 80-100 by the XFree86 unmap_vmas
issue, as the times match and these are by far the most common reported
in latency_trace.

There are a number of samples above 700us.  I am working with a period
time of 666 usecs, and since there are 2 periods per buffer, we would
have to hit two > 666 usec latencies in a row for an xrun - it appears
that there are many individual latencies above 666, certainly more than
there are xruns.  So, maybe the mlockall issue is not a result of
triggering a single large latency, but of increasing the frequency of
these higher latencies so that we are more likely to hit 2 in a row.

IIRC ksoftirqd will defer more work under load, and ksoftirqd is one of
the more common offenders to hit the extract_entropy latency.  Maybe
mlockall causes more softirqs to be deferred, thus increaing the change
that we will have to do more than 666 usecs worth of work on 2
successive wakeups.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> There are a number of samples above 700us.  I am working with a period
> time of 666 usecs, and since there are 2 periods per buffer, we would
> have to hit two > 666 usec latencies in a row for an xrun - it appears
> that there are many individual latencies above 666, certainly more
> than there are xruns.  So, maybe the mlockall issue is not a result of
> triggering a single large latency, but of increasing the frequency of
> these higher latencies so that we are more likely to hit 2 in a row.

hm, it seems the mlockall() issue is too deterministic for it to be a
statistical-only phenomenon. Also, isnt that xrun on the order of 15
msecs? That's way too big too.

> IIRC ksoftirqd will defer more work under load, and ksoftirqd is one
> of the more common offenders to hit the extract_entropy latency. 
> Maybe mlockall causes more softirqs to be deferred, thus increaing the
> change that we will have to do more than 666 usecs worth of work on 2
> successive wakeups.

there should be no relation between softirqs and mlockall().

this is truly a mind-boggling latency. mlockall() is fully preemptible. 
All it does is to pre-fault the whole range of pages that a process has.

could you try another thing: modify mlockall-test.cc to use mlock() on
the freshly allocated anonymous pages? Does this produce the same
latencies? mlockall() prefaults _all_ pages the process currently has. 
Maybe mlockall() touches some page that is mapped both by jackd and
mlockall-test and thus somehow interacts with jackd's scheduling.

the anonymous pages themselves can have no IPC-alike connection to any
page jackd owns. It is unlikely for that to be any connection between
jackd and mlockall-test - other than both map glibc. To further exclude
any possibility of resource sharing between jackd and mlockall-test,
could you compile the later with -static?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> http://krustophenia.net/testresults.php?dataset=2.6.8.1-P0

nice. (What is the difference between the left-hand and the right-hand
graphs - why is the right-hand side one 'wider'?)

> The peaks on this graph should correspond directly to the length of
> the non-preemptible critical section reported by Ingo's latency
> tracer.  I think the large peak around 580-600usecs is caused by the
> extract_entropy issue (which can be hit by regular processes and
> ksoftirqd), and the large peak around 80-100 by the XFree86 unmap_vmas
> issue, as the times match and these are by far the most common
> reported in latency_trace.

just to check this theory, could you make __check_and_rekey() an empty
function? This should still produce a working random driver, albeit at
much reduced entropy. If these latencies have a relationship to the
mlockall() issue then this change should have an effect.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 22:36, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > There are a number of samples above 700us.  I am working with a period
> > time of 666 usecs, and since there are 2 periods per buffer, we would
> > have to hit two > 666 usec latencies in a row for an xrun - it appears
> > that there are many individual latencies above 666, certainly more
> > than there are xruns.  So, maybe the mlockall issue is not a result of
> > triggering a single large latency, but of increasing the frequency of
> > these higher latencies so that we are more likely to hit 2 in a row.
> 
> hm, it seems the mlockall() issue is too deterministic for it to be a
> statistical-only phenomenon. Also, isnt that xrun on the order of 15
> msecs? That's way too big too.
> 

I believe the constant-time behavior that I reported was an artifact of
ALSA xrun debugging.  Now it seems like the latency produced *does*
correspond directly to the amount of memory being mlockall'ed.  If
./mlockall-test 1500 triggers an xrun at all it's ~0.2ms.  3000 triggers
a ~1ms xrun, and 10000 a ~3 ms xrun.

> > IIRC ksoftirqd will defer more work under load, and ksoftirqd is one
> > of the more common offenders to hit the extract_entropy latency. 
> > Maybe mlockall causes more softirqs to be deferred, thus increaing the
> > change that we will have to do more than 666 usecs worth of work on 2
> > successive wakeups.
> 
> there should be no relation between softirqs and mlockall().
> 
> this is truly a mind-boggling latency. mlockall() is fully preemptible. 
> All it does is to pre-fault the whole range of pages that a process has.
> 
> could you try another thing: modify mlockall-test.cc to use mlock() on
> the freshly allocated anonymous pages? Does this produce the same
> latencies? mlockall() prefaults _all_ pages the process currently has. 
> Maybe mlockall() touches some page that is mapped both by jackd and
> mlockall-test and thus somehow interacts with jackd's scheduling.
> 

I don't know C++, Florian wrote this program.  Can you provide a
pseudo-patch?

> the anonymous pages themselves can have no IPC-alike connection to any
> page jackd owns. It is unlikely for that to be any connection between
> jackd and mlockall-test - other than both map glibc. To further exclude
> any possibility of resource sharing between jackd and mlockall-test,
> could you compile the later with -static?

Sure, I will try this.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 22:43, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > http://krustophenia.net/testresults.php?dataset=2.6.8.1-P0
> 
> nice. (What is the difference between the left-hand and the right-hand
> graphs - why is the right-hand side one 'wider'?)
> 

The right hand graph is logarithmically scaled on the y axis (set
logscale y in gnuplot).  Some of the latencies are rare enough to not
show up at all on the linear scale, like the peak in the 400s.

> > The peaks on this graph should correspond directly to the length of
> > the non-preemptible critical section reported by Ingo's latency
> > tracer.  I think the large peak around 580-600usecs is caused by the
> > extract_entropy issue (which can be hit by regular processes and
> > ksoftirqd), and the large peak around 80-100 by the XFree86 unmap_vmas
> > issue, as the times match and these are by far the most common
> > reported in latency_trace.
> 
> just to check this theory, could you make __check_and_rekey() an empty
> function? This should still produce a working random driver, albeit at
> much reduced entropy. If these latencies have a relationship to the
> mlockall() issue then this change should have an effect.
> 

Sure, will try this next.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 22:43, Lee Revell wrote:

> I don't know C++, Florian wrote this program.  Can you provide a
> pseudo-patch?
> 

Never mind, figured it out.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> I believe the constant-time behavior that I reported was an artifact
> of ALSA xrun debugging.  Now it seems like the latency produced *does*
> correspond directly to the amount of memory being mlockall'ed.  If
> ./mlockall-test 1500 triggers an xrun at all it's ~0.2ms.  3000
> triggers a ~1ms xrun, and 10000 a ~3 ms xrun.

ah ...

could this be some DMA starvation effect? Or is this xrun calculated
from arrival of the audio interrupt (hence DMA completion) to the actual
running of jackd?

> > could you try another thing: modify mlockall-test.cc to use mlock() on
> > the freshly allocated anonymous pages? Does this produce the same
> > latencies? mlockall() prefaults _all_ pages the process currently has. 
> > Maybe mlockall() touches some page that is mapped both by jackd and
> > mlockall-test and thus somehow interacts with jackd's scheduling.
> 
> I don't know C++, Florian wrote this program.  Can you provide a
> pseudo-patch?

your above observation wrt. linearity of the xrun makes it less likely
that the issue is caused by page sharing between jackd and
mlockall-test. (if then they share a constant amount of pages.)

but it would be nice to test it anyway, i've attached mlock-test.cc.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> > I believe the constant-time behavior that I reported was an artifact
> > of ALSA xrun debugging.  Now it seems like the latency produced *does*
> > correspond directly to the amount of memory being mlockall'ed.  If
> > ./mlockall-test 1500 triggers an xrun at all it's ~0.2ms.  3000
> > triggers a ~1ms xrun, and 10000 a ~3 ms xrun.
> 
> ah ...
> 
> could this be some DMA starvation effect? Or is this xrun calculated
> from arrival of the audio interrupt (hence DMA completion) to the
> actual running of jackd?

i've attached mlock-test2.cc that should test this theory. The code
breaks up the mlock-ed region into 8 equal pieces and does mlock() on
them separately. It's basically a lock-break done in user-space. Does
this change the nature of xruns?

if it doesnt change the xruns then it shows that it's not the locking of
make_pages_present() that interacts with jackd, but it's what it does
that interacts with it (or with the audio driver).

assuming the DMA-starvation theory isnt excluded via mlock-test2.c:

prefaulting is quite memory-bandwidth-intense. It might even be that the
CPU, internally, deals with pagetable related memory fetches (and
writebacks) differently - e.g. gives it a higher priority on the bus. 
Does your audio card have a maximum PCI latency setting already?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> could this be some DMA starvation effect? Or is this xrun calculated
> from arrival of the audio interrupt (hence DMA completion) to the
> actual running of jackd?

would there be a way to find out what portion of the xrun is caused by
latencies of the audio card (DMA, etc.) vs. latency from the point jackd
is woken up by the sound-driver to the point jackd preempts the mlock
process and runs?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 22:58, Ingo Molnar wrote:
> * Ingo Molnar <mingo@elte.hu> wrote:
> i've attached mlock-test2.cc that should test this theory. The code
> breaks up the mlock-ed region into 8 equal pieces and does mlock() on
> them separately. It's basically a lock-break done in user-space. Does
> this change the nature of xruns?

-ENOATTACHMENT

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

[-- Attachment #1: Type: text/plain, Size: 452 bytes --]


* Lee Revell <rlrevell@joe-job.com> wrote:

> On Sun, 2004-08-15 at 22:58, Ingo Molnar wrote:
> > * Ingo Molnar <mingo@elte.hu> wrote:
> > i've attached mlock-test2.cc that should test this theory. The code
> > breaks up the mlock-ed region into 8 equal pieces and does mlock() on
> > them separately. It's basically a lock-break done in user-space. Does
> > this change the nature of xruns?
> 
> -ENOATTACHMENT

-ESTILLTOOEARLY. attached now.

	Ingo

[-- Attachment #2: mlock-test2.cc --]
[-- Type: text/plain, Size: 978 bytes --]

// here is the code i used to test the mlockall caused xruns
#include <sys/mman.h>
#include <iostream>
#include <sstream>
#include <unistd.h>

#define INTERVALS 8

int main (int argc, char *argv[]) {
        if (argc < 2) {
                std::cout << "how many kbytes you want allocated and mlockall'ed?" << std::endl;
        }

        std::stringstream stream(argv[1]);
        int kbytes, error = 0, i, chunk;

        stream >> kbytes;
        char *mem = new char[kbytes*1024];
        std::cout << "filling with 0's" << std::endl;
        for (int i = 0; i < kbytes*1024; ++i) {
                mem[i] = 0;
        }

        std::cout << "ok, you want " << kbytes << "kb of memory mlocked.  going for it.." << std::endl;
	chunk = kbytes*1024/INTERVALS;
	for (i = 0; i < INTERVALS; i++)
        	error |= mlock(mem + i*chunk, chunk);
        if (error != 0) { std::cout << "mlock error" << std::endl; }
        else { std::cout << "mlock successfull" << std::endl;}
}

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> just to check this theory, could you make __check_and_rekey() an empty
> function? This should still produce a working random driver, albeit at
> much reduced entropy. If these latencies have a relationship to the
> mlockall() issue then this change should have an effect.

hm, could you disable the random driver in the .config rather? It seems
that adding to the entropy pool (from hardirq context) alone is quite
expensive too.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 22:58, Ingo Molnar wrote:
> * Ingo Molnar <mingo@elte.hu> wrote:

> i've attached mlock-test2.cc that should test this theory. The code
> breaks up the mlock-ed region into 8 equal pieces and does mlock() on
> them separately. It's basically a lock-break done in user-space. Does
> this change the nature of xruns?
> 

This does change the xruns.  I have to `mlock-test2 8000' to produce any
xrun at all, and that only produces a 2-300 usec xrun.  With
mlockall-test the threshold to produce an xrun was ~1500.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > i've attached mlock-test2.cc that should test this theory. The code
> > breaks up the mlock-ed region into 8 equal pieces and does mlock() on
> > them separately. It's basically a lock-break done in user-space. Does
> > this change the nature of xruns?
> 
> This does change the xruns.  I have to `mlock-test2 8000' to produce
> any xrun at all, and that only produces a 2-300 usec xrun.  With
> mlockall-test the threshold to produce an xrun was ~1500.

did you try mlock-test.cc too? Just to make sure that mlock-test.cc is
equivalent to mlockall-test.cc.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 23:14, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > i've attached mlock-test2.cc that should test this theory. The code
> > > breaks up the mlock-ed region into 8 equal pieces and does mlock() on
> > > them separately. It's basically a lock-break done in user-space. Does
> > > this change the nature of xruns?
> > 
> > This does change the xruns.  I have to `mlock-test2 8000' to produce
> > any xrun at all, and that only produces a 2-300 usec xrun.  With
> > mlockall-test the threshold to produce an xrun was ~1500.
> 
> did you try mlock-test.cc too? Just to make sure that mlock-test.cc is
> equivalent to mlockall-test.cc.
> 

That attachment was also missing.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 22:58, Ingo Molnar wrote:
> * Ingo Molnar <mingo@elte.hu> wrote:

> if it doesnt change the xruns then it shows that it's not the locking of
> make_pages_present() that interacts with jackd, but it's what it does
> that interacts with it (or with the audio driver).
> 
> assuming the DMA-starvation theory isnt excluded via mlock-test2.c:

Sorry, you lost me here.  Does the behavior of mlock-test2 point to the
locking of make_pages_present interfering with jackd, or with the audio
driver?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

[-- Attachment #1: Type: text/plain, Size: 217 bytes --]


* Lee Revell <rlrevell@joe-job.com> wrote:

> > did you try mlock-test.cc too? Just to make sure that mlock-test.cc is
> > equivalent to mlockall-test.cc.
> 
> That attachment was also missing.

attached now.

	Ingo

[-- Attachment #2: mlock-test.cc --]
[-- Type: text/plain, Size: 868 bytes --]

// here is the code i used to test the mlockall caused xruns
#include <sys/mman.h>
#include <iostream>
#include <sstream>
#include <unistd.h>

int main (int argc, char *argv[]) {
        if (argc < 2) {
                std::cout << "how many kbytes you want allocated and mlockall'ed?" << std::endl;
        }

        std::stringstream stream(argv[1]);
        int kbytes;
        stream >> kbytes;
        char *mem = new char[kbytes*1024];
        std::cout << "filling with 0's" << std::endl;
        for (int i = 0; i < kbytes*1024; ++i) {
                mem[i] = 0;
        }

        std::cout << "ok, you want " << kbytes << "kb of memory mlocked.  going for it.." << std::endl;
        int error = mlock(mem, kbytes*1024);
        if (error != 0) { std::cout << "mlock error" << std::endl; }
        else { std::cout << "mlock successfull" << std::endl;}
}

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> On Sun, 2004-08-15 at 22:58, Ingo Molnar wrote:
> > * Ingo Molnar <mingo@elte.hu> wrote:
> 
> > if it doesnt change the xruns then it shows that it's not the locking of
> > make_pages_present() that interacts with jackd, but it's what it does
> > that interacts with it (or with the audio driver).
> > 
> > assuming the DMA-starvation theory isnt excluded via mlock-test2.c:
> 
> Sorry, you lost me here.  Does the behavior of mlock-test2 point to
> the locking of make_pages_present interfering with jackd, or with the
> audio driver?

it's rather pointing in the direction of locking, away from DMA issues. 
Any DMA issue should not depend if we do the activity in 8 chunks or in
one go. Locking on the other hand depends on the length of a single
chunk, not on the length of the total activity.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 23:20, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > did you try mlock-test.cc too? Just to make sure that mlock-test.cc is
> > > equivalent to mlockall-test.cc.
> > 
> > That attachment was also missing.
> 

mlock-test.cc does seem equivalent to mlockall-test.cc.  `mlock-test
10000' produces a ~5ms xrun.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> I believe the constant-time behavior that I reported was an artifact
> of ALSA xrun debugging.  Now it seems like the latency produced *does*
> correspond directly to the amount of memory being mlockall'ed.  If
> ./mlockall-test 1500 triggers an xrun at all it's ~0.2ms.  3000
> triggers a ~1ms xrun, and 10000 a ~3 ms xrun.

could you try more extreme values - e.g. does 100 MB cause a 30 msec
xrun?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 23:00, Ingo Molnar wrote:
> * Ingo Molnar <mingo@elte.hu> wrote:
> 
> > could this be some DMA starvation effect? Or is this xrun calculated
> > from arrival of the audio interrupt (hence DMA completion) to the
> > actual running of jackd?
> 
> would there be a way to find out what portion of the xrun is caused by
> latencies of the audio card (DMA, etc.) vs. latency from the point jackd
> is woken up by the sound-driver to the point jackd preempts the mlock
> process and runs?
> 

WHat is being measured is the length of time between jackd is woken up
and when it runs - the xrun report prints the difference between the
current time and the timestamp returned by
snd_pcm_status_get_trigger_tstamp (updated by the interrupt handler) and
the current time.

jackd prints a different error message if it detects that the interrupt
handler was delayed, this is a separate issue than an xrun.  I have seen
this happen before, but I don't think this is going on here.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

doh - i think i found a brown-paperbag bug.

could you change this code in kernel/sched.c:

 int __sched voluntary_resched(void)
 {
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
         __might_sleep(__FILE__, __LINE__);
 #endif
         if (kernel_preemption || !voluntary_preemption)
                 return 0;

to:

 int __sched voluntary_resched(void)
 {
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
         __might_sleep(__FILE__, __LINE__);
 #endif
         if (!voluntary_preemption)
                 return 0;

Does the mlockall xrun still occur?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 23:28, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > I believe the constant-time behavior that I reported was an artifact
> > of ALSA xrun debugging.  Now it seems like the latency produced *does*
> > correspond directly to the amount of memory being mlockall'ed.  If
> > ./mlockall-test 1500 triggers an xrun at all it's ~0.2ms.  3000
> > triggers a ~1ms xrun, and 10000 a ~3 ms xrun.
> 
> could you try more extreme values - e.g. does 100 MB cause a 30 msec
> xrun?
> 

This causes a series of xruns, ranging from maybe 2 to 10ms.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 23:36, Ingo Molnar wrote:
> doh - i think i found a brown-paperbag bug.

Heh, this would explain all those latency traces with repeated 
calls to voluntary_resched...

Oh well, happens to the best of us.  See also Linux 2.6.8.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> On Sun, 2004-08-15 at 23:36, Ingo Molnar wrote:
> > doh - i think i found a brown-paperbag bug.
> 
> Heh, this would explain all those latency traces with repeated 
> calls to voluntary_resched...

hm, didnt those traces show preempt_schedule()?

this bug also fooled the latency tracer because the preempt count went
down to zero (hence the latency got reset), but in reality we skipped
the reschedule.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 23:46, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > On Sun, 2004-08-15 at 23:36, Ingo Molnar wrote:
> > > doh - i think i found a brown-paperbag bug.
> > 
> > Heh, this would explain all those latency traces with repeated 
> > calls to voluntary_resched...
> 
> hm, didnt those traces show preempt_schedule()?
> 

Yes, they did.

Anyway, the change to sched.c fixes the mlockall bug, it works perfectly
now.  Thanks!

I will try next with /dev/random disabled.  Don't most/many new machines
have a hardware RNG that would eliminate the need for this code?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> I will try next with /dev/random disabled.  Don't most/many new
> machines have a hardware RNG that would eliminate the need for this
> code?

The C3 does have one IIRC, but do Intel CPUs have it too? Also, there's
the question of trust - how random it truly is. Is it a partly
pseudo-RNG masked via encryption? /dev/random i know is random, driven
by random timings of real disks and real network packets. The CPU's HRNG
is much more encapsulated and can only be blackbox-tested.

	Ingo

[patch] voluntary-preempt-2.6.8.1-P1

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Anyway, the change to sched.c fixes the mlockall bug, it works
> perfectly now.  Thanks!

great! This fix also means that we've got one more lock-break in the
ext3 journalling code and one more lock-break in dcache.c. I've released
-P1 with the fix included:

 http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Mon, 2004-08-16 at 00:01, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > I will try next with /dev/random disabled.  Don't most/many new
> > machines have a hardware RNG that would eliminate the need for this
> > code?
> 
> The C3 does have one IIRC, but do Intel CPUs have it too? Also, there's
> the question of trust - how random it truly is. Is it a partly
> pseudo-RNG masked via encryption? /dev/random i know is random, driven
> by random timings of real disks and real network packets. The CPU's HRNG
> is much more encapsulated and can only be blackbox-tested.

According to menuconfig, it looks like Linux supports hardware RNGs from
AMD, Intel, and Via.  Via at least has published a cryptanalysis of
theirs: http://www.via.com.tw/en/viac3/via_c3_padlock_evaluation.pdf.

I am not a crypto expert so I can't comment much more.  Also I have one
of the older C3s without this feature so it's beside the point.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Sun, 2004-08-15 at 23:08, Ingo Molnar wrote:
> * Ingo Molnar <mingo@elte.hu> wrote:
> 
> > just to check this theory, could you make __check_and_rekey() an empty
> > function? This should still produce a working random driver, albeit at
> > much reduced entropy. If these latencies have a relationship to the
> > mlockall() issue then this change should have an effect.
> 
> hm, could you disable the random driver in the .config rather? It seems
> that adding to the entropy pool (from hardirq context) alone is quite
> expensive too.
> 

Can this be disabled in the .config?  I can't find an option for it.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Lee Revell]

On Mon, 2004-08-16 at 00:05, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Anyway, the change to sched.c fixes the mlockall bug, it works
> > perfectly now.  Thanks!
> 
> great! This fix also means that we've got one more lock-break in the
> ext3 journalling code and one more lock-break in dcache.c. I've released
> -P1 with the fix included:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1
> 

The highest latency I am seeing now is the rhine_check_duplex problem. 
Should I try making mdio_delay an NOOP?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > > just to check this theory, could you make __check_and_rekey() an empty
> > > function? This should still produce a working random driver, albeit at
> > > much reduced entropy. If these latencies have a relationship to the
> > > mlockall() issue then this change should have an effect.
> > 
> > hm, could you disable the random driver in the .config rather? It seems
> > that adding to the entropy pool (from hardirq context) alone is quite
> > expensive too.
> > 
> 
> Can this be disabled in the .config?  I can't find an option for it.

oh well, indeed it cannot be disabled. Then i'd suggest to return early
from extract_entropy(), without doing anything. That is the function
that seems to introduce the worst overhead.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Mon, 2004-08-16 at 00:26, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > > just to check this theory, could you make __check_and_rekey() an empty
> > > > function? This should still produce a working random driver, albeit at
> > > > much reduced entropy. If these latencies have a relationship to the
> > > > mlockall() issue then this change should have an effect.
> > > 
> > > hm, could you disable the random driver in the .config rather? It seems
> > > that adding to the entropy pool (from hardirq context) alone is quite
> > > expensive too.
> > > 
> > 
> > Can this be disabled in the .config?  I can't find an option for it.
> 
> oh well, indeed it cannot be disabled. Then i'd suggest to return early
> from extract_entropy(), without doing anything. That is the function
> that seems to introduce the worst overhead.
> 

Yes, I really seem to remember this could be disabled in the past...
maybe I am thinking of BSD/OS, or maybe this CONFIG_ option was removed.

This was caused by 'Actions -> Run -> rxvt':

preemption latency trace v1.0
-----------------------------
 latency: 409 us, entries: 300 (300)
 process: gnome-panel/7575, uid: 1000
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): __vma_link_rb (copy_mm)
 0.000ms (+0.000ms): rb_insert_color (copy_mm)
 0.000ms (+0.000ms): __rb_rotate_left (rb_insert_color)
 0.000ms (+0.000ms): copy_page_range (copy_mm)
 0.001ms (+0.000ms): pte_alloc_map (copy_page_range)
 0.205ms (+0.204ms): do_IRQ (common_interrupt)
 0.206ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.209ms (+0.003ms): generic_redirect_hardirq (do_IRQ)
 0.210ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.210ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 0.210ms (+0.000ms): mark_offset_tsc (timer_interrupt)
 0.217ms (+0.006ms): do_timer (timer_interrupt)
 0.217ms (+0.000ms): update_process_times (do_timer)
 0.217ms (+0.000ms): update_one_process (update_process_times)
 0.217ms (+0.000ms): run_local_timers (update_process_times)
 0.218ms (+0.000ms): raise_softirq (update_process_times)
 0.218ms (+0.000ms): scheduler_tick (update_process_times)
 0.218ms (+0.000ms): sched_clock (scheduler_tick)
 0.219ms (+0.001ms): task_timeslice (scheduler_tick)
 0.220ms (+0.000ms): update_wall_time (do_timer)
 0.220ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 0.221ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.221ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.222ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.223ms (+0.001ms): do_softirq (do_IRQ)
 0.223ms (+0.000ms): __do_softirq (do_softirq)
 0.224ms (+0.000ms): wake_up_process (do_softirq)
 0.224ms (+0.000ms): try_to_wake_up (wake_up_process)
 0.224ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.225ms (+0.000ms): activate_task (try_to_wake_up)
 0.225ms (+0.000ms): sched_clock (activate_task)
 0.225ms (+0.000ms): recalc_task_prio (activate_task)
 0.226ms (+0.000ms): effective_prio (recalc_task_prio)
 0.226ms (+0.000ms): enqueue_task (activate_task)
 0.227ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.228ms (+0.000ms): preempt_schedule (copy_page_range)

[...]

 0.399ms (+0.000ms): preempt_schedule (copy_page_range)
 0.400ms (+0.000ms): check_preempt_timing (touch_preempt_timing)

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > > Anyway, the change to sched.c fixes the mlockall bug, it works
> > > perfectly now.  Thanks!
> > 
> > great! This fix also means that we've got one more lock-break in the
> > ext3 journalling code and one more lock-break in dcache.c. I've released
> > -P1 with the fix included:
> > 
> >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1
> > 
> 
> The highest latency I am seeing now is the rhine_check_duplex problem. 
> Should I try making mdio_delay an NOOP?

there's no mdio_delay() in via-rhine.c AFAICS. Could you add a pair of
touch-latency calls to around this code in mdio_read():

+       touch_preempt_timing();
        /* Wait for a previous command to complete. */
        while ((readb(ioaddr + MIICmd) & 0x60) && --boguscnt > 0)
+       touch_preempt_timing();

i suspect it's this one that introduces the biggest delay. Also:

+	touch_preempt_timing();
        while ((readb(ioaddr + MIICmd) & 0x40) && --boguscnt > 0)
                ;
+	touch_preempt_timing();

assuming that the latencies still show up even if delimited like this. 
(note that this only changes the way the latency is tracked - the
latency itself is still there so this isnt a fix.)

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Mon, 2004-08-16 at 00:26, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > > just to check this theory, could you make __check_and_rekey() an empty
> > > > function? This should still produce a working random driver, albeit at
> > > > much reduced entropy. If these latencies have a relationship to the
> > > > mlockall() issue then this change should have an effect.
> > > 
> > > hm, could you disable the random driver in the .config rather? It seems
> > > that adding to the entropy pool (from hardirq context) alone is quite
> > > expensive too.
> > > 
> > 
> > Can this be disabled in the .config?  I can't find an option for it.
> 
> oh well, indeed it cannot be disabled.

Hmm, what happens if CONFIG_HW_RANDOM is set then?  I would call it a
bug if having a hardware RNG didn't disable the software /dev/random
driver.  This implies that the software RNG can be easily disabled.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

[-- Attachment #1: Type: text/plain, Size: 801 bytes --]


* Lee Revell <rlrevell@joe-job.com> wrote:

> This was caused by 'Actions -> Run -> rxvt':

>  0.001ms (+0.000ms): pte_alloc_map (copy_page_range)
>  0.205ms (+0.204ms): do_IRQ (common_interrupt)

>  0.228ms (+0.000ms): preempt_schedule (copy_page_range)

>  0.399ms (+0.000ms): preempt_schedule (copy_page_range)
>  0.400ms (+0.000ms): check_preempt_timing (touch_preempt_timing)

seems we need a lock-break in the innermost loop of copy_page_range(). 
That loop processes up to 1024 pages currently, before the lock-break in
the outer loop happens. Large GUI processes are more likely to have full
4MB regions mapped & populated.

i suspect you could trigger a similarly bad latency by doing a fork() in
mlockall-test.cc - the attached mlockall-test2.cc does this. Do you get
bad latencies?

	Ingo

[-- Attachment #2: mlockall-test2.cc --]
[-- Type: text/plain, Size: 875 bytes --]

// here is the code i used to test the mlockall caused xruns
#include <sys/mman.h>
#include <iostream>
#include <sstream>
#include <unistd.h>

int main (int argc, char *argv[]) {
        if (argc < 2) {
                std::cout << "how many kbytes you want allocated and mlockall'ed?" << std::endl;
        }

        std::stringstream stream(argv[1]);
        int kbytes;
        stream >> kbytes;
        char *mem = new char[kbytes*1024];
        std::cout << "filling with 0's" << std::endl;
        for (int i = 0; i < kbytes*1024; ++i) {
                mem[i] = 0;
        }

        std::cout << "ok, you want " << kbytes << "kb of memory mlocked.  going for it.." << std::endl;
        int error = mlockall(MCL_CURRENT);
        if (error != 0) { std::cout << "mlock error" << std::endl; }
        else { std::cout << "mlock successfull" << std::endl;}
	fork();
}

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Lee Revell]

On Mon, 2004-08-16 at 00:33, Ingo Molnar wrote:
> there's no mdio_delay() in via-rhine.c AFAICS. Could you add a pair of
> touch-latency calls to around this code in mdio_read():
> 
> +       touch_preempt_timing();
>         /* Wait for a previous command to complete. */
>         while ((readb(ioaddr + MIICmd) & 0x60) && --boguscnt > 0)
> +       touch_preempt_timing();
> 
> i suspect it's this one that introduces the biggest delay. Also:
> 
> +	touch_preempt_timing();
>         while ((readb(ioaddr + MIICmd) & 0x40) && --boguscnt > 0)
>                 ;
> +	touch_preempt_timing();
> 
> assuming that the latencies still show up even if delimited like this. 
> (note that this only changes the way the latency is tracked - the
> latency itself is still there so this isnt a fix.)
> 

Sure, but, what would this accomplish, if the latency is still there? 
Are we just trying to track down exactly where in the network driver
this is triggered?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Mon, 2004-08-16 at 00:26, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > > just to check this theory, could you make __check_and_rekey() an empty
> > > > function? This should still produce a working random driver, albeit at
> > > > much reduced entropy. If these latencies have a relationship to the
> > > > mlockall() issue then this change should have an effect.
> > > 
> > > hm, could you disable the random driver in the .config rather? It seems
> > > that adding to the entropy pool (from hardirq context) alone is quite
> > > expensive too.
> > > 
> > 
> > Can this be disabled in the .config?  I can't find an option for it.
> 
> oh well, indeed it cannot be disabled. Then i'd suggest to return early
> from extract_entropy(), without doing anything. That is the function
> that seems to introduce the worst overhead.
> 

OK, I tried the quick hack of just returning 0 before taking the
spinlock in extract_entropy, but this broke /dev/random which prevented
me from logging in.  I guess we will have to properly fake it here.

Alas, it looks like the software RNG is not disabled when
CONFIG_HW_RANDOM is set.  I would call this a bug - imagine incurring
the overhead of software 3D acceleration even with hardware 3D
acceleration enabled.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > +	touch_preempt_timing();
> >         while ((readb(ioaddr + MIICmd) & 0x40) && --boguscnt > 0)
> >                 ;
> > +	touch_preempt_timing();
> > 
> > assuming that the latencies still show up even if delimited like this. 
> > (note that this only changes the way the latency is tracked - the
> > latency itself is still there so this isnt a fix.)
> > 
> 
> Sure, but, what would this accomplish, if the latency is still there? 
> Are we just trying to track down exactly where in the network driver
> this is triggered?

yeah. If it's the first chunk then we could perhaps avoid it by doing it
outside of the lock.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > oh well, indeed it cannot be disabled. Then i'd suggest to return early
> > from extract_entropy(), without doing anything. That is the function
> > that seems to introduce the worst overhead.
> 
> OK, I tried the quick hack of just returning 0 before taking the
> spinlock in extract_entropy, but this broke /dev/random which
> prevented me from logging in.  I guess we will have to properly fake
> it here.

you should return nbytes, not zero.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Lee Revell]

On Mon, 2004-08-16 at 00:33, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > > Anyway, the change to sched.c fixes the mlockall bug, it works
> > > > perfectly now.  Thanks!
> > > 
> > > great! This fix also means that we've got one more lock-break in the
> > > ext3 journalling code and one more lock-break in dcache.c. I've released
> > > -P1 with the fix included:
> > > 
> > >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1
> > > 
> > 
> > The highest latency I am seeing now is the rhine_check_duplex problem. 
> > Should I try making mdio_delay an NOOP?
> 
> there's no mdio_delay() in via-rhine.c AFAICS. Could you add a pair of
> touch-latency calls to around this code in mdio_read():
> 
> +       touch_preempt_timing();
>         /* Wait for a previous command to complete. */
>         while ((readb(ioaddr + MIICmd) & 0x60) && --boguscnt > 0)
> +       touch_preempt_timing();
> 
> i suspect it's this one that introduces the biggest delay. Also:
> 
> +	touch_preempt_timing();
>         while ((readb(ioaddr + MIICmd) & 0x40) && --boguscnt > 0)
>                 ;
> +	touch_preempt_timing();
> 

Just to clarify, the touch_preempt_timing should be after, not inside
the while loop in both cases, right?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Mon, 2004-08-16 at 00:51, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:

> seems we need a lock-break in the innermost loop of copy_page_range(). 
> That loop processes up to 1024 pages currently, before the lock-break in
> the outer loop happens. Large GUI processes are more likely to have full
> 4MB regions mapped & populated.
> 
> i suspect you could trigger a similarly bad latency by doing a fork() in
> mlockall-test.cc - the attached mlockall-test2.cc does this. Do you get
> bad latencies?
> 

Yes, mlockall-test2 with 10+MB of memory produces a 200us-3ms xrun about
half the time.

Lee 

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Lee Revell]

On Mon, 2004-08-16 at 00:33, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > > Anyway, the change to sched.c fixes the mlockall bug, it works
> > > > perfectly now.  Thanks!
> > > 
> > > great! This fix also means that we've got one more lock-break in the
> > > ext3 journalling code and one more lock-break in dcache.c. I've released
> > > -P1 with the fix included:
> > > 
> > >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1
> > > 
> > 
> > The highest latency I am seeing now is the rhine_check_duplex problem. 
> > Should I try making mdio_delay an NOOP?
> 
> there's no mdio_delay() in via-rhine.c AFAICS. Could you add a pair of
> touch-latency calls to around this code in mdio_read():
> 

Adding this code causes the rhine-related latency reports to go away. 
Should I try removing the second pair, to verify it's the first chunk
that's responsible?

With this, and the extract_entropy hack, the biggest common latency I am
now seeing (the copy_page_one is bigger but rarer) is the XFree86
unmap_vmas issue.  This one actually occurs so often that I can't tell
what #2 is.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Lee Revell]

On Mon, 2004-08-16 at 01:02, Ingo Molnar wrote:
> yeah. If it's the first chunk then we could perhaps avoid it by doing it
> outside of the lock.
> 

Hmm, this is odd:

preemption latency trace v1.0
-----------------------------
 latency: 71 us, entries: 6 (6)
 process: XFree86/518, uid: 0
 nice: -10, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): sched_clock (schedule)
 0.000ms (+0.000ms): deactivate_task (schedule)
 0.000ms (+0.000ms): dequeue_task (deactivate_task)
 0.001ms (+0.000ms): __switch_to (schedule)
 0.068ms (+0.066ms): finish_task_switch (schedule)
 0.069ms (+0.000ms): check_preempt_timing (sub_preempt_count)

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Lee Revell]

On Mon, 2004-08-16 at 00:05, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Anyway, the change to sched.c fixes the mlockall bug, it works
> > perfectly now.  Thanks!
> 
> great! This fix also means that we've got one more lock-break in the
> ext3 journalling code and one more lock-break in dcache.c. I've released
> -P1 with the fix included:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1
> 

Ok, there is a visible difference from P0 to P1 with the extract_entropy
and via-rhine hacks:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P0
http://krustophenia.net/testresults.php?dataset=2.6.8.1-P1

The left spike is bigger with P1, which is more easily visible on linear
scaled graph on the left, as we are hitting the fast path more often. 
Also the smaller spike at 400 usecs disappears.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Roger Luethi]

On Sun, 15 Aug 2004 19:24:24 -0400, Lee Revell wrote:
>  0.000ms (+0.000ms): rhine_check_duplex (rhine_timer)
>  0.000ms (+0.000ms): mdio_read (rhine_check_duplex)
>  0.067ms (+0.067ms): mdio_read (rhine_timer)
>  0.139ms (+0.071ms): check_preempt_timing (sub_preempt_count)
> 
> This looks like the exact same problem Florian had, in his case it was 
> the sis900 driver.  Your recommendation was:
> 
>         #define mdio_delay() do { } while (0)
> 
> Should I try this?

If you want to try it, you are looking for IOSYNC (not medio_delay)
in this driver. However, doing this can break write ordering and thus
any driver you are treating that way.

> Also, isn't there a better solution than for network drivers to actively 
> poll for changes in link status?  Can't they just register a callback that 

For the Rhine, there is, but making it work requires some extra black
magic we didn't know until a few months ago. It's fixed in -mm now and
will probably be in 2.6.9. That doesn't mean the media_check is gone,
though. It just means it only happens on actual events.

Roger

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Takashi Iwai]

Hi,

(sorry for late reply, just back from vacation)

At Fri, 06 Aug 2004 22:54:26 -0400,
Lee Revell wrote:
> 
> On Mon, 2004-08-02 at 11:19, Takashi Iwai wrote:
> > At Fri, 30 Jul 2004 18:54:39 -0400,
> > Lee Revell wrote:
> > > 
> > > I discovered that a few of the XRUN traces were spurious - jackd
> > > apparently does something while stopping and starting that produces an
> > > XRUN trace but that jackd does not consider an error.  I will fix this
> > > in jackd.  The msync() related XRUN triggered by apt-get is definitely
> > > real.
> > 
> > Yes.  There is a bogus report at stopping (snd_pcm_drain is called).
> > It was fixed in the recent ALSA cvs tree, but seems not propagated to
> > bk yet...
> > 
> 
> It also seems to produce an xrun at startup.  This is with the latest
> ALSA CVS.  Is this behavior by design?

No, this is not.  It should be a real XRUN, I believe.


Takashi

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Mon, 2004-08-16 at 06:38, Takashi Iwai wrote:
> Hi,
> 
> (sorry for late reply, just back from vacation)
> 
> At Fri, 06 Aug 2004 22:54:26 -0400,
> Lee Revell wrote:
> > 
> > On Mon, 2004-08-02 at 11:19, Takashi Iwai wrote:
> > > At Fri, 30 Jul 2004 18:54:39 -0400,
> > > Lee Revell wrote:
> > > > 
> > > > I discovered that a few of the XRUN traces were spurious - jackd
> > > > apparently does something while stopping and starting that produces an
> > > > XRUN trace but that jackd does not consider an error.  I will fix this
> > > > in jackd.  The msync() related XRUN triggered by apt-get is definitely
> > > > real.
> > > 
> > > Yes.  There is a bogus report at stopping (snd_pcm_drain is called).
> > > It was fixed in the recent ALSA cvs tree, but seems not propagated to
> > > bk yet...
> > > 
> > 
> > It also seems to produce an xrun at startup.  This is with the latest
> > ALSA CVS.  Is this behavior by design?
> 
> No, this is not.  It should be a real XRUN, I believe.
> 

This one has still defied explanation.  The working theory was that it
was the same bug causing an xrun if an unrelated process called
mlockall, but now that bug has been fixed, and this xrun at startup
still happens.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> With this, and the extract_entropy hack, the biggest common latency I
> am now seeing (the copy_page_one is bigger but rarer) is the XFree86
> unmap_vmas issue.  This one actually occurs so often that I can't tell
> what #2 is.

found the unmap_vmas() latency: pages get queued up for TLB flush (and
subsequent freeing) and the lock-break in unmap_vmas() didnt account for
this. When there's a preemption request and we do the lock-break it's
already too late: we first have to free possibly thousands of pages,
resulting in the latency. The solution is to make the lock-break (and
hence, the queue-flush) periodically, regardless of preemption requests. 
Will release -P2 soon.

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > No, this is not.  It should be a real XRUN, I believe.
> 
> This one has still defied explanation.  The working theory was that it
> was the same bug causing an xrun if an unrelated process called
> mlockall, but now that bug has been fixed, and this xrun at startup
> still happens.

does the first xrun happen right during startup, or only when the first
jack application uses jackd to do audio?

if the former then does jackd set itself up (does an mlockall, etc.) 
before it opens the audio device? If the audio device has an event for
jackd the moment the device is opened, and jackd opens the audio device
early during startup, then jackd might not be able to process this event
until it has started up (which can take milliseconds).

	Ingo

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Mon, 2004-08-16 at 06:48, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > No, this is not.  It should be a real XRUN, I believe.
> > 
> > This one has still defied explanation.  The working theory was that it
> > was the same bug causing an xrun if an unrelated process called
> > mlockall, but now that bug has been fixed, and this xrun at startup
> > still happens.
> 
> does the first xrun happen right during startup, or only when the first
> jack application uses jackd to do audio?
> 

It happens when the jackd server starts up, before any clients connect.

> if the former then does jackd set itself up (does an mlockall, etc.) 
> before it opens the audio device? If the audio device has an event for
> jackd the moment the device is opened, and jackd opens the audio device
> early during startup, then jackd might not be able to process this event
> until it has started up (which can take milliseconds).

This is probably what is happening, the kernel-side issue seems fixed,
but I will have to test some more and look at the source tomorrow, it's
getting late.

Lee

Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Mon, 2004-08-16 at 06:52, Lee Revell wrote:
> On Mon, 2004-08-16 at 06:48, Ingo Molnar wrote:
>  if the former then does jackd set itself up (does an mlockall, etc.) 
> > before it opens the audio device? If the audio device has an event for
> > jackd the moment the device is opened, and jackd opens the audio device
> > early during startup, then jackd might not be able to process this event
> > until it has started up (which can take milliseconds).
> 
> This is probably what is happening, the kernel-side issue seems fixed,

It looks like this is what happens - jackd calls snd_pcm_start, then
does several other thinks like malloc'ing memory for the array of fd's
to poll() before entering the polling loop, by which time there has been
data ready for a while.  This may or may not be worth fixing, I am
adding jackit-devel to the cc: list.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Thomas Charbonnel]

Ingo Molnar wrote :
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Anyway, the change to sched.c fixes the mlockall bug, it works
> > perfectly now.  Thanks!
> 
> great! This fix also means that we've got one more lock-break in the
> ext3 journalling code and one more lock-break in dcache.c. I've released
> -P1 with the fix included:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1
> 
> 	Ingo

Tested P1. The biggest offender on my system is still reiserfs'
search_by_key. The trace is made of a very long loop of :
 0.016ms (+0.000ms): reiserfs_in_journal (scan_bitmap_block)
 0.016ms (+0.000ms): find_next_zero_bit (scan_bitmap_block)
(...)
 0.977ms (+0.000ms): reiserfs_in_journal (scan_bitmap_block)
 0.977ms (+0.000ms): find_next_zero_bit (scan_bitmap_block)
with interrupts showing up in the trace from time to time.
Do you have plans to fix this, or should I switch to ext3 ?

Another point was the ACPI induced latency I was experiencing. I think
that P1 could help track this down. After a fresh boot on an ACPI
enabled kernel, once the initialization-related latency spikes are over,
I reset /proc/sys/kernel/preempt_max_latency to 100, and then I get a
100us latency_trace attributed to swapper/1:

preemption latency trace v1.0
-----------------------------
 latency: 100 us, entries: 4000 (14253)
 process: swapper/1, uid: 0
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): startup_8259A_irq (probe_irq_on)
 0.000ms (+0.000ms): enable_8259A_irq (startup_8259A_irq)
 0.001ms (+0.001ms): startup_8259A_irq (probe_irq_on)
 0.001ms (+0.000ms): enable_8259A_irq (startup_8259A_irq)
 0.002ms (+0.001ms): startup_8259A_irq (probe_irq_on)
 0.002ms (+0.000ms): enable_8259A_irq (startup_8259A_irq)
(...)

but the weird thing is that the latency sum goes way above those 100us,
the culprit being do_IRQ, regularly chewing up to 1ms !

(...)
 0.009ms (+0.000ms): enable_8259A_irq (startup_8259A_irq)
 0.011ms (+0.001ms): startup_8259A_irq (probe_irq_on)
 0.011ms (+0.000ms): enable_8259A_irq (startup_8259A_irq)
 0.954ms (+0.943ms): do_IRQ (common_interrupt)
-------------^
 0.954ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.956ms (+0.002ms): generic_redirect_hardirq (do_IRQ)
 0.956ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.957ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 0.957ms (+0.000ms): mark_offset_tsc (timer_interrupt)
 0.962ms (+0.005ms): do_timer (timer_interrupt)
(...)
 119.027ms (+0.000ms): preempt_schedule (timer_interrupt)
 119.027ms (+0.000ms): preempt_schedule (timer_interrupt)
 119.027ms (+0.000ms): generic_note_interrupt (do_IRQ)
 119.027ms (+0.000ms): end_8259A_irq (do_IRQ)
 119.028ms (+0.000ms): enable_8259A_irq (do_IRQ)
 119.028ms (+0.000ms): preempt_schedule (do_IRQ)
 119.029ms (+0.000ms): preempt_schedule (do_IRQ)
 119.029ms (+0.000ms): do_softirq (do_IRQ)
 119.029ms (+0.000ms): __do_softirq (do_softirq)
 120.017ms (+0.987ms): do_IRQ (common_interrupt)
---------------^
 120.017ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 120.019ms (+0.002ms): preempt_schedule (do_IRQ)
 120.019ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
 120.019ms (+0.000ms): preempt_schedule (do_IRQ)
 120.020ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
(...)

You can find the full trace here :
http://www.undata.org/~thomas/swapper.trace

Do you have an idea of what's happening here ?

Thomas

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Florian Schmidt]

On Mon, 16 Aug 2004 06:05:15 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> 
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Anyway, the change to sched.c fixes the mlockall bug, it works
> > perfectly now.  Thanks!
> 
> great! This fix also means that we've got one more lock-break in the
> ext3 journalling code and one more lock-break in dcache.c. I've
> released-P1 with the fix included:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1

Hi,

cool, i can mlockall_test 500000  without an xrun in jackd! :) 

But it seems that this wasn't the only thing causing an xrun on jackd
client startup. I will try to take another look at the jackd source..

flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Lee Revell]

On Mon, 2004-08-16 at 07:13, Florian Schmidt wrote:
> On Mon, 16 Aug 2004 06:05:15 +0200
> Ingo Molnar <mingo@elte.hu> wrote:
> 
> > 
> > * Lee Revell <rlrevell@joe-job.com> wrote:
> > 
> > > Anyway, the change to sched.c fixes the mlockall bug, it works
> > > perfectly now.  Thanks!
> > 
> > great! This fix also means that we've got one more lock-break in the
> > ext3 journalling code and one more lock-break in dcache.c. I've
> > released-P1 with the fix included:
> > 
> >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1
> 
> Hi,
> 
> cool, i can mlockall_test 500000  without an xrun in jackd! :) 
> 
> But it seems that this wasn't the only thing causing an xrun on jackd
> client startup. I will try to take another look at the jackd source..
> 

Ingo mentioned that possibly the mlockall issue resulted from both
processes mapping some of the same pages, which was ruled out by using
small test programs, but maybe that is what is going on here.  A jack
client and server by definition have to map some of the same pages.

Would it be worthwhile to compile the jack client -static?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Ingo Molnar]

* Thomas Charbonnel <thomas@undata.org> wrote:

> Tested P1. The biggest offender on my system is still reiserfs'
> search_by_key. The trace is made of a very long loop of :
>  0.016ms (+0.000ms): reiserfs_in_journal (scan_bitmap_block)
>  0.016ms (+0.000ms): find_next_zero_bit (scan_bitmap_block)
> (...)
>  0.977ms (+0.000ms): reiserfs_in_journal (scan_bitmap_block)
>  0.977ms (+0.000ms): find_next_zero_bit (scan_bitmap_block)
> with interrupts showing up in the trace from time to time. Do you have
> plans to fix this, or should I switch to ext3 ?

i took a quick look and the reiserfs locking rules in that place do not
seem to be easily fixable - this is the tree-lookup code which i suspect
cannot be preempted. The reiser journalling code also makes use of the
big kernel lock. I'd suggest reporting this to the reiserfs folks and
(if it's not too much effort to migrate) use ext3 meanwhile.

> but the weird thing is that the latency sum goes way above those 100us,
> the culprit being do_IRQ, regularly chewing up to 1ms !

>  0.011ms (+0.000ms): enable_8259A_irq (startup_8259A_irq)
>  0.954ms (+0.943ms): do_IRQ (common_interrupt)

weird. This has the looks of a preempt-timing bug (we get a timer IRQ
every 1 msec) - but there should be no preempt-timing when we are in the
idle task (swapper).

> http://www.undata.org/~thomas/swapper.trace

i'll upload -P2 in a couple of minutes, it will trace the code that
do_IRQ() interrupted too - that would be quite useful in your case.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Hugh Dickins]

On Mon, 16 Aug 2004, Ingo Molnar wrote:
> 
> found the unmap_vmas() latency: pages get queued up for TLB flush (and
> subsequent freeing) and the lock-break in unmap_vmas() didnt account for
> this. When there's a preemption request and we do the lock-break it's
> already too late: we first have to free possibly thousands of pages,

Thousands?  Or FREE_PTE_NR 506?  Ah, ia64 has FREE_PTE_NR 2048: thousands.

Hugh

[patch] voluntary-preempt-2.6.8.1-P2

[Ingo Molnar]

here's -P2:

 http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P2

Changes since -P1:

 - trace interrupted kernel code (via hardirqs, NMIs and pagefaults)

 - yet another shot at trying to fix the IO-APIC/USB issues.

 - mcount speedups - tracing should be faster

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Florian Schmidt]

[-- Attachment #1: Type: text/plain, Size: 1816 bytes --]

On Mon, 16 Aug 2004 07:17:10 -0400
Lee Revell <rlrevell@joe-job.com> wrote:

> > But it seems that this wasn't the only thing causing an xrun on
> > jackd client startup. I will try to take another look at the jackd
> > source..
> > 
> 
> Ingo mentioned that possibly the mlockall issue resulted from both
> processes mapping some of the same pages, which was ruled out by using
> small test programs, but maybe that is what is going on here.  A jack
> client and server by definition have to map some of the same pages.
> 
> Would it be worthwhile to compile the jack client -static?

Here's a minimal jack client which does _not_ produce an xrun on startup
for me (it doesn't really do anything either).. Maybe the xruns are the
other clients fault and not really determined by the jack mechanisms.. I
will extend it step by step to do something functional.. maybe i'll find
out what change introduces xruns. compile with

g++ jack_test.cc `pkg-config jack --libs` -o jack_test


jack_test.cc:
----------------------
#include <jack/jack.h>
#include <iostream>

jack_client_t *client;
jack_port_t *port;

int process(jack_nframes_t frames, void *arg) {
        return 0;
}

int main(int argc, char *argv[]) {
        std::cout << "client_new" << std::endl;
        client = jack_client_new("foo");

        std::cout << "port_register." << std::endl;
        port = jack_port_register(client, "foobar",
JACK_DEFAULT_AUDIO_TYPE, JackPortIsOutput|JackPortIsTerminal, 0);

        std::cout << "set_process_callback" << std::endl;
        jack_set_process_callback(client, process, 0);

        std::cout << "activate" << std::endl;
        jack_activate(client);

        std::cout << "running" << std::endl;
        while(1) {sleep(1);};
}
-----------------------


-- 
Palimm Palimm!
http://affenbande.org/~tapas/


[-- Attachment #2: jack_test.cc --]
[-- Type: text/x-c++src, Size: 721 bytes --]

#include <jack/jack.h>
#include <iostream>

jack_client_t *client;
jack_port_t *port;

int process(jack_nframes_t frames, void *arg) {
        return 0;
}

int main(int argc, char *argv[]) {
        std::cout << "client_new" << std::endl;
        client = jack_client_new("foo");

        std::cout << "port_register." << std::endl;
        port = jack_port_register(client, "foobar", JACK_DEFAULT_AUDIO_TYPE, JackPortIsOutput|JackPortIsTerminal, 0);

        std::cout << "set_process_callback" << std::endl;
        jack_set_process_callback(client, process, 0);

        std::cout << "activate" << std::endl;
        jack_activate(client);

        std::cout << "running" << std::endl;
        while(1) {sleep(1);};
}

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Thomas Charbonnel]

Ingo Molnar wrote :
> here's -P2:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P2
> 
> Changes since -P1:
> 
>  - trace interrupted kernel code (via hardirqs, NMIs and pagefaults)
> 
>  - yet another shot at trying to fix the IO-APIC/USB issues.
> 
>  - mcount speedups - tracing should be faster
> 
> 	Ingo

Same do_IRQ problem with P2, trace is here :
http://www.undata.org/~thomas/swapper-P2.trace

Thomas

Re: [patch] voluntary-preempt-2.6.8-rc3-O4

[Takashi Iwai]

At Tue, 10 Aug 2004 11:22:49 +0200,
Ingo Molnar wrote:
> 
> 
> * Ingo Molnar <mingo@elte.hu> wrote:
> 
> > another idea: you are running this on a C3, using CONFIG_MCYRIXIII,
> > correct? That is one of the rare configs that triggers X86_USE_3DNOW
> > and MMX ops. If 3dnow is in any way handicapped in that CPU then that
> > could cause trouble. Could you compile for e.g. CONFIG_M586TSC? [that
> > option should be fully compatible with a C3.] - this will exclude the
> > MMX page clearing ops.
> 
> another (more remote) possibility is that the timestamp counter gets
> somehow messed up during MMX ops. Does the ALSA detector use the
> timestamp counter, or does it only use jiffies?

No, neither of them.  It checks the DMA buffer position, so the
accuracy of the latency measurement depends on the size of DMA buffer 
fragment (called "period" in ALSA).

The only thing related with the time in the pcm layer is the call of
do_gettimeofday() when status ioctl is called (or at each DMA
interrupt, depending on the running mode).


Takashi

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Thomas Charbonnel]

I wrote :
> Ingo Molnar wrote :
> > here's -P2:
> > 
> >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P2
> > 
> > Changes since -P1:
> > 
> >  - trace interrupted kernel code (via hardirqs, NMIs and pagefaults)
> > 
> >  - yet another shot at trying to fix the IO-APIC/USB issues.
> > 
> >  - mcount speedups - tracing should be faster
> > 
> > 	Ingo
> 
> Same do_IRQ problem with P2, trace is here :
> http://www.undata.org/~thomas/swapper-P2.trace
> 
> Thomas
> 

Ok, maybe that was a false positive. In fact the trace corresponds to
some preempt violation occurring during the boot process :
preemption latency trace v1.0
-----------------------------
 latency: 136095 us, entries: 4000 (14818)
 process: swapper/1, uid: 0
 nice: 0, policy: 0, rt_priority: 0
=======>
When I reset preempt_max_latency to 100, /proc/latency_trace stays the
same, but the latency time reported is updated to reflect the new
preemtp_max_latency :
root@satellite thomas # diff trace1-P2 trace2-P2
3c3
<  latency: 136095 us, entries: 4000 (14818)
---
>  latency: 100 us, entries: 4000 (14818)

There still are weird things happening with irq handling, though. how
can generic_redirect_hardirq eat half a millisecond :

preemption latency trace v1.0
-----------------------------
 latency: 481 us, entries: 32 (32)
 process: swapper/0, uid: 0
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): do_IRQ (default_idle)
 0.000ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.459ms (+0.459ms): generic_redirect_hardirq (do_IRQ)
 0.459ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.459ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 0.459ms (+0.000ms): mark_offset_tsc (timer_interrupt)
 0.465ms (+0.005ms): do_timer (timer_interrupt)
 0.465ms (+0.000ms): update_process_times (do_timer)
 0.465ms (+0.000ms): update_one_process (update_process_times)
 0.465ms (+0.000ms): run_local_timers (update_process_times)
 0.465ms (+0.000ms): raise_softirq (update_process_times)
 0.466ms (+0.000ms): scheduler_tick (update_process_times)
 0.466ms (+0.000ms): sched_clock (scheduler_tick)
 0.466ms (+0.000ms): update_wall_time (do_timer)
 0.466ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 0.466ms (+0.000ms): profile_hook (timer_interrupt)
 0.466ms (+0.000ms): notifier_call_chain (profile_hook)
 0.467ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.467ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.467ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.468ms (+0.000ms): do_softirq (do_IRQ)
 0.468ms (+0.000ms): __do_softirq (do_softirq)
 0.468ms (+0.000ms): wake_up_process (do_softirq)
 0.468ms (+0.000ms): try_to_wake_up (wake_up_process)
 0.468ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.468ms (+0.000ms): activate_task (try_to_wake_up)
 0.469ms (+0.000ms): sched_clock (activate_task)
 0.469ms (+0.000ms): recalc_task_prio (activate_task)
 0.469ms (+0.000ms): effective_prio (recalc_task_prio)
 0.469ms (+0.000ms): enqueue_task (activate_task)
 0.469ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.469ms (+0.000ms): check_preempt_timing (sub_preempt_count)

Is there any source of interruption not covered by your P2 patch ?

Thomas

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Thomas Charbonnel]

I wrote :
> I wrote :
> > Ingo Molnar wrote :
> > > here's -P2:
> > > 
> > >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P2
> > > 
> > > Changes since -P1:
> > > 
> > >  - trace interrupted kernel code (via hardirqs, NMIs and pagefaults)
> > > 
> > >  - yet another shot at trying to fix the IO-APIC/USB issues.
> > > 
> > >  - mcount speedups - tracing should be faster
> > > 
> > > 	Ingo
> > 
> > Same do_IRQ problem with P2, trace is here :
> > http://www.undata.org/~thomas/swapper-P2.trace
> > 
> > Thomas
> > 
(...)
> There still are weird things happening with irq handling, though. how
> can generic_redirect_hardirq eat half a millisecond :
> 
(...)
> Is there any source of interruption not covered by your P2 patch ?
(...)

More on this : 

preemption latency trace v1.0
-----------------------------
 latency: 611 us, entries: 21 (21)
 process: ksoftirqd/0/2, uid: 0
 nice: -10, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): sched_clock (schedule)
 0.000ms (+0.000ms): dequeue_task (schedule)
 0.000ms (+0.000ms): recalc_task_prio (schedule)
 0.590ms (+0.589ms): effective_prio (recalc_task_prio)
 0.590ms (+0.000ms): enqueue_task (schedule)
 0.590ms (+0.000ms): __switch_to (schedule)
 0.590ms (+0.000ms): finish_task_switch (schedule)
 0.591ms (+0.001ms): do_IRQ (finish_task_switch)
(...)

and

preemption latency trace v1.0
-----------------------------
 latency: 481 us, entries: 32 (32)
 process: swapper/0, uid: 0
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): do_IRQ (default_idle)
 0.000ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.459ms (+0.459ms): generic_redirect_hardirq (do_IRQ)
 0.459ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.459ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 0.459ms (+0.000ms): mark_offset_tsc (timer_interrupt)
 0.465ms (+0.005ms): do_timer (timer_interrupt)
 0.465ms (+0.000ms): update_process_times (do_timer)
 0.465ms (+0.000ms): update_one_process (update_process_times)
 0.465ms (+0.000ms): run_local_timers (update_process_times)
 0.465ms (+0.000ms): raise_softirq (update_process_times)
 0.466ms (+0.000ms): scheduler_tick (update_process_times)
 0.466ms (+0.000ms): sched_clock (scheduler_tick)
 0.466ms (+0.000ms): update_wall_time (do_timer)
 0.466ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 0.466ms (+0.000ms): profile_hook (timer_interrupt)
 0.466ms (+0.000ms): notifier_call_chain (profile_hook)
 0.467ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.467ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.467ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.468ms (+0.000ms): do_softirq (do_IRQ)
 0.468ms (+0.000ms): __do_softirq (do_softirq)
 0.468ms (+0.000ms): wake_up_process (do_softirq)
 0.468ms (+0.000ms): try_to_wake_up (wake_up_process)
 0.468ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.468ms (+0.000ms): activate_task (try_to_wake_up)
 0.469ms (+0.000ms): sched_clock (activate_task)
 0.469ms (+0.000ms): recalc_task_prio (activate_task)
 0.469ms (+0.000ms): effective_prio (recalc_task_prio)
 0.469ms (+0.000ms): enqueue_task (activate_task)
 0.469ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.469ms (+0.000ms): check_preempt_timing (sub_preempt_count)

It definitely looks like the kernel is interrupted by some interrupt
source not covered by the patch.

Thomas

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Ingo Molnar]

* Thomas Charbonnel <thomas@undata.org> wrote:

>  0.000ms (+0.000ms): do_IRQ (default_idle)
>  0.000ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
>  0.459ms (+0.459ms): generic_redirect_hardirq (do_IRQ)
>  0.459ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
>  0.459ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)

> It definitely looks like the kernel is interrupted by some interrupt
> source not covered by the patch.

the only possibility is SMM, which is not handled by Linux. (but by the
BIOS.) Otherwise we track everything - including NMIs.

can you reproduce this using an UP kernel too?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Thomas Charbonnel]

Ingo Molnar wrote :
> * Thomas Charbonnel <thomas@undata.org> wrote:
> 
> >  0.000ms (+0.000ms): do_IRQ (default_idle)
> >  0.000ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
> >  0.459ms (+0.459ms): generic_redirect_hardirq (do_IRQ)
> >  0.459ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
> >  0.459ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
> 
> > It definitely looks like the kernel is interrupted by some interrupt
> > source not covered by the patch.
> 
> the only possibility is SMM, which is not handled by Linux. (but by the
> BIOS.) Otherwise we track everything - including NMIs.
> 
> can you reproduce this using an UP kernel too?
> 
> 	Ingo

I'm currently not using SMP :

root@satellite linux # grep SMP .config
CONFIG_BROKEN_ON_SMP=y
# CONFIG_X86_BIGSMP is not set
# CONFIG_SMP is not set

Whole .config here:
http://www.undata.org/~thomas/config-2.6.8-P2

This would confirm the hypothesis of a buggy BIOS, I'm afraid.

Thomas

Re: [Jackit-devel] Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Jack O'Quin]

Lee Revell <rlrevell@joe-job.com> writes:

> On Mon, 2004-08-16 at 06:52, Lee Revell wrote:
> > On Mon, 2004-08-16 at 06:48, Ingo Molnar wrote:
> >  if the former then does jackd set itself up (does an mlockall, etc.) 
> > > before it opens the audio device? If the audio device has an event for
> > > jackd the moment the device is opened, and jackd opens the audio device
> > > early during startup, then jackd might not be able to process this event
> > > until it has started up (which can take milliseconds).
> > 
> > This is probably what is happening, the kernel-side issue seems fixed,
> 
> It looks like this is what happens - jackd calls snd_pcm_start, then
> does several other thinks like malloc'ing memory for the array of fd's
> to poll() before entering the polling loop, by which time there has been
> data ready for a while.  This may or may not be worth fixing, I am
> adding jackit-devel to the cc: list.

Yep.  This looks like a bug to me.  While jackd, itself, seems to
allocate everything before calling driver->start(), the ALSA driver
internally calls malloc() *after* calling snd_pcm_start().  I doubt
anyone has ever made a concerted effort to clean up this path for
realtime safety.

I think it should be fixed (I need to study the code in more detail).
There's probably nothing to prevent us moving the free() and malloc()
calls up nearer the top of alsa_driver_start().  That will probably
require an extra error test and free in case snd_pcm_start() fails.
-- 
  joq

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Ingo Molnar]

* Thomas Charbonnel <thomas@undata.org> wrote:

> > >  0.000ms (+0.000ms): do_IRQ (default_idle)
> > >  0.000ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
> > >  0.459ms (+0.459ms): generic_redirect_hardirq (do_IRQ)
> > >  0.459ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
> > >  0.459ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
> > 
> > > It definitely looks like the kernel is interrupted by some interrupt
> > > source not covered by the patch.
> > 
> > the only possibility is SMM, which is not handled by Linux. (but by the
> > BIOS.) Otherwise we track everything - including NMIs.

> This would confirm the hypothesis of a buggy BIOS, I'm afraid.

there are still other (and more likely) possible reasons like a bug in
the latency tracer. (Or a broken TSC - albeit this is less likely.)

can you reproduce this phenomenon at will? Does it go away if you turn 
ACPI/APM off (both in the kernel and in the BIOS). Does it go away if 
you use idle=poll?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Thomas Charbonnel]

Ingo Molnar wrote :
> * Thomas Charbonnel <thomas@undata.org> wrote:
> 
> > > >  0.000ms (+0.000ms): do_IRQ (default_idle)
> > > >  0.000ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
> > > >  0.459ms (+0.459ms): generic_redirect_hardirq (do_IRQ)
> > > >  0.459ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
> > > >  0.459ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
> > > 
> > > > It definitely looks like the kernel is interrupted by some interrupt
> > > > source not covered by the patch.
> > > 
> > > the only possibility is SMM, which is not handled by Linux. (but by the
> > > BIOS.) Otherwise we track everything - including NMIs.
> 
> > This would confirm the hypothesis of a buggy BIOS, I'm afraid.
> 
> there are still other (and more likely) possible reasons like a bug in
> the latency tracer. (Or a broken TSC - albeit this is less likely.)
> 

I don't think the latency tracer is involved. I think this is the same
phenomenon as the regular latency spikes I reported earlier.
At some point I thought of a clock issue, indeed, so I switched to
pmtmr, but this is broken with voluntary preempt (I think it relates to
the sched_clock code when not using TSC -> precision of the reported
preempt threshold violations was 1ms). Worth mentioning is that the
latency spikes interval was the same even if I compiled cpufreq in and
switched the processor speed at runtime.

> can you reproduce this phenomenon at will? Does it go away if you turn 
> ACPI/APM off (both in the kernel and in the BIOS).

I can't turn PM off in the BIOS, but turning ACPI off in the kernel
suppresses the problem.

>  Does it go away if 
> you use idle=poll?
> 

I don't think so (the spikes where still here when you first merged
wli's preempt timing patch and forced idle=poll).

So basically I think the spikes are BIOS related, they've been detected
so far by the alsa xrun_debug mechanism, by the latency-test suite, and
now by the latency tracer, which shows that they're not linked to any
particular code section.

Thomas

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Hans Reiser]

Ingo Molnar wrote:

>* Thomas Charbonnel <thomas@undata.org> wrote:
>
>  
>
>>Tested P1. The biggest offender on my system is still reiserfs'
>>search_by_key. The trace is made of a very long loop of :
>> 0.016ms (+0.000ms): reiserfs_in_journal (scan_bitmap_block)
>> 0.016ms (+0.000ms): find_next_zero_bit (scan_bitmap_block)
>>(...)
>> 0.977ms (+0.000ms): reiserfs_in_journal (scan_bitmap_block)
>> 0.977ms (+0.000ms): find_next_zero_bit (scan_bitmap_block)
>>with interrupts showing up in the trace from time to time. Do you have
>>plans to fix this, or should I switch to ext3 ?
>>    
>>
>
>i took a quick look and the reiserfs locking rules in that place do not
>seem to be easily fixable - this is the tree-lookup code which i suspect
>cannot be preempted. The reiser journalling code also makes use of the
>big kernel lock. I'd suggest reporting this to the reiserfs folks
>
the fix to reiserfs doing single threaded balancing/searching is called 
reiser4;-)  It was not a trivial fix.  It is however released.

> and
>(if it's not too much effort to migrate) use ext3 meanwhile.
>
>  
>
>>but the weird thing is that the latency sum goes way above those 100us,
>>the culprit being do_IRQ, regularly chewing up to 1ms !
>>    
>>
>
>  
>
>> 0.011ms (+0.000ms): enable_8259A_irq (startup_8259A_irq)
>> 0.954ms (+0.943ms): do_IRQ (common_interrupt)
>>    
>>
>
>weird. This has the looks of a preempt-timing bug (we get a timer IRQ
>every 1 msec) - but there should be no preempt-timing when we are in the
>idle task (swapper).
>
>  
>
>>http://www.undata.org/~thomas/swapper.trace
>>    
>>
>
>i'll upload -P2 in a couple of minutes, it will trace the code that
>do_IRQ() interrupted too - that would be quite useful in your case.
>
>	Ingo
>-
>To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
>the body of a message to majordomo@vger.kernel.org
>More majordomo info at  http://vger.kernel.org/majordomo-info.html
>Please read the FAQ at  http://www.tux.org/lkml/
>
>
>  
>

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Lee Revell]

On Mon, 2004-08-16 at 15:20, Hans Reiser wrote:
> Ingo Molnar wrote:
> >
> >i took a quick look and the reiserfs locking rules in that place do not
> >seem to be easily fixable - this is the tree-lookup code which i suspect
> >cannot be preempted. The reiser journalling code also makes use of the
> >big kernel lock. I'd suggest reporting this to the reiserfs folks
> >
> the fix to reiserfs doing single threaded balancing/searching is called 
> reiser4;-)  It was not a trivial fix.  It is however released.
> 

When will reiser4 will be in the stock kernel?

Lee 

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Mon, 2004-08-16 at 04:07, Roger Luethi wrote:
> On Sun, 15 Aug 2004 19:24:24 -0400, Lee Revell wrote:
> > Also, isn't there a better solution than for network drivers to actively 
> > poll for changes in link status?  Can't they just register a callback that 
> 
> For the Rhine, there is, but making it work requires some extra black
> magic we didn't know until a few months ago. It's fixed in -mm now and
> will probably be in 2.6.9. That doesn't mean the media_check is gone,
> though. It just means it only happens on actual events.
> 

OK, this would certainly be an improvement, I was using the driver from
-mm until 2.6.8.1 anyway due to that damn rhine_power_init bug, with no
problems whatsoever.

What do you think of Ingo's solution of trying to move the problematic
call to mdio_read out of the spinlocked section?  It does seem that the
slowness of mdio_read causes the rp->lock spinlock to be held for an
awfully long time.  In a live audio setting you would actually get lots
of media events.

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Fri, 2004-08-13 at 06:31, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8-rc4-O6
> > 
> > Ugh, this is a bad one:
> > 
> > preemption latency trace v1.0
> > -----------------------------
> >  latency: 506 us, entries: 157 (157)
> >  process: evolution/3461, uid: 1000
> >  nice: 0, policy: 0, rt_priority: 0
> > =======>
> >  0.000ms (+0.000ms): get_random_bytes (__check_and_rekey)
> [...]
> >  0.493ms (+0.001ms): local_bh_enable (__check_and_rekey)
> 
> indeed this is a new one. Entropy rekeying every 300 seconds. Most of
> the overhead comes from the memcpy's - 10 usecs a pop!
> 

I have attached a patch that effectively disables extract_entropy.  I am
adding Theodore T'so to the cc: list as he is the author of the code in
question.

For the time being this hack is required to avoid ~0.5 ms
non-preemptible sections caused by the excessive memcpy's in
extract_entropy.

I am not a crypto expert, but it seems like there would be a zero-copy
solution.  Alternatively, a way to tell the system that we just don't
need THAT much entropy would be acceptable.  I would gladly trade
predictable TCP sequence numbers for the ability to do low latency
audio.

Ingo, can you add this to -P3, unless someone proposes a better way?

Lee

--- linux-2.6.8.1/drivers/char/random.c	2004-08-14 06:54:48.000000000 -0400
+++ linux-2.6.8.1-P2/drivers/char/random.c	2004-08-16 19:26:29.000000000 -0400
@@ -1354,6 +1354,8 @@
 static ssize_t extract_entropy(struct entropy_store *r, void * buf,
 			       size_t nbytes, int flags)
 {
+	return nbytes;
+    
 	ssize_t ret, i;
 	__u32 tmp[TMP_BUF_SIZE];
 	__u32 x;

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Lee Revell]

On Mon, 2004-08-16 at 10:12, Thomas Charbonnel wrote:
> I wrote :
> > Ingo Molnar wrote :
> > > here's -P2:
> > > 
> > >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P2
> > > 
> > > Changes since -P1:
> > > 
> > >  - trace interrupted kernel code (via hardirqs, NMIs and pagefaults)
> > > 
> > >  - yet another shot at trying to fix the IO-APIC/USB issues.
> > > 
> > >  - mcount speedups - tracing should be faster
> > > 
> > > 	Ingo
> > 
> > Same do_IRQ problem with P2, trace is here :
> > http://www.undata.org/~thomas/swapper-P2.trace
> > 
> > Thomas
> > 
> 
> Ok, maybe that was a false positive. In fact the trace corresponds to
> some preempt violation occurring during the boot process :
> preemption latency trace v1.0
> -----------------------------
>  latency: 136095 us, entries: 4000 (14818)
>  process: swapper/1, uid: 0
>  nice: 0, policy: 0, rt_priority: 0

Yes, I get this one too.  Maybe the tracer should ignore such huge
values, as they seem to only happen during boot and there dosn't seem to
be a reason to fix them.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Lee Revell]

On Mon, 2004-08-16 at 20:14, Florian Schmidt wrote:
> On Mon, 16 Aug 2004 06:05:15 +0200
> Ingo Molnar <mingo@elte.hu> wrote:
> 
> > 
> > * Lee Revell <rlrevell@joe-job.com> wrote:
> > 
> > > Anyway, the change to sched.c fixes the mlockall bug, it works
> > > perfectly now.  Thanks!
> > 
> > great! This fix also means that we've got one more lock-break in the
> > ext3 journalling code and one more lock-break in dcache.c. I've
> > released-P1 with the fix included:
> > 
> >  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1
> 
> Hi,
> 
> i don't know if this was mentioned before, but i sometimes see traces
> like this where half the entries are "preempt_schedule
> (copy_page_range)". I just wanted to ask if this is normal and expected
> behaviour.
> 

Yes, Ingo identified an issue with copy_page_range, I don't think it's
fixed yet.  See the voluntary-preempt-2.6.8.1-P0 thread.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Florian Schmidt]

On Mon, 16 Aug 2004 06:05:15 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> 
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Anyway, the change to sched.c fixes the mlockall bug, it works
> > perfectly now.  Thanks!
> 
> great! This fix also means that we've got one more lock-break in the
> ext3 journalling code and one more lock-break in dcache.c. I've
> released-P1 with the fix included:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P1

Hi,

i don't know if this was mentioned before, but i sometimes see traces
like this where half the entries are "preempt_schedule
(copy_page_range)". I just wanted to ask if this is normal and expected
behaviour.


preemption latency trace v1.0
-----------------------------
 latency: 162 us, entries: 361 (361)
 process: qjackctl/828, uid: 1000
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): __vma_link_rb (copy_mm)
 0.000ms (+0.000ms): rb_insert_color (copy_mm)
 0.000ms (+0.000ms): __rb_rotate_left (rb_insert_color)
 0.000ms (+0.000ms): copy_page_range (copy_mm)
 0.000ms (+0.000ms): pte_alloc_map (copy_page_range)
 0.053ms (+0.053ms): do_IRQ (common_interrupt)
 0.054ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.056ms (+0.001ms): generic_redirect_hardirq (do_IRQ)
 0.056ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.056ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 0.057ms (+0.000ms): mark_offset_tsc (timer_interrupt)
 0.064ms (+0.006ms): do_timer (timer_interrupt)
 0.064ms (+0.000ms): update_process_times (do_timer)
 0.064ms (+0.000ms): update_one_process (update_process_times)
 0.064ms (+0.000ms): run_local_timers (update_process_times)
 0.064ms (+0.000ms): raise_softirq (update_process_times)
 0.065ms (+0.000ms): scheduler_tick (update_process_times)
 0.065ms (+0.000ms): sched_clock (scheduler_tick)
 0.066ms (+0.001ms): task_timeslice (scheduler_tick)
 0.066ms (+0.000ms): update_wall_time (do_timer)
 0.066ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 0.068ms (+0.001ms): generic_note_interrupt (do_IRQ)
 0.068ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.069ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.070ms (+0.000ms): do_softirq (do_IRQ)
 0.070ms (+0.000ms): __do_softirq (do_softirq)
 0.070ms (+0.000ms): wake_up_process (do_softirq)
 0.070ms (+0.000ms): try_to_wake_up (wake_up_process)
 0.070ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.071ms (+0.000ms): activate_task (try_to_wake_up)
 0.071ms (+0.000ms): sched_clock (activate_task)
 0.071ms (+0.000ms): recalc_task_prio (activate_task)
 0.071ms (+0.000ms): effective_prio (recalc_task_prio)
 0.071ms (+0.000ms): enqueue_task (activate_task)
 0.072ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.074ms (+0.000ms): preempt_schedule (copy_page_range)
 0.074ms (+0.000ms): preempt_schedule (copy_page_range)
 0.074ms (+0.000ms): preempt_schedule (copy_page_range)
 0.074ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.086ms (+0.000ms): preempt_schedule (copy_page_range)
 0.086ms (+0.000ms): preempt_schedule (copy_page_range)
 0.086ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.092ms (+0.000ms): preempt_schedule (copy_page_range)
 0.092ms (+0.000ms): preempt_schedule (copy_page_range)
 0.092ms (+0.000ms): preempt_schedule (copy_page_range)
 0.092ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.094ms (+0.000ms): preempt_schedule (copy_page_range)
 0.094ms (+0.000ms): preempt_schedule (copy_page_range)
 0.094ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.105ms (+0.000ms): preempt_schedule (copy_page_range)
 0.105ms (+0.000ms): preempt_schedule (copy_page_range)
 0.105ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.109ms (+0.000ms): preempt_schedule (copy_page_range)
 0.109ms (+0.000ms): preempt_schedule (copy_page_range)
 0.109ms (+0.000ms): preempt_schedule (copy_page_range)
 0.109ms (+0.000ms): preempt_schedule (copy_page_range)
 0.110ms (+0.000ms): preempt_schedule (copy_page_range)
 0.110ms (+0.000ms): preempt_schedule (copy_page_range)
 0.110ms (+0.000ms): preempt_schedule (copy_page_range)
 0.110ms (+0.000ms): preempt_schedule (copy_page_range)
 0.110ms (+0.000ms): preempt_schedule (copy_page_range)
 0.111ms (+0.000ms): preempt_schedule (copy_page_range)
 0.111ms (+0.000ms): preempt_schedule (copy_page_range)
 0.111ms (+0.000ms): preempt_schedule (copy_page_range)
 0.111ms (+0.000ms): preempt_schedule (copy_page_range)
 0.112ms (+0.000ms): preempt_schedule (copy_page_range)
 0.112ms (+0.000ms): preempt_schedule (copy_page_range)
 0.112ms (+0.000ms): preempt_schedule (copy_page_range)
 0.112ms (+0.000ms): preempt_schedule (copy_page_range)
 0.113ms (+0.000ms): preempt_schedule (copy_page_range)
 0.113ms (+0.000ms): preempt_schedule (copy_page_range)
 0.113ms (+0.000ms): preempt_schedule (copy_page_range)
 0.114ms (+0.000ms): preempt_schedule (copy_page_range)
 0.114ms (+0.000ms): preempt_schedule (copy_page_range)
 0.114ms (+0.000ms): preempt_schedule (copy_page_range)
 0.114ms (+0.000ms): preempt_schedule (copy_page_range)
 0.114ms (+0.000ms): preempt_schedule (copy_page_range)
 0.115ms (+0.000ms): preempt_schedule (copy_page_range)
 0.115ms (+0.000ms): preempt_schedule (copy_page_range)
 0.115ms (+0.000ms): preempt_schedule (copy_page_range)
 0.116ms (+0.000ms): preempt_schedule (copy_page_range)
 0.116ms (+0.000ms): preempt_schedule (copy_page_range)
 0.116ms (+0.000ms): preempt_schedule (copy_page_range)
 0.117ms (+0.000ms): preempt_schedule (copy_page_range)
 0.117ms (+0.000ms): preempt_schedule (copy_page_range)
 0.117ms (+0.000ms): preempt_schedule (copy_page_range)
 0.117ms (+0.000ms): preempt_schedule (copy_page_range)
 0.117ms (+0.000ms): preempt_schedule (copy_page_range)
 0.117ms (+0.000ms): preempt_schedule (copy_page_range)
 0.118ms (+0.000ms): preempt_schedule (copy_page_range)
 0.118ms (+0.000ms): preempt_schedule (copy_page_range)
 0.118ms (+0.000ms): preempt_schedule (copy_page_range)
 0.118ms (+0.000ms): preempt_schedule (copy_page_range)
 0.119ms (+0.000ms): preempt_schedule (copy_page_range)
 0.119ms (+0.000ms): preempt_schedule (copy_page_range)
 0.119ms (+0.000ms): preempt_schedule (copy_page_range)
 0.120ms (+0.000ms): preempt_schedule (copy_page_range)
 0.120ms (+0.000ms): preempt_schedule (copy_page_range)
 0.120ms (+0.000ms): preempt_schedule (copy_page_range)
 0.120ms (+0.000ms): preempt_schedule (copy_page_range)
 0.120ms (+0.000ms): preempt_schedule (copy_page_range)
 0.121ms (+0.000ms): preempt_schedule (copy_page_range)
 0.121ms (+0.000ms): preempt_schedule (copy_page_range)
 0.121ms (+0.000ms): preempt_schedule (copy_page_range)
 0.122ms (+0.000ms): preempt_schedule (copy_page_range)
 0.122ms (+0.000ms): preempt_schedule (copy_page_range)
 0.122ms (+0.000ms): preempt_schedule (copy_page_range)
 0.123ms (+0.000ms): preempt_schedule (copy_page_range)
 0.123ms (+0.000ms): preempt_schedule (copy_page_range)
 0.123ms (+0.000ms): preempt_schedule (copy_page_range)
 0.123ms (+0.000ms): preempt_schedule (copy_page_range)
 0.124ms (+0.000ms): preempt_schedule (copy_page_range)
 0.124ms (+0.000ms): preempt_schedule (copy_page_range)
 0.124ms (+0.000ms): preempt_schedule (copy_page_range)
 0.124ms (+0.000ms): preempt_schedule (copy_page_range)
 0.125ms (+0.000ms): preempt_schedule (copy_page_range)
 0.125ms (+0.000ms): preempt_schedule (copy_page_range)
 0.125ms (+0.000ms): preempt_schedule (copy_page_range)
 0.125ms (+0.000ms): preempt_schedule (copy_page_range)
 0.126ms (+0.000ms): preempt_schedule (copy_page_range)
 0.126ms (+0.000ms): preempt_schedule (copy_page_range)
 0.126ms (+0.000ms): preempt_schedule (copy_page_range)
 0.126ms (+0.000ms): preempt_schedule (copy_page_range)
 0.126ms (+0.000ms): preempt_schedule (copy_page_range)
 0.127ms (+0.000ms): preempt_schedule (copy_page_range)
 0.127ms (+0.000ms): preempt_schedule (copy_page_range)
 0.127ms (+0.000ms): preempt_schedule (copy_page_range)
 0.127ms (+0.000ms): preempt_schedule (copy_page_range)
 0.128ms (+0.000ms): preempt_schedule (copy_page_range)
 0.128ms (+0.000ms): preempt_schedule (copy_page_range)
 0.128ms (+0.000ms): preempt_schedule (copy_page_range)
 0.128ms (+0.000ms): preempt_schedule (copy_page_range)
 0.129ms (+0.000ms): preempt_schedule (copy_page_range)
 0.129ms (+0.000ms): preempt_schedule (copy_page_range)
 0.130ms (+0.000ms): preempt_schedule (copy_page_range)
 0.130ms (+0.000ms): preempt_schedule (copy_page_range)
 0.130ms (+0.000ms): preempt_schedule (copy_page_range)
 0.130ms (+0.000ms): preempt_schedule (copy_page_range)
 0.130ms (+0.000ms): preempt_schedule (copy_page_range)
 0.131ms (+0.000ms): preempt_schedule (copy_page_range)
 0.131ms (+0.000ms): preempt_schedule (copy_page_range)
 0.131ms (+0.000ms): preempt_schedule (copy_page_range)
 0.132ms (+0.000ms): preempt_schedule (copy_page_range)
 0.132ms (+0.000ms): preempt_schedule (copy_page_range)
 0.132ms (+0.000ms): preempt_schedule (copy_page_range)
 0.133ms (+0.000ms): preempt_schedule (copy_page_range)
 0.133ms (+0.000ms): preempt_schedule (copy_page_range)
 0.133ms (+0.000ms): preempt_schedule (copy_page_range)
 0.133ms (+0.000ms): preempt_schedule (copy_page_range)
 0.134ms (+0.000ms): preempt_schedule (copy_page_range)
 0.134ms (+0.000ms): preempt_schedule (copy_page_range)
 0.134ms (+0.000ms): preempt_schedule (copy_page_range)
 0.135ms (+0.000ms): preempt_schedule (copy_page_range)
 0.135ms (+0.000ms): preempt_schedule (copy_page_range)
 0.135ms (+0.000ms): preempt_schedule (copy_page_range)
 0.136ms (+0.000ms): preempt_schedule (copy_page_range)
 0.136ms (+0.000ms): preempt_schedule (copy_page_range)
 0.136ms (+0.000ms): preempt_schedule (copy_page_range)
 0.136ms (+0.000ms): preempt_schedule (copy_page_range)
 0.137ms (+0.000ms): preempt_schedule (copy_page_range)
 0.137ms (+0.000ms): preempt_schedule (copy_page_range)
 0.137ms (+0.000ms): preempt_schedule (copy_page_range)
 0.137ms (+0.000ms): preempt_schedule (copy_page_range)
 0.137ms (+0.000ms): preempt_schedule (copy_page_range)
 0.138ms (+0.000ms): preempt_schedule (copy_page_range)
 0.138ms (+0.000ms): preempt_schedule (copy_page_range)
 0.139ms (+0.000ms): preempt_schedule (copy_page_range)
 0.139ms (+0.000ms): preempt_schedule (copy_page_range)
 0.139ms (+0.000ms): preempt_schedule (copy_page_range)
 0.139ms (+0.000ms): preempt_schedule (copy_page_range)
 0.140ms (+0.000ms): preempt_schedule (copy_page_range)
 0.140ms (+0.000ms): preempt_schedule (copy_page_range)
 0.140ms (+0.000ms): preempt_schedule (copy_page_range)
 0.141ms (+0.000ms): preempt_schedule (copy_page_range)
 0.141ms (+0.000ms): preempt_schedule (copy_page_range)
 0.141ms (+0.000ms): preempt_schedule (copy_page_range)
 0.141ms (+0.000ms): preempt_schedule (copy_page_range)
 0.142ms (+0.000ms): preempt_schedule (copy_page_range)
 0.142ms (+0.000ms): preempt_schedule (copy_page_range)
 0.143ms (+0.000ms): preempt_schedule (copy_page_range)
 0.143ms (+0.000ms): preempt_schedule (copy_page_range)
 0.143ms (+0.000ms): preempt_schedule (copy_page_range)
 0.144ms (+0.000ms): preempt_schedule (copy_page_range)
 0.144ms (+0.000ms): preempt_schedule (copy_page_range)
 0.144ms (+0.000ms): preempt_schedule (copy_page_range)
 0.144ms (+0.000ms): preempt_schedule (copy_page_range)
 0.144ms (+0.000ms): preempt_schedule (copy_page_range)
 0.145ms (+0.000ms): preempt_schedule (copy_page_range)
 0.145ms (+0.000ms): preempt_schedule (copy_page_range)
 0.145ms (+0.000ms): preempt_schedule (copy_page_range)
 0.146ms (+0.000ms): preempt_schedule (copy_page_range)
 0.146ms (+0.000ms): preempt_schedule (copy_page_range)
 0.147ms (+0.000ms): preempt_schedule (copy_page_range)
 0.147ms (+0.000ms): preempt_schedule (copy_page_range)
 0.147ms (+0.000ms): preempt_schedule (copy_page_range)
 0.147ms (+0.000ms): preempt_schedule (copy_page_range)
 0.148ms (+0.000ms): preempt_schedule (copy_page_range)
 0.148ms (+0.000ms): preempt_schedule (copy_page_range)
 0.148ms (+0.000ms): preempt_schedule (copy_page_range)
 0.148ms (+0.000ms): preempt_schedule (copy_page_range)
 0.149ms (+0.000ms): preempt_schedule (copy_page_range)
 0.149ms (+0.000ms): preempt_schedule (copy_page_range)
 0.149ms (+0.000ms): preempt_schedule (copy_page_range)
 0.149ms (+0.000ms): preempt_schedule (copy_page_range)
 0.149ms (+0.000ms): preempt_schedule (copy_page_range)
 0.150ms (+0.000ms): preempt_schedule (copy_page_range)
 0.150ms (+0.000ms): preempt_schedule (copy_page_range)
 0.150ms (+0.000ms): preempt_schedule (copy_page_range)
 0.151ms (+0.000ms): preempt_schedule (copy_page_range)
 0.151ms (+0.000ms): preempt_schedule (copy_page_range)
 0.151ms (+0.000ms): preempt_schedule (copy_page_range)
 0.152ms (+0.000ms): preempt_schedule (copy_page_range)
 0.152ms (+0.000ms): preempt_schedule (copy_page_range)
 0.152ms (+0.000ms): preempt_schedule (copy_page_range)
 0.152ms (+0.000ms): preempt_schedule (copy_page_range)
 0.153ms (+0.000ms): preempt_schedule (copy_page_range)
 0.153ms (+0.000ms): preempt_schedule (copy_page_range)
 0.153ms (+0.000ms): preempt_schedule (copy_page_range)
 0.154ms (+0.000ms): preempt_schedule (copy_page_range)
 0.154ms (+0.000ms): preempt_schedule (copy_page_range)
 0.154ms (+0.000ms): preempt_schedule (copy_page_range)
 0.154ms (+0.000ms): preempt_schedule (copy_page_range)
 0.154ms (+0.000ms): preempt_schedule (copy_page_range)
 0.155ms (+0.000ms): preempt_schedule (copy_page_range)
 0.155ms (+0.000ms): preempt_schedule (copy_page_range)
 0.155ms (+0.000ms): preempt_schedule (copy_page_range)
 0.155ms (+0.000ms): preempt_schedule (copy_page_range)
 0.156ms (+0.000ms): preempt_schedule (copy_page_range)
 0.156ms (+0.000ms): preempt_schedule (copy_page_range)
 0.156ms (+0.000ms): preempt_schedule (copy_page_range)
 0.157ms (+0.000ms): preempt_schedule (copy_page_range)
 0.157ms (+0.000ms): preempt_schedule (copy_page_range)
 0.157ms (+0.000ms): preempt_schedule (copy_page_range)
 0.157ms (+0.000ms): preempt_schedule (copy_page_range)
 0.158ms (+0.000ms): preempt_schedule (copy_page_range)
 0.158ms (+0.000ms): preempt_schedule (copy_page_range)
 0.159ms (+0.000ms): check_preempt_timing (touch_preempt_timing)








preemption latency trace v1.0
-----------------------------
 latency: 258 us, entries: 722 (722)
 process: sylpheed-claws/796, uid: 1000
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): do_IRQ (common_interrupt)
 0.000ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.002ms (+0.001ms): generic_redirect_hardirq (do_IRQ)
 0.002ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.002ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 0.002ms (+0.000ms): mark_offset_tsc (timer_interrupt)
 0.009ms (+0.007ms): do_timer (timer_interrupt)
 0.010ms (+0.000ms): update_process_times (do_timer)
 0.010ms (+0.000ms): update_one_process (update_process_times)
 0.010ms (+0.000ms): run_local_timers (update_process_times)
 0.011ms (+0.000ms): raise_softirq (update_process_times)
 0.011ms (+0.000ms): scheduler_tick (update_process_times)
 0.011ms (+0.000ms): sched_clock (scheduler_tick)
 0.012ms (+0.000ms): task_timeslice (scheduler_tick)
 0.013ms (+0.000ms): update_wall_time (do_timer)
 0.013ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 0.014ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.014ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.015ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.015ms (+0.000ms): do_softirq (do_IRQ)
 0.015ms (+0.000ms): __do_softirq (do_softirq)
 0.016ms (+0.000ms): wake_up_process (do_softirq)
 0.016ms (+0.000ms): try_to_wake_up (wake_up_process)
 0.016ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.016ms (+0.000ms): activate_task (try_to_wake_up)
 0.016ms (+0.000ms): sched_clock (activate_task)
 0.016ms (+0.000ms): recalc_task_prio (activate_task)
 0.016ms (+0.000ms): effective_prio (recalc_task_prio)
 0.017ms (+0.000ms): enqueue_task (activate_task)
 0.017ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.017ms (+0.000ms): preempt_schedule (copy_page_range)
 0.018ms (+0.000ms): preempt_schedule (copy_page_range)
 0.018ms (+0.000ms): preempt_schedule (copy_page_range)
 0.018ms (+0.000ms): preempt_schedule (copy_page_range)
 0.018ms (+0.000ms): preempt_schedule (copy_page_range)
 0.019ms (+0.000ms): preempt_schedule (copy_page_range)
 0.019ms (+0.000ms): preempt_schedule (copy_page_range)
 0.019ms (+0.000ms): preempt_schedule (copy_page_range)
 0.019ms (+0.000ms): preempt_schedule (copy_page_range)
 0.020ms (+0.000ms): preempt_schedule (copy_page_range)
 0.020ms (+0.000ms): preempt_schedule (copy_page_range)
 0.020ms (+0.000ms): preempt_schedule (copy_page_range)
 0.020ms (+0.000ms): preempt_schedule (copy_page_range)
 0.020ms (+0.000ms): preempt_schedule (copy_page_range)
 0.021ms (+0.000ms): preempt_schedule (copy_page_range)
 0.021ms (+0.000ms): preempt_schedule (copy_page_range)
 0.021ms (+0.000ms): preempt_schedule (copy_page_range)
 0.021ms (+0.000ms): preempt_schedule (copy_page_range)
 0.022ms (+0.000ms): preempt_schedule (copy_page_range)
 0.022ms (+0.000ms): preempt_schedule (copy_page_range)
 0.022ms (+0.000ms): preempt_schedule (copy_page_range)
 0.022ms (+0.000ms): preempt_schedule (copy_page_range)
 0.023ms (+0.000ms): preempt_schedule (copy_page_range)
 0.023ms (+0.000ms): preempt_schedule (copy_page_range)
 0.023ms (+0.000ms): preempt_schedule (copy_page_range)
 0.023ms (+0.000ms): preempt_schedule (copy_page_range)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.025ms (+0.000ms): preempt_schedule (copy_page_range)
 0.025ms (+0.000ms): preempt_schedule (copy_page_range)
 0.025ms (+0.000ms): preempt_schedule (copy_page_range)
 0.025ms (+0.000ms): preempt_schedule (copy_page_range)
 0.026ms (+0.000ms): preempt_schedule (copy_page_range)
 0.026ms (+0.000ms): preempt_schedule (copy_page_range)
 0.026ms (+0.000ms): preempt_schedule (copy_page_range)
 0.026ms (+0.000ms): preempt_schedule (copy_page_range)
 0.026ms (+0.000ms): preempt_schedule (copy_page_range)
 0.027ms (+0.000ms): preempt_schedule (copy_page_range)
 0.027ms (+0.000ms): preempt_schedule (copy_page_range)
 0.027ms (+0.000ms): preempt_schedule (copy_page_range)
 0.028ms (+0.000ms): preempt_schedule (copy_page_range)
 0.028ms (+0.000ms): preempt_schedule (copy_page_range)
 0.028ms (+0.000ms): preempt_schedule (copy_page_range)
 0.028ms (+0.000ms): preempt_schedule (copy_page_range)
 0.029ms (+0.000ms): preempt_schedule (copy_page_range)
 0.029ms (+0.000ms): preempt_schedule (copy_page_range)
 0.029ms (+0.000ms): preempt_schedule (copy_page_range)
 0.029ms (+0.000ms): preempt_schedule (copy_page_range)
 0.029ms (+0.000ms): preempt_schedule (copy_page_range)
 0.030ms (+0.000ms): preempt_schedule (copy_page_range)
 0.030ms (+0.000ms): preempt_schedule (copy_page_range)
 0.030ms (+0.000ms): preempt_schedule (copy_page_range)
 0.030ms (+0.000ms): preempt_schedule (copy_page_range)
 0.031ms (+0.000ms): preempt_schedule (copy_page_range)
 0.031ms (+0.000ms): preempt_schedule (copy_page_range)
 0.031ms (+0.000ms): preempt_schedule (copy_page_range)
 0.032ms (+0.000ms): preempt_schedule (copy_page_range)
 0.032ms (+0.000ms): preempt_schedule (copy_page_range)
 0.032ms (+0.000ms): preempt_schedule (copy_page_range)
 0.032ms (+0.000ms): preempt_schedule (copy_page_range)
 0.032ms (+0.000ms): preempt_schedule (copy_page_range)
 0.033ms (+0.000ms): preempt_schedule (copy_page_range)
 0.033ms (+0.000ms): preempt_schedule (copy_page_range)
 0.033ms (+0.000ms): preempt_schedule (copy_page_range)
 0.033ms (+0.000ms): preempt_schedule (copy_page_range)
 0.034ms (+0.000ms): preempt_schedule (copy_page_range)
 0.034ms (+0.000ms): preempt_schedule (copy_page_range)
 0.034ms (+0.000ms): preempt_schedule (copy_page_range)
 0.034ms (+0.000ms): preempt_schedule (copy_page_range)
 0.034ms (+0.000ms): preempt_schedule (copy_page_range)
 0.035ms (+0.000ms): preempt_schedule (copy_page_range)
 0.035ms (+0.000ms): preempt_schedule (copy_page_range)
 0.035ms (+0.000ms): preempt_schedule (copy_page_range)
 0.036ms (+0.000ms): preempt_schedule (copy_page_range)
 0.036ms (+0.000ms): preempt_schedule (copy_page_range)
 0.036ms (+0.000ms): preempt_schedule (copy_page_range)
 0.036ms (+0.000ms): preempt_schedule (copy_page_range)
 0.036ms (+0.000ms): preempt_schedule (copy_page_range)
 0.037ms (+0.000ms): preempt_schedule (copy_page_range)
 0.037ms (+0.000ms): preempt_schedule (copy_page_range)
 0.037ms (+0.000ms): preempt_schedule (copy_page_range)
 0.037ms (+0.000ms): preempt_schedule (copy_page_range)
 0.038ms (+0.000ms): preempt_schedule (copy_page_range)
 0.038ms (+0.000ms): preempt_schedule (copy_page_range)
 0.038ms (+0.000ms): preempt_schedule (copy_page_range)
 0.038ms (+0.000ms): preempt_schedule (copy_page_range)
 0.039ms (+0.000ms): preempt_schedule (copy_page_range)
 0.039ms (+0.000ms): preempt_schedule (copy_page_range)
 0.039ms (+0.000ms): preempt_schedule (copy_page_range)
 0.040ms (+0.000ms): preempt_schedule (copy_page_range)
 0.040ms (+0.000ms): preempt_schedule (copy_page_range)
 0.040ms (+0.000ms): preempt_schedule (copy_page_range)
 0.040ms (+0.000ms): preempt_schedule (copy_page_range)
 0.041ms (+0.000ms): preempt_schedule (copy_page_range)
 0.041ms (+0.000ms): preempt_schedule (copy_page_range)
 0.041ms (+0.000ms): preempt_schedule (copy_page_range)
 0.041ms (+0.000ms): preempt_schedule (copy_page_range)
 0.041ms (+0.000ms): preempt_schedule (copy_page_range)
 0.042ms (+0.000ms): preempt_schedule (copy_page_range)
 0.042ms (+0.000ms): preempt_schedule (copy_page_range)
 0.042ms (+0.000ms): preempt_schedule (copy_page_range)
 0.042ms (+0.000ms): preempt_schedule (copy_page_range)
 0.043ms (+0.000ms): preempt_schedule (copy_page_range)
 0.043ms (+0.000ms): preempt_schedule (copy_page_range)
 0.043ms (+0.000ms): preempt_schedule (copy_page_range)
 0.044ms (+0.000ms): preempt_schedule (copy_page_range)
 0.044ms (+0.000ms): preempt_schedule (copy_page_range)
 0.044ms (+0.000ms): preempt_schedule (copy_page_range)
 0.044ms (+0.000ms): preempt_schedule (copy_page_range)
 0.044ms (+0.000ms): preempt_schedule (copy_page_range)
 0.045ms (+0.000ms): preempt_schedule (copy_page_range)
 0.045ms (+0.000ms): preempt_schedule (copy_page_range)
 0.045ms (+0.000ms): preempt_schedule (copy_page_range)
 0.045ms (+0.000ms): preempt_schedule (copy_page_range)
 0.046ms (+0.000ms): preempt_schedule (copy_page_range)
 0.046ms (+0.000ms): preempt_schedule (copy_page_range)
 0.046ms (+0.000ms): preempt_schedule (copy_page_range)
 0.046ms (+0.000ms): preempt_schedule (copy_page_range)
 0.047ms (+0.000ms): preempt_schedule (copy_page_range)
 0.047ms (+0.000ms): preempt_schedule (copy_page_range)
 0.047ms (+0.000ms): preempt_schedule (copy_page_range)
 0.047ms (+0.000ms): preempt_schedule (copy_page_range)
 0.048ms (+0.000ms): preempt_schedule (copy_page_range)
 0.048ms (+0.000ms): preempt_schedule (copy_page_range)
 0.048ms (+0.000ms): preempt_schedule (copy_page_range)
 0.048ms (+0.000ms): preempt_schedule (copy_page_range)
 0.049ms (+0.000ms): preempt_schedule (copy_page_range)
 0.049ms (+0.000ms): preempt_schedule (copy_page_range)
 0.049ms (+0.000ms): preempt_schedule (copy_page_range)
 0.049ms (+0.000ms): preempt_schedule (copy_page_range)
 0.050ms (+0.000ms): preempt_schedule (copy_page_range)
 0.050ms (+0.000ms): preempt_schedule (copy_page_range)
 0.050ms (+0.000ms): preempt_schedule (copy_page_range)
 0.050ms (+0.000ms): preempt_schedule (copy_page_range)
 0.051ms (+0.000ms): preempt_schedule (copy_page_range)
 0.051ms (+0.000ms): preempt_schedule (copy_page_range)
 0.051ms (+0.000ms): preempt_schedule (copy_page_range)
 0.051ms (+0.000ms): preempt_schedule (copy_page_range)
 0.052ms (+0.000ms): preempt_schedule (copy_page_range)
 0.052ms (+0.000ms): preempt_schedule (copy_page_range)
 0.052ms (+0.000ms): preempt_schedule (copy_page_range)
 0.052ms (+0.000ms): preempt_schedule (copy_page_range)
 0.053ms (+0.000ms): preempt_schedule (copy_page_range)
 0.053ms (+0.000ms): preempt_schedule (copy_page_range)
 0.053ms (+0.000ms): preempt_schedule (copy_page_range)
 0.053ms (+0.000ms): preempt_schedule (copy_page_range)
 0.054ms (+0.000ms): preempt_schedule (copy_page_range)
 0.054ms (+0.000ms): preempt_schedule (copy_page_range)
 0.054ms (+0.000ms): preempt_schedule (copy_page_range)
 0.054ms (+0.000ms): preempt_schedule (copy_page_range)
 0.055ms (+0.000ms): preempt_schedule (copy_page_range)
 0.055ms (+0.000ms): preempt_schedule (copy_page_range)
 0.055ms (+0.000ms): preempt_schedule (copy_page_range)
 0.055ms (+0.000ms): preempt_schedule (copy_page_range)
 0.056ms (+0.000ms): preempt_schedule (copy_page_range)
 0.056ms (+0.000ms): preempt_schedule (copy_page_range)
 0.056ms (+0.000ms): preempt_schedule (copy_page_range)
 0.056ms (+0.000ms): preempt_schedule (copy_page_range)
 0.056ms (+0.000ms): preempt_schedule (copy_page_range)
 0.057ms (+0.000ms): preempt_schedule (copy_page_range)
 0.057ms (+0.000ms): preempt_schedule (copy_page_range)
 0.057ms (+0.000ms): preempt_schedule (copy_page_range)
 0.057ms (+0.000ms): preempt_schedule (copy_page_range)
 0.058ms (+0.000ms): preempt_schedule (copy_page_range)
 0.058ms (+0.000ms): preempt_schedule (copy_page_range)
 0.058ms (+0.000ms): preempt_schedule (copy_page_range)
 0.058ms (+0.000ms): preempt_schedule (copy_page_range)
 0.059ms (+0.000ms): preempt_schedule (copy_page_range)
 0.059ms (+0.000ms): preempt_schedule (copy_page_range)
 0.059ms (+0.000ms): preempt_schedule (copy_page_range)
 0.059ms (+0.000ms): preempt_schedule (copy_page_range)
 0.060ms (+0.000ms): preempt_schedule (copy_page_range)
 0.060ms (+0.000ms): preempt_schedule (copy_page_range)
 0.060ms (+0.000ms): preempt_schedule (copy_page_range)
 0.060ms (+0.000ms): preempt_schedule (copy_page_range)
 0.061ms (+0.000ms): preempt_schedule (copy_page_range)
 0.061ms (+0.000ms): preempt_schedule (copy_page_range)
 0.061ms (+0.000ms): preempt_schedule (copy_page_range)
 0.061ms (+0.000ms): preempt_schedule (copy_page_range)
 0.062ms (+0.000ms): preempt_schedule (copy_page_range)
 0.062ms (+0.000ms): preempt_schedule (copy_page_range)
 0.062ms (+0.000ms): preempt_schedule (copy_page_range)
 0.062ms (+0.000ms): preempt_schedule (copy_page_range)
 0.063ms (+0.000ms): preempt_schedule (copy_page_range)
 0.063ms (+0.000ms): preempt_schedule (copy_page_range)
 0.063ms (+0.000ms): preempt_schedule (copy_page_range)
 0.063ms (+0.000ms): preempt_schedule (copy_page_range)
 0.064ms (+0.000ms): preempt_schedule (copy_page_range)
 0.064ms (+0.000ms): preempt_schedule (copy_page_range)
 0.064ms (+0.000ms): preempt_schedule (copy_page_range)
 0.064ms (+0.000ms): preempt_schedule (copy_page_range)
 0.065ms (+0.000ms): preempt_schedule (copy_page_range)
 0.065ms (+0.000ms): preempt_schedule (copy_page_range)
 0.065ms (+0.000ms): preempt_schedule (copy_page_range)
 0.065ms (+0.000ms): preempt_schedule (copy_page_range)
 0.066ms (+0.000ms): preempt_schedule (copy_page_range)
 0.066ms (+0.000ms): preempt_schedule (copy_page_range)
 0.066ms (+0.000ms): preempt_schedule (copy_page_range)
 0.067ms (+0.000ms): preempt_schedule (copy_page_range)
 0.067ms (+0.000ms): preempt_schedule (copy_page_range)
 0.067ms (+0.000ms): preempt_schedule (copy_page_range)
 0.067ms (+0.000ms): preempt_schedule (copy_page_range)
 0.067ms (+0.000ms): preempt_schedule (copy_page_range)
 0.068ms (+0.000ms): preempt_schedule (copy_page_range)
 0.068ms (+0.000ms): preempt_schedule (copy_page_range)
 0.068ms (+0.000ms): preempt_schedule (copy_page_range)
 0.068ms (+0.000ms): preempt_schedule (copy_page_range)
 0.069ms (+0.000ms): preempt_schedule (copy_page_range)
 0.069ms (+0.000ms): preempt_schedule (copy_page_range)
 0.069ms (+0.000ms): preempt_schedule (copy_page_range)
 0.069ms (+0.000ms): preempt_schedule (copy_page_range)
 0.070ms (+0.000ms): preempt_schedule (copy_page_range)
 0.070ms (+0.000ms): preempt_schedule (copy_page_range)
 0.070ms (+0.000ms): preempt_schedule (copy_page_range)
 0.070ms (+0.000ms): preempt_schedule (copy_page_range)
 0.071ms (+0.000ms): preempt_schedule (copy_page_range)
 0.071ms (+0.000ms): preempt_schedule (copy_page_range)
 0.071ms (+0.000ms): preempt_schedule (copy_page_range)
 0.071ms (+0.000ms): preempt_schedule (copy_page_range)
 0.072ms (+0.000ms): preempt_schedule (copy_page_range)
 0.072ms (+0.000ms): preempt_schedule (copy_page_range)
 0.072ms (+0.000ms): preempt_schedule (copy_page_range)
 0.072ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.074ms (+0.000ms): preempt_schedule (copy_page_range)
 0.074ms (+0.000ms): preempt_schedule (copy_page_range)
 0.074ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.086ms (+0.000ms): preempt_schedule (copy_page_range)
 0.086ms (+0.000ms): preempt_schedule (copy_page_range)
 0.086ms (+0.000ms): preempt_schedule (copy_page_range)
 0.086ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.092ms (+0.000ms): preempt_schedule (copy_page_range)
 0.092ms (+0.000ms): preempt_schedule (copy_page_range)
 0.092ms (+0.000ms): preempt_schedule (copy_page_range)
 0.092ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.094ms (+0.000ms): preempt_schedule (copy_page_range)
 0.094ms (+0.000ms): preempt_schedule (copy_page_range)
 0.094ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.105ms (+0.000ms): preempt_schedule (copy_page_range)
 0.105ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.109ms (+0.000ms): preempt_schedule (copy_page_range)
 0.109ms (+0.000ms): preempt_schedule (copy_page_range)
 0.109ms (+0.000ms): preempt_schedule (copy_page_range)
 0.109ms (+0.000ms): preempt_schedule (copy_page_range)
 0.110ms (+0.000ms): preempt_schedule (copy_page_range)
 0.110ms (+0.000ms): preempt_schedule (copy_page_range)
 0.110ms (+0.000ms): preempt_schedule (copy_page_range)
 0.111ms (+0.000ms): preempt_schedule (copy_page_range)
 0.111ms (+0.000ms): preempt_schedule (copy_page_range)
 0.111ms (+0.000ms): preempt_schedule (copy_page_range)
 0.111ms (+0.000ms): preempt_schedule (copy_page_range)
 0.112ms (+0.000ms): preempt_schedule (copy_page_range)
 0.112ms (+0.000ms): preempt_schedule (copy_page_range)
 0.112ms (+0.000ms): preempt_schedule (copy_page_range)
 0.113ms (+0.000ms): preempt_schedule (copy_page_range)
 0.113ms (+0.000ms): preempt_schedule (copy_page_range)
 0.113ms (+0.000ms): preempt_schedule (copy_page_range)
 0.114ms (+0.000ms): preempt_schedule (copy_page_range)
 0.114ms (+0.000ms): preempt_schedule (copy_page_range)
 0.114ms (+0.000ms): preempt_schedule (copy_page_range)
 0.114ms (+0.000ms): preempt_schedule (copy_page_range)
 0.115ms (+0.000ms): preempt_schedule (copy_page_range)
 0.115ms (+0.000ms): preempt_schedule (copy_page_range)
 0.115ms (+0.000ms): preempt_schedule (copy_page_range)
 0.116ms (+0.000ms): preempt_schedule (copy_page_range)
 0.116ms (+0.000ms): preempt_schedule (copy_page_range)
 0.116ms (+0.000ms): preempt_schedule (copy_page_range)
 0.116ms (+0.000ms): preempt_schedule (copy_page_range)
 0.117ms (+0.000ms): preempt_schedule (copy_page_range)
 0.117ms (+0.000ms): preempt_schedule (copy_page_range)
 0.117ms (+0.000ms): preempt_schedule (copy_page_range)
 0.118ms (+0.000ms): preempt_schedule (copy_page_range)
 0.118ms (+0.000ms): preempt_schedule (copy_page_range)
 0.118ms (+0.000ms): preempt_schedule (copy_page_range)
 0.119ms (+0.000ms): preempt_schedule (copy_page_range)
 0.119ms (+0.000ms): preempt_schedule (copy_page_range)
 0.119ms (+0.000ms): preempt_schedule (copy_page_range)
 0.119ms (+0.000ms): preempt_schedule (copy_page_range)
 0.119ms (+0.000ms): preempt_schedule (copy_page_range)
 0.119ms (+0.000ms): preempt_schedule (copy_page_range)
 0.119ms (+0.000ms): preempt_schedule (copy_page_range)
 0.120ms (+0.000ms): preempt_schedule (copy_page_range)
 0.120ms (+0.000ms): preempt_schedule (copy_page_range)
 0.120ms (+0.000ms): preempt_schedule (copy_page_range)
 0.120ms (+0.000ms): preempt_schedule (copy_page_range)
 0.121ms (+0.000ms): preempt_schedule (copy_page_range)
 0.121ms (+0.000ms): preempt_schedule (copy_page_range)
 0.121ms (+0.000ms): preempt_schedule (copy_page_range)
 0.122ms (+0.001ms): do_IRQ (common_interrupt)
 0.122ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.124ms (+0.001ms): preempt_schedule (do_IRQ)
 0.124ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
 0.124ms (+0.000ms): preempt_schedule (do_IRQ)
 0.124ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.125ms (+0.000ms): snd_cs46xx_interrupt (generic_handle_IRQ_event)
 0.127ms (+0.001ms): snd_pcm_period_elapsed (snd_cs46xx_interrupt)
 0.128ms (+0.001ms): snd_cs46xx_playback_direct_pointer
(snd_pcm_period_elapsed)
 0.131ms (+0.002ms): __wake_up (snd_pcm_period_elapsed)
 0.131ms (+0.000ms): __wake_up_common (__wake_up)
 0.131ms (+0.000ms): default_wake_function (__wake_up_common)
 0.131ms (+0.000ms): try_to_wake_up (__wake_up_common)
 0.131ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.131ms (+0.000ms): activate_task (try_to_wake_up)
 0.131ms (+0.000ms): sched_clock (activate_task)
 0.131ms (+0.000ms): recalc_task_prio (activate_task)
 0.132ms (+0.000ms): effective_prio (recalc_task_prio)
 0.132ms (+0.000ms): enqueue_task (activate_task)
 0.132ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.132ms (+0.000ms): preempt_schedule (snd_pcm_period_elapsed)
 0.132ms (+0.000ms): preempt_schedule (snd_pcm_period_elapsed)
 0.133ms (+0.000ms): preempt_schedule (snd_pcm_period_elapsed)
 0.133ms (+0.000ms): kill_fasync (snd_pcm_period_elapsed)
 0.133ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.134ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.134ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.134ms (+0.000ms): preempt_schedule (do_IRQ)
 0.134ms (+0.000ms): preempt_schedule (do_IRQ)
 0.135ms (+0.000ms): do_softirq (do_IRQ)
 0.135ms (+0.000ms): __do_softirq (do_softirq)
 0.135ms (+0.000ms): preempt_schedule (copy_page_range)
 0.135ms (+0.000ms): preempt_schedule (copy_page_range)
 0.135ms (+0.000ms): preempt_schedule (copy_page_range)
 0.135ms (+0.000ms): preempt_schedule (copy_page_range)
 0.136ms (+0.000ms): preempt_schedule (copy_page_range)
 0.136ms (+0.000ms): preempt_schedule (copy_page_range)
 0.136ms (+0.000ms): preempt_schedule (copy_page_range)
 0.136ms (+0.000ms): preempt_schedule (copy_page_range)
 0.136ms (+0.000ms): preempt_schedule (copy_page_range)
 0.137ms (+0.000ms): preempt_schedule (copy_page_range)
 0.137ms (+0.000ms): preempt_schedule (copy_page_range)
 0.137ms (+0.000ms): preempt_schedule (copy_page_range)
 0.137ms (+0.000ms): preempt_schedule (copy_page_range)
 0.138ms (+0.000ms): preempt_schedule (copy_page_range)
 0.138ms (+0.000ms): preempt_schedule (copy_page_range)
 0.138ms (+0.000ms): preempt_schedule (copy_page_range)
 0.139ms (+0.000ms): preempt_schedule (copy_page_range)
 0.139ms (+0.000ms): preempt_schedule (copy_page_range)
 0.139ms (+0.000ms): preempt_schedule (copy_page_range)
 0.140ms (+0.000ms): preempt_schedule (copy_page_range)
 0.140ms (+0.000ms): preempt_schedule (copy_page_range)
 0.140ms (+0.000ms): preempt_schedule (copy_page_range)
 0.141ms (+0.000ms): do_IRQ (common_interrupt)
 0.141ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.142ms (+0.001ms): preempt_schedule (do_IRQ)
 0.143ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
 0.143ms (+0.000ms): preempt_schedule (do_IRQ)
 0.143ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.143ms (+0.000ms): snd_cs46xx_interrupt (generic_handle_IRQ_event)
 0.144ms (+0.001ms): snd_pcm_period_elapsed (snd_cs46xx_interrupt)
 0.145ms (+0.001ms): snd_cs46xx_capture_indirect_pointer
(snd_pcm_period_elapsed)
 0.146ms (+0.000ms): snd_cs46xx_capture_transfer
(snd_cs46xx_capture_indirect_pointer)
 0.148ms (+0.002ms): __wake_up (snd_pcm_period_elapsed)
 0.148ms (+0.000ms): __wake_up_common (__wake_up)
 0.149ms (+0.000ms): default_wake_function (__wake_up_common)
 0.149ms (+0.000ms): try_to_wake_up (__wake_up_common)
 0.149ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.149ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.149ms (+0.000ms): preempt_schedule (snd_pcm_period_elapsed)
 0.149ms (+0.000ms): preempt_schedule (snd_pcm_period_elapsed)
 0.149ms (+0.000ms): preempt_schedule (snd_pcm_period_elapsed)
 0.150ms (+0.000ms): kill_fasync (snd_pcm_period_elapsed)
 0.150ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.150ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.150ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.151ms (+0.000ms): preempt_schedule (do_IRQ)
 0.151ms (+0.000ms): preempt_schedule (do_IRQ)
 0.151ms (+0.000ms): do_softirq (do_IRQ)
 0.151ms (+0.000ms): __do_softirq (do_softirq)
 0.151ms (+0.000ms): preempt_schedule (copy_page_range)
 0.152ms (+0.000ms): preempt_schedule (copy_page_range)
 0.152ms (+0.000ms): preempt_schedule (copy_page_range)
 0.152ms (+0.000ms): preempt_schedule (copy_page_range)
 0.152ms (+0.000ms): preempt_schedule (copy_page_range)
 0.153ms (+0.000ms): preempt_schedule (copy_page_range)
 0.153ms (+0.000ms): preempt_schedule (copy_page_range)
 0.153ms (+0.000ms): preempt_schedule (copy_page_range)
 0.153ms (+0.000ms): preempt_schedule (copy_page_range)
 0.153ms (+0.000ms): preempt_schedule (copy_page_range)
 0.154ms (+0.000ms): preempt_schedule (copy_page_range)
 0.154ms (+0.000ms): preempt_schedule (copy_page_range)
 0.154ms (+0.000ms): preempt_schedule (copy_page_range)
 0.155ms (+0.000ms): preempt_schedule (copy_page_range)
 0.155ms (+0.000ms): preempt_schedule (copy_page_range)
 0.155ms (+0.000ms): preempt_schedule (copy_page_range)
 0.155ms (+0.000ms): preempt_schedule (copy_page_range)
 0.156ms (+0.000ms): preempt_schedule (copy_page_range)
 0.156ms (+0.000ms): preempt_schedule (copy_page_range)
 0.156ms (+0.000ms): preempt_schedule (copy_page_range)
 0.156ms (+0.000ms): preempt_schedule (copy_page_range)
 0.156ms (+0.000ms): preempt_schedule (copy_page_range)
 0.156ms (+0.000ms): preempt_schedule (copy_page_range)
 0.157ms (+0.000ms): preempt_schedule (copy_page_range)
 0.157ms (+0.000ms): preempt_schedule (copy_page_range)
 0.157ms (+0.000ms): preempt_schedule (copy_page_range)
 0.157ms (+0.000ms): preempt_schedule (copy_page_range)
 0.158ms (+0.000ms): preempt_schedule (copy_page_range)
 0.158ms (+0.000ms): preempt_schedule (copy_page_range)
 0.158ms (+0.000ms): preempt_schedule (copy_page_range)
 0.158ms (+0.000ms): preempt_schedule (copy_page_range)
 0.159ms (+0.000ms): preempt_schedule (copy_page_range)
 0.159ms (+0.000ms): preempt_schedule (copy_page_range)
 0.159ms (+0.000ms): preempt_schedule (copy_page_range)
 0.159ms (+0.000ms): preempt_schedule (copy_page_range)
 0.159ms (+0.000ms): preempt_schedule (copy_page_range)
 0.160ms (+0.000ms): preempt_schedule (copy_page_range)
 0.160ms (+0.000ms): preempt_schedule (copy_page_range)
 0.160ms (+0.000ms): preempt_schedule (copy_page_range)
 0.160ms (+0.000ms): preempt_schedule (copy_page_range)
 0.160ms (+0.000ms): preempt_schedule (copy_page_range)
 0.161ms (+0.000ms): preempt_schedule (copy_page_range)
 0.161ms (+0.000ms): preempt_schedule (copy_page_range)
 0.161ms (+0.000ms): preempt_schedule (copy_page_range)
 0.161ms (+0.000ms): preempt_schedule (copy_page_range)
 0.162ms (+0.000ms): preempt_schedule (copy_page_range)
 0.162ms (+0.000ms): preempt_schedule (copy_page_range)
 0.162ms (+0.000ms): preempt_schedule (copy_page_range)
 0.162ms (+0.000ms): preempt_schedule (copy_page_range)
 0.163ms (+0.000ms): preempt_schedule (copy_page_range)
 0.163ms (+0.000ms): preempt_schedule (copy_page_range)
 0.163ms (+0.000ms): preempt_schedule (copy_page_range)
 0.163ms (+0.000ms): preempt_schedule (copy_page_range)
 0.163ms (+0.000ms): preempt_schedule (copy_page_range)
 0.164ms (+0.000ms): preempt_schedule (copy_page_range)
 0.164ms (+0.000ms): preempt_schedule (copy_page_range)
 0.164ms (+0.000ms): preempt_schedule (copy_page_range)
 0.164ms (+0.000ms): preempt_schedule (copy_page_range)
 0.165ms (+0.000ms): preempt_schedule (copy_page_range)
 0.165ms (+0.000ms): preempt_schedule (copy_page_range)
 0.165ms (+0.000ms): preempt_schedule (copy_page_range)
 0.165ms (+0.000ms): preempt_schedule (copy_page_range)
 0.166ms (+0.000ms): preempt_schedule (copy_page_range)
 0.166ms (+0.000ms): preempt_schedule (copy_page_range)
 0.166ms (+0.000ms): preempt_schedule (copy_page_range)
 0.166ms (+0.000ms): preempt_schedule (copy_page_range)
 0.167ms (+0.000ms): preempt_schedule (copy_page_range)
 0.167ms (+0.000ms): preempt_schedule (copy_page_range)
 0.167ms (+0.000ms): preempt_schedule (copy_page_range)
 0.167ms (+0.000ms): preempt_schedule (copy_page_range)
 0.167ms (+0.000ms): preempt_schedule (copy_page_range)
 0.168ms (+0.000ms): preempt_schedule (copy_page_range)
 0.168ms (+0.000ms): preempt_schedule (copy_page_range)
 0.168ms (+0.000ms): preempt_schedule (copy_page_range)
 0.168ms (+0.000ms): preempt_schedule (copy_page_range)
 0.169ms (+0.000ms): preempt_schedule (copy_page_range)
 0.169ms (+0.000ms): preempt_schedule (copy_page_range)
 0.169ms (+0.000ms): preempt_schedule (copy_page_range)
 0.169ms (+0.000ms): preempt_schedule (copy_page_range)
 0.170ms (+0.000ms): preempt_schedule (copy_page_range)
 0.170ms (+0.000ms): preempt_schedule (copy_page_range)
 0.170ms (+0.000ms): preempt_schedule (copy_page_range)
 0.170ms (+0.000ms): preempt_schedule (copy_page_range)
 0.170ms (+0.000ms): preempt_schedule (copy_page_range)
 0.170ms (+0.000ms): preempt_schedule (copy_page_range)
 0.171ms (+0.000ms): preempt_schedule (copy_page_range)
 0.171ms (+0.000ms): preempt_schedule (copy_page_range)
 0.171ms (+0.000ms): preempt_schedule (copy_page_range)
 0.171ms (+0.000ms): preempt_schedule (copy_page_range)
 0.172ms (+0.000ms): preempt_schedule (copy_page_range)
 0.172ms (+0.000ms): preempt_schedule (copy_page_range)
 0.172ms (+0.000ms): preempt_schedule (copy_page_range)
 0.172ms (+0.000ms): preempt_schedule (copy_page_range)
 0.173ms (+0.000ms): preempt_schedule (copy_page_range)
 0.173ms (+0.000ms): preempt_schedule (copy_page_range)
 0.173ms (+0.000ms): preempt_schedule (copy_page_range)
 0.173ms (+0.000ms): preempt_schedule (copy_page_range)
 0.174ms (+0.000ms): preempt_schedule (copy_page_range)
 0.174ms (+0.000ms): preempt_schedule (copy_page_range)
 0.174ms (+0.000ms): preempt_schedule (copy_page_range)
 0.174ms (+0.000ms): preempt_schedule (copy_page_range)
 0.175ms (+0.000ms): preempt_schedule (copy_page_range)
 0.175ms (+0.000ms): preempt_schedule (copy_page_range)
 0.175ms (+0.000ms): preempt_schedule (copy_page_range)
 0.175ms (+0.000ms): preempt_schedule (copy_page_range)
 0.175ms (+0.000ms): preempt_schedule (copy_page_range)
 0.176ms (+0.000ms): preempt_schedule (copy_page_range)
 0.176ms (+0.000ms): preempt_schedule (copy_page_range)
 0.176ms (+0.000ms): preempt_schedule (copy_page_range)
 0.176ms (+0.000ms): preempt_schedule (copy_page_range)
 0.177ms (+0.000ms): preempt_schedule (copy_page_range)
 0.177ms (+0.000ms): preempt_schedule (copy_page_range)
 0.177ms (+0.000ms): preempt_schedule (copy_page_range)
 0.177ms (+0.000ms): preempt_schedule (copy_page_range)
 0.177ms (+0.000ms): preempt_schedule (copy_page_range)
 0.178ms (+0.000ms): preempt_schedule (copy_page_range)
 0.178ms (+0.000ms): preempt_schedule (copy_page_range)
 0.178ms (+0.000ms): preempt_schedule (copy_page_range)
 0.179ms (+0.000ms): preempt_schedule (copy_page_range)
 0.179ms (+0.000ms): preempt_schedule (copy_page_range)
 0.179ms (+0.000ms): preempt_schedule (copy_page_range)
 0.179ms (+0.000ms): preempt_schedule (copy_page_range)
 0.180ms (+0.000ms): preempt_schedule (copy_page_range)
 0.180ms (+0.000ms): preempt_schedule (copy_page_range)
 0.180ms (+0.000ms): preempt_schedule (copy_page_range)
 0.180ms (+0.000ms): preempt_schedule (copy_page_range)
 0.181ms (+0.000ms): preempt_schedule (copy_page_range)
 0.181ms (+0.000ms): preempt_schedule (copy_page_range)
 0.181ms (+0.000ms): preempt_schedule (copy_page_range)
 0.181ms (+0.000ms): preempt_schedule (copy_page_range)
 0.181ms (+0.000ms): preempt_schedule (copy_page_range)
 0.182ms (+0.000ms): preempt_schedule (copy_page_range)
 0.182ms (+0.000ms): preempt_schedule (copy_page_range)
 0.182ms (+0.000ms): preempt_schedule (copy_page_range)
 0.182ms (+0.000ms): preempt_schedule (copy_page_range)
 0.182ms (+0.000ms): preempt_schedule (copy_page_range)
 0.182ms (+0.000ms): preempt_schedule (copy_page_range)
 0.183ms (+0.000ms): preempt_schedule (copy_page_range)
 0.183ms (+0.000ms): preempt_schedule (copy_page_range)
 0.183ms (+0.000ms): preempt_schedule (copy_page_range)
 0.184ms (+0.000ms): preempt_schedule (copy_page_range)
 0.184ms (+0.000ms): preempt_schedule (copy_page_range)
 0.184ms (+0.000ms): preempt_schedule (copy_page_range)
 0.184ms (+0.000ms): preempt_schedule (copy_page_range)
 0.185ms (+0.000ms): preempt_schedule (copy_page_range)
 0.185ms (+0.000ms): preempt_schedule (copy_page_range)
 0.185ms (+0.000ms): preempt_schedule (copy_page_range)
 0.185ms (+0.000ms): preempt_schedule (copy_page_range)
 0.186ms (+0.000ms): preempt_schedule (copy_page_range)
 0.186ms (+0.000ms): preempt_schedule (copy_page_range)
 0.186ms (+0.000ms): preempt_schedule (copy_page_range)
 0.186ms (+0.000ms): preempt_schedule (copy_page_range)
 0.186ms (+0.000ms): preempt_schedule (copy_page_range)
 0.187ms (+0.000ms): preempt_schedule (copy_page_range)
 0.187ms (+0.000ms): preempt_schedule (copy_page_range)
 0.187ms (+0.000ms): preempt_schedule (copy_page_range)
 0.187ms (+0.000ms): preempt_schedule (copy_page_range)
 0.188ms (+0.000ms): preempt_schedule (copy_page_range)
 0.188ms (+0.000ms): preempt_schedule (copy_page_range)
 0.188ms (+0.000ms): preempt_schedule (copy_page_range)
 0.188ms (+0.000ms): preempt_schedule (copy_page_range)
 0.188ms (+0.000ms): preempt_schedule (copy_page_range)
 0.189ms (+0.000ms): preempt_schedule (copy_page_range)
 0.189ms (+0.000ms): preempt_schedule (copy_page_range)
 0.189ms (+0.000ms): preempt_schedule (copy_page_range)
 0.189ms (+0.000ms): preempt_schedule (copy_page_range)
 0.189ms (+0.000ms): preempt_schedule (copy_page_range)
 0.190ms (+0.000ms): preempt_schedule (copy_page_range)
 0.190ms (+0.000ms): preempt_schedule (copy_page_range)
 0.190ms (+0.000ms): preempt_schedule (copy_page_range)
 0.190ms (+0.000ms): preempt_schedule (copy_page_range)
 0.191ms (+0.000ms): preempt_schedule (copy_page_range)
 0.191ms (+0.000ms): preempt_schedule (copy_page_range)
 0.191ms (+0.000ms): preempt_schedule (copy_page_range)
 0.191ms (+0.000ms): preempt_schedule (copy_page_range)
 0.191ms (+0.000ms): preempt_schedule (copy_page_range)
 0.192ms (+0.000ms): preempt_schedule (copy_page_range)
 0.192ms (+0.000ms): preempt_schedule (copy_page_range)
 0.192ms (+0.000ms): preempt_schedule (copy_page_range)
 0.192ms (+0.000ms): preempt_schedule (copy_page_range)
 0.193ms (+0.000ms): preempt_schedule (copy_page_range)
 0.193ms (+0.000ms): preempt_schedule (copy_page_range)
 0.193ms (+0.000ms): preempt_schedule (copy_page_range)
 0.193ms (+0.000ms): preempt_schedule (copy_page_range)
 0.193ms (+0.000ms): preempt_schedule (copy_page_range)
 0.194ms (+0.000ms): preempt_schedule (copy_page_range)
 0.194ms (+0.000ms): preempt_schedule (copy_page_range)
 0.194ms (+0.000ms): check_preempt_timing (touch_preempt_timing)






preemption latency trace v1.0
-----------------------------
 latency: 212 us, entries: 402 (402)
 process: sylpheed-claws/796, uid: 1000
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): do_IRQ (common_interrupt)
 0.000ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 0.001ms (+0.001ms): generic_redirect_hardirq (do_IRQ)
 0.001ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.001ms (+0.000ms): snd_cs46xx_interrupt (generic_handle_IRQ_event)
 0.003ms (+0.001ms): snd_pcm_period_elapsed (snd_cs46xx_interrupt)
 0.004ms (+0.000ms): snd_cs46xx_capture_indirect_pointer
(snd_pcm_period_elapsed)
 0.005ms (+0.001ms): snd_cs46xx_capture_transfer
(snd_cs46xx_capture_indirect_pointer)
 0.007ms (+0.001ms): __wake_up (snd_pcm_period_elapsed)
 0.007ms (+0.000ms): __wake_up_common (__wake_up)
 0.007ms (+0.000ms): default_wake_function (__wake_up_common)
 0.007ms (+0.000ms): try_to_wake_up (__wake_up_common)
 0.007ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.007ms (+0.000ms): activate_task (try_to_wake_up)
 0.007ms (+0.000ms): sched_clock (activate_task)
 0.007ms (+0.000ms): recalc_task_prio (activate_task)
 0.007ms (+0.000ms): effective_prio (recalc_task_prio)
 0.008ms (+0.000ms): enqueue_task (activate_task)
 0.008ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.008ms (+0.000ms): preempt_schedule (snd_pcm_period_elapsed)
 0.008ms (+0.000ms): preempt_schedule (snd_pcm_period_elapsed)
 0.008ms (+0.000ms): preempt_schedule (snd_pcm_period_elapsed)
 0.008ms (+0.000ms): kill_fasync (snd_pcm_period_elapsed)
 0.009ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.009ms (+0.000ms): end_8259A_irq (do_IRQ)
 0.009ms (+0.000ms): enable_8259A_irq (do_IRQ)
 0.010ms (+0.000ms): preempt_schedule (do_IRQ)
 0.010ms (+0.000ms): preempt_schedule (do_IRQ)
 0.010ms (+0.000ms): preempt_schedule (copy_page_range)
 0.011ms (+0.000ms): preempt_schedule (copy_page_range)
 0.011ms (+0.000ms): preempt_schedule (copy_page_range)
 0.011ms (+0.000ms): preempt_schedule (copy_page_range)
 0.011ms (+0.000ms): preempt_schedule (copy_page_range)
 0.012ms (+0.000ms): preempt_schedule (copy_page_range)
 0.012ms (+0.000ms): preempt_schedule (copy_page_range)
 0.012ms (+0.000ms): preempt_schedule (copy_page_range)
 0.012ms (+0.000ms): preempt_schedule (copy_page_range)
 0.012ms (+0.000ms): preempt_schedule (copy_page_range)
 0.013ms (+0.000ms): preempt_schedule (copy_page_range)
 0.013ms (+0.000ms): preempt_schedule (copy_page_range)
 0.013ms (+0.000ms): preempt_schedule (copy_page_range)
 0.013ms (+0.000ms): preempt_schedule (copy_page_range)
 0.014ms (+0.000ms): preempt_schedule (copy_page_range)
 0.014ms (+0.000ms): preempt_schedule (copy_page_range)
 0.014ms (+0.000ms): preempt_schedule (copy_page_range)
 0.014ms (+0.000ms): preempt_schedule (copy_page_range)
 0.015ms (+0.000ms): preempt_schedule (copy_page_range)
 0.015ms (+0.000ms): preempt_schedule (copy_page_range)
 0.015ms (+0.000ms): preempt_schedule (copy_page_range)
 0.016ms (+0.000ms): preempt_schedule (copy_page_range)
 0.016ms (+0.000ms): preempt_schedule (copy_page_range)
 0.016ms (+0.000ms): preempt_schedule (copy_page_range)
 0.017ms (+0.000ms): preempt_schedule (copy_page_range)
 0.017ms (+0.000ms): preempt_schedule (copy_page_range)
 0.017ms (+0.000ms): preempt_schedule (copy_page_range)
 0.017ms (+0.000ms): preempt_schedule (copy_page_range)
 0.017ms (+0.000ms): preempt_schedule (copy_page_range)
 0.018ms (+0.000ms): preempt_schedule (copy_page_range)
 0.018ms (+0.000ms): preempt_schedule (copy_page_range)
 0.018ms (+0.000ms): preempt_schedule (copy_page_range)
 0.018ms (+0.000ms): preempt_schedule (copy_page_range)
 0.019ms (+0.000ms): preempt_schedule (copy_page_range)
 0.019ms (+0.000ms): preempt_schedule (copy_page_range)
 0.019ms (+0.000ms): preempt_schedule (copy_page_range)
 0.019ms (+0.000ms): preempt_schedule (copy_page_range)
 0.020ms (+0.000ms): preempt_schedule (copy_page_range)
 0.020ms (+0.000ms): preempt_schedule (copy_page_range)
 0.020ms (+0.000ms): preempt_schedule (copy_page_range)
 0.020ms (+0.000ms): preempt_schedule (copy_page_range)
 0.021ms (+0.000ms): preempt_schedule (copy_page_range)
 0.021ms (+0.000ms): preempt_schedule (copy_page_range)
 0.021ms (+0.000ms): preempt_schedule (copy_page_range)
 0.021ms (+0.000ms): preempt_schedule (copy_page_range)
 0.022ms (+0.000ms): preempt_schedule (copy_page_range)
 0.022ms (+0.000ms): preempt_schedule (copy_page_range)
 0.022ms (+0.000ms): preempt_schedule (copy_page_range)
 0.022ms (+0.000ms): preempt_schedule (copy_page_range)
 0.022ms (+0.000ms): preempt_schedule (copy_page_range)
 0.022ms (+0.000ms): preempt_schedule (copy_page_range)
 0.023ms (+0.000ms): preempt_schedule (copy_page_range)
 0.023ms (+0.000ms): preempt_schedule (copy_page_range)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.024ms (+0.000ms): preempt_schedule (copy_page_range)
 0.025ms (+0.000ms): preempt_schedule (copy_page_range)
 0.025ms (+0.000ms): preempt_schedule (copy_page_range)
 0.025ms (+0.000ms): preempt_schedule (copy_page_range)
 0.025ms (+0.000ms): preempt_schedule (copy_page_range)
 0.025ms (+0.000ms): preempt_schedule (copy_page_range)
 0.026ms (+0.000ms): preempt_schedule (copy_page_range)
 0.026ms (+0.000ms): preempt_schedule (copy_page_range)
 0.026ms (+0.000ms): preempt_schedule (copy_page_range)
 0.026ms (+0.000ms): preempt_schedule (copy_page_range)
 0.027ms (+0.000ms): preempt_schedule (copy_page_range)
 0.027ms (+0.000ms): preempt_schedule (copy_page_range)
 0.027ms (+0.000ms): preempt_schedule (copy_page_range)
 0.028ms (+0.000ms): preempt_schedule (copy_page_range)
 0.028ms (+0.000ms): preempt_schedule (copy_page_range)
 0.028ms (+0.000ms): preempt_schedule (copy_page_range)
 0.028ms (+0.000ms): preempt_schedule (copy_page_range)
 0.028ms (+0.000ms): preempt_schedule (copy_page_range)
 0.029ms (+0.000ms): preempt_schedule (copy_page_range)
 0.029ms (+0.000ms): preempt_schedule (copy_page_range)
 0.029ms (+0.000ms): preempt_schedule (copy_page_range)
 0.029ms (+0.000ms): preempt_schedule (copy_page_range)
 0.030ms (+0.000ms): preempt_schedule (copy_page_range)
 0.030ms (+0.000ms): preempt_schedule (copy_page_range)
 0.030ms (+0.000ms): preempt_schedule (copy_page_range)
 0.030ms (+0.000ms): preempt_schedule (copy_page_range)
 0.031ms (+0.000ms): preempt_schedule (copy_page_range)
 0.031ms (+0.000ms): preempt_schedule (copy_page_range)
 0.031ms (+0.000ms): preempt_schedule (copy_page_range)
 0.031ms (+0.000ms): preempt_schedule (copy_page_range)
 0.032ms (+0.000ms): preempt_schedule (copy_page_range)
 0.032ms (+0.000ms): preempt_schedule (copy_page_range)
 0.032ms (+0.000ms): preempt_schedule (copy_page_range)
 0.032ms (+0.000ms): preempt_schedule (copy_page_range)
 0.033ms (+0.000ms): preempt_schedule (copy_page_range)
 0.033ms (+0.000ms): preempt_schedule (copy_page_range)
 0.033ms (+0.000ms): preempt_schedule (copy_page_range)
 0.033ms (+0.000ms): preempt_schedule (copy_page_range)
 0.033ms (+0.000ms): preempt_schedule (copy_page_range)
 0.034ms (+0.000ms): preempt_schedule (copy_page_range)
 0.034ms (+0.000ms): preempt_schedule (copy_page_range)
 0.034ms (+0.000ms): preempt_schedule (copy_page_range)
 0.034ms (+0.000ms): preempt_schedule (copy_page_range)
 0.035ms (+0.000ms): preempt_schedule (copy_page_range)
 0.035ms (+0.000ms): preempt_schedule (copy_page_range)
 0.035ms (+0.000ms): preempt_schedule (copy_page_range)
 0.035ms (+0.000ms): preempt_schedule (copy_page_range)
 0.036ms (+0.000ms): preempt_schedule (copy_page_range)
 0.036ms (+0.000ms): preempt_schedule (copy_page_range)
 0.036ms (+0.000ms): preempt_schedule (copy_page_range)
 0.036ms (+0.000ms): preempt_schedule (copy_page_range)
 0.036ms (+0.000ms): preempt_schedule (copy_page_range)
 0.037ms (+0.000ms): preempt_schedule (copy_page_range)
 0.037ms (+0.000ms): preempt_schedule (copy_page_range)
 0.037ms (+0.000ms): preempt_schedule (copy_page_range)
 0.037ms (+0.000ms): preempt_schedule (copy_page_range)
 0.038ms (+0.000ms): preempt_schedule (copy_page_range)
 0.038ms (+0.000ms): preempt_schedule (copy_page_range)
 0.038ms (+0.000ms): preempt_schedule (copy_page_range)
 0.039ms (+0.000ms): preempt_schedule (copy_page_range)
 0.039ms (+0.000ms): preempt_schedule (copy_page_range)
 0.039ms (+0.000ms): preempt_schedule (copy_page_range)
 0.039ms (+0.000ms): preempt_schedule (copy_page_range)
 0.040ms (+0.000ms): preempt_schedule (copy_page_range)
 0.040ms (+0.000ms): preempt_schedule (copy_page_range)
 0.040ms (+0.000ms): preempt_schedule (copy_page_range)
 0.040ms (+0.000ms): preempt_schedule (copy_page_range)
 0.040ms (+0.000ms): preempt_schedule (copy_page_range)
 0.041ms (+0.000ms): preempt_schedule (copy_page_range)
 0.041ms (+0.000ms): preempt_schedule (copy_page_range)
 0.041ms (+0.000ms): preempt_schedule (copy_page_range)
 0.042ms (+0.000ms): preempt_schedule (copy_page_range)
 0.042ms (+0.000ms): preempt_schedule (copy_page_range)
 0.042ms (+0.000ms): preempt_schedule (copy_page_range)
 0.043ms (+0.000ms): preempt_schedule (copy_page_range)
 0.043ms (+0.000ms): preempt_schedule (copy_page_range)
 0.044ms (+0.000ms): preempt_schedule (copy_page_range)
 0.044ms (+0.000ms): preempt_schedule (copy_page_range)
 0.044ms (+0.000ms): preempt_schedule (copy_page_range)
 0.045ms (+0.000ms): preempt_schedule (copy_page_range)
 0.045ms (+0.000ms): preempt_schedule (copy_page_range)
 0.045ms (+0.000ms): preempt_schedule (copy_page_range)
 0.046ms (+0.000ms): preempt_schedule (copy_page_range)
 0.046ms (+0.000ms): preempt_schedule (copy_page_range)
 0.046ms (+0.000ms): preempt_schedule (copy_page_range)
 0.046ms (+0.000ms): preempt_schedule (copy_page_range)
 0.047ms (+0.000ms): preempt_schedule (copy_page_range)
 0.047ms (+0.000ms): preempt_schedule (copy_page_range)
 0.047ms (+0.000ms): preempt_schedule (copy_page_range)
 0.047ms (+0.000ms): preempt_schedule (copy_page_range)
 0.048ms (+0.000ms): preempt_schedule (copy_page_range)
 0.048ms (+0.000ms): preempt_schedule (copy_page_range)
 0.049ms (+0.000ms): preempt_schedule (copy_page_range)
 0.049ms (+0.000ms): preempt_schedule (copy_page_range)
 0.049ms (+0.000ms): preempt_schedule (copy_page_range)
 0.049ms (+0.000ms): preempt_schedule (copy_page_range)
 0.050ms (+0.000ms): preempt_schedule (copy_page_range)
 0.050ms (+0.000ms): preempt_schedule (copy_page_range)
 0.050ms (+0.000ms): preempt_schedule (copy_page_range)
 0.051ms (+0.000ms): preempt_schedule (copy_page_range)
 0.051ms (+0.000ms): preempt_schedule (copy_page_range)
 0.051ms (+0.000ms): preempt_schedule (copy_page_range)
 0.052ms (+0.000ms): preempt_schedule (copy_page_range)
 0.052ms (+0.000ms): preempt_schedule (copy_page_range)
 0.052ms (+0.000ms): preempt_schedule (copy_page_range)
 0.053ms (+0.000ms): preempt_schedule (copy_page_range)
 0.053ms (+0.000ms): preempt_schedule (copy_page_range)
 0.054ms (+0.000ms): preempt_schedule (copy_page_range)
 0.054ms (+0.000ms): preempt_schedule (copy_page_range)
 0.054ms (+0.000ms): preempt_schedule (copy_page_range)
 0.054ms (+0.000ms): preempt_schedule (copy_page_range)
 0.055ms (+0.000ms): preempt_schedule (copy_page_range)
 0.055ms (+0.000ms): preempt_schedule (copy_page_range)
 0.055ms (+0.000ms): preempt_schedule (copy_page_range)
 0.055ms (+0.000ms): preempt_schedule (copy_page_range)
 0.056ms (+0.000ms): preempt_schedule (copy_page_range)
 0.056ms (+0.000ms): preempt_schedule (copy_page_range)
 0.056ms (+0.000ms): preempt_schedule (copy_page_range)
 0.057ms (+0.000ms): preempt_schedule (copy_page_range)
 0.057ms (+0.000ms): preempt_schedule (copy_page_range)
 0.057ms (+0.000ms): preempt_schedule (copy_page_range)
 0.057ms (+0.000ms): preempt_schedule (copy_page_range)
 0.058ms (+0.000ms): preempt_schedule (copy_page_range)
 0.058ms (+0.000ms): preempt_schedule (copy_page_range)
 0.058ms (+0.000ms): preempt_schedule (copy_page_range)
 0.059ms (+0.000ms): preempt_schedule (copy_page_range)
 0.059ms (+0.000ms): preempt_schedule (copy_page_range)
 0.059ms (+0.000ms): preempt_schedule (copy_page_range)
 0.060ms (+0.000ms): preempt_schedule (copy_page_range)
 0.060ms (+0.000ms): preempt_schedule (copy_page_range)
 0.060ms (+0.000ms): preempt_schedule (copy_page_range)
 0.061ms (+0.000ms): preempt_schedule (copy_page_range)
 0.061ms (+0.000ms): preempt_schedule (copy_page_range)
 0.061ms (+0.000ms): preempt_schedule (copy_page_range)
 0.061ms (+0.000ms): preempt_schedule (copy_page_range)
 0.061ms (+0.000ms): preempt_schedule (copy_page_range)
 0.062ms (+0.000ms): preempt_schedule (copy_page_range)
 0.062ms (+0.000ms): preempt_schedule (copy_page_range)
 0.062ms (+0.000ms): preempt_schedule (copy_page_range)
 0.063ms (+0.000ms): preempt_schedule (copy_page_range)
 0.063ms (+0.000ms): preempt_schedule (copy_page_range)
 0.063ms (+0.000ms): preempt_schedule (copy_page_range)
 0.063ms (+0.000ms): preempt_schedule (copy_page_range)
 0.064ms (+0.000ms): preempt_schedule (copy_page_range)
 0.064ms (+0.000ms): preempt_schedule (copy_page_range)
 0.064ms (+0.000ms): preempt_schedule (copy_page_range)
 0.064ms (+0.000ms): preempt_schedule (copy_page_range)
 0.064ms (+0.000ms): preempt_schedule (copy_page_range)
 0.065ms (+0.000ms): preempt_schedule (copy_page_range)
 0.065ms (+0.000ms): preempt_schedule (copy_page_range)
 0.065ms (+0.000ms): preempt_schedule (copy_page_range)
 0.065ms (+0.000ms): preempt_schedule (copy_page_range)
 0.066ms (+0.000ms): preempt_schedule (copy_page_range)
 0.066ms (+0.000ms): preempt_schedule (copy_page_range)
 0.066ms (+0.000ms): preempt_schedule (copy_page_range)
 0.067ms (+0.000ms): preempt_schedule (copy_page_range)
 0.067ms (+0.000ms): preempt_schedule (copy_page_range)
 0.067ms (+0.000ms): preempt_schedule (copy_page_range)
 0.067ms (+0.000ms): preempt_schedule (copy_page_range)
 0.067ms (+0.000ms): preempt_schedule (copy_page_range)
 0.068ms (+0.000ms): preempt_schedule (copy_page_range)
 0.068ms (+0.000ms): preempt_schedule (copy_page_range)
 0.068ms (+0.000ms): preempt_schedule (copy_page_range)
 0.069ms (+0.000ms): preempt_schedule (copy_page_range)
 0.069ms (+0.000ms): preempt_schedule (copy_page_range)
 0.069ms (+0.000ms): preempt_schedule (copy_page_range)
 0.069ms (+0.000ms): preempt_schedule (copy_page_range)
 0.070ms (+0.000ms): preempt_schedule (copy_page_range)
 0.070ms (+0.000ms): preempt_schedule (copy_page_range)
 0.070ms (+0.000ms): preempt_schedule (copy_page_range)
 0.070ms (+0.000ms): preempt_schedule (copy_page_range)
 0.071ms (+0.000ms): preempt_schedule (copy_page_range)
 0.071ms (+0.000ms): preempt_schedule (copy_page_range)
 0.071ms (+0.000ms): preempt_schedule (copy_page_range)
 0.072ms (+0.000ms): preempt_schedule (copy_page_range)
 0.072ms (+0.000ms): preempt_schedule (copy_page_range)
 0.072ms (+0.000ms): preempt_schedule (copy_page_range)
 0.072ms (+0.000ms): preempt_schedule (copy_page_range)
 0.072ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.073ms (+0.000ms): preempt_schedule (copy_page_range)
 0.074ms (+0.000ms): preempt_schedule (copy_page_range)
 0.074ms (+0.000ms): preempt_schedule (copy_page_range)
 0.074ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.075ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.076ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.077ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.078ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.079ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.080ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.081ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.082ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.083ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.084ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.085ms (+0.000ms): preempt_schedule (copy_page_range)
 0.086ms (+0.000ms): preempt_schedule (copy_page_range)
 0.086ms (+0.000ms): preempt_schedule (copy_page_range)
 0.086ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.087ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.088ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.089ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.090ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.091ms (+0.000ms): preempt_schedule (copy_page_range)
 0.092ms (+0.000ms): preempt_schedule (copy_page_range)
 0.092ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.093ms (+0.000ms): preempt_schedule (copy_page_range)
 0.094ms (+0.000ms): preempt_schedule (copy_page_range)
 0.094ms (+0.000ms): preempt_schedule (copy_page_range)
 0.094ms (+0.000ms): preempt_schedule (copy_page_range)
 0.094ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.095ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.096ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.097ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.098ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.099ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.100ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.101ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.102ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.103ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.104ms (+0.000ms): preempt_schedule (copy_page_range)
 0.105ms (+0.000ms): preempt_schedule (copy_page_range)
 0.105ms (+0.000ms): preempt_schedule (copy_page_range)
 0.105ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.106ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.107ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.108ms (+0.000ms): preempt_schedule (copy_page_range)
 0.109ms (+0.000ms): check_preempt_timing (touch_preempt_timing)

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [Jackit-devel] Re: [patch] voluntary-preempt-2.6.8-rc2-M5

[Lee Revell]

On Mon, 2004-08-16 at 11:33, Jack O'Quin wrote:
> Lee Revell <rlrevell@joe-job.com> writes:
> 
> > On Mon, 2004-08-16 at 06:52, Lee Revell wrote:
> > > On Mon, 2004-08-16 at 06:48, Ingo Molnar wrote:
> > >  if the former then does jackd set itself up (does an mlockall, etc.) 
> > > > before it opens the audio device? If the audio device has an event for
> > > > jackd the moment the device is opened, and jackd opens the audio device
> > > > early during startup, then jackd might not be able to process this event
> > > > until it has started up (which can take milliseconds).
> > > 
> > > This is probably what is happening, the kernel-side issue seems fixed,
> > 
> > It looks like this is what happens - jackd calls snd_pcm_start, then
> > does several other thinks like malloc'ing memory for the array of fd's
> > to poll() before entering the polling loop, by which time there has been
> > data ready for a while.  This may or may not be worth fixing, I am
> > adding jackit-devel to the cc: list.
> 
> Yep.  This looks like a bug to me.  While jackd, itself, seems to
> allocate everything before calling driver->start(), the ALSA driver
> internally calls malloc() *after* calling snd_pcm_start().  I doubt
> anyone has ever made a concerted effort to clean up this path for
> realtime safety.
> 
> I think it should be fixed (I need to study the code in more detail).
> There's probably nothing to prevent us moving the free() and malloc()
> calls up nearer the top of alsa_driver_start().  That will probably
> require an extra error test and free in case snd_pcm_start() fails.

I made this change and it seems to work fine, snd_pcm_start is now the
very last thing that happens in alsa_driver_start, and we free() if
snd_pcm_start fails.

I still get the xruns at startup, but they were ~100ms before, now they
are more like 20ms.  Unfortunately the OO-ness of the code makes it hard
for me to figure out where alsa_driver_start gets called from, so I'm
not sure what else happens before we enter the polling loop.

Lee  

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Lee Revell]

On Mon, 2004-08-16 at 08:09, Ingo Molnar wrote:
> here's -P2:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P2
> 
> Changes since -P1:
> 
>  - trace interrupted kernel code (via hardirqs, NMIs and pagefaults)
> 
>  - yet another shot at trying to fix the IO-APIC/USB issues.
> 
>  - mcount speedups - tracing should be faster
> 

P2 will not boot for me.  It hangs right after detecting my (surprise!)
USB mouse.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> P2 will not boot for me.  It hangs right after detecting my
> (surprise!) USB mouse.

hm. Could you take -P1's arch/i386/io_apic.c and put it into the P2 tree
- does that fix your bootup? (and does your USB mouse work after that?)

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Lee Revell]

On Tue, 2004-08-17 at 03:30, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > P2 will not boot for me.  It hangs right after detecting my
> > (surprise!) USB mouse.
> 
> hm. Could you take -P1's arch/i386/io_apic.c and put it into the P2 tree
> - does that fix your bootup? (and does your USB mouse work after that?)
> 

I think this is an unrelated problem, still testing.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > i don't know if this was mentioned before, but i sometimes see traces
> > like this where half the entries are "preempt_schedule
> > (copy_page_range)". I just wanted to ask if this is normal and expected
> > behaviour.
> 
> Yes, Ingo identified an issue with copy_page_range, I don't think it's
> fixed yet.  See the voluntary-preempt-2.6.8.1-P0 thread.

right, it's not fixed yet. It's not a trivial critical section - we are
holding two locks and are mapping two atomic kmaps.

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> I have attached a patch that effectively disables extract_entropy.  I
> am adding Theodore T'so to the cc: list as he is the author of the
> code in question.
> 
> For the time being this hack is required to avoid ~0.5 ms
> non-preemptible sections caused by the excessive memcpy's in
> extract_entropy.

i'm not 100% sure it's the memcpy's - but it's extract_entropy()
overhead. Might be the algorithmic slowness of SHATransform().

> +	return nbytes;
> +    

since this effectively disables the random driver i cannot add it to the
patch.

there's another thing you could try: various SHA_CODE_SIZE values in
drivers/char/random.c. Could you try 1, 2 and 3, does it change the
overhead as seen in the trace?

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Tue, 2004-08-17 at 03:48, Ingo Molnar wrote:
> > +	return nbytes;
> > +    
> 
> since this effectively disables the random driver i cannot add it to the
> patch.
> 

Yes, I figured as much, this is more of a temporary workaround, so we
can continue to identify other issues - otherwise this one shows up
constantly in latency_trace.

> there's another thing you could try: various SHA_CODE_SIZE values in
> drivers/char/random.c. Could you try 1, 2 and 3, does it change the
> overhead as seen in the trace?
> 

Sure, I will test this.

Lee

[patch] voluntary-preempt-2.6.8.1-P3

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> P2 will not boot for me.  It hangs right after detecting my
> (surprise!) USB mouse.

i've uploaded -P3 with the IO-APIC changes reverted to -P1:

 http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P3

it has no other changes.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Lee Revell]

On Tue, 2004-08-17 at 03:30, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > P2 will not boot for me.  It hangs right after detecting my
> > (surprise!) USB mouse.
> 
> hm. Could you take -P1's arch/i386/io_apic.c and put it into the P2 tree
> - does that fix your bootup? (and does your USB mouse work after that?)
> 

OK, I am still not sure that P2 was the problem - reverting from ALSA
1.0.6a to ALSA cvs from a few weeks ago seemed to fix the problem. 
Ugh.  Still testing.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Hans Reiser]

Lee Revell wrote:

>
>When will reiser4 will be in the stock kernel?
>
>Lee 
>
>
>
>  
>
I hope for 3-6 weeks from now, but it is not my decision.....

We are going into -mm this week.

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> > Yes, Ingo identified an issue with copy_page_range, I don't think it's
> > fixed yet.  See the voluntary-preempt-2.6.8.1-P0 thread.
> 
> right, it's not fixed yet. It's not a trivial critical section - we
> are holding two locks and are mapping two atomic kmaps.

fortunately it's easier than i thought - neither the source pmd nor the
target pmd can go away so we can simply drop the locks, reschedule, and
remap. Does the patch below (ontop of -P3) fix the copy_page_range()
latencies you are seeing?

	Ingo

--- linux/mm/memory.c.orig	
+++ linux/mm/memory.c	
@@ -337,6 +337,15 @@ cont_copy_pte_range_noset:
 				}
 				src_pte++;
 				dst_pte++;
+				if (voluntary_need_resched()) {
+					pte_unmap_nested(src_pte);
+					pte_unmap(dst_pte);
+					spin_unlock(&src->page_table_lock);
+					cond_resched_lock(&dst->page_table_lock);
+					spin_lock(&src->page_table_lock);
+					dst_pte = pte_offset_map(dst_pmd, address);
+					src_pte = pte_offset_map_nested(src_pmd, address);
+				}
 			} while ((unsigned long)src_pte & PTE_TABLE_MASK);
 			pte_unmap_nested(src_pte-1);
 			pte_unmap(dst_pte-1);

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Thomas Charbonnel]

Ingo Molnar wrote :
> here's -P2:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P2
> 
> Changes since -P1:
> 
>  - trace interrupted kernel code (via hardirqs, NMIs and pagefaults)
> 
>  - yet another shot at trying to fix the IO-APIC/USB issues.
> 
>  - mcount speedups - tracing should be faster
> 
> 	Ingo

I think I stumbled across some bugs/false positives.
Those tests were run with acpi=off, so that this specific issue doesn't
interfere. voluntary_preemption is set to 3 throughout.

The first problem was already reported by Lee Revell. Creating a new tab
in gnome-terminal gives :

[...]
 0.064ms (+0.000ms): preempt_schedule (try_to_wake_up)
 0.065ms (+0.000ms): preempt_schedule (copy_page_range)
 0.065ms (+0.000ms): preempt_schedule (copy_page_range)
[... plenty of preempt_schedule (copy_page_range) skipped ...]
 0.202ms (+0.000ms): preempt_schedule (copy_page_range)
 0.202ms (+0.000ms): preempt_schedule (copy_page_range)
 0.202ms (+0.000ms): check_preempt_timing (touch_preempt_timing)

This is induced by kernel_preemption=0 and disappears with
kernel_preemption=1. It seems to be a side-effect of the current design.
Is there a way to avoid this ?

The second one still involves creating a new tab in gnome-terminal, but
with kernel_preemption=1 :

preemption latency trace v1.0
-----------------------------
 latency: 130 us, entries: 6 (6)
 process: gnome-terminal/14381, uid: 1000
 nice: 0, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): __vma_link_rb (copy_mm)
 0.000ms (+0.000ms): rb_insert_color (copy_mm)
 0.000ms (+0.000ms): __rb_rotate_left (rb_insert_color)
 0.000ms (+0.000ms): copy_page_range (copy_mm)
 0.000ms (+0.000ms): pte_alloc_map (copy_page_range)
 0.127ms (+0.126ms): check_preempt_timing (touch_preempt_timing)

When entering check_preempt_timing, preempt_thresh was 0, and
preempt_max_latency had been freshly reset to 100. It should have
triggered this code :
		if (latency < preempt_max_latency)
			goto out;
but for some reason it didn't (or there is a problem in the tracing
code, not showing events that would have increased 'latency').

Thomas

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Theodore Ts'o]

On Tue, Aug 17, 2004 at 09:48:26AM +0200, Ingo Molnar wrote:
> > +	return nbytes;
> > +    
> 
> since this effectively disables the random driver i cannot add it to the
> patch.

The problem isn't extract_entropy(), as much as the fact that we're
calling it from check_and_rekey(), which in turn is being called by
the secure TCP sequence number generation code.  If you are looking
for a more localized way of getting rid of the probably while
minimizing the amount of code that's being disabled, you can simply do
this instead:

--- random.c	2004-08-09 02:26:29 -0400
+++ random.c.new	2004-08-17 14:04:56 -0400
@@ -2207,7 +2207,8 @@
 #undef K3
 
 /* This should not be decreased so low that ISNs wrap too fast. */
-#define REKEY_INTERVAL	300
+#define REKEY_INTERVAL	((time_t) -1)
+
 /*
  * Bit layout of the tcp sequence numbers (before adding current time):
  * bit 24-31: increased after every key exchange

WARNING: this will effectively disable the secure TCP sequence
generation, thus making your system more vulnerable to TCP hijacking
attacks --- but if you cared about such attacks, you'd be using Real
Crypto in your application progams anyway...

> there's another thing you could try: various SHA_CODE_SIZE values in
> drivers/char/random.c. Could you try 1, 2 and 3, does it change the
> overhead as seen in the trace?

I doubt SHA_CODE_SIZE will make a sufficient difference to avoid the
latency problems.  What we would need to do is to change the code so
that the rekey operation in __check_and_rekey takes place in a
workqueue.  Say, something like this (warning, I haven't tested this
patch; if it breaks, you get to keep both pieces):

===== drivers/char/random.c 1.45 vs edited =====
--- 1.45/drivers/char/random.c	2004-08-08 02:43:40 -04:00
+++ edited/drivers/char/random.c	2004-08-17 15:15:57 -04:00
@@ -2241,30 +2241,35 @@
 static spinlock_t ip_lock = SPIN_LOCK_UNLOCKED;
 static unsigned int ip_cnt;
 
-static struct keydata *__check_and_rekey(time_t time)
+static void rekey_seq_generator(void *private_)
 {
 	struct keydata *keyptr;
+	struct timeval 	tv;
+
+	do_gettimeofday(&tv);
+
 	spin_lock_bh(&ip_lock);
 	keyptr = &ip_keydata[ip_cnt&1];
-	if (!keyptr->rekey_time || (time - keyptr->rekey_time) > REKEY_INTERVAL) {
-		keyptr = &ip_keydata[1^(ip_cnt&1)];
-		keyptr->rekey_time = time;
-		get_random_bytes(keyptr->secret, sizeof(keyptr->secret));
-		keyptr->count = (ip_cnt&COUNT_MASK)<<HASH_BITS;
-		mb();
-		ip_cnt++;
-	}
+
+	keyptr = &ip_keydata[1^(ip_cnt&1)];
+	keyptr->rekey_time = tv.tv_sec;
+	get_random_bytes(keyptr->secret, sizeof(keyptr->secret));
+	keyptr->count = (ip_cnt&COUNT_MASK)<<HASH_BITS;
+	mb();
+	ip_cnt++;
+
 	spin_unlock_bh(&ip_lock);
-	return keyptr;
 }
 
+static DECLARE_WORK(rekey_work, rekey_seq_generator, NULL);
+
 static inline struct keydata *check_and_rekey(time_t time)
 {
 	struct keydata *keyptr = &ip_keydata[ip_cnt&1];
 
 	rmb();
 	if (!keyptr->rekey_time || (time - keyptr->rekey_time) > REKEY_INTERVAL) {
-		keyptr = __check_and_rekey(time);
+		schedule_work(&rekey_work);
 	}
 
 	return keyptr;

						- Ted

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Roger Luethi]

On Mon, 16 Aug 2004 18:53:56 -0400, Lee Revell wrote:
> What do you think of Ingo's solution of trying to move the problematic
> call to mdio_read out of the spinlocked section?  It does seem that the

Can't comment on that, I missed it. I am aware that locking in via-rhine
needs work, though, it's one of the things I haven't touched.

> awfully long time.  In a live audio setting you would actually get lots
> of media events.

Don't trip over the network cables. Duh.

Roger

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Tue, 2004-08-17 at 16:52, Roger Luethi wrote:
> On Mon, 16 Aug 2004 18:53:56 -0400, Lee Revell wrote:
> > What do you think of Ingo's solution of trying to move the problematic
> > call to mdio_read out of the spinlocked section?  It does seem that the
> 
> Can't comment on that, I missed it. I am aware that locking in via-rhine
> needs work, though, it's one of the things I haven't touched.
> 
> > awfully long time.  In a live audio setting you would actually get lots
> > of media events.
> 
> Don't trip over the network cables. Duh.
> 

You might want to intentionally plug or unplug them.  Live music != a
server room.  Think laptop DJs.  It would be bad if plugging into the
network caused a click in your sound output - this could be VERY loud
depnding on the setting!

This will become more important once Linux has good
audio/midi-over-ethernet support.  You might be playing sound out of
your sound interface, and sending the same sound over the network as UDP
packets for recording or additional processing.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Lee Revell]

On Mon, 2004-08-16 at 04:07, Roger Luethi wrote:

> > Also, isn't there a better solution than for network drivers to actively 
> > poll for changes in link status?  Can't they just register a callback that 
> 
> For the Rhine, there is, but making it work requires some extra black
> magic we didn't know until a few months ago. It's fixed in -mm now and
> will probably be in 2.6.9.

Which of the broken out patches from -mm should I apply to get this
fix?  Should I just apply all the ones that touch via-rhine.c?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Lee Revell]

On Tue, 2004-08-17 at 04:18, Ingo Molnar wrote:
> * Ingo Molnar <mingo@elte.hu> wrote:
> 
> > > Yes, Ingo identified an issue with copy_page_range, I don't think it's
> > > fixed yet.  See the voluntary-preempt-2.6.8.1-P0 thread.
> > 
> > right, it's not fixed yet. It's not a trivial critical section - we
> > are holding two locks and are mapping two atomic kmaps.
> 
> fortunately it's easier than i thought - neither the source pmd nor the
> target pmd can go away so we can simply drop the locks, reschedule, and
> remap. Does the patch below (ontop of -P3) fix the copy_page_range()
> latencies you are seeing?
> 

Yes, this fixes it.  Now the most frequent issue is the xfree86
unmap_vmas latency:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P3

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P3

[Florian Schmidt]

On Tue, 17 Aug 2004 10:05:12 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> 
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > P2 will not boot for me.  It hangs right after detecting my
> > (surprise!) USB mouse.
> 
> i've uploaded -P3 with the IO-APIC changes reverted to -P1:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P3
> 
> it has no other changes.
> 
> 	Ingo
> 

Hi, it applied against 2.6.8.1 with some offsets and some buzz [?]. Well anyways it compiled fine and the copy_page_range latency is gone.. Now i also see the extracty entropy thing, too..

Btw: one question: at one point in time the IRQ handlers were in the SCHED_FIFO scheduling class. Why has this changed?

[top extract]:

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND 
   11 root       5 -10     0    0    0 S  0.0  0.0   0:00.00 IRQ 8    


preemption latency trace v1.0
-----------------------------
 latency: 186 us, entries: 275 (275)
 process: ksoftirqd/0/2, uid: 0
 nice: -10, policy: 0, rt_priority: 0
=======>
 0.000ms (+0.000ms): ip_rcv (netif_receive_skb)
 0.001ms (+0.001ms): ip_route_input (ip_rcv)
 0.002ms (+0.000ms): rt_hash_code (ip_route_input)
 0.003ms (+0.001ms): ip_route_input_slow (ip_rcv)
 0.004ms (+0.000ms): rt_hash_code (ip_route_input_slow)
 0.005ms (+0.001ms): fn_hash_lookup (ip_route_input_slow)
 0.006ms (+0.001ms): fib_semantic_match (fn_hash_lookup)
 0.008ms (+0.002ms): fib_validate_source (ip_route_input_slow)
 0.009ms (+0.000ms): fn_hash_lookup (fib_validate_source)
 0.010ms (+0.001ms): fn_hash_lookup (fib_validate_source)
 0.012ms (+0.001ms): fib_semantic_match (fn_hash_lookup)
 0.013ms (+0.000ms): __fib_res_prefsrc (fib_validate_source)
 0.013ms (+0.000ms): inet_select_addr (__fib_res_prefsrc)
 0.015ms (+0.001ms): dst_alloc (ip_route_input_slow)
 0.015ms (+0.000ms): kmem_cache_alloc (dst_alloc)
 0.016ms (+0.000ms): cache_alloc_refill (kmem_cache_alloc)
 0.019ms (+0.003ms): rt_intern_hash (ip_route_input_slow)
 0.020ms (+0.000ms): local_bh_enable (rt_intern_hash)
 0.021ms (+0.001ms): ip_local_deliver (ip_rcv)
 0.022ms (+0.001ms): icmp_rcv (ip_local_deliver)
 0.023ms (+0.000ms): skb_checksum (icmp_rcv)
 0.025ms (+0.001ms): icmp_echo (icmp_rcv)
 0.025ms (+0.000ms): icmp_reply (icmp_echo)
 0.026ms (+0.000ms): ip_options_echo (icmp_reply)
 0.026ms (+0.000ms): icmp_out_count (icmp_reply)
 0.028ms (+0.001ms): ip_route_output_key (icmp_reply)
 0.028ms (+0.000ms): __ip_route_output_key (ip_route_output_key)
 0.028ms (+0.000ms): rt_hash_code (__ip_route_output_key)
 0.029ms (+0.001ms): ip_route_output_slow (ip_route_output_key)
 0.030ms (+0.000ms): ip_dev_find (ip_route_output_slow)
 0.031ms (+0.000ms): fn_hash_lookup (ip_dev_find)
 0.031ms (+0.000ms): fib_semantic_match (fn_hash_lookup)
 0.032ms (+0.000ms): fn_hash_lookup (ip_route_output_slow)
 0.032ms (+0.000ms): fn_hash_lookup (ip_route_output_slow)
 0.032ms (+0.000ms): fib_semantic_match (fn_hash_lookup)
 0.033ms (+0.000ms): fn_hash_select_default (ip_route_output_slow)
 0.035ms (+0.001ms): dst_alloc (ip_route_output_slow)
 0.035ms (+0.000ms): kmem_cache_alloc (dst_alloc)
 0.036ms (+0.001ms): rt_set_nexthop (ip_route_output_slow)
 0.036ms (+0.000ms): memcpy (rt_set_nexthop)
 0.037ms (+0.000ms): set_class_tag (rt_set_nexthop)
 0.037ms (+0.000ms): rt_hash_code (ip_route_output_slow)
 0.037ms (+0.000ms): rt_intern_hash (ip_route_output_slow)
 0.038ms (+0.000ms): arp_bind_neighbour (rt_intern_hash)
 0.038ms (+0.000ms): neigh_lookup (arp_bind_neighbour)
 0.039ms (+0.000ms): arp_hash (neigh_lookup)
 0.040ms (+0.001ms): local_bh_enable (neigh_lookup)
 0.040ms (+0.000ms): local_bh_enable (rt_intern_hash)
 0.040ms (+0.000ms): icmp_push_reply (icmp_reply)
 0.041ms (+0.000ms): ip_append_data (icmp_push_reply)
 0.043ms (+0.002ms): sock_alloc_send_skb (ip_append_data)
 0.043ms (+0.000ms): sock_alloc_send_pskb (sock_alloc_send_skb)
 0.044ms (+0.000ms): alloc_skb (sock_alloc_send_pskb)
 0.044ms (+0.000ms): kmem_cache_alloc (alloc_skb)
 0.044ms (+0.000ms): __kmalloc (alloc_skb)
 0.046ms (+0.002ms): ip_push_pending_frames (icmp_reply)
 0.048ms (+0.001ms): __ip_select_ident (ip_push_pending_frames)
 0.048ms (+0.000ms): rt_bind_peer (__ip_select_ident)
 0.049ms (+0.000ms): inet_getpeer (rt_bind_peer)
 0.049ms (+0.000ms): local_bh_enable (inet_getpeer)
 0.049ms (+0.000ms): kmem_cache_alloc (inet_getpeer)
 0.050ms (+0.000ms): cache_alloc_refill (kmem_cache_alloc)
 0.051ms (+0.001ms): secure_ip_id (inet_getpeer)
 0.052ms (+0.000ms): __check_and_rekey (secure_ip_id)
 0.053ms (+0.000ms): get_random_bytes (__check_and_rekey)
 0.053ms (+0.000ms): extract_entropy (get_random_bytes)
 0.054ms (+0.000ms): extract_entropy (extract_entropy)
 0.055ms (+0.000ms): SHATransform (extract_entropy)
 0.055ms (+0.000ms): memcpy (SHATransform)
 0.058ms (+0.002ms): add_entropy_words (extract_entropy)
 0.059ms (+0.001ms): SHATransform (extract_entropy)
 0.059ms (+0.000ms): memcpy (SHATransform)
 0.061ms (+0.001ms): add_entropy_words (extract_entropy)
 0.061ms (+0.000ms): SHATransform (extract_entropy)
 0.061ms (+0.000ms): memcpy (SHATransform)
 0.063ms (+0.001ms): add_entropy_words (extract_entropy)
 0.063ms (+0.000ms): SHATransform (extract_entropy)
 0.063ms (+0.000ms): memcpy (SHATransform)
 0.065ms (+0.001ms): add_entropy_words (extract_entropy)
 0.065ms (+0.000ms): SHATransform (extract_entropy)
 0.065ms (+0.000ms): memcpy (SHATransform)
 0.066ms (+0.001ms): add_entropy_words (extract_entropy)
 0.067ms (+0.000ms): SHATransform (extract_entropy)
 0.067ms (+0.000ms): memcpy (SHATransform)
 0.068ms (+0.001ms): add_entropy_words (extract_entropy)
 0.068ms (+0.000ms): SHATransform (extract_entropy)
 0.069ms (+0.000ms): memcpy (SHATransform)
 0.070ms (+0.001ms): add_entropy_words (extract_entropy)
 0.070ms (+0.000ms): SHATransform (extract_entropy)
 0.070ms (+0.000ms): memcpy (SHATransform)
 0.072ms (+0.001ms): add_entropy_words (extract_entropy)
 0.073ms (+0.000ms): SHATransform (extract_entropy)
 0.073ms (+0.000ms): memcpy (SHATransform)
 0.074ms (+0.001ms): add_entropy_words (extract_entropy)
 0.074ms (+0.000ms): SHATransform (extract_entropy)
 0.075ms (+0.000ms): memcpy (SHATransform)
 0.076ms (+0.001ms): add_entropy_words (extract_entropy)
 0.076ms (+0.000ms): SHATransform (extract_entropy)
 0.076ms (+0.000ms): memcpy (SHATransform)
 0.078ms (+0.001ms): add_entropy_words (extract_entropy)
 0.078ms (+0.000ms): SHATransform (extract_entropy)
 0.078ms (+0.000ms): memcpy (SHATransform)
 0.080ms (+0.001ms): add_entropy_words (extract_entropy)
 0.080ms (+0.000ms): SHATransform (extract_entropy)
 0.080ms (+0.000ms): memcpy (SHATransform)
 0.082ms (+0.001ms): add_entropy_words (extract_entropy)
 0.082ms (+0.000ms): SHATransform (extract_entropy)
 0.082ms (+0.000ms): memcpy (SHATransform)
 0.084ms (+0.001ms): add_entropy_words (extract_entropy)
 0.084ms (+0.000ms): SHATransform (extract_entropy)
 0.084ms (+0.000ms): memcpy (SHATransform)
 0.086ms (+0.001ms): add_entropy_words (extract_entropy)
 0.086ms (+0.000ms): SHATransform (extract_entropy)
 0.086ms (+0.000ms): memcpy (SHATransform)
 0.088ms (+0.001ms): add_entropy_words (extract_entropy)
 0.088ms (+0.000ms): SHATransform (extract_entropy)
 0.088ms (+0.000ms): memcpy (SHATransform)
 0.090ms (+0.001ms): add_entropy_words (extract_entropy)
 0.090ms (+0.000ms): SHATransform (extract_entropy)
 0.090ms (+0.000ms): memcpy (SHATransform)
 0.091ms (+0.001ms): add_entropy_words (extract_entropy)
 0.092ms (+0.000ms): SHATransform (extract_entropy)
 0.092ms (+0.000ms): memcpy (SHATransform)
 0.093ms (+0.001ms): add_entropy_words (extract_entropy)
 0.094ms (+0.000ms): SHATransform (extract_entropy)
 0.094ms (+0.000ms): memcpy (SHATransform)
 0.095ms (+0.001ms): add_entropy_words (extract_entropy)
 0.095ms (+0.000ms): SHATransform (extract_entropy)
 0.096ms (+0.000ms): memcpy (SHATransform)
 0.097ms (+0.001ms): add_entropy_words (extract_entropy)
 0.097ms (+0.000ms): SHATransform (extract_entropy)
 0.097ms (+0.000ms): memcpy (SHATransform)
 0.099ms (+0.001ms): add_entropy_words (extract_entropy)
 0.099ms (+0.000ms): SHATransform (extract_entropy)
 0.099ms (+0.000ms): memcpy (SHATransform)
 0.101ms (+0.001ms): add_entropy_words (extract_entropy)
 0.101ms (+0.000ms): SHATransform (extract_entropy)
 0.101ms (+0.000ms): memcpy (SHATransform)
 0.103ms (+0.001ms): add_entropy_words (extract_entropy)
 0.103ms (+0.000ms): SHATransform (extract_entropy)
 0.103ms (+0.000ms): memcpy (SHATransform)
 0.105ms (+0.001ms): add_entropy_words (extract_entropy)
 0.105ms (+0.000ms): SHATransform (extract_entropy)
 0.105ms (+0.000ms): memcpy (SHATransform)
 0.107ms (+0.001ms): add_entropy_words (extract_entropy)
 0.107ms (+0.000ms): SHATransform (extract_entropy)
 0.107ms (+0.000ms): memcpy (SHATransform)
 0.109ms (+0.001ms): add_entropy_words (extract_entropy)
 0.109ms (+0.000ms): SHATransform (extract_entropy)
 0.109ms (+0.000ms): memcpy (SHATransform)
 0.110ms (+0.001ms): add_entropy_words (extract_entropy)
 0.111ms (+0.000ms): SHATransform (extract_entropy)
 0.111ms (+0.000ms): memcpy (SHATransform)
 0.112ms (+0.001ms): add_entropy_words (extract_entropy)
 0.112ms (+0.000ms): SHATransform (extract_entropy)
 0.112ms (+0.000ms): memcpy (SHATransform)
 0.114ms (+0.001ms): add_entropy_words (extract_entropy)
 0.114ms (+0.000ms): SHATransform (extract_entropy)
 0.114ms (+0.000ms): memcpy (SHATransform)
 0.116ms (+0.001ms): add_entropy_words (extract_entropy)
 0.116ms (+0.000ms): SHATransform (extract_entropy)
 0.116ms (+0.000ms): memcpy (SHATransform)
 0.118ms (+0.001ms): add_entropy_words (extract_entropy)
 0.118ms (+0.000ms): SHATransform (extract_entropy)
 0.118ms (+0.000ms): memcpy (SHATransform)
 0.120ms (+0.001ms): add_entropy_words (extract_entropy)
 0.120ms (+0.000ms): SHATransform (extract_entropy)
 0.120ms (+0.000ms): memcpy (SHATransform)
 0.122ms (+0.001ms): add_entropy_words (extract_entropy)
 0.122ms (+0.000ms): SHATransform (extract_entropy)
 0.122ms (+0.000ms): memcpy (SHATransform)
 0.124ms (+0.001ms): add_entropy_words (extract_entropy)
 0.124ms (+0.000ms): SHATransform (extract_entropy)
 0.124ms (+0.000ms): memcpy (SHATransform)
 0.125ms (+0.001ms): add_entropy_words (extract_entropy)
 0.126ms (+0.000ms): SHATransform (extract_entropy)
 0.126ms (+0.000ms): memcpy (SHATransform)
 0.127ms (+0.001ms): add_entropy_words (extract_entropy)
 0.127ms (+0.000ms): SHATransform (extract_entropy)
 0.128ms (+0.000ms): memcpy (SHATransform)
 0.129ms (+0.001ms): add_entropy_words (extract_entropy)
 0.129ms (+0.000ms): SHATransform (extract_entropy)
 0.129ms (+0.000ms): memcpy (SHATransform)
 0.131ms (+0.001ms): add_entropy_words (extract_entropy)
 0.131ms (+0.000ms): SHATransform (extract_entropy)
 0.131ms (+0.000ms): memcpy (SHATransform)
 0.133ms (+0.001ms): add_entropy_words (extract_entropy)
 0.133ms (+0.000ms): add_entropy_words (extract_entropy)
 0.136ms (+0.002ms): credit_entropy_store (extract_entropy)
 0.137ms (+0.000ms): __wake_up (extract_entropy)
 0.137ms (+0.000ms): __wake_up_common (__wake_up)
 0.137ms (+0.000ms): SHATransform (extract_entropy)
 0.137ms (+0.000ms): memcpy (SHATransform)
 0.139ms (+0.001ms): add_entropy_words (extract_entropy)
 0.139ms (+0.000ms): SHATransform (extract_entropy)
 0.139ms (+0.000ms): memcpy (SHATransform)
 0.141ms (+0.001ms): add_entropy_words (extract_entropy)
 0.141ms (+0.000ms): SHATransform (extract_entropy)
 0.141ms (+0.000ms): memcpy (SHATransform)
 0.143ms (+0.001ms): add_entropy_words (extract_entropy)
 0.143ms (+0.000ms): SHATransform (extract_entropy)
 0.143ms (+0.000ms): memcpy (SHATransform)
 0.145ms (+0.001ms): add_entropy_words (extract_entropy)
 0.145ms (+0.000ms): SHATransform (extract_entropy)
 0.145ms (+0.000ms): memcpy (SHATransform)
 0.147ms (+0.001ms): add_entropy_words (extract_entropy)
 0.147ms (+0.000ms): SHATransform (extract_entropy)
 0.147ms (+0.000ms): memcpy (SHATransform)
 0.149ms (+0.001ms): add_entropy_words (extract_entropy)
 0.149ms (+0.000ms): SHATransform (extract_entropy)
 0.149ms (+0.000ms): memcpy (SHATransform)
 0.151ms (+0.001ms): add_entropy_words (extract_entropy)
 0.151ms (+0.000ms): SHATransform (extract_entropy)
 0.151ms (+0.000ms): memcpy (SHATransform)
 0.152ms (+0.001ms): add_entropy_words (extract_entropy)
 0.153ms (+0.000ms): SHATransform (extract_entropy)
 0.153ms (+0.000ms): memcpy (SHATransform)
 0.154ms (+0.001ms): add_entropy_words (extract_entropy)
 0.155ms (+0.000ms): SHATransform (extract_entropy)
 0.155ms (+0.000ms): memcpy (SHATransform)
 0.156ms (+0.001ms): add_entropy_words (extract_entropy)
 0.157ms (+0.000ms): local_bh_enable (__check_and_rekey)
 0.157ms (+0.000ms): halfMD4Transform (secure_ip_id)
 0.159ms (+0.001ms): peer_avl_rebalance (inet_getpeer)
 0.160ms (+0.000ms): local_bh_enable (inet_getpeer)
 0.160ms (+0.000ms): local_bh_enable (rt_bind_peer)
 0.161ms (+0.000ms): local_bh_enable (__ip_select_ident)
 0.162ms (+0.000ms): ip_output (ip_push_pending_frames)
 0.162ms (+0.000ms): ip_finish_output (ip_push_pending_frames)
 0.163ms (+0.000ms): dev_queue_xmit (ip_finish_output)
 0.164ms (+0.001ms): pfifo_fast_enqueue (dev_queue_xmit)
 0.165ms (+0.000ms): qdisc_restart (dev_queue_xmit)
 0.165ms (+0.000ms): pfifo_fast_dequeue (qdisc_restart)
 0.166ms (+0.001ms): ppp_start_xmit (qdisc_restart)
 0.167ms (+0.000ms): skb_queue_tail (ppp_start_xmit)
 0.167ms (+0.000ms): ppp_xmit_process (ppp_start_xmit)
 0.167ms (+0.000ms): ppp_push (ppp_xmit_process)
 0.168ms (+0.000ms): skb_dequeue (ppp_xmit_process)
 0.168ms (+0.000ms): ppp_send_frame (ppp_xmit_process)
 0.169ms (+0.001ms): ppp_push (ppp_send_frame)
 0.170ms (+0.000ms): ppp_async_send (ppp_push)
 0.170ms (+0.000ms): ppp_async_push (ppp_async_send)
 0.171ms (+0.000ms): local_bh_enable (ppp_async_push)
 0.171ms (+0.000ms): ppp_async_push (ppp_async_send)
 0.171ms (+0.000ms): ppp_async_encode (ppp_async_push)
 0.173ms (+0.002ms): __kfree_skb (ppp_async_encode)
 0.173ms (+0.000ms): sock_wfree (__kfree_skb)
 0.173ms (+0.000ms): sock_def_write_space (sock_wfree)
 0.174ms (+0.000ms): kfree_skbmem (__kfree_skb)
 0.174ms (+0.000ms): skb_release_data (kfree_skbmem)
 0.174ms (+0.000ms): kfree (kfree_skbmem)
 0.174ms (+0.000ms): kmem_cache_free (kfree_skbmem)
 0.174ms (+0.000ms): pty_write (ppp_async_push)
 0.175ms (+0.000ms): n_tty_receive_room (pty_write)
 0.175ms (+0.000ms): n_tty_receive_buf (pty_write)
 0.177ms (+0.001ms): kill_fasync (n_tty_receive_buf)
 0.177ms (+0.000ms): n_tty_receive_room (n_tty_receive_buf)
 0.177ms (+0.000ms): local_bh_enable (ppp_async_push)
 0.177ms (+0.000ms): local_bh_enable (ppp_send_frame)
 0.177ms (+0.000ms): skb_dequeue (ppp_xmit_process)
 0.178ms (+0.000ms): raise_softirq_irqoff (ppp_xmit_process)
 0.178ms (+0.000ms): local_bh_enable (ppp_start_xmit)
 0.178ms (+0.000ms): qdisc_restart (dev_queue_xmit)
 0.178ms (+0.000ms): pfifo_fast_dequeue (qdisc_restart)
 0.179ms (+0.000ms): local_bh_enable (dev_queue_xmit)
 0.180ms (+0.000ms): icmp_xmit_unlock (icmp_reply)
 0.180ms (+0.000ms): local_bh_enable (icmp_reply)
 0.180ms (+0.000ms): do_softirq (local_bh_enable)
 0.180ms (+0.000ms): __do_softirq (do_softirq)
 0.180ms (+0.000ms): __kfree_skb (icmp_rcv)
 0.180ms (+0.000ms): kfree_skbmem (__kfree_skb)
 0.180ms (+0.000ms): skb_release_data (kfree_skbmem)
 0.181ms (+0.000ms): kfree (kfree_skbmem)
 0.181ms (+0.000ms): kmem_cache_free (kfree_skbmem)
 0.181ms (+0.000ms): check_preempt_timing (sub_preempt_count)



-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Thomas Charbonnel]

Ingo Molnar wrote :
> here's -P2:
> 
>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P2
> 
> Changes since -P1:
> 
>  - trace interrupted kernel code (via hardirqs, NMIs and pagefaults)
> 
>  - yet another shot at trying to fix the IO-APIC/USB issues.
> 
>  - mcount speedups - tracing should be faster
> 
> 	Ingo
> -

The next problem I have relates to irq sharing. 
On my laptop I can't avoid it :
 10:    1070631          XT-PIC  yenta, yenta, uhci_hcd, Intel
82801CA-ICH3, hdsp, eth0
If I set the sound card's interrupt to be non threaded, then I get a
rather long non preemptible section :
http://www.undata.org/~thomas/irq_sharing.trace

As a side note, and this has already been reported here several times,
the SA_INTERRUPT flag set notably by the sound card drivers handlers is
not honored on current kernels if the device is not the first one to be
registered. A simple fix would be to add SA_INTERRUPT handlers at the
beginning instead of the end of the irq queue in setup_irq.

Similarly, when using SA_SAMPLE_RANDOM, all devices on the given irq
contribute to the entropy, even those that have a predictable interrupt
rate (e.g. sound cards), and/or for which the number of interrupts could
outweight the number of interrupts of the original SA_SAMPLE_RANDOM
driver. 

Thomas

Re: [patch] voluntary-preempt-2.6.8.1-P1

[Thomas Charbonnel]

Ingo Molnar wrote :
> * Ingo Molnar <mingo@elte.hu> wrote:
> 
> > > Yes, Ingo identified an issue with copy_page_range, I don't think it's
> > > fixed yet.  See the voluntary-preempt-2.6.8.1-P0 thread.
> > 
> > right, it's not fixed yet. It's not a trivial critical section - we
> > are holding two locks and are mapping two atomic kmaps.
> 
> fortunately it's easier than i thought - neither the source pmd nor the
> target pmd can go away so we can simply drop the locks, reschedule, and
> remap. Does the patch below (ontop of -P3) fix the copy_page_range()
> latencies you are seeing?
> 
> 	Ingo
> 

Ooops, sorry about my previous post, I had missed this one. It fixes the
problem, thanks.

Thomas

Re: [patch] voluntary-preempt-2.6.8.1-P3

[Ingo Molnar]

[-- Attachment #1: Type: text/plain, Size: 707 bytes --]


* Florian Schmidt <mista.tapas@gmx.net> wrote:

> Hi, it applied against 2.6.8.1 with some offsets and some buzz [?].
> Well anyways it compiled fine and the copy_page_range latency is
> gone.. Now i also see the extracty entropy thing, too..

could you try the attached patch that changes SHA_CODE_SIZE to 3 - does
this reduce the latency caused by extract_entropy?

> Btw: one question: at one point in time the IRQ handlers were in the
> SCHED_FIFO scheduling class. Why has this changed?

so that they dont starve the audio threads by default - the audio IRQ
has to get another priority anyway. Maybe we could try a default
SCHED_FIFO prio lower than the typical rt_priority of jackd - e.g. 30?

	Ingo

[-- Attachment #2: 1 --]
[-- Type: text/plain, Size: 358 bytes --]

--- linux/drivers/char/random.c.orig	
+++ linux/drivers/char/random.c	
@@ -942,7 +942,7 @@ EXPORT_SYMBOL(add_disk_randomness);
 #define HASH_TRANSFORM SHATransform
 
 /* Various size/speed tradeoffs are available.  Choose 0..3. */
-#define SHA_CODE_SIZE 0
+#define SHA_CODE_SIZE 3
 
 /*
  * SHA transform algorithm, taken from code written by Peter Gutmann,

Re: [patch] voluntary-preempt-2.6.8.1-P3

[Florian Schmidt]

On Wed, 18 Aug 2004 14:27:03 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> 
> * Florian Schmidt <mista.tapas@gmx.net> wrote:
> 
> > Hi, it applied against 2.6.8.1 with some offsets and some buzz [?].
> > Well anyways it compiled fine and the copy_page_range latency is
> > gone.. Now i also see the extracty entropy thing, too..
> 
> could you try the attached patch that changes SHA_CODE_SIZE to 3 -
> does this reduce the latency caused by extract_entropy?

will do..

> 
> > Btw: one question: at one point in time the IRQ handlers were in the
> > SCHED_FIFO scheduling class. Why has this changed?
> 
> so that they dont starve the audio threads by default - the audio IRQ
> has to get another priority anyway. Maybe we could try a default
> SCHED_FIFO prio lower than the typical rt_priority of jackd - e.g. 30?

Oh, upon rereading the chrt manpage i found out why i failed to set them
to SCHED_FIFO manually. So it was my error. I thought the
scheduling of the IRQ handlers was not changable at runtime. Thus my
question to make them SCHED_FIFO by default.

Well, i still think they should be SCHED_FIFO by default, so no user
process that is not itself SCHED_FIFO can starve them [X11 was able to
starve mouse irq's on my system with the defualt IRQ handlers
running SCHED_OTHER FWIW]. To make starving of user-SCHED_FIFO processes
unprobably maybe use a default static prio of 0.

Afaik jackd uses priorities > 0 for its audio threads when runing
SCHED_FIFO anyways..

But since the user will have to tweak his IRQ handlers manually anyways
[set soundcard irq higher prio than the rest, etc..], it doesn't really
make a difference.

Flo

-- 
Palimm Palimm!
http://affenbande.org/~tapas/

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Takashi Iwai]

At Wed, 18 Aug 2004 14:22:07 +0200,
Thomas Charbonnel wrote:
> 
> As a side note, and this has already been reported here several times,
> the SA_INTERRUPT flag set notably by the sound card drivers handlers is
> not honored on current kernels if the device is not the first one to be
> registered. A simple fix would be to add SA_INTERRUPT handlers at the
> beginning instead of the end of the irq queue in setup_irq.
> 
> Similarly, when using SA_SAMPLE_RANDOM, all devices on the given irq
> contribute to the entropy, even those that have a predictable interrupt
> rate (e.g. sound cards), and/or for which the number of interrupts could
> outweight the number of interrupts of the original SA_SAMPLE_RANDOM
> driver. 

Hmm, something like this?


Takashi

--- linux/arch/i386/kernel/irq.c	2004-08-18 15:15:18.272067276 +0200
+++ linux/arch/i386/kernel/irq.c	2004-08-18 15:26:13.513979698 +0200
@@ -219,15 +219,22 @@ inline void synchronize_irq(unsigned int
 asmlinkage int handle_IRQ_event(unsigned int irq,
 		struct pt_regs *regs, struct irqaction *action)
 {
-	int status = 1;	/* Force the "do bottom halves" bit */
-	int retval = 0;
-
-	if (!(action->flags & SA_INTERRUPT))
+	int status;
+	int ret, retval = 0;
+	struct irqaction *ac;
+
+	status = 0;
+	for (ac = action; ac; ac = ac->next)
+		status |= ac->flags;
+	if (!(status & SA_INTERRUPT))
 		local_irq_enable();
 
+	status = 0;
 	do {
-		status |= action->flags;
-		retval |= action->handler(irq, action->dev_id, regs);
+		ret = action->handler(irq, action->dev_id, regs);
+		if (ret)
+			status |= action->flags;
+		retval |= ret;
 		action = action->next;
 	} while (action);
 	if (status & SA_SAMPLE_RANDOM)

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Thomas Charbonnel]

Takashi Iwai wrote :
> At Wed, 18 Aug 2004 14:22:07 +0200,
> Thomas Charbonnel wrote:
> > 
> > As a side note, and this has already been reported here several times,
> > the SA_INTERRUPT flag set notably by the sound card drivers handlers is
> > not honored on current kernels if the device is not the first one to be
> > registered. A simple fix would be to add SA_INTERRUPT handlers at the
> > beginning instead of the end of the irq queue in setup_irq.
> > 
> > Similarly, when using SA_SAMPLE_RANDOM, all devices on the given irq
> > contribute to the entropy, even those that have a predictable interrupt
> > rate (e.g. sound cards), and/or for which the number of interrupts could
> > outweight the number of interrupts of the original SA_SAMPLE_RANDOM
> > driver. 
> 
> Hmm, something like this?
> 
> 
> Takashi
> 
> --- linux/arch/i386/kernel/irq.c	2004-08-18 15:15:18.272067276 +0200
> +++ linux/arch/i386/kernel/irq.c	2004-08-18 15:26:13.513979698 +0200
> @@ -219,15 +219,22 @@ inline void synchronize_irq(unsigned int
>  asmlinkage int handle_IRQ_event(unsigned int irq,
>  		struct pt_regs *regs, struct irqaction *action)
>  {
> -	int status = 1;	/* Force the "do bottom halves" bit */
> -	int retval = 0;
> -
> -	if (!(action->flags & SA_INTERRUPT))
> +	int status;
> +	int ret, retval = 0;
> +	struct irqaction *ac;
> +
> +	status = 0;
> +	for (ac = action; ac; ac = ac->next)
> +		status |= ac->flags;
> +	if (!(status & SA_INTERRUPT))
>  		local_irq_enable();
>  
> +	status = 0;
>  	do {
> -		status |= action->flags;
> -		retval |= action->handler(irq, action->dev_id, regs);
> +		ret = action->handler(irq, action->dev_id, regs);
> +		if (ret)
> +			status |= action->flags;
> +		retval |= ret;
>  		action = action->next;
>  	} while (action);
>  	if (status & SA_SAMPLE_RANDOM)

I was thinking more of something like this :

--- irq.c.orig  2004-08-16 14:26:34.000000000 +0200
+++ irq.c       2004-08-18 17:23:18.011059064 +0200
@@ -955,11 +955,16 @@
                        return -EBUSY;
                }

-               /* add new interrupt at end of irq queue */
-               do {
-                       p = &old->next;
-                       old = *p;
-               } while (old);
+               if (new->flags & SA_INTERRUPT)
+                       /* add interrupt at the start of the queue */
+                       new->next = old;
+               else
+                       /* add new interrupt at end of irq queue */
+                       do {
+                               p = &old->next;
+                               old = *p;
+                       } while (old);
+
                shared = 1;
        }

Thomas

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Takashi Iwai]

At Wed, 18 Aug 2004 17:26:16 +0200,
Thomas Charbonnel wrote:
> 
> I was thinking more of something like this :
> 
> --- irq.c.orig  2004-08-16 14:26:34.000000000 +0200
> +++ irq.c       2004-08-18 17:23:18.011059064 +0200
> @@ -955,11 +955,16 @@
>                         return -EBUSY;
>                 }
> 
> -               /* add new interrupt at end of irq queue */
> -               do {
> -                       p = &old->next;
> -                       old = *p;
> -               } while (old);
> +               if (new->flags & SA_INTERRUPT)
> +                       /* add interrupt at the start of the queue */
> +                       new->next = old;
> +               else
> +                       /* add new interrupt at end of irq queue */
> +                       do {
> +                               p = &old->next;
> +                               old = *p;
> +                       } while (old);
> +
>                 shared = 1;
>         }

Ok, how about this one?  Together with your patch, the irq is turned
on for handlers without SA_INTERRUPT.


Takashi

--- linux/arch/i386/kernel/irq.c	2004-08-18 15:15:18.000000000 +0200
+++ linux/arch/i386/kernel/irq.c	2004-08-18 17:42:57.454819403 +0200
@@ -219,15 +219,23 @@ inline void synchronize_irq(unsigned int
 asmlinkage int handle_IRQ_event(unsigned int irq,
 		struct pt_regs *regs, struct irqaction *action)
 {
-	int status = 1;	/* Force the "do bottom halves" bit */
-	int retval = 0;
-
-	if (!(action->flags & SA_INTERRUPT))
-		local_irq_enable();
+	int status;
+	int ret, retval = 0;
+	int irq_off = 1;
 
+	status = 0;
 	do {
-		status |= action->flags;
-		retval |= action->handler(irq, action->dev_id, regs);
+		/* Assume that all SA_INTERRUPT handlers are at the head
+		 * of the irq queue
+		 */
+		if (irq_off && ! (action->flags & SA_INTERRUPT)) {
+			local_irq_enable();
+			irq_off = 0;
+		}
+		ret = action->handler(irq, action->dev_id, regs);
+		if (ret)
+			status |= action->flags;
+		retval |= ret;
 		action = action->next;
 	} while (action);
 	if (status & SA_SAMPLE_RANDOM)
@@ -955,11 +963,16 @@ int setup_irq(unsigned int irq, struct i
 			return -EBUSY;
 		}
 
-		/* add new interrupt at end of irq queue */
-		do {
-			p = &old->next;
-			old = *p;
-		} while (old);
+		if (new->flags & SA_INTERRUPT)
+			/* add interrupt at the start of the queue */
+			new->next = old;
+		else
+			/* add new interrupt at end of irq queue */
+			do {
+				p = &old->next;
+				old = *p;
+			} while (old);
+
 		shared = 1;
 	}
 

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Thomas Charbonnel]

Takashi Iwai wrote :

[...]
> Ok, how about this one?  Together with your patch, the irq is turned
> on for handlers without SA_INTERRUPT.
> 
> 
> Takashi
> 
> --- linux/arch/i386/kernel/irq.c	2004-08-18 15:15:18.000000000 +0200
> +++ linux/arch/i386/kernel/irq.c	2004-08-18 17:42:57.454819403 +0200
> @@ -219,15 +219,23 @@ inline void synchronize_irq(unsigned int
>  asmlinkage int handle_IRQ_event(unsigned int irq,
>  		struct pt_regs *regs, struct irqaction *action)
>  {
> -	int status = 1;	/* Force the "do bottom halves" bit */
> -	int retval = 0;
> -
> -	if (!(action->flags & SA_INTERRUPT))
> -		local_irq_enable();
> +	int status;
> +	int ret, retval = 0;
> +	int irq_off = 1;
>  
> +	status = 0;
>  	do {
> -		status |= action->flags;
> -		retval |= action->handler(irq, action->dev_id, regs);
> +		/* Assume that all SA_INTERRUPT handlers are at the head
> +		 * of the irq queue
> +		 */
> +		if (irq_off && ! (action->flags & SA_INTERRUPT)) {
> +			local_irq_enable();
> +			irq_off = 0;
> +		}
> +		ret = action->handler(irq, action->dev_id, regs);
> +		if (ret)
> +			status |= action->flags;
> +		retval |= ret;
>  		action = action->next;
>  	} while (action);
>  	if (status & SA_SAMPLE_RANDOM)
> @@ -955,11 +963,16 @@ int setup_irq(unsigned int irq, struct i
>  			return -EBUSY;
>  		}
>  
> -		/* add new interrupt at end of irq queue */
> -		do {
> -			p = &old->next;
> -			old = *p;
> -		} while (old);
> +		if (new->flags & SA_INTERRUPT)
> +			/* add interrupt at the start of the queue */
> +			new->next = old;
> +		else
> +			/* add new interrupt at end of irq queue */
> +			do {
> +				p = &old->next;
> +				old = *p;
> +			} while (old);
> +
>  		shared = 1;
>  	}
>  

You're right, of course, my mistake. It looks good now.

Thomas

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Roger Luethi]

On Tue, 17 Aug 2004 20:01:06 -0400, Lee Revell wrote:
> > For the Rhine, there is, but making it work requires some extra black
> > magic we didn't know until a few months ago. It's fixed in -mm now and
> > will probably be in 2.6.9.
> 
> Which of the broken out patches from -mm should I apply to get this
> fix?  Should I just apply all the ones that touch via-rhine.c?

I don't know what Andrew called them. But applying all via-rhine patches
in -mm should work. Check for a function called rhine_enable_linkmon.

Roger

Re: [patch] voluntary-preempt-2.6.8.1-P3

[Elladan]

On Wed, Aug 18, 2004 at 03:01:48PM +0200, Florian Schmidt wrote:
> On Wed, 18 Aug 2004 14:27:03 +0200
> Ingo Molnar <mingo@elte.hu> wrote:
> > 
> > > Btw: one question: at one point in time the IRQ handlers were in the
> > > SCHED_FIFO scheduling class. Why has this changed?
> > 
> > so that they dont starve the audio threads by default - the audio IRQ
> > has to get another priority anyway. Maybe we could try a default
> > SCHED_FIFO prio lower than the typical rt_priority of jackd - e.g. 30?
> 
> Oh, upon rereading the chrt manpage i found out why i failed to set them
> to SCHED_FIFO manually. So it was my error. I thought the
> scheduling of the IRQ handlers was not changable at runtime. Thus my
> question to make them SCHED_FIFO by default.
> 
> Well, i still think they should be SCHED_FIFO by default, so no user
> process that is not itself SCHED_FIFO can starve them [X11 was able to
> starve mouse irq's on my system with the defualt IRQ handlers
> running SCHED_OTHER FWIW]. To make starving of user-SCHED_FIFO processes
> unprobably maybe use a default static prio of 0.
> 
> Afaik jackd uses priorities > 0 for its audio threads when runing
> SCHED_FIFO anyways..
> 
> But since the user will have to tweak his IRQ handlers manually anyways
> [set soundcard irq higher prio than the rest, etc..], it doesn't really
> make a difference.

This doesn't take into account SCHED_RR processes, which expect to
behave just like SCHED_FIFO unless there's another SCHED_RR process of
the same priority...

-J

[patch] voluntary-preempt-2.6.8.1-P4

[Ingo Molnar]

i've uploaded the -P4 patch:

  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4

the main change is a more robust latency tracer - the previous one was
not 100% correct for interrupts. People who have exprienced those weird
~1msec latencies in the idle task please re-check and re-post latency
traces.

Changes since -P3:

 - changed SHA_CODE_SIZE from 0 to 3 to reduce RNG overhead

 - fixed the copy_page_range latency

 - improved the latency tracer

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Ingo Molnar]

* Thomas Charbonnel <thomas@undata.org> wrote:

> The next problem I have relates to irq sharing. 
> On my laptop I can't avoid it :
>  10:    1070631          XT-PIC  yenta, yenta, uhci_hcd, Intel
> 82801CA-ICH3, hdsp, eth0
> If I set the sound card's interrupt to be non threaded, then I get a
> rather long non preemptible section :
> http://www.undata.org/~thomas/irq_sharing.trace

i'm not sure the IRQ sharing problem can be solved.

we could execute certain handlers immediately, and defer others to an
IRQ thread. But when we defer an IRQ we must keep the IRQ masked - which
prevents further interrupts (possibly from a high-prio non-threaded
handler) to be executed. So we'd see similar (or in fact worse, due to
the redirection cost) latencies than with the current 'all or nothing'
approach.

now in theory we only have to keep the IRQ line masked for level
triggered interrupts (most APIC interrupts are level-triggered). 
Edge-triggered interrupts (such as the XT-PIC ones you have) could be
acked immediately. I'll try to do something later, but right now there
are still some IRQ problems (USB issues, PS2 mouse/keyboard issues) so
i'd not like to complicate the design just yet.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P2

[Ingo Molnar]

* Thomas Charbonnel <thomas@undata.org> wrote:

> When entering check_preempt_timing, preempt_thresh was 0, and
> preempt_max_latency had been freshly reset to 100. It should have
> triggered this code :
>
> 		if (latency < preempt_max_latency)
> 			goto out;
>
> but for some reason it didn't (or there is a problem in the tracing
> code, not showing events that would have increased 'latency').

there is one case where we could 'miss' a new latency: when
/proc/latency_trace is accessed. For the duration of /proc/latency_trace
access, the updating of the max latency is stopped:

        if (down_trylock(&max_mutex))
                goto out;

this is not really a practical problem and fixing it would be quite 
complex.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P0

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > > What do you think of Ingo's solution of trying to move the problematic
> > > call to mdio_read out of the spinlocked section?  It does seem that the
> > 
> > Can't comment on that, I missed it. I am aware that locking in via-rhine
> > needs work, though, it's one of the things I haven't touched.
> > 
> > > awfully long time.  In a live audio setting you would actually get lots
> > > of media events.
> > 
> > Don't trip over the network cables. Duh.
> > 
> 
> You might want to intentionally plug or unplug them.  Live music != a
> server room.  Think laptop DJs.  It would be bad if plugging into the
> network caused a click in your sound output - this could be VERY loud
> depnding on the setting!

right now the mdio_read() latency causes a latency of 140 usecs which
isnt that bad. But looking at it in isolation, mdio_read() is 70 usecs a
pop which is excessive for a kernel function.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Thu, 2004-08-19 at 03:32, Ingo Molnar wrote:
> i've uploaded the -P4 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4
> 

Any comments on the unmap_vmas issue?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> On Thu, 2004-08-19 at 03:32, Ingo Molnar wrote:
> > i've uploaded the -P4 patch:
> > 
> >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4
> 
> Any comments on the unmap_vmas issue?

wli indicated he's working on the pagetable zapping critical section
issue - wli?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Thu, 2004-08-19 at 04:40, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > On Thu, 2004-08-19 at 03:32, Ingo Molnar wrote:
> > > i've uploaded the -P4 patch:
> > > 
> > >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4
> > 
> > Any comments on the unmap_vmas issue?
> 
> wli indicated he's working on the pagetable zapping critical section
> issue - wli?
> 

In the meantime, can we easily do a touch_preempt_timing() here, to
disable reporting of this issue, so we can continue to identify others? 
This one is frequent enough that I have not been able to identify any
more.

Lee 

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > wli indicated he's working on the pagetable zapping critical section
> > issue - wli?
> 
> In the meantime, can we easily do a touch_preempt_timing() here, to
> disable reporting of this issue, so we can continue to identify
> others?  This one is frequent enough that I have not been able to
> identify any more.

yeah, you can add touch_preempt_timing() there. (i'd rather not do it in
the patch, to get real results.) It is safe to add it anywhere in the
source, it only has side-effects on tracing.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Ingo Molnar]

> > > Any comments on the unmap_vmas issue?
> > 
> > wli indicated he's working on the pagetable zapping critical section
> > issue - wli?

actually - the unmap_vmas() issue might be different. Could you change
the '32' in this part of mm/memory.c:

# define ZAP_BLOCK_SIZE \
                (voluntary_preemption ? (32 * PAGE_SIZE) : __ZAP_BLOCK_SIZE)

to 16? Does that roughly halve the ~180 usec latency?

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* James Courtier-Dutton <James@superbug.demon.co.uk> wrote:

> Could the patch be adjusted to make the syslog and the
> /proc/latency_trace produce the same output?

to further unify the formats i've added the start/end entries to
latency_trace in -P4. Saving the stacktrace would be too much trouble
though.

	Ingo

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Ingo Molnar]

* James Courtier-Dutton <James@superbug.demon.co.uk> wrote:

> I have found a new problem though:
> 
> # cat latency_trace
> preemption latency trace v1.0
> -----------------------------
>  latency: 1883455195 us, entries: 1 (1)
>  process: ksoftirqd/1/5, uid: 0
>  nice: -10, policy: 0, rt_priority: 0
> =======>
>  0.000ms (+0.000ms): cond_resched_softirq (___do_softirq)
> 
> That looks bad to me. The user did not notice any latency, but 1883
> seconds seems like a high latency to me!

yeah, it doesnt look healthy. This could be some sort of tracing
underflow - what is the 'grep MHz /proc/cpuinfo' value of your box?

Also, could you try the -P4 patch, it has a more robust timestamping
mechanism.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P4

[William Lee Irwin III]

* Lee Revell <rlrevell@joe-job.com> wrote:
>> Any comments on the unmap_vmas issue?

On Thu, Aug 19, 2004 at 10:40:01AM +0200, Ingo Molnar wrote:
> wli indicated he's working on the pagetable zapping critical section
> issue - wli?

What I have is meant to remove clear_page_tables() by incrementally
pruning at munmap() -time. As far as unmap_vmas() goes, it's not
directly pertinent. It rather affects (and is meant to remove)
clear_page_tables().


-- wli

Re: [patch] voluntary-preempt-2.6.8.1-P3

[Florian Schmidt]

On Wed, 18 Aug 2004 14:27:03 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> 
> * Florian Schmidt <mista.tapas@gmx.net> wrote:
> 
> > Hi, it applied against 2.6.8.1 with some offsets and some buzz [?].
> > Well anyways it compiled fine and the copy_page_range latency is
> > gone.. Now i also see the extracty entropy thing, too..
> 
> could you try the attached patch that changes SHA_CODE_SIZE to 3 -
> does this reduce the latency caused by extract_entropy?

sorry, my box got fsck'ed. rebuilding system now. will be a day before i
can resume testing..

flo

Re: [patch] voluntary-preempt-2.6.8.1-P3

[Lee Revell]

On Thu, 2004-08-19 at 05:54, Florian Schmidt wrote:
> On Wed, 18 Aug 2004 14:27:03 +0200
> Ingo Molnar <mingo@elte.hu> wrote:
> 
> > 
> > * Florian Schmidt <mista.tapas@gmx.net> wrote:
> > 
> > > Hi, it applied against 2.6.8.1 with some offsets and some buzz [?].
> > > Well anyways it compiled fine and the copy_page_range latency is
> > > gone.. Now i also see the extracty entropy thing, too..
> > 
> > could you try the attached patch that changes SHA_CODE_SIZE to 3 -
> > does this reduce the latency caused by extract_entropy?
> 
> sorry, my box got fsck'ed. rebuilding system now. will be a day before i
> can resume testing..
> 

By any chance did you try the hack I posted to disable the random
driver?  I suspect that had subtle effects that hosed my machine.  I did
not have to rebuild it from scratch, but it was close.  Things would
break and then start working again after a reboot or two for no reason.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Thu, 2004-08-19 at 04:56, Ingo Molnar wrote:
> > > > Any comments on the unmap_vmas issue?
> > > 
> > > wli indicated he's working on the pagetable zapping critical section
> > > issue - wli?
> 
> actually - the unmap_vmas() issue might be different. Could you change
> the '32' in this part of mm/memory.c:
> 
> # define ZAP_BLOCK_SIZE \
>                 (voluntary_preemption ? (32 * PAGE_SIZE) : __ZAP_BLOCK_SIZE)
> 
> to 16? Does that roughly halve the ~180 usec latency?
> 

Yes, this takes care of it.  Now the dominant latency is the 142us
latency from the via-rhine driver, which is fixed by using the driver
from -mm (specifically it's fixed in bk-netdev.patch).

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Thu, 2004-08-19 at 06:29, Lee Revell wrote:

> Yes, this takes care of it.  Now the dominant latency is the 142us
> latency from the via-rhine driver, which is fixed by using the driver
> from -mm (specifically it's fixed in bk-netdev.patch).
> 

OK, this is a new one:

preemption latency trace v1.0.1
-------------------------------
 latency: 104 us, entries: 26 (26) 
    -----------------
    | task: pdflush/33, uid:0 nice:0 policy:0 rt_prio:0
    -----------------
 => started at: find_get_pages_tag+0x19/0x90
 => ended at:   find_get_pages_tag+0x61/0x90
=======>
 0.000ms (+0.000ms): find_get_pages_tag (pagevec_lookup_tag)
 0.000ms (+0.000ms): radix_tree_gang_lookup_tag (find_get_pages_tag)
 0.000ms (+0.000ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.004ms (+0.004ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.008ms (+0.004ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.015ms (+0.006ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.019ms (+0.004ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.022ms (+0.002ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.026ms (+0.003ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.030ms (+0.004ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.034ms (+0.003ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.038ms (+0.003ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.044ms (+0.005ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.048ms (+0.003ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.055ms (+0.007ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.057ms (+0.002ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.060ms (+0.003ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.066ms (+0.005ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.070ms (+0.004ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.074ms (+0.003ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.082ms (+0.008ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.083ms (+0.000ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.088ms (+0.004ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.092ms (+0.004ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.095ms (+0.003ms): __lookup_tag (radix_tree_gang_lookup_tag)
 0.102ms (+0.007ms): sub_preempt_count (find_get_pages_tag)

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Thu, 2004-08-19 at 06:38, Lee Revell wrote:
> On Thu, 2004-08-19 at 06:29, Lee Revell wrote:
> 
> > Yes, this takes care of it.  Now the dominant latency is the 142us
> > latency from the via-rhine driver, which is fixed by using the driver
> > from -mm (specifically it's fixed in bk-netdev.patch).
> > 
> 
> OK, this is a new one:
> 

This is the other common one:

preemption latency trace v1.0.1
-------------------------------
 latency: 74 us, entries: 9 (9)
    -----------------
    | task: XFree86/493, uid:0 nice:-10 policy:0 rt_prio:0
    -----------------
 => started at: schedule+0x5b/0x590
 => ended at:   schedule+0x2f0/0x590
=======>
 0.000ms (+0.000ms): schedule (work_resched)
 0.000ms (+0.000ms): sched_clock (schedule)
 0.001ms (+0.001ms): dequeue_task (schedule)
 0.001ms (+0.000ms): recalc_task_prio (schedule)
 0.002ms (+0.000ms): effective_prio (recalc_task_prio)
 0.002ms (+0.000ms): enqueue_task (schedule)
 0.004ms (+0.002ms): __switch_to (schedule)
 0.072ms (+0.067ms): finish_task_switch (schedule)
 0.073ms (+0.000ms): sub_preempt_count (schedule)

Not sure what to make of this one.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Thu, 2004-08-19 at 06:43, Lee Revell wrote:
> On Thu, 2004-08-19 at 06:38, Lee Revell wrote:
> > On Thu, 2004-08-19 at 06:29, Lee Revell wrote:
> > 
> > > Yes, this takes care of it.  Now the dominant latency is the 142us
> > > latency from the via-rhine driver, which is fixed by using the driver
> > > from -mm (specifically it's fixed in bk-netdev.patch).
> > > 
> > 
> > OK, this is a new one:
> > 
> 
> This is the other common one:
> 

Ouch, just got a 384usec latency from extract_entropy:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4

I think I will try the untested patch from Theodore T'so that pushes
this off into a workqueue.

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Tue, 2004-08-17 at 15:18, Theodore Ts'o wrote:
> On Tue, Aug 17, 2004 at 09:48:26AM +0200, Ingo Molnar wrote:
> > there's another thing you could try: various SHA_CODE_SIZE values in
> > drivers/char/random.c. Could you try 1, 2 and 3, does it change the
> > overhead as seen in the trace?
> 
> I doubt SHA_CODE_SIZE will make a sufficient difference to avoid the
> latency problems.

Correct, the difference is less than 50%.  I will try the patch.

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Tue, 2004-08-17 at 15:18, Theodore Ts'o wrote:
> On Tue, Aug 17, 2004 at 09:48:26AM +0200, Ingo Molnar wrote:
> > > +	return nbytes;
> > > +    
> > 
> > since this effectively disables the random driver i cannot add it to the
> > patch.
> 
> I doubt SHA_CODE_SIZE will make a sufficient difference to avoid the
> latency problems.  What we would need to do is to change the code so
> that the rekey operation in __check_and_rekey takes place in a
> workqueue.  Say, something like this (warning, I haven't tested this
> patch; if it breaks, you get to keep both pieces):
> 

Tested, works for me.  This should probably be pushed upstream, as well
as added to -P5, correct?  Is there any disadvantage to doing it this
way?

Lee  

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Thu, 2004-08-19 at 07:28, Florian Schmidt wrote:
> On Thu, 19 Aug 2004 09:32:47 +0200
> Ingo Molnar <mingo@elte.hu> wrote:
> 
> > 
> > i've uploaded the -P4 patch:
> > 
> >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4
> 
> Hi, this patch doesn't really apply cleanly. Do i need to worry about
> the "fuzz" and the offsets?
> 

Does not seem to hurt anything.  Ingo probably has some differences in
his tree, the 'fuzz' just means that patch was smart enough to figure
out that they don't matter.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Florian Schmidt]

On Thu, 19 Aug 2004 09:32:47 +0200
Ingo Molnar <mingo@elte.hu> wrote:

> 
> i've uploaded the -P4 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4

Hi, this patch doesn't really apply cleanly. Do i need to worry about
the "fuzz" and the offsets?

flo

patch -p1 --dry-run < /home/tapas/downloads/voluntary-preempt-2.6.8.1-P4
patching file arch/i386/kernel/entry.S
patching file arch/i386/kernel/i386_ksyms.c
patching file arch/i386/kernel/i8259.c
patching file arch/i386/kernel/io_apic.c
patching file arch/i386/kernel/irq.c
Hunk #12 succeeded at 656 with fuzz 2.
patching file arch/i386/kernel/apic.c
patching file arch/i386/kernel/smp.c
patching file arch/i386/kernel/traps.c
patching file arch/i386/mm/fault.c
patching file arch/i386/Kconfig
patching file arch/i386/lib/usercopy.c
patching file arch/i386/mach-visws/setup.c
patching file arch/i386/mach-visws/visws_apic.c
patching file arch/i386/mach-voyager/setup.c
patching file arch/i386/boot/compressed/misc.c
patching file arch/i386/mach-default/setup.c
patching file include/asm-ppc64/uaccess.h
patching file include/asm-m68k/uaccess.h
patching file include/asm-parisc/uaccess.h
patching file include/asm-mips/uaccess.h
patching file include/asm-ppc/uaccess.h
patching file include/asm-x86_64/uaccess.h
patching file include/asm-ia64/uaccess.h
patching file include/linux/irq.h
patching file include/linux/sched.h
patching file include/linux/blkdev.h
patching file include/linux/buffer_head.h
patching file include/linux/interrupt.h
patching file include/linux/kernel.h
patching file include/linux/kernel_stat.h
patching file include/linux/pagemap.h
patching file include/linux/preempt.h
patching file include/linux/sysctl.h
patching file include/linux/writeback.h
patching file include/asm-m68knommu/uaccess.h
patching file include/asm-um/uaccess.h
patching file include/asm-arm/uaccess.h
patching file include/asm-sparc/uaccess.h
patching file include/asm-alpha/uaccess.h
patching file include/asm-h8300/uaccess.h
patching file include/asm-i386/irq.h
patching file include/asm-i386/checksum.h
patching file include/asm-i386/hardirq.h
patching file include/asm-i386/hw_irq.h
patching file include/asm-i386/io_apic.h
patching file include/asm-i386/signal.h
patching file include/asm-i386/uaccess.h
patching file include/asm-i386/system.h
patching file include/asm-s390/uaccess.h
patching file include/asm-cris/uaccess.h
patching file include/asm-sparc64/uaccess.h
patching file include/asm-arm26/uaccess.h
patching file include/asm-v850/uaccess.h
patching file include/asm-sh/uaccess.h
patching file include/asm-sh64/uaccess.h
patching file drivers/char/tty_io.c
patching file drivers/char/random.c
Hunk #1 succeeded at 947 (offset 5 lines).
patching file drivers/block/ll_rw_blk.c
Hunk #3 succeeded at 2400 (offset 1 line).
Hunk #4 succeeded at 2448 (offset 1 line).
Hunk #5 succeeded at 3065 (offset 3 lines).
Hunk #6 succeeded at 3132 (offset 3 lines).
patching file drivers/input/keyboard/atkbd.c
patching file drivers/input/mouse/psmouse-base.c
patching file drivers/ide/ide-io.c
patching file drivers/md/dm-table.c
patching file drivers/md/linear.c
patching file drivers/md/multipath.c
patching file drivers/md/raid0.c
patching file drivers/md/raid1.c
patching file fs/ext3/ialloc.c
patching file fs/ext3/inode.c
patching file fs/ext3/namei.c
patching file fs/ext3/super.c
patching file fs/proc/proc_misc.c
patching file fs/proc/task_mmu.c
patching file fs/jbd/checkpoint.c
patching file fs/jbd/commit.c
patching file fs/jbd/revoke.c
patching file fs/buffer.c
patching file fs/dcache.c
patching file fs/exec.c
patching file fs/file_table.c
patching file fs/fs-writeback.c
patching file fs/inode.c
patching file fs/select.c
patching file net/ipv4/route.c
patching file net/ipv4/tcp_minisocks.c
patching file net/core/dev.c
patching file net/core/sock.c
patching file kernel/Makefile
patching file kernel/exit.c
patching file kernel/hardirq.c
patching file kernel/kthread.c
patching file kernel/printk.c
patching file kernel/profile.c
patching file kernel/rcupdate.c
patching file kernel/sched.c
patching file kernel/softirq.c
patching file kernel/sysctl.c
patching file kernel/timer.c
patching file kernel/latency.c
patching file kernel/kallsyms.c
patching file mm/filemap.c
patching file mm/memory.c
patching file mm/mempool.c
patching file mm/mmap.c
patching file mm/slab.c
patching file mm/vmscan.c
patching file mm/msync.c
patching file init/Kconfig
patching file init/main.c
patching file Makefile

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4
> > 
> > Hi, this patch doesn't really apply cleanly. Do i need to worry about
> > the "fuzz" and the offsets?
> 
> Does not seem to hurt anything.  Ingo probably has some differences in
> his tree, the 'fuzz' just means that patch was smart enough to figure
> out that they don't matter.

yeah, it shouldnt be an issue, until there's no reject. (I'll rebuild my
tree because i thought i have a vanilla tree, but apparently not.)

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Matthew Frost]

--- Ingo Molnar <mingo@elte.hu> wrote: 
> i've uploaded the -P4 patch:
> 
>  
> http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4
> 
> the main change is a more robust latency tracer - the previous one was
> not 100% correct for interrupts. People who have exprienced those weird
> ~1msec latencies in the idle task please re-check and re-post latency
> traces.

I receive the following errors on compile, after make has gotten through
building all config options:

-- SNIP --

  GEN     .version
  CHK     include/linux/compile.h
  UPD     include/linux/compile.h
  CC      init/version.o
  LD      init/built-in.o
  LD      .tmp_vmlinux1
arch/i386/kernel/built-in.o(.text+0x398e): In function `do_nmi':
: undefined reference to `__trace'
arch/i386/kernel/built-in.o(.text+0x4101): In function `do_IRQ':
: undefined reference to `__trace'
arch/i386/kernel/built-in.o(.text+0xe1e9): In function
`smp_apic_timer_interrupt':
: undefined reference to `__trace'
arch/i386/mm/built-in.o(.text+0x71b): In function `do_page_fault':
: undefined reference to `__trace'
make: *** [.tmp_vmlinux1] Error 1


-P2 and -P3 gave me the same errors except for
`smp_apic_timer_interrupt'.
I have tried my .config, default, allyes, and allno, and I get the same
result regardless of config.  Tree is fresh, patch applied with -Np1
<../[patchname].  I also get fuzz and offsets, but they don't seem to
bear on the problem directly.  I'm currently running -P1, which doesn't
have this problem.  gcc is 3.3.4.

With thanks for a smoother system feel,
Matthew Frost
artusemrys at sbcglobal dot net

Re: [patch] voluntary-preempt-2.6.8.1-P4

[karl.vogel]

When logging into my Fedora Core 2 box on the console (normal vga text console),
I get the following latency trace: 

(preempt=1 voluntary-preempt=3 elevator=cfq)


preemption latency trace v1.0.1
-------------------------------
 latency: 3568 us, entries: 137 (137)
    -----------------
    | task: setfont/2209, uid:500 nice:0 policy:0 rt_prio:0
    -----------------
 => started at: voluntary_resched+0x31/0x60
 => ended at:   voluntary_resched+0x31/0x60
=======>
 0.000ms (+0.000ms): touch_preempt_timing (voluntary_resched)
 0.000ms (+0.000ms): vgacon_font_set (con_font_set)
 0.000ms (+0.000ms): vgacon_do_font_op (vgacon_font_set)
 0.057ms (+0.056ms): smp_apic_timer_interrupt (vgacon_do_font_op)
 0.057ms (+0.000ms): profile_hook (smp_apic_timer_interrupt)
 0.057ms (+0.000ms): notifier_call_chain (profile_hook)
 0.120ms (+0.063ms): do_IRQ (vgacon_do_font_op)
 0.121ms (+0.000ms): ack_edge_ioapic_irq (do_IRQ)
 0.121ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
 0.121ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 0.121ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 0.121ms (+0.000ms): mark_offset_pmtmr (timer_interrupt)
 0.128ms (+0.006ms): do_timer (timer_interrupt)
 0.128ms (+0.000ms): update_process_times (do_timer)
 0.128ms (+0.000ms): update_one_process (update_process_times)
 0.128ms (+0.000ms): run_local_timers (update_process_times)
 0.128ms (+0.000ms): raise_softirq (update_process_times)
 0.128ms (+0.000ms): scheduler_tick (update_process_times)
 0.128ms (+0.000ms): sched_clock (scheduler_tick)
 0.128ms (+0.000ms): rcu_check_callbacks (scheduler_tick)
 0.128ms (+0.000ms): idle_cpu (rcu_check_callbacks)
 0.129ms (+0.000ms): __tasklet_schedule (scheduler_tick)
 0.129ms (+0.000ms): update_wall_time (do_timer)
 0.129ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 0.129ms (+0.000ms): generic_note_interrupt (do_IRQ)
 0.129ms (+0.000ms): end_edge_ioapic_irq (do_IRQ)
 0.130ms (+0.000ms): do_softirq (do_IRQ)
 0.130ms (+0.000ms): __do_softirq (do_softirq)
 0.130ms (+0.000ms): wake_up_process (do_softirq)
 0.130ms (+0.000ms): try_to_wake_up (wake_up_process)
 0.130ms (+0.000ms): task_rq_lock (try_to_wake_up)
 0.130ms (+0.000ms): activate_task (try_to_wake_up)
 0.130ms (+0.000ms): sched_clock (activate_task)
 0.130ms (+0.000ms): recalc_task_prio (activate_task)
 0.130ms (+0.000ms): effective_prio (recalc_task_prio)
 0.130ms (+0.000ms): enqueue_task (activate_task)
 1.058ms (+0.927ms): smp_apic_timer_interrupt (vgacon_do_font_op)
 1.058ms (+0.000ms): profile_hook (smp_apic_timer_interrupt)
 1.058ms (+0.000ms): notifier_call_chain (profile_hook)
 1.058ms (+0.000ms): preempt_schedule (smp_apic_timer_interrupt)
 1.058ms (+0.000ms): do_softirq (smp_apic_timer_interrupt)
 1.058ms (+0.000ms): __do_softirq (do_softirq)
 1.122ms (+0.063ms): do_IRQ (vgacon_do_font_op)
 1.122ms (+0.000ms): ack_edge_ioapic_irq (do_IRQ)
 1.122ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
 1.122ms (+0.000ms): preempt_schedule (do_IRQ)
 1.122ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 1.122ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 1.123ms (+0.000ms): mark_offset_pmtmr (timer_interrupt)
 1.129ms (+0.006ms): do_timer (timer_interrupt)
 1.129ms (+0.000ms): update_process_times (do_timer)
 1.129ms (+0.000ms): update_one_process (update_process_times)
 1.129ms (+0.000ms): run_local_timers (update_process_times)
 1.129ms (+0.000ms): raise_softirq (update_process_times)
 1.129ms (+0.000ms): scheduler_tick (update_process_times)
 1.129ms (+0.000ms): sched_clock (scheduler_tick)
 1.129ms (+0.000ms): rcu_check_callbacks (scheduler_tick)
 1.129ms (+0.000ms): idle_cpu (rcu_check_callbacks)
 1.130ms (+0.000ms): update_wall_time (do_timer)
 1.130ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 1.130ms (+0.000ms): generic_note_interrupt (do_IRQ)
 1.130ms (+0.000ms): end_edge_ioapic_irq (do_IRQ)
 1.130ms (+0.000ms): preempt_schedule (do_IRQ)
 1.130ms (+0.000ms): do_softirq (do_IRQ)
 1.130ms (+0.000ms): __do_softirq (do_softirq)
 2.059ms (+0.928ms): smp_apic_timer_interrupt (vgacon_do_font_op)
 2.059ms (+0.000ms): profile_hook (smp_apic_timer_interrupt)
 2.059ms (+0.000ms): notifier_call_chain (profile_hook)
 2.059ms (+0.000ms): preempt_schedule (smp_apic_timer_interrupt)
 2.059ms (+0.000ms): do_softirq (smp_apic_timer_interrupt)
 2.059ms (+0.000ms): __do_softirq (do_softirq)
 2.123ms (+0.063ms): do_IRQ (vgacon_do_font_op)
 2.123ms (+0.000ms): ack_edge_ioapic_irq (do_IRQ)
 2.123ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
 2.123ms (+0.000ms): preempt_schedule (do_IRQ)
 2.123ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 2.123ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 2.123ms (+0.000ms): mark_offset_pmtmr (timer_interrupt)
 2.130ms (+0.006ms): do_timer (timer_interrupt)
 2.130ms (+0.000ms): update_process_times (do_timer)
 2.130ms (+0.000ms): update_one_process (update_process_times)
 2.130ms (+0.000ms): run_local_timers (update_process_times)
 2.130ms (+0.000ms): raise_softirq (update_process_times)
 2.130ms (+0.000ms): scheduler_tick (update_process_times)
 2.130ms (+0.000ms): sched_clock (scheduler_tick)
 2.130ms (+0.000ms): rcu_check_callbacks (scheduler_tick)
 2.131ms (+0.000ms): idle_cpu (rcu_check_callbacks)
 2.131ms (+0.000ms): dequeue_task (scheduler_tick)
 2.131ms (+0.000ms): effective_prio (scheduler_tick)
 2.131ms (+0.000ms): task_timeslice (scheduler_tick)
 2.131ms (+0.000ms): enqueue_task (scheduler_tick)
 2.131ms (+0.000ms): update_wall_time (do_timer)
 2.131ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 2.131ms (+0.000ms): generic_note_interrupt (do_IRQ)
 2.132ms (+0.000ms): end_edge_ioapic_irq (do_IRQ)
 2.132ms (+0.000ms): preempt_schedule (do_IRQ)
 2.132ms (+0.000ms): do_softirq (do_IRQ)
 2.132ms (+0.000ms): __do_softirq (do_softirq)
 3.060ms (+0.927ms): smp_apic_timer_interrupt (vgacon_do_font_op)
 3.060ms (+0.000ms): profile_hook (smp_apic_timer_interrupt)
 3.060ms (+0.000ms): notifier_call_chain (profile_hook)
 3.060ms (+0.000ms): preempt_schedule (smp_apic_timer_interrupt)
 3.060ms (+0.000ms): do_softirq (smp_apic_timer_interrupt)
 3.060ms (+0.000ms): __do_softirq (do_softirq)
 3.124ms (+0.063ms): do_IRQ (vgacon_do_font_op)
 3.124ms (+0.000ms): ack_edge_ioapic_irq (do_IRQ)
 3.124ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
 3.124ms (+0.000ms): preempt_schedule (do_IRQ)
 3.124ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 3.124ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 3.124ms (+0.000ms): mark_offset_pmtmr (timer_interrupt)
 3.131ms (+0.006ms): do_timer (timer_interrupt)
 3.131ms (+0.000ms): update_process_times (do_timer)
 3.131ms (+0.000ms): update_one_process (update_process_times)
 3.131ms (+0.000ms): run_local_timers (update_process_times)
 3.131ms (+0.000ms): raise_softirq (update_process_times)
 3.131ms (+0.000ms): scheduler_tick (update_process_times)
 3.131ms (+0.000ms): sched_clock (scheduler_tick)
 3.131ms (+0.000ms): rcu_check_callbacks (scheduler_tick)
 3.131ms (+0.000ms): idle_cpu (rcu_check_callbacks)
 3.131ms (+0.000ms): update_wall_time (do_timer)
 3.131ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 3.132ms (+0.000ms): generic_note_interrupt (do_IRQ)
 3.132ms (+0.000ms): end_edge_ioapic_irq (do_IRQ)
 3.132ms (+0.000ms): preempt_schedule (do_IRQ)
 3.132ms (+0.000ms): do_softirq (do_IRQ)
 3.132ms (+0.000ms): __do_softirq (do_softirq)
 3.485ms (+0.353ms): preempt_schedule (vgacon_do_font_op)
 3.486ms (+0.000ms): vgacon_adjust_height (vgacon_font_set)
 3.486ms (+0.000ms): release_console_sem (con_font_set)
 3.486ms (+0.000ms): preempt_schedule (release_console_sem)
 3.486ms (+0.000ms): kfree (con_font_set)
 3.487ms (+0.000ms): copy_to_user (vt_ioctl)
 3.487ms (+0.000ms): __might_sleep (copy_to_user)
 3.487ms (+0.000ms): voluntary_resched (copy_to_user)
 3.487ms (+0.000ms): __might_sleep (voluntary_resched)
 3.487ms (+0.000ms): touch_preempt_timing (voluntary_resched)


$ cat /proc/interrupts
           CPU0
  0:    1286100    IO-APIC-edge  timer
  1:       4572    IO-APIC-edge  i8042
  8:          1    IO-APIC-edge  rtc
  9:         10   IO-APIC-level  acpi
 12:      86351    IO-APIC-edge  i8042
 14:      14089    IO-APIC-edge  ide0
 15:        477    IO-APIC-edge  ide1
 19:      13568   IO-APIC-level  eth0
 20:         94   IO-APIC-level  ehci_hcd, ohci_hcd
 21:          0   IO-APIC-level  ohci_hcd
 22:        593   IO-APIC-level  NVidia nForce3
NMI:          0
LOC:    1286057
ERR:          0
MIS:          0


Machine is an ACER Aspire 1510 notebook
(AMD64 processor running a 32-bit kernel)

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Theodore Ts'o]

On Thu, Aug 19, 2004 at 07:19:58AM -0400, Lee Revell wrote:
> > I doubt SHA_CODE_SIZE will make a sufficient difference to avoid the
> > latency problems.  What we would need to do is to change the code so
> > that the rekey operation in __check_and_rekey takes place in a
> > workqueue.  Say, something like this (warning, I haven't tested this
> > patch; if it breaks, you get to keep both pieces):
> > 
> 
> Tested, works for me.  This should probably be pushed upstream, as well
> as added to -P5, correct?  Is there any disadvantage to doing it this
> way?

Great, I will be pushing this upstream very shortly.

						- Ted

Re: [patch] voluntary-preempt-2.6.8.1-P4

[karl.vogel]

The following latency trace is generated each time the sound driver is opened
by an application on my box.

# lspci -s 00:06.0
00:06.0 Multimedia audio controller: nVidia Corporation nForce3 Audio (rev a2)

Audio driver:
# lsmod|grep snd
snd_seq                51856  0
snd_intel8x0           30600  0
snd_ac97_codec         66820  1 snd_intel8x0
snd_pcm                88264  1 snd_intel8x0
snd_timer              22596  2 snd_seq,snd_pcm
snd_page_alloc          9288  2 snd_intel8x0,snd_pcm
snd_mpu401_uart         6656  1 snd_intel8x0
snd_rawmidi            21156  1 snd_mpu401_uart
snd_seq_device          7176  2 snd_seq,snd_rawmidi
snd                    48804  8 snd_seq,snd_intel8x0,snd_ac97_codec,snd_pcm,snd_timer,snd_mpu401_uart,snd_rawmidi,snd_seq_device
soundcore               7840  1 snd



preemption latency trace v1.0.1
-------------------------------
 latency: 50752 us, entries: 267 (267)
    -----------------
    | task: artsd/2502, uid:500 nice:0 policy:0 rt_prio:0
    -----------------
 => started at: voluntary_resched+0x31/0x60
 => ended at:   sys_ioctl+0xf4/0x2b0
=======>
 0.000ms (+0.000ms): touch_preempt_timing (voluntary_resched)
 0.000ms (+0.000ms): snd_power_wait (snd_pcm_prepare)
 0.000ms (+0.000ms): snd_pcm_action_lock_irq (snd_pcm_prepare)
 0.001ms (+0.000ms): snd_pcm_action_single (snd_pcm_action_lock_irq)
 0.001ms (+0.000ms): snd_pcm_pre_prepare (snd_pcm_action_single)
 0.001ms (+0.000ms): snd_pcm_do_prepare (snd_pcm_action_single)
 0.001ms (+0.000ms): snd_intel8x0_pcm_prepare (snd_pcm_do_prepare)
 0.002ms (+0.000ms): snd_intel8x0_setup_pcm_out (snd_intel8x0_pcm_prepare)
 0.002ms (+0.000ms): igetdword (snd_intel8x0_setup_pcm_out)
 0.002ms (+0.000ms): iputdword (snd_intel8x0_setup_pcm_out)
 0.003ms (+0.000ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 0.003ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 0.003ms (+0.000ms): delay_pmtmr (__delay)
 0.995ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 0.995ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 0.995ms (+0.000ms): delay_pmtmr (__delay)
 1.986ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 1.986ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 1.986ms (+0.000ms): delay_pmtmr (__delay)
 2.978ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 2.978ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 2.978ms (+0.000ms): delay_pmtmr (__delay)
 3.969ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 3.969ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 3.970ms (+0.000ms): delay_pmtmr (__delay)
 4.961ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 4.961ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 4.961ms (+0.000ms): delay_pmtmr (__delay)
 5.953ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 5.953ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 5.953ms (+0.000ms): delay_pmtmr (__delay)
 6.944ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 6.944ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 6.944ms (+0.000ms): delay_pmtmr (__delay)
 7.936ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 7.936ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 7.936ms (+0.000ms): delay_pmtmr (__delay)
 8.927ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 8.927ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 8.927ms (+0.000ms): delay_pmtmr (__delay)
 9.919ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 9.919ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 9.919ms (+0.000ms): delay_pmtmr (__delay)
 10.910ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 10.910ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 10.911ms (+0.000ms): delay_pmtmr (__delay)
 11.902ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 11.902ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 11.902ms (+0.000ms): delay_pmtmr (__delay)
 12.893ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 12.894ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 12.894ms (+0.000ms): delay_pmtmr (__delay)
 13.885ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 13.885ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 13.885ms (+0.000ms): delay_pmtmr (__delay)
 14.877ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 14.877ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 14.877ms (+0.000ms): delay_pmtmr (__delay)
 15.868ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 15.868ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 15.868ms (+0.000ms): delay_pmtmr (__delay)
 16.860ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 16.860ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 16.860ms (+0.000ms): delay_pmtmr (__delay)
 17.851ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 17.851ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 17.851ms (+0.000ms): delay_pmtmr (__delay)
 18.843ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 18.843ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 18.843ms (+0.000ms): delay_pmtmr (__delay)
 19.834ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 19.835ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 19.835ms (+0.000ms): delay_pmtmr (__delay)
 20.826ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 20.826ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 20.826ms (+0.000ms): delay_pmtmr (__delay)
 21.818ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 21.818ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 21.818ms (+0.000ms): delay_pmtmr (__delay)
 22.809ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 22.809ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 22.809ms (+0.000ms): delay_pmtmr (__delay)
 23.801ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 23.801ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 23.801ms (+0.000ms): delay_pmtmr (__delay)
 24.792ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 24.792ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 24.792ms (+0.000ms): delay_pmtmr (__delay)
 25.784ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 25.784ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 25.784ms (+0.000ms): delay_pmtmr (__delay)
 26.775ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 26.775ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 26.776ms (+0.000ms): delay_pmtmr (__delay)
 27.767ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 27.767ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 27.767ms (+0.000ms): delay_pmtmr (__delay)
 28.759ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 28.759ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 28.759ms (+0.000ms): delay_pmtmr (__delay)
 29.750ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 29.750ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 29.750ms (+0.000ms): delay_pmtmr (__delay)
 30.742ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 30.742ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 30.742ms (+0.000ms): delay_pmtmr (__delay)
 31.733ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 31.733ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 31.733ms (+0.000ms): delay_pmtmr (__delay)
 32.725ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 32.725ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 32.725ms (+0.000ms): delay_pmtmr (__delay)
 33.716ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 33.716ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 33.717ms (+0.000ms): delay_pmtmr (__delay)
 34.708ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 34.708ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 34.708ms (+0.000ms): delay_pmtmr (__delay)
 35.700ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 35.700ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 35.700ms (+0.000ms): delay_pmtmr (__delay)
 36.691ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 36.691ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 36.691ms (+0.000ms): delay_pmtmr (__delay)
 37.683ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 37.683ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 37.683ms (+0.000ms): delay_pmtmr (__delay)
 38.674ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 38.674ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 38.674ms (+0.000ms): delay_pmtmr (__delay)
 39.666ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 39.666ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 39.666ms (+0.000ms): delay_pmtmr (__delay)
 40.657ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 40.657ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 40.657ms (+0.000ms): delay_pmtmr (__delay)
 41.649ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 41.649ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 41.649ms (+0.000ms): delay_pmtmr (__delay)
 42.640ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 42.641ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 42.641ms (+0.000ms): delay_pmtmr (__delay)
 43.632ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 43.632ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 43.632ms (+0.000ms): delay_pmtmr (__delay)
 44.624ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 44.624ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 44.624ms (+0.000ms): delay_pmtmr (__delay)
 45.615ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 45.615ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 45.615ms (+0.000ms): delay_pmtmr (__delay)
 46.607ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 46.607ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 46.607ms (+0.000ms): delay_pmtmr (__delay)
 47.598ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 47.598ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 47.598ms (+0.000ms): delay_pmtmr (__delay)
 48.590ms (+0.991ms): __const_udelay (snd_intel8x0_setup_pcm_out)
 48.590ms (+0.000ms): __delay (snd_intel8x0_setup_pcm_out)
 48.590ms (+0.000ms): delay_pmtmr (__delay)
 49.581ms (+0.991ms): iputdword (snd_intel8x0_setup_pcm_out)
 49.582ms (+0.000ms): snd_intel8x0_setup_periods (snd_intel8x0_pcm_prepare)
 49.582ms (+0.000ms): iputdword (snd_intel8x0_setup_periods)
 49.585ms (+0.002ms): iputbyte (snd_intel8x0_setup_periods)
 49.585ms (+0.000ms): iputbyte (snd_intel8x0_setup_periods)
 49.586ms (+0.000ms): iputbyte (snd_intel8x0_setup_periods)
 49.586ms (+0.000ms): snd_pcm_do_reset (snd_pcm_action_single)
 49.586ms (+0.000ms): snd_pcm_lib_ioctl (snd_pcm_do_reset)
 49.586ms (+0.000ms): snd_pcm_lib_ioctl_reset (snd_pcm_do_reset)
 49.587ms (+0.000ms): snd_pcm_post_prepare (snd_pcm_action_single)
 49.587ms (+0.000ms): smp_apic_timer_interrupt (snd_pcm_action_lock_irq)
 49.587ms (+0.000ms): profile_hook (smp_apic_timer_interrupt)
 49.588ms (+0.000ms): notifier_call_chain (profile_hook)
 49.588ms (+0.000ms): do_IRQ (snd_pcm_action_lock_irq)
 49.589ms (+0.000ms): ack_edge_ioapic_irq (do_IRQ)
 49.589ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
 49.589ms (+0.000ms): wake_up_process (generic_redirect_hardirq)
 49.589ms (+0.000ms): try_to_wake_up (wake_up_process)
 49.589ms (+0.000ms): task_rq_lock (try_to_wake_up)
 49.589ms (+0.000ms): activate_task (try_to_wake_up)
 49.589ms (+0.000ms): sched_clock (activate_task)
 49.590ms (+0.000ms): recalc_task_prio (activate_task)
 49.590ms (+0.000ms): effective_prio (recalc_task_prio)
 49.590ms (+0.000ms): enqueue_task (activate_task)
 49.590ms (+0.000ms): preempt_schedule (try_to_wake_up)
 49.590ms (+0.000ms): preempt_schedule (do_IRQ)
 49.591ms (+0.000ms): do_IRQ (snd_pcm_action_lock_irq)
 49.591ms (+0.000ms): ack_edge_ioapic_irq (do_IRQ)
 49.591ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
 49.591ms (+0.000ms): preempt_schedule (do_IRQ)
 49.591ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 49.591ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
 49.592ms (+0.000ms): mark_offset_pmtmr (timer_interrupt)
 49.598ms (+0.006ms): do_timer (timer_interrupt)
 49.598ms (+0.000ms): update_process_times (do_timer)
 49.598ms (+0.000ms): update_one_process (update_process_times)
 49.598ms (+0.000ms): run_local_timers (update_process_times)
 49.598ms (+0.000ms): raise_softirq (update_process_times)
 49.598ms (+0.000ms): scheduler_tick (update_process_times)
 49.598ms (+0.000ms): sched_clock (scheduler_tick)
 49.599ms (+0.000ms): task_timeslice (scheduler_tick)
 49.599ms (+0.000ms): update_wall_time (do_timer)
 49.599ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.599ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.599ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.599ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.599ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.599ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.600ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.601ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.602ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.603ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.603ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.603ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.603ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.603ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.603ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.603ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.603ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.603ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
 49.603ms (+0.000ms): generic_note_interrupt (do_IRQ)
 49.604ms (+0.000ms): end_edge_ioapic_irq (do_IRQ)
 49.604ms (+0.000ms): preempt_schedule (do_IRQ)
 49.604ms (+0.000ms): do_softirq (do_IRQ)
 49.604ms (+0.000ms): __do_softirq (do_softirq)
 49.604ms (+0.000ms): wake_up_process (do_softirq)
 49.604ms (+0.000ms): try_to_wake_up (wake_up_process)
 49.604ms (+0.000ms): task_rq_lock (try_to_wake_up)
 49.604ms (+0.000ms): activate_task (try_to_wake_up)
 49.604ms (+0.000ms): sched_clock (activate_task)
 49.604ms (+0.000ms): recalc_task_prio (activate_task)
 49.605ms (+0.000ms): effective_prio (recalc_task_prio)
 49.605ms (+0.000ms): enqueue_task (activate_task)
 49.605ms (+0.000ms): preempt_schedule (snd_pcm_action_lock_irq)
 49.605ms (+0.000ms): sub_preempt_count (sys_ioctl)


sound/pci/intel8x0.c contains:

                if (chip->device_type == DEVICE_NFORCE) {
                        /* reset to 2ch once to keep the 6 channel data in alignment,
                         * to start from Front Left always
                         */
                        iputdword(chip, ICHREG(GLOB_CNT), (cnt & 0xcfffff));
                        mdelay(50); /* grrr... */
                } else if (chip->device_type == DEVICE_INTEL_ICH4) {

So I guess it's that mdelay(50) which is generating these (each time the
device is opened).

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-19 at 15:30, Theodore Ts'o wrote:
> On Thu, Aug 19, 2004 at 07:19:58AM -0400, Lee Revell wrote:
> > > I doubt SHA_CODE_SIZE will make a sufficient difference to avoid the
> > > latency problems.  What we would need to do is to change the code so
> > > that the rekey operation in __check_and_rekey takes place in a
> > > workqueue.  Say, something like this (warning, I haven't tested this
> > > patch; if it breaks, you get to keep both pieces):
> > > 
> > 
> > Tested, works for me.  This should probably be pushed upstream, as well
> > as added to -P5, correct?  Is there any disadvantage to doing it this
> > way?
> 
> Great, I will be pushing this upstream very shortly.
> 

Hmm, turns out that this does not fix the problem:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/extract_entropy_latency_trace.txt

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-19 at 18:32, Lee Revell wrote:
> On Thu, 2004-08-19 at 15:30, Theodore Ts'o wrote:
> > On Thu, Aug 19, 2004 at 07:19:58AM -0400, Lee Revell wrote:
> > > > I doubt SHA_CODE_SIZE will make a sufficient difference to avoid the
> > > > latency problems.  What we would need to do is to change the code so
> > > > that the rekey operation in __check_and_rekey takes place in a
> > > > workqueue.  Say, something like this (warning, I haven't tested this
> > > > patch; if it breaks, you get to keep both pieces):
> > > > 
> > > 
> > > Tested, works for me.  This should probably be pushed upstream, as well
> > > as added to -P5, correct?  Is there any disadvantage to doing it this
> > > way?
> > 
> > Great, I will be pushing this upstream very shortly.
> > 
> 
> Hmm, turns out that this does not fix the problem:
> 
> http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/extract_entropy_latency_trace.txt
> 

Looking at the code, it's obvious why this doesn't fix the problem - you
are still doing 380 usecs (with SHA_CODE_SIZE 3) to 780 usecs (with
SHA_CODE_SIZE 0) worth of work inside the spinlock, it is just being
deferred via a workqueue, rather than executing right away.  So this
improves the average case, but does not help the worst case scenario at
all.  380 usecs is too long to hold a spinlock.

Lee

Re: [patch] Latency Tracer, voluntary-preempt-2.6.8-rc4-O6

[Lee Revell]

On Thu, 2004-08-19 at 18:32, Lee Revell wrote:
> On Thu, 2004-08-19 at 15:30, Theodore Ts'o wrote:
> > On Thu, Aug 19, 2004 at 07:19:58AM -0400, Lee Revell wrote:
> > > > I doubt SHA_CODE_SIZE will make a sufficient difference to avoid the
> > > > latency problems.  What we would need to do is to change the code so
> > > > that the rekey operation in __check_and_rekey takes place in a
> > > > workqueue.  Say, something like this (warning, I haven't tested this
> > > > patch; if it breaks, you get to keep both pieces):
> > > > 
> > > 
> > > Tested, works for me.  This should probably be pushed upstream, as well
> > > as added to -P5, correct?  Is there any disadvantage to doing it this
> > > way?
> > 
> > Great, I will be pushing this upstream very shortly.
> > 
> 
> Hmm, turns out that this does not fix the problem:
> 

Here is another new one:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/kswapd_refill_inactive_zone_latency_trace.txt

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Thu, 2004-08-19 at 03:32, Ingo Molnar wrote:
> i've uploaded the -P4 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4

I think I am seeing those weird ~1 ms latencies still (actually I am not
sure I saw before now):

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/kswapd_latency_trace.txt

This was the result of doing `make -j12' with a large C++ program (I
know, I wanted to see what would happen).  This gradually slowed the
machine to a crawl - for a while I could watch the /proc/latency_trace
fiels and see the latency gradually climbing.  Eventually tha machine
ground almost to a halt, with lots of disk activity., and trying to
switch from X to a console window resulted in a blank screen for 2-3
minutes.  I did eventually get a text console and was able to kill all
the cc1plus processes, and the machine went back to normal.

The link above is the highest latency recorded during this episode.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Thu, 2004-08-19 at 16:37, karl.vogel@seagha.com wrote:
> The following latency trace is generated each time the sound driver is opened
> by an application on my box.
> 

This is a pretty big trace.  Please try to trim these, especially if a
few lines repeat hundreds of times (common).

The comment seems to imply that the author didn't like the mdelay but it
didn't work otherwise.  What happens if you get rid of the mdelay?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4
> 
> I think I am seeing those weird ~1 ms latencies still (actually I am not
> sure I saw before now):
> 
> http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/kswapd_latency_trace.txt

this is a 9 msec latency in fact. I know what's going on - it's the
get_swap_page() locking - i looked at it once already but there's no
immediate silver arrow. As we allocate more and more swap entries the
longer the scan gets.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Fri, 2004-08-20 at 04:13, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P4
> > 
> > I think I am seeing those weird ~1 ms latencies still (actually I am not
> > sure I saw before now):
> > 
> > http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/kswapd_latency_trace.txt
> 
> this is a 9 msec latency in fact. I know what's going on - it's the
> get_swap_page() locking - i looked at it once already but there's no
> immediate silver arrow. As we allocate more and more swap entries the
> longer the scan gets.
> 

This is an extreme load situation, so I don't think it will be a
real-world problem.  I have not seen it under any normal workload.

What about this one:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/netif_receive_skb_latency_trace.txt

This appears during normal use.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> This is an extreme load situation, so I don't think it will be a
> real-world problem.  I have not seen it under any normal workload.

well, 9 msecs is still not nice. I've been able to trigger larger than
10msec latencies too on a 2 GHz box.

> What about this one:
> 
> http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/netif_receive_skb_latency_trace.txt
> 
> This appears during normal use.

hm, tcp_collapse() in net/ipv4/tcp_input.c. Could you try to just return
from that function?  Collapsing skbs of a given socket is not a
necessary functionality (it is only a 'nice' thing to have in OOM
situations) and it indeed can introduce quite high latencies.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Fri, 2004-08-20 at 06:27, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > This is an extreme load situation, so I don't think it will be a
> > real-world problem.  I have not seen it under any normal workload.
> 
> well, 9 msecs is still not nice. I've been able to trigger larger than
> 10msec latencies too on a 2 GHz box.
> 

If this is the case then should a make -j12 have ground the machine 
to a halt the way it did?

> hm, tcp_collapse() in net/ipv4/tcp_input.c. Could you try to just return
> from that function?  Collapsing skbs of a given socket is not a
> necessary functionality (it is only a 'nice' thing to have in OOM
> situations) and it indeed can introduce quite high latencies.
> 

OK, will test this.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > well, 9 msecs is still not nice. I've been able to trigger larger than
> > 10msec latencies too on a 2 GHz box.
> 
> If this is the case then should a make -j12 have ground the machine to
> a halt the way it did?

well, does it slow down that much with tracing and voluntary-preempt 
disabled too?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Fri, 2004-08-20 at 06:41, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > well, 9 msecs is still not nice. I've been able to trigger larger than
> > > 10msec latencies too on a 2 GHz box.
> > 
> > If this is the case then should a make -j12 have ground the machine to
> > a halt the way it did?
> 
> well, does it slow down that much with tracing and voluntary-preempt 
> disabled too?
> 

Yup. turns out, this is enough to use all available RAM and swap space
(1G total).  I probably went OOM.

Good stress test though.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Fri, 2004-08-20 at 06:27, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > This is an extreme load situation, so I don't think it will be a
> > real-world problem.  I have not seen it under any normal workload.
> 
> well, 9 msecs is still not nice. I've been able to trigger larger than
> 10msec latencies too on a 2 GHz box.
> 

Here is another one:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/cache_reap_latency_trace.txt

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P4

[Lee Revell]

On Fri, 2004-08-20 at 06:27, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > This is an extreme load situation, so I don't think it will be a
> > real-world problem.  I have not seen it under any normal workload.
> 
> well, 9 msecs is still not nice. I've been able to trigger larger than
> 10msec latencies too on a 2 GHz box.
> 
> > What about this one:
> > 
> > http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/netif_receive_skb_latency_trace.txt
> > 
> > This appears during normal use.
> 

Here is another:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P4#/var/www/2.6.8.1-P4/rt_run_flush_latency_trace.txt

Lee

[patch] voluntary-preempt-2.6.8.1-P5

[Ingo Molnar]

i've uploaded the -P5 patch:

  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P5

Changes since -P4:

 - increase PERCPU_ENOUGH_ROOM to avoid percpu overflow on SMP.
   (Mark H Johnson)

 - reduce ZAP_BLOCK_SIZE to 16 when vp != 0. This pushes the exit
   latency down to below 100 usecs on Lee Revell's box.

 - added a preempt_count field to /proc/latency_trace. This makes it
   easier to spot IRQ contexts and generally it gives a nice overview of
   how the preemption depth changes. It should also help us debug
   those 900usec weirdnesses related to cpu_idle. (if they still occur)

 - made the tcp packet-queue collapsing dependent on VOLUNTARY_PREEMPT.

 - fixed 10-20 msec latencies triggered by 'netstat', which occur when
   there are lots of sockets on a box.

 - rediffed against 2.6.8.1 for the patch to apply without fuzz

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P5

[K.R. Foley]

Ingo Molnar wrote:
 > i've uploaded the -P5 patch:
 >
 >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P5
 >
<snip>

I have been running the voluntary-preempt patches on one of my slower
(450) servers at home. The question is would latency traces from such a
slow system be useful to you in this testing. About the most load that
gets generated on this system, usually, is compiling the kernel or a
very large app. that I do development on. What I tend to do to load the
system is just run the stress-kernel suite and sometimes Andrew's amlat
program to provide RT scheduling pressure.

If this would be useful, let me know as I have an interest in seeing
latencies reduced as much as possible. I am building with the -P5 patch
right now. If it would be more useful to try this on a faster system or
to stess the system in a different manner, I could do that as well.

kr

[patch] intel8x0 latency fix

[karl.vogel]

karl.vogel@seagha.com writes:

> # lspci -s 00:06.0
> 00:06.0 Multimedia audio controller: nVidia Corporation nForce3 Audio (rev a2)
>
>  latency: 50752 us, entries: 267 (267)


Following patch fixes it for 2 channel devices (like my notebook). Although I'm
not sure if this is all that useful... if there are workloads other than audio
work that need low latency, then this might be useful.

Without this patch, my machine was generating the latency each time artsd (KDE)
re-opened the audio device.


--- a/sound/pci/intel8x0.c	2004-08-20 18:05:46.006717144 +0200
+++ b/sound/pci/intel8x0.c	2004-08-20 17:52:02.759869688 +0200
@@ -1021,8 +1021,10 @@
 			/* reset to 2ch once to keep the 6 channel data in alignment,
 			 * to start from Front Left always
 			 */
-			iputdword(chip, ICHREG(GLOB_CNT), (cnt & 0xcfffff));
-			mdelay(50); /* grrr... */
+			if (chip->multi4 || chip->multi6) {
+				iputdword(chip, ICHREG(GLOB_CNT), (cnt & 0xcfffff));
+				mdelay(50); /* grrr... */
+			}
 		} else if (chip->device_type == DEVICE_INTEL_ICH4) {
 			if (sample_bits > 16)
 				cnt |= ICH_PCM_20BIT;

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Ingo Molnar]

* K.R. Foley <kr@cybsft.com> wrote:

> Ingo Molnar wrote:
> > i've uploaded the -P5 patch:
> >
> >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P5
> >
> <snip>
> 
> I have been running the voluntary-preempt patches on one of my slower
> (450) servers at home. The question is would latency traces from such
> a slow system be useful to you in this testing. About the most load
> that gets generated on this system, usually, is compiling the kernel
> or a very large app. that I do development on. What I tend to do to
> load the system is just run the stress-kernel suite and sometimes
> Andrew's amlat program to provide RT scheduling pressure.
> 
> If this would be useful, let me know as I have an interest in seeing
> latencies reduced as much as possible. I am building with the -P5
> patch right now. If it would be more useful to try this on a faster
> system or to stess the system in a different manner, I could do that
> as well.

sure, all traces are interesting! On your system the latencies will be a
bit larger than on others, but that's all - the maximum latency points
should be the same (or similar) critical sections. So it's just as
useful as the other traces.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P5

[K.R. Foley]

Ingo Molnar wrote:
 > i've uploaded the -P5 patch:
 >
 >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P5
 >

Here is a latency trace generated by the NFS compile in the
stress-kernel. It is ~1.7 ms which doesn't really surprise me for the
NFS compile. I am going to take the NFS compile out of the test now and
see what else falls out.

If it would be useful for me to provide additional information with
this, please let me know.

http://www.cybsft.com/testresults/2.6.8.1-P5/latency_trace1.txt

kr

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Ingo Molnar]

* K.R. Foley <kr@cybsft.com> wrote:

> Here is a latency trace generated by the NFS compile in the
> stress-kernel. It is ~1.7 ms which doesn't really surprise me for the
> NFS compile. I am going to take the NFS compile out of the test now
> and see what else falls out.

NFS uses lock_kernel() for most of its locking. This means that the
sharing and locking rules and assumptions are not clear (it's all within
this opaque lock). I tried to figure out whether it would be safe to do
a lock-break in nfs_cached_lookup() but it's quite hard ... Please bug
the NFS-client maintainer(s) about this. To break this latency we'd have
to do a lock-break in fs/nfs/dir.c:nfs_cached_lookup()'s main loop:

        for(;(page = find_get_page(dir->i_mapping, desc.page_index)); desc.page_index++) {

                res = -EIO;

you could try this at the end of the loop:

                if (res == 0)
                        goto out_found;
                if (res != -EAGAIN)
                        break;
+               cond_resched();
        }

but i cannot guarantee that this is correct ... It might seem to work 
now but break later under load, etc. It's the NFS maintainer's call.

> If it would be useful for me to provide additional information with
> this, please let me know.

reports like this are perfect for most latencies. I tried to include
everything that is needed normally in /proc/latency_trace itself.

	Ingo

[patch] voluntary-preempt-2.6.8.1-P6

[Ingo Molnar]

i've uploaded the -P6 patch:

  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P6

i'm releasing another patch today because the mystic 900 usec latency
seems to be nailed finally, with the help of Mark H Johnson's traces:
there were two places that set ->preempt_count back to 0 without the
tracer noticing it: preempt_schedule() and entry.S's return path.

other changes since -P5:

 - generic kernel fix: do not ignore idle=poll on non-P4 CPUs.
   (Mark H Johnson)

 - fix the __trace link bug reported by Martin Rumori. All combinations
   of CONFIG_PREEMPT_TIMING & CONFIG_LATENCY_TRACE should work now.

 - make kernel_preemption=1 the default. Most people use this anyway.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Lee Revell]

On Fri, 2004-08-20 at 14:37, K.R. Foley wrote:
> Ingo Molnar wrote:
>  > i've uploaded the -P5 patch:
>  >
>  >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P5
>  >
> 
> Here is a latency trace generated by the NFS compile in the
> stress-kernel. It is ~1.7 ms which doesn't really surprise me for the
> NFS compile. I am going to take the NFS compile out of the test now and
> see what else falls out.
> 
> If it would be useful for me to provide additional information with
> this, please let me know.
> 
> http://www.cybsft.com/testresults/2.6.8.1-P5/latency_trace1.txt
> 

Ugh, XDR rears its ugly head...

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P6

[Lee Revell]

On Fri, 2004-08-20 at 15:55, Ingo Molnar wrote:
> i've uploaded the -P6 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P6
> 

Traces here (I have given up on giving them descriptive names).  Max is
about 120 usec:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Linh Dang]

K. R. Foley <kr@cybsft.com> wrote:

> Ingo Molnar wrote:
>> i've uploaded the -P5 patch:
>>
>> http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P5
>>
>
> Here is a latency trace generated by the NFS compile in the
> stress-kernel. It is ~1.7 ms which doesn't really surprise me for
> the NFS compile. I am going to take the NFS compile out of the test
> now and see what else falls out.
>
> If it would be useful for me to provide additional information with
> this, please let me know.
>
> http://www.cybsft.com/testresults/2.6.8.1-P5/latency_trace1.txt


You don't need to do a NFS compilation to trigger it. just to a "find"
on a huge NFS-mounted tree and you will see it.

Another candidate for cond_resched() is the loop in nfs_readdir(). It
seems safe to do so be cause it calls readdir_search_pagecache() which
would call schedule().


Regards
-- 
Linh Dang

Re: [patch] voluntary-preempt-2.6.8.1-P6

[Lee Revell]

On Fri, 2004-08-20 at 15:55, Ingo Molnar wrote:
> i've uploaded the -P6 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P6
> 

What is the number in the left column?

04000000 0.000ms (+0.000ms): preempt_schedule (unix_stream_sendmsg)
04000000 0.000ms (+0.000ms): schedule (preempt_schedule)
04000001 0.001ms (+0.000ms): sched_clock (schedule)
04000002 0.002ms (+0.001ms): dequeue_task (schedule)

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Lee Revell]

On Fri, 2004-08-20 at 09:30, Ingo Molnar wrote:
> i've uploaded the -P5 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P5

>  - added a preempt_count field to /proc/latency_trace. This makes it
>    easier to spot IRQ contexts and generally it gives a nice overview of
>    how the preemption depth changes. It should also help us debug
>    those 900usec weirdnesses related to cpu_idle. (if they still occur)
> 

Ok, disregard my question about what this field was.

>  - made the tcp packet-queue collapsing dependent on VOLUNTARY_PREEMPT.
> 

I am seeing ~180usec latencies related to netif_receive_skb:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6#/var/www/2.6.8.1-P6/trace5.txt

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Lee Revell]

On Fri, 2004-08-20 at 09:30, Ingo Molnar wrote:

>  - reduce ZAP_BLOCK_SIZE to 16 when vp != 0. This pushes the exit
>    latency down to below 100 usecs on Lee Revell's box.
> 

Almost:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6#/var/www/2.6.8.1-P6/trace8.txt

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P6

[Lee Revell]

On Fri, 2004-08-20 at 15:55, Ingo Molnar wrote:
> i've uploaded the -P6 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P6
> 

Here's a 171 usec latency from ext3_free_blocks:

preemption latency trace v1.0.1
-------------------------------
 latency: 171 us, entries: 2 (2)
    -----------------
    | task: evolution/863, uid:1000 nice:0 policy:0 rt_prio:0
    -----------------
 => started at: ext3_free_blocks+0x1d0/0x4b0
 => ended at:   ext3_free_blocks+0x229/0x4b0
=======>
00000001 0.000ms (+0.000ms): ext3_free_blocks (ext3_free_data)
00000001 0.167ms (+0.167ms): sub_preempt_count (ext3_free_blocks)

Also I have noticed a pattern with the XFree86 schedule() latencies,
they all have a section like this:

04000002 0.003ms (+0.000ms): effective_prio (recalc_task_prio)
04000002 0.003ms (+0.000ms): enqueue_task (schedule)
00000002 0.006ms (+0.003ms): __switch_to (schedule)
00000002 0.088ms (+0.082ms): finish_task_switch (schedule)
00010002 0.090ms (+0.001ms): do_IRQ (finish_task_switch)
00010003 0.091ms (+0.000ms): mask_and_ack_8259A (do_IRQ)

04000002 0.002ms (+0.000ms): effective_prio (recalc_task_prio)
04000002 0.002ms (+0.000ms): enqueue_task (schedule)
00000002 0.005ms (+0.002ms): __switch_to (schedule)
00000002 0.067ms (+0.062ms): finish_task_switch (schedule)
00010002 0.068ms (+0.001ms): do_IRQ (finish_task_switch)
00010003 0.069ms (+0.000ms): mask_and_ack_8259A (do_IRQ)

I presume the 04000002 -> 00000002 is some interrupt being unmasked (or
interrupts being globally enabled), then there's a 60-80 usec latency in
schedule().

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P6

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Also I have noticed a pattern with the XFree86 schedule() latencies,
> they all have a section like this:
> 
> 04000002 0.003ms (+0.000ms): effective_prio (recalc_task_prio)
> 04000002 0.003ms (+0.000ms): enqueue_task (schedule)
> 00000002 0.006ms (+0.003ms): __switch_to (schedule)
> 00000002 0.088ms (+0.082ms): finish_task_switch (schedule)
> 00010002 0.090ms (+0.001ms): do_IRQ (finish_task_switch)

> I presume the 04000002 -> 00000002 is some interrupt being unmasked
> (or interrupts being globally enabled), then there's a 60-80 usec
> latency in schedule().

0x04000000 is PREEMPT_ACTIVE - which is just a bit we set to make sure
we dont try to preempt recursively.

but the XFree86 latency is interesting indeed. It could be the effect of
the now-enlarged ioperm() bitmap! 80 usecs is excessive.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> On Fri, 2004-08-20 at 09:30, Ingo Molnar wrote:
> 
> >  - reduce ZAP_BLOCK_SIZE to 16 when vp != 0. This pushes the exit
> >    latency down to below 100 usecs on Lee Revell's box.
> > 
> 
> Almost:
> 
> http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6#/var/www/2.6.8.1-P6/trace8.txt

i suspect if you turn off tracing (or disable it in the kernel
completely) then you'd see below-100 usec latencies here. A trace entry
costs ~0.3 usecs on your box, so this 50-entry trace has roughly 15
usecs of tracing overhead :-)

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Lee Revell]

On Sat, 2004-08-21 at 05:13, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > On Fri, 2004-08-20 at 09:30, Ingo Molnar wrote:
> > 
> > >  - reduce ZAP_BLOCK_SIZE to 16 when vp != 0. This pushes the exit
> > >    latency down to below 100 usecs on Lee Revell's box.
> > > 
> > 
> > Almost:
> > 
> > http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6#/var/www/2.6.8.1-P6/trace8.txt
> 
> i suspect if you turn off tracing (or disable it in the kernel
> completely) then you'd see below-100 usec latencies here. A trace entry
> costs ~0.3 usecs on your box, so this 50-entry trace has roughly 15
> usecs of tracing overhead :-)
> 

FWIW, I did see one of these over 200 usecs.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > > http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6#/var/www/2.6.8.1-P6/trace8.txt
> > 
> > i suspect if you turn off tracing (or disable it in the kernel
> > completely) then you'd see below-100 usec latencies here. A trace entry
> > costs ~0.3 usecs on your box, so this 50-entry trace has roughly 15
> > usecs of tracing overhead :-)
> > 
> 
> FWIW, I did see one of these over 200 usecs.

which trace is this?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Lee Revell]

On Sat, 2004-08-21 at 05:18, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > > http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6#/var/www/2.6.8.1-P6/trace8.txt
> > > 
> > > i suspect if you turn off tracing (or disable it in the kernel
> > > completely) then you'd see below-100 usec latencies here. A trace entry
> > > costs ~0.3 usecs on your box, so this 50-entry trace has roughly 15
> > > usecs of tracing overhead :-)
> > > 
> > 
> > FWIW, I did see one of these over 200 usecs.
> 
> which trace is this?
> 

I don't think I was able to save this one.  I will post it if I see it
again.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Lee Revell]

On Sat, 2004-08-21 at 05:18, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > > http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6#/var/www/2.6.8.1-P6/trace8.txt
> > > 
> > > i suspect if you turn off tracing (or disable it in the kernel
> > > completely) then you'd see below-100 usec latencies here. A trace entry
> > > costs ~0.3 usecs on your box, so this 50-entry trace has roughly 15
> > > usecs of tracing overhead :-)
> > > 
> > 
> > FWIW, I did see one of these over 200 usecs.
> 
> which trace is this?
> 

Sorry, I was thinking of this one:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6#/var/www/2.6.8.1-P6/trace10.txt

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P6

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Here's a 171 usec latency from ext3_free_blocks:
> 
> preemption latency trace v1.0.1
> -------------------------------
>  latency: 171 us, entries: 2 (2)
>     -----------------
>     | task: evolution/863, uid:1000 nice:0 policy:0 rt_prio:0
>     -----------------
>  => started at: ext3_free_blocks+0x1d0/0x4b0
>  => ended at:   ext3_free_blocks+0x229/0x4b0
> =======>
> 00000001 0.000ms (+0.000ms): ext3_free_blocks (ext3_free_data)
> 00000001 0.167ms (+0.167ms): sub_preempt_count (ext3_free_blocks)

ok, i broke this lock in my tree, will show up in -P7.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> Sorry, I was thinking of this one:
> 
> http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6#/var/www/2.6.8.1-P6/trace10.txt

ok, reduced the zap block size to 8 pages a time. It should not be a big
issue to have this at a low value because we dont retry anything and we
do a full TLB flush only once. (subsequent flushes are caught by the
tlb->need_flush optimization.)

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Lee Revell]

On Sat, 2004-08-21 at 05:31, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > Sorry, I was thinking of this one:
> > 
> > http://krustophenia.net/testresults.php?dataset=2.6.8.1-P6#/var/www/2.6.8.1-P6/trace10.txt
> 
> ok, reduced the zap block size to 8 pages a time. It should not be a big
> issue to have this at a low value because we dont retry anything and we
> do a full TLB flush only once. (subsequent flushes are caught by the
> tlb->need_flush optimization.)
> 

This one is interesting.  What is going on here?

preemption latency trace v1.0.1
-------------------------------
 latency: 146 us, entries: 45 (45)
    -----------------
    | task: pdflush/33, uid:0 nice:0 policy:0 rt_prio:0
    -----------------
 => started at: find_get_pages_tag+0x19/0x90
 => ended at:   find_get_pages_tag+0x61/0x90
=======>
00000001 0.000ms (+0.000ms): find_get_pages_tag (pagevec_lookup_tag)
00000001 0.000ms (+0.000ms): radix_tree_gang_lookup_tag (find_get_pages_tag)
00000001 0.001ms (+0.000ms): __lookup_tag (radix_tree_gang_lookup_tag)
00000001 0.007ms (+0.005ms): __lookup_tag (radix_tree_gang_lookup_tag)

[ 20-30 more of these ]

00000001 0.119ms (+0.003ms): __lookup_tag (radix_tree_gang_lookup_tag)
00000001 0.125ms (+0.005ms): __lookup_tag (radix_tree_gang_lookup_tag)
00010001 0.132ms (+0.007ms): do_IRQ (find_get_pages_tag)
00010002 0.133ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
00010002 0.137ms (+0.004ms): generic_redirect_hardirq (do_IRQ)
00010002 0.138ms (+0.000ms): wake_up_process (generic_redirect_hardirq)
00010002 0.138ms (+0.000ms): try_to_wake_up (wake_up_process)
00010002 0.138ms (+0.000ms): task_rq_lock (try_to_wake_up)
00010003 0.139ms (+0.000ms): activate_task (try_to_wake_up)
00010003 0.139ms (+0.000ms): sched_clock (activate_task)
00010003 0.140ms (+0.000ms): recalc_task_prio (activate_task)
00010003 0.141ms (+0.000ms): effective_prio (recalc_task_prio)
00010003 0.141ms (+0.000ms): enqueue_task (activate_task)
00010002 0.141ms (+0.000ms): preempt_schedule (try_to_wake_up)
00010001 0.142ms (+0.000ms): preempt_schedule (do_IRQ)
00000001 0.143ms (+0.001ms): sub_preempt_count (find_get_pages_tag)

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P6

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> but the XFree86 latency is interesting indeed. It could be the effect
> of the now-enlarged ioperm() bitmap! 80 usecs is excessive.

yeah, that's the reason. The quick fix is to reduce IO_BITMAP_BITS to
1024 in include/asm-i386/processor.h. I'm working on a real fix.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> 
> This one is interesting.  What is going on here?
> 
> preemption latency trace v1.0.1
> -------------------------------
>  latency: 146 us, entries: 45 (45)
>     -----------------
>     | task: pdflush/33, uid:0 nice:0 policy:0 rt_prio:0
>     -----------------
>  => started at: find_get_pages_tag+0x19/0x90
>  => ended at:   find_get_pages_tag+0x61/0x90
> =======>
> 00000001 0.000ms (+0.000ms): find_get_pages_tag (pagevec_lookup_tag)
> 00000001 0.000ms (+0.000ms): radix_tree_gang_lookup_tag (find_get_pages_tag)
> 00000001 0.001ms (+0.000ms): __lookup_tag (radix_tree_gang_lookup_tag)
> 00000001 0.007ms (+0.005ms): __lookup_tag (radix_tree_gang_lookup_tag)
> 
> [ 20-30 more of these ]

this is the pagevec code, triggering the radix tree multi-entry-lookup
code. Could you try to reduce PAGEVEC_SIZE from 16 to 8 (or 4) in
include/linux/pagevec.h - does this reduce these particular latencies?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P5

[Karl Vogel]

I'm regularly seeing acpi_ec_read latencies. (e.g. when closing the lid of
my notebook, when the processor cooler activates, ...)

I'm guessing there isn't much that can be done about this, right?!


preemption latency trace v1.0.1
-------------------------------
 latency: 1151 us, entries: 94 (94)
    -----------------
    | task: kacpid/6, uid:0 nice:-10 policy:0 rt_prio:0
    -----------------
 => started at: acpi_ec_read+0x6b/0xf7
 => ended at:   acpi_ec_read+0xc8/0xf7
=======>
00000001 0.000ms (+0.000ms): acpi_ec_read (acpi_ec_space_handler)
00000001 0.000ms (+0.000ms): acpi_hw_low_level_write (acpi_ec_read)
00000001 0.000ms (+0.000ms): acpi_os_write_port (acpi_hw_low_level_write)
00000001 0.001ms (+0.001ms): acpi_ec_wait (acpi_ec_read)
00000001 0.001ms (+0.000ms): acpi_hw_low_level_read (acpi_ec_wait)
00000001 0.001ms (+0.000ms): acpi_os_read_port (acpi_hw_low_level_read)
00000001 0.003ms (+0.001ms): __const_udelay (acpi_ec_wait)
00000001 0.003ms (+0.000ms): __delay (acpi_ec_wait)
00000001 0.003ms (+0.000ms): delay_tsc (__delay)
[...]
00000001 1.113ms (+0.001ms): smp_apic_timer_interrupt (acpi_ec_read)
00010001 1.113ms (+0.000ms): profile_hook (smp_apic_timer_interrupt)
00010002 1.113ms (+0.000ms): notifier_call_chain (profile_hook)
00000002 1.113ms (+0.000ms): do_softirq (smp_apic_timer_interrupt)
00000100 1.113ms (+0.000ms): __do_softirq (do_softirq)
00010001 1.113ms (+0.000ms): do_IRQ (acpi_ec_read)
00010002 1.113ms (+0.000ms): ack_edge_ioapic_irq (do_IRQ)
00010002 1.113ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
00010000 1.113ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
00010000 1.113ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
00010001 1.113ms (+0.000ms): mark_offset_tsc (timer_interrupt)
00010001 1.123ms (+0.009ms): do_timer (timer_interrupt)
00010001 1.123ms (+0.000ms): update_process_times (do_timer)
00010001 1.123ms (+0.000ms): update_one_process (update_process_times)
00010001 1.123ms (+0.000ms): run_local_timers (update_process_times)
00010001 1.123ms (+0.000ms): raise_softirq (update_process_times)
00010001 1.123ms (+0.000ms): scheduler_tick (update_process_times)
00010001 1.123ms (+0.000ms): sched_clock (scheduler_tick)
00010002 1.123ms (+0.000ms): task_timeslice (scheduler_tick)
00010001 1.123ms (+0.000ms): update_wall_time (do_timer)
00010001 1.123ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
00010002 1.123ms (+0.000ms): generic_note_interrupt (do_IRQ)
00010002 1.123ms (+0.000ms): end_edge_ioapic_irq (do_IRQ)
00000002 1.123ms (+0.000ms): do_softirq (do_IRQ)
00000100 1.123ms (+0.000ms): __do_softirq (do_softirq)
00000001 1.124ms (+0.000ms): sub_preempt_count (acpi_ec_read)

Full trace output available on:
 http://users.telenet.be/kvogel/ec-latency.txt

Culprit is the following code in drivers/acpi/ec.c
function acpi_ec_wait (edited for mail):

----
#define ACPI_EC_UDELAY          100     /* Poll @ 100us increments */
#define ACPI_EC_UDELAY_COUNT    1000    /* Wait 10ms max. during EC ops */

u32                     i = ACPI_EC_UDELAY_COUNT;

do {
    acpi_hw_low_level_read(8, &acpi_ec_status, &ec->status_addr);
    if (acpi_ec_status & ACPI_EC_FLAG_OBF)
            return 0;
    udelay(ACPI_EC_UDELAY);
} while (--i>0);
----

NOTE: isn't the comment on the ACPI_EC_UDELAY_COUNT wrong?! Or is my
arithmetic not what it used to be. (1000 loops of 100us = 100ms instead of
10ms, no?)

Re: [patch] voluntary-preempt-2.6.8.1-P6

[Karl Vogel]

preemption latency trace v1.0.1
-------------------------------
 latency: 408 us, entries: 2273 (2273)
    -----------------
    | task: ksoftirqd/0/2, uid:0 nice:-10 policy:0 rt_prio:0
    -----------------
 => started at: netif_receive_skb+0x74/0x1f0
 => ended at:   netif_receive_skb+0x171/0x1f0
=======>

This one is hard to trim to post here, so full trace is posted on:
  http://users.telenet.be/kvogel/network.txt

.config at:
  http://users.telenet.be/kvogel/config.txt

[patch] voluntary-preempt-2.6.8.1-P7

[Ingo Molnar]

i've uploaded the -P7 patch:

  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P7

Changes since -P6:

- fixed the XFree86/X.org context-switch latency. (let me know if you
  see any weirdness like X not starting up while it did before.)

- halved the pagevec size, to reduce the radix gang-lookup costs.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P6 -- False positive?!

[Karl Vogel]

I'm not sure if the following is a false positive or not:

preemption latency trace v1.0.1
-------------------------------
 latency: 3022 us, entries: 15 (15)
    -----------------
    | task: swapper/0, uid:0 nice:0 policy:0 rt_prio:0
    -----------------
 => started at: do_IRQ+0x27/0x230
 => ended at:   do_IRQ+0x1a3/0x230
=======>
00010000 0.000ms (+0.000ms): do_IRQ (common_interrupt)
00010000 0.000ms (+0.000ms): do_IRQ (acpi_processor_idle)
00010001 0.000ms (+0.000ms): ack_edge_ioapic_irq (do_IRQ)
00010001 0.000ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
00010001 2.953ms (+2.952ms): wake_up_process (generic_redirect_hardirq)
00010001 2.953ms (+0.000ms): try_to_wake_up (wake_up_process)
00010001 2.953ms (+0.000ms): task_rq_lock (try_to_wake_up)
00010002 2.953ms (+0.000ms): activate_task (try_to_wake_up)
00010002 2.953ms (+0.000ms): sched_clock (activate_task)
00010002 2.953ms (+0.000ms): recalc_task_prio (activate_task)
00010002 2.953ms (+0.000ms): effective_prio (recalc_task_prio)
00010002 2.953ms (+0.000ms): enqueue_task (activate_task)
00010001 2.954ms (+0.000ms): preempt_schedule (try_to_wake_up)
00010000 2.954ms (+0.000ms): preempt_schedule (do_IRQ)
00000001 2.954ms (+0.000ms): sub_preempt_count (do_IRQ)

The following looks similar:

 latency: 517 us, entries: 29 (29)
    -----------------
    | task: swapper/0, uid:0 nice:0 policy:0 rt_prio:0
    -----------------
 => started at: do_IRQ+0x27/0x230
 => ended at:   do_IRQ+0x1a3/0x230
=======>
00010000 0.000ms (+0.000ms): do_IRQ (common_interrupt)
00010000 0.000ms (+0.000ms): do_IRQ (acpi_processor_idle)
00010001 0.000ms (+0.000ms): ack_edge_ioapic_irq (do_IRQ)
00010001 0.000ms (+0.000ms): generic_redirect_hardirq (do_IRQ)
00010000 0.000ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
00010000 0.000ms (+0.000ms): timer_interrupt (generic_handle_IRQ_event)
00010001 0.000ms (+0.000ms): mark_offset_tsc (timer_interrupt)
00010001 0.009ms (+0.008ms): do_timer (timer_interrupt)
00010001 0.503ms (+0.493ms): update_process_times (do_timer)
00010001 0.503ms (+0.000ms): update_one_process (update_process_times)
00010001 0.503ms (+0.000ms): run_local_timers (update_process_times)
00010001 0.503ms (+0.000ms): raise_softirq (update_process_times)
00010001 0.503ms (+0.000ms): scheduler_tick (update_process_times)
00010001 0.504ms (+0.000ms): sched_clock (scheduler_tick)
00010001 0.504ms (+0.000ms): update_wall_time (do_timer)
00010001 0.504ms (+0.000ms): update_wall_time_one_tick (update_wall_time)
00010001 0.504ms (+0.000ms): generic_note_interrupt (do_IRQ)
00010001 0.504ms (+0.000ms): end_edge_ioapic_irq (do_IRQ)
00000001 0.504ms (+0.000ms): do_softirq (do_IRQ)
00000100 0.505ms (+0.000ms): __do_softirq (do_softirq)
00000100 0.505ms (+0.000ms): wake_up_process (do_softirq)
00000100 0.505ms (+0.000ms): try_to_wake_up (wake_up_process)
00000100 0.505ms (+0.000ms): task_rq_lock (try_to_wake_up)
00000101 0.505ms (+0.000ms): activate_task (try_to_wake_up)
00000101 0.505ms (+0.000ms): sched_clock (activate_task)
00000101 0.505ms (+0.000ms): recalc_task_prio (activate_task)
00000101 0.505ms (+0.000ms): effective_prio (recalc_task_prio)
00000101 0.505ms (+0.000ms): enqueue_task (activate_task)
00000001 0.506ms (+0.000ms): sub_preempt_count (do_IRQ)

Re: [patch] voluntary-preempt-2.6.8.1-P6 -- False positive?!

[Karl Vogel]

I think this may have been my own mistake. In between all the kernel
compiles, I seem to have managed to do the following:

# CONFIG_X86_PM_TIMER is not set

Which was probably the cause. (was getting other side effects too, 
like keys stuck in repeat, 1-2 second freezes etc etc)

Re: [patch] voluntary-preempt-2.6.8.1-P7

[Lee Revell]

On Sat, 2004-08-21 at 10:05, Ingo Molnar wrote:
> i've uploaded the -P7 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P7
> 
> Changes since -P6:
> 
> - fixed the XFree86/X.org context-switch latency. (let me know if you
>   see any weirdness like X not starting up while it did before.)
> 
> - halved the pagevec size, to reduce the radix gang-lookup costs.
> 

Great, this is a significant improvement.  Most of the worst case
latencies (~150 usec) seem related to the TCP stack now, and a minor one
(51 usec) in the ext3 journaling:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P7

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P7

[K.R. Foley]

Lee Revell wrote:
 > On Sat, 2004-08-21 at 10:05, Ingo Molnar wrote:
 >
 >>i've uploaded the -P7 patch:
 >>
 >>  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P7
 >>
 >>Changes since -P6:
 >>
 >>- fixed the XFree86/X.org context-switch latency. (let me know if you
 >>  see any weirdness like X not starting up while it did before.)
 >>
 >>- halved the pagevec size, to reduce the radix gang-lookup costs.
 >>
 >
 >
 > Great, this is a significant improvement.  Most of the worst case
 > latencies (~150 usec) seem related to the TCP stack now, and a minor one
 > (51 usec) in the ext3 journaling:
 >
 > http://krustophenia.net/testresults.php?dataset=2.6.8.1-P7
 >
 > Lee
 >

I just posted a similar trace of ~4141 usec from P6 here:

http://www.cybsft.com/testresults/2.6.8.1-P6/latency-trace1.txt

This was part of a run from yesterday evening. After just rebooting, I
too am seeing several of these. With the system not being hammered by
the stress tests though, the max is only up to ~103 usec. I will be
updating to P7 shortly.

kr

Re: [patch] voluntary-preempt-2.6.8.1-P7

[Lee Revell]

On Sat, 2004-08-21 at 20:06, K.R. Foley wrote:
> I just posted a similar trace of ~4141 usec from P6 here:
> 
> http://www.cybsft.com/testresults/2.6.8.1-P6/latency-trace1.txt
> 

This looks wrong:

00000003 0.008ms (+0.001ms): dummy_socket_sock_rcv_skb (tcp_v4_rcv)
00000004 0.008ms (+0.000ms): tcp_v4_do_rcv (tcp_v4_rcv)
00000004 0.009ms (+0.000ms): tcp_rcv_established (tcp_v4_do_rcv)
00010004 3.998ms (+3.989ms): do_IRQ (tcp_rcv_established)
00010005 3.999ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
00010005 4.001ms (+0.002ms): generic_redirect_hardirq (do_IRQ)
00010004 4.002ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)

Probably a false positive, Ingo would know better.  What kind of stress
testing were you doing?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P7

[K.R. Foley]

Lee Revell wrote:
 > On Sat, 2004-08-21 at 20:06, K.R. Foley wrote:
 >
 >>I just posted a similar trace of ~4141 usec from P6 here:
 >>
 >>http://www.cybsft.com/testresults/2.6.8.1-P6/latency-trace1.txt
 >>
 >
 >
 > This looks wrong:
 >
 > 00000003 0.008ms (+0.001ms): dummy_socket_sock_rcv_skb (tcp_v4_rcv)
 > 00000004 0.008ms (+0.000ms): tcp_v4_do_rcv (tcp_v4_rcv)
 > 00000004 0.009ms (+0.000ms): tcp_rcv_established (tcp_v4_do_rcv)
 > 00010004 3.998ms (+3.989ms): do_IRQ (tcp_rcv_established)
 > 00010005 3.999ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 > 00010005 4.001ms (+0.002ms): generic_redirect_hardirq (do_IRQ)
 > 00010004 4.002ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 >
 > Probably a false positive, Ingo would know better.  What kind of stress
 > testing were you doing?
 >
 > Lee
 >

This is while running the stress-kernel suite. I don't know about it
being a false possitive, it very well may be. Looking back through the
logs though I am not sure this is a valid latency anyway. This trace was
from 06:37 this morning. About 19:54 last night I got an oops in kswapd
and this morning around 10:30ish I had stuff getting killed right and
left by oom. So, I am thinking that this probably is not a very reliable
test.

kr

Re: [patch] voluntary-preempt-2.6.8.1-P7

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> On Sat, 2004-08-21 at 20:06, K.R. Foley wrote:
> > I just posted a similar trace of ~4141 usec from P6 here:
> > 
> > http://www.cybsft.com/testresults/2.6.8.1-P6/latency-trace1.txt
> > 
> 
> This looks wrong:
> 
> 00000003 0.008ms (+0.001ms): dummy_socket_sock_rcv_skb (tcp_v4_rcv)
> 00000004 0.008ms (+0.000ms): tcp_v4_do_rcv (tcp_v4_rcv)
> 00000004 0.009ms (+0.000ms): tcp_rcv_established (tcp_v4_do_rcv)
> 00010004 3.998ms (+3.989ms): do_IRQ (tcp_rcv_established)
> 00010005 3.999ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
> 00010005 4.001ms (+0.002ms): generic_redirect_hardirq (do_IRQ)
> 00010004 4.002ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
> 
> Probably a false positive, Ingo would know better.  What kind of
> stress testing were you doing?

indeed this looks suspect. Is this an SMP system?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P7

[Lee Revell]

On Sat, 2004-08-21 at 10:05, Ingo Molnar wrote:
> i've uploaded the -P7 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P7

Here's a pretty bad one, over 500 usecs in rt_garbage_collect:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P7#/var/www/2.6.8.1-P7/trace7.txt

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P7

[K.R. Foley]

Ingo Molnar wrote:
 > * Lee Revell <rlrevell@joe-job.com> wrote:
 >
 >
 >>On Sat, 2004-08-21 at 20:06, K.R. Foley wrote:
 >>
 >>>I just posted a similar trace of ~4141 usec from P6 here:
 >>>
 >>>http://www.cybsft.com/testresults/2.6.8.1-P6/latency-trace1.txt
 >>>
 >>
 >>This looks wrong:
 >>
 >>00000003 0.008ms (+0.001ms): dummy_socket_sock_rcv_skb (tcp_v4_rcv)
 >>00000004 0.008ms (+0.000ms): tcp_v4_do_rcv (tcp_v4_rcv)
 >>00000004 0.009ms (+0.000ms): tcp_rcv_established (tcp_v4_do_rcv)
 >>00010004 3.998ms (+3.989ms): do_IRQ (tcp_rcv_established)
 >>00010005 3.999ms (+0.000ms): mask_and_ack_8259A (do_IRQ)
 >>00010005 4.001ms (+0.002ms): generic_redirect_hardirq (do_IRQ)
 >>00010004 4.002ms (+0.000ms): generic_handle_IRQ_event (do_IRQ)
 >>
 >>Probably a false positive, Ingo would know better.  What kind of
 >>stress testing were you doing?
 >
 >
 > indeed this looks suspect. Is this an SMP system?
 >
 > 	Ingo
 >

Actually no. It is an SMP ready system, but with a single PII 450. As I
responded to Lee's response, I am not sure that I completely trust the
results of this trace anyway.

I would like to know why you guys think this may be a false positive. Is
it just the extremely long latency? Or is there something else that
makes it look suspect?

By the way I just posted two more traces, one that I caught last night
and one from this morning:

This is another one similar to the last but much more reasonably latency:

http://www.cybsft.com/testresults/2.6.8.1-P7/2.6.8.1-P7-1.txt

And this one this morning appears to be from updatedb running, while the
  tests were running. It's worth noting that this one appears to have
happened about the same time today that the other ~4100+ one happened
yesterday. Also worth noting is that the system was probably swapping
pretty good when this occurred.

http://www.cybsft.com/testresults/2.6.8.1-P7/2.6.8.1-P7-2.txt

kr

Re: [patch] voluntary-preempt-2.6.8.1-P7

[R. J. Wysocki]

[-- Attachment #1: Type: text/plain, Size: 1469 bytes --]

On Saturday 21 of August 2004 16:05, Ingo Molnar wrote:
> i've uploaded the -P7 patch:
>
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P7
>
> Changes since -P6:
>
> - fixed the XFree86/X.org context-switch latency. (let me know if you
>   see any weirdness like X not starting up while it did before.)
>
> - halved the pagevec size, to reduce the radix gang-lookup costs.

It does not compile on x86-64.  I get this:

  CC      arch/x86_64/ia32/syscall32.o
kernel/hardirq.c: In function `recalculate_desc_flags':
kernel/hardirq.c:309: error: `SA_NODELAY' undeclared (first use in this 
function)
kernel/hardirq.c:309: error: (Each undeclared identifier is reported only once
kernel/hardirq.c:309: error: for each function it appears in.)
kernel/hardirq.c: In function `generic_setup_irq':
kernel/hardirq.c:339: error: `SA_NODELAY' undeclared (first use in this 
function)
kernel/hardirq.c: In function `threaded_read_proc':
kernel/hardirq.c:611: error: `SA_NODELAY' undeclared (first use in this 
function)
kernel/hardirq.c: In function `threaded_write_proc':
kernel/hardirq.c:629: error: `SA_NODELAY' undeclared (first use in this 
function)
make[1]: *** [kernel/hardirq.o] Error 1
make: *** [kernel] Error 2

The kernel config is attached.

Greetings,
-- 
Rafael J. Wysocki
----------------------------
For a successful technology, reality must take precedence over public 
relations, for nature cannot be fooled.
					-- Richard P. Feynman

[-- Attachment #2: 2.6.8.1-P7.config --]
[-- Type: text/plain, Size: 28209 bytes --]

#
# Automatically generated make config: don't edit
#
CONFIG_X86_64=y
CONFIG_64BIT=y
CONFIG_X86=y
CONFIG_MMU=y
CONFIG_RWSEM_GENERIC_SPINLOCK=y
CONFIG_X86_CMPXCHG=y
CONFIG_EARLY_PRINTK=y
CONFIG_HPET_TIMER=y
CONFIG_HPET_EMULATE_RTC=y
CONFIG_GENERIC_ISA_DMA=y

#
# Code maturity level options
#
CONFIG_EXPERIMENTAL=y
CONFIG_CLEAN_COMPILE=y

#
# General setup
#
CONFIG_SWAP=y
CONFIG_SYSVIPC=y
CONFIG_POSIX_MQUEUE=y
CONFIG_BSD_PROCESS_ACCT=y
CONFIG_BSD_PROCESS_ACCT_V3=y
CONFIG_SYSCTL=y
CONFIG_AUDIT=y
CONFIG_AUDITSYSCALL=y
CONFIG_LOG_BUF_SHIFT=15
CONFIG_HOTPLUG=y
# CONFIG_IKCONFIG is not set
# CONFIG_EMBEDDED is not set
CONFIG_KALLSYMS=y
# CONFIG_KALLSYMS_ALL is not set
# CONFIG_KALLSYMS_EXTRA_PASS is not set
CONFIG_FUTEX=y
CONFIG_EPOLL=y
CONFIG_IOSCHED_NOOP=y
CONFIG_IOSCHED_AS=y
CONFIG_IOSCHED_DEADLINE=y
CONFIG_IOSCHED_CFQ=y
# CONFIG_CC_OPTIMIZE_FOR_SIZE is not set

#
# Loadable module support
#
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
# CONFIG_MODULE_FORCE_UNLOAD is not set
CONFIG_OBSOLETE_MODPARM=y
CONFIG_MODVERSIONS=y
CONFIG_KMOD=y
CONFIG_STOP_MACHINE=y

#
# Processor type and features
#
CONFIG_MK8=y
# CONFIG_MPSC is not set
# CONFIG_GENERIC_CPU is not set
CONFIG_X86_L1_CACHE_BYTES=64
CONFIG_X86_L1_CACHE_SHIFT=6
CONFIG_X86_TSC=y
CONFIG_X86_GOOD_APIC=y
# CONFIG_MICROCODE is not set
CONFIG_X86_MSR=y
CONFIG_X86_CPUID=y
CONFIG_X86_IO_APIC=y
CONFIG_X86_LOCAL_APIC=y
CONFIG_MTRR=y
CONFIG_SMP=y
# CONFIG_PREEMPT is not set
# CONFIG_SCHED_SMT is not set
CONFIG_K8_NUMA=y
CONFIG_DISCONTIGMEM=y
CONFIG_NUMA=y
CONFIG_HAVE_DEC_LOCK=y
CONFIG_NR_CPUS=2
CONFIG_GART_IOMMU=y
CONFIG_SWIOTLB=y
CONFIG_X86_MCE=y

#
# Power management options
#
CONFIG_PM=y
# CONFIG_SOFTWARE_SUSPEND is not set

#
# ACPI (Advanced Configuration and Power Interface) Support
#
CONFIG_ACPI=y
CONFIG_ACPI_BOOT=y
CONFIG_ACPI_INTERPRETER=y
# CONFIG_ACPI_SLEEP is not set
CONFIG_ACPI_AC=y
# CONFIG_ACPI_BATTERY is not set
CONFIG_ACPI_BUTTON=y
CONFIG_ACPI_FAN=y
CONFIG_ACPI_PROCESSOR=y
CONFIG_ACPI_THERMAL=y
# CONFIG_ACPI_ASUS is not set
# CONFIG_ACPI_TOSHIBA is not set
# CONFIG_ACPI_DEBUG is not set
CONFIG_ACPI_BUS=y
CONFIG_ACPI_EC=y
CONFIG_ACPI_POWER=y
CONFIG_ACPI_PCI=y
CONFIG_ACPI_SYSTEM=y

#
# CPU Frequency scaling
#
CONFIG_CPU_FREQ=y
# CONFIG_CPU_FREQ_PROC_INTF is not set
CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
# CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE is not set
CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
# CONFIG_CPU_FREQ_GOV_POWERSAVE is not set
# CONFIG_CPU_FREQ_GOV_USERSPACE is not set
CONFIG_CPU_FREQ_TABLE=y

#
# CPUFreq processor drivers
#
CONFIG_X86_POWERNOW_K8=m

#
# Bus options (PCI etc.)
#
CONFIG_PCI=y
CONFIG_PCI_DIRECT=y
CONFIG_PCI_MMCONFIG=y
# CONFIG_PCI_MSI is not set
CONFIG_PCI_LEGACY_PROC=y
CONFIG_PCI_NAMES=y

#
# PCMCIA/CardBus support
#
# CONFIG_PCMCIA is not set

#
# PCI Hotplug Support
#
# CONFIG_HOTPLUG_PCI is not set

#
# Executable file formats / Emulations
#
CONFIG_BINFMT_ELF=y
CONFIG_BINFMT_MISC=m
CONFIG_IA32_EMULATION=y
CONFIG_IA32_AOUT=y
CONFIG_COMPAT=y
CONFIG_SYSVIPC_COMPAT=y
CONFIG_UID16=y

#
# Device Drivers
#

#
# Generic Driver Options
#
CONFIG_STANDALONE=y
CONFIG_PREVENT_FIRMWARE_BUILD=y
# CONFIG_FW_LOADER is not set
# CONFIG_DEBUG_DRIVER is not set

#
# Memory Technology Devices (MTD)
#
# CONFIG_MTD is not set

#
# Parallel port support
#
CONFIG_PARPORT=m
CONFIG_PARPORT_PC=m
CONFIG_PARPORT_PC_CML1=m
# CONFIG_PARPORT_SERIAL is not set
CONFIG_PARPORT_PC_FIFO=y
CONFIG_PARPORT_PC_SUPERIO=y
CONFIG_PARPORT_OTHER=y
CONFIG_PARPORT_1284=y

#
# Plug and Play support
#

#
# Block devices
#
CONFIG_BLK_DEV_FD=m
# CONFIG_PARIDE is not set
# CONFIG_BLK_CPQ_DA is not set
# CONFIG_BLK_CPQ_CISS_DA is not set
# CONFIG_BLK_DEV_DAC960 is not set
# CONFIG_BLK_DEV_UMEM is not set
CONFIG_BLK_DEV_LOOP=y
CONFIG_BLK_DEV_CRYPTOLOOP=m
# CONFIG_BLK_DEV_NBD is not set
# CONFIG_BLK_DEV_SX8 is not set
# CONFIG_BLK_DEV_RAM is not set
CONFIG_LBD=y

#
# ATA/ATAPI/MFM/RLL support
#
CONFIG_IDE=y
CONFIG_BLK_DEV_IDE=y

#
# Please see Documentation/ide.txt for help/info on IDE drives
#
# CONFIG_BLK_DEV_IDE_SATA is not set
# CONFIG_BLK_DEV_HD_IDE is not set
CONFIG_BLK_DEV_IDEDISK=y
CONFIG_IDEDISK_MULTI_MODE=y
CONFIG_BLK_DEV_IDECD=m
# CONFIG_BLK_DEV_IDETAPE is not set
# CONFIG_BLK_DEV_IDEFLOPPY is not set
CONFIG_BLK_DEV_IDESCSI=m
# CONFIG_IDE_TASK_IOCTL is not set
# CONFIG_IDE_TASKFILE_IO is not set

#
# IDE chipset support/bugfixes
#
CONFIG_IDE_GENERIC=y
# CONFIG_BLK_DEV_CMD640 is not set
CONFIG_BLK_DEV_IDEPCI=y
CONFIG_IDEPCI_SHARE_IRQ=y
CONFIG_BLK_DEV_OFFBOARD=y
CONFIG_BLK_DEV_GENERIC=y
# CONFIG_BLK_DEV_OPTI621 is not set
# CONFIG_BLK_DEV_RZ1000 is not set
CONFIG_BLK_DEV_IDEDMA_PCI=y
# CONFIG_BLK_DEV_IDEDMA_FORCED is not set
CONFIG_IDEDMA_PCI_AUTO=y
# CONFIG_IDEDMA_ONLYDISK is not set
CONFIG_BLK_DEV_ADMA=y
# CONFIG_BLK_DEV_AEC62XX is not set
# CONFIG_BLK_DEV_ALI15X3 is not set
CONFIG_BLK_DEV_AMD74XX=y
# CONFIG_BLK_DEV_ATIIXP is not set
# CONFIG_BLK_DEV_CMD64X is not set
# CONFIG_BLK_DEV_TRIFLEX is not set
# CONFIG_BLK_DEV_CY82C693 is not set
# CONFIG_BLK_DEV_CS5520 is not set
# CONFIG_BLK_DEV_CS5530 is not set
# CONFIG_BLK_DEV_HPT34X is not set
# CONFIG_BLK_DEV_HPT366 is not set
# CONFIG_BLK_DEV_SC1200 is not set
# CONFIG_BLK_DEV_PIIX is not set
# CONFIG_BLK_DEV_NS87415 is not set
# CONFIG_BLK_DEV_PDC202XX_OLD is not set
# CONFIG_BLK_DEV_PDC202XX_NEW is not set
# CONFIG_BLK_DEV_SVWKS is not set
# CONFIG_BLK_DEV_SIIMAGE is not set
# CONFIG_BLK_DEV_SIS5513 is not set
# CONFIG_BLK_DEV_SLC90E66 is not set
# CONFIG_BLK_DEV_TRM290 is not set
# CONFIG_BLK_DEV_VIA82CXXX is not set
# CONFIG_IDE_ARM is not set
CONFIG_BLK_DEV_IDEDMA=y
CONFIG_IDEDMA_IVB=y
CONFIG_IDEDMA_AUTO=y
# CONFIG_BLK_DEV_HD is not set

#
# SCSI device support
#
CONFIG_SCSI=y
CONFIG_SCSI_PROC_FS=y

#
# SCSI support type (disk, tape, CD-ROM)
#
CONFIG_BLK_DEV_SD=y
# CONFIG_CHR_DEV_ST is not set
# CONFIG_CHR_DEV_OSST is not set
CONFIG_BLK_DEV_SR=m
CONFIG_BLK_DEV_SR_VENDOR=y
CONFIG_CHR_DEV_SG=m

#
# Some SCSI devices (e.g. CD jukebox) support multiple LUNs
#
# CONFIG_SCSI_MULTI_LUN is not set
CONFIG_SCSI_CONSTANTS=y
CONFIG_SCSI_LOGGING=y

#
# SCSI Transport Attributes
#
CONFIG_SCSI_SPI_ATTRS=y
# CONFIG_SCSI_FC_ATTRS is not set

#
# SCSI low-level drivers
#
CONFIG_BLK_DEV_3W_XXXX_RAID=y
# CONFIG_SCSI_3W_9XXX is not set
# CONFIG_SCSI_ACARD is not set
# CONFIG_SCSI_AACRAID is not set
# CONFIG_SCSI_AIC7XXX is not set
# CONFIG_SCSI_AIC7XXX_OLD is not set
# CONFIG_SCSI_AIC79XX is not set
# CONFIG_SCSI_MEGARAID is not set
CONFIG_SCSI_SATA=y
# CONFIG_SCSI_SATA_SVW is not set
# CONFIG_SCSI_ATA_PIIX is not set
# CONFIG_SCSI_SATA_NV is not set
# CONFIG_SCSI_SATA_PROMISE is not set
# CONFIG_SCSI_SATA_SX4 is not set
CONFIG_SCSI_SATA_SIL=y
# CONFIG_SCSI_SATA_SIS is not set
# CONFIG_SCSI_SATA_VIA is not set
# CONFIG_SCSI_SATA_VITESSE is not set
# CONFIG_SCSI_BUSLOGIC is not set
# CONFIG_SCSI_DMX3191D is not set
# CONFIG_SCSI_EATA is not set
# CONFIG_SCSI_EATA_PIO is not set
# CONFIG_SCSI_FUTURE_DOMAIN is not set
# CONFIG_SCSI_GDTH is not set
# CONFIG_SCSI_IPS is not set
# CONFIG_SCSI_INIA100 is not set
# CONFIG_SCSI_PPA is not set
# CONFIG_SCSI_IMM is not set
CONFIG_SCSI_SYM53C8XX_2=y
CONFIG_SCSI_SYM53C8XX_DMA_ADDRESSING_MODE=1
CONFIG_SCSI_SYM53C8XX_DEFAULT_TAGS=16
CONFIG_SCSI_SYM53C8XX_MAX_TAGS=64
# CONFIG_SCSI_SYM53C8XX_IOMAPPED is not set
# CONFIG_SCSI_IPR is not set
# CONFIG_SCSI_QLOGIC_ISP is not set
# CONFIG_SCSI_QLOGIC_FC is not set
# CONFIG_SCSI_QLOGIC_1280 is not set
CONFIG_SCSI_QLA2XXX=y
# CONFIG_SCSI_QLA21XX is not set
# CONFIG_SCSI_QLA22XX is not set
# CONFIG_SCSI_QLA2300 is not set
# CONFIG_SCSI_QLA2322 is not set
# CONFIG_SCSI_QLA6312 is not set
# CONFIG_SCSI_QLA6322 is not set
# CONFIG_SCSI_DC395x is not set
# CONFIG_SCSI_DC390T is not set
CONFIG_SCSI_DEBUG=m

#
# Multi-device support (RAID and LVM)
#
CONFIG_MD=y
CONFIG_BLK_DEV_MD=y
# CONFIG_MD_LINEAR is not set
# CONFIG_MD_RAID0 is not set
CONFIG_MD_RAID1=y
CONFIG_MD_RAID5=y
# CONFIG_MD_RAID6 is not set
CONFIG_MD_MULTIPATH=m
CONFIG_BLK_DEV_DM=m
CONFIG_DM_CRYPT=m
CONFIG_DM_SNAPSHOT=m
CONFIG_DM_MIRROR=m
CONFIG_DM_ZERO=m

#
# Fusion MPT device support
#
# CONFIG_FUSION is not set

#
# IEEE 1394 (FireWire) support
#
CONFIG_IEEE1394=m

#
# Subsystem Options
#
# CONFIG_IEEE1394_VERBOSEDEBUG is not set
# CONFIG_IEEE1394_OUI_DB is not set
CONFIG_IEEE1394_EXTRA_CONFIG_ROMS=y
CONFIG_IEEE1394_CONFIG_ROM_IP1394=y

#
# Device Drivers
#

#
# Texas Instruments PCILynx requires I2C
#
CONFIG_IEEE1394_OHCI1394=m

#
# Protocol Drivers
#
CONFIG_IEEE1394_VIDEO1394=m
CONFIG_IEEE1394_SBP2=m
# CONFIG_IEEE1394_SBP2_PHYS_DMA is not set
CONFIG_IEEE1394_ETH1394=m
CONFIG_IEEE1394_DV1394=m
CONFIG_IEEE1394_RAWIO=m
CONFIG_IEEE1394_CMP=m
CONFIG_IEEE1394_AMDTP=m

#
# I2O device support
#
# CONFIG_I2O is not set

#
# Networking support
#
CONFIG_NET=y

#
# Networking options
#
CONFIG_PACKET=m
CONFIG_PACKET_MMAP=y
# CONFIG_NETLINK_DEV is not set
CONFIG_UNIX=y
CONFIG_NET_KEY=y
CONFIG_INET=y
# CONFIG_IP_MULTICAST is not set
# CONFIG_IP_ADVANCED_ROUTER is not set
# CONFIG_IP_PNP is not set
# CONFIG_NET_IPIP is not set
# CONFIG_NET_IPGRE is not set
# CONFIG_ARPD is not set
CONFIG_SYN_COOKIES=y
CONFIG_INET_AH=y
CONFIG_INET_ESP=y
CONFIG_INET_IPCOMP=y

#
# IP: Virtual Server Configuration
#
# CONFIG_IP_VS is not set
# CONFIG_IPV6 is not set
CONFIG_NETFILTER=y
CONFIG_NETFILTER_DEBUG=y

#
# IP: Netfilter Configuration
#
CONFIG_IP_NF_CONNTRACK=y
CONFIG_IP_NF_FTP=y
CONFIG_IP_NF_IRC=m
CONFIG_IP_NF_TFTP=m
CONFIG_IP_NF_AMANDA=m
CONFIG_IP_NF_QUEUE=m
CONFIG_IP_NF_IPTABLES=m
CONFIG_IP_NF_MATCH_LIMIT=m
CONFIG_IP_NF_MATCH_IPRANGE=m
CONFIG_IP_NF_MATCH_MAC=m
CONFIG_IP_NF_MATCH_PKTTYPE=m
CONFIG_IP_NF_MATCH_MARK=m
CONFIG_IP_NF_MATCH_MULTIPORT=m
CONFIG_IP_NF_MATCH_TOS=m
CONFIG_IP_NF_MATCH_RECENT=m
CONFIG_IP_NF_MATCH_ECN=m
CONFIG_IP_NF_MATCH_DSCP=m
CONFIG_IP_NF_MATCH_AH_ESP=m
CONFIG_IP_NF_MATCH_LENGTH=m
CONFIG_IP_NF_MATCH_TTL=m
CONFIG_IP_NF_MATCH_TCPMSS=m
CONFIG_IP_NF_MATCH_HELPER=m
CONFIG_IP_NF_MATCH_STATE=m
CONFIG_IP_NF_MATCH_CONNTRACK=m
CONFIG_IP_NF_MATCH_OWNER=m
CONFIG_IP_NF_FILTER=m
CONFIG_IP_NF_TARGET_REJECT=m
CONFIG_IP_NF_NAT=m
CONFIG_IP_NF_NAT_NEEDED=y
CONFIG_IP_NF_TARGET_MASQUERADE=m
CONFIG_IP_NF_TARGET_REDIRECT=m
CONFIG_IP_NF_TARGET_NETMAP=m
CONFIG_IP_NF_TARGET_SAME=m
CONFIG_IP_NF_NAT_LOCAL=y
CONFIG_IP_NF_NAT_SNMP_BASIC=m
CONFIG_IP_NF_NAT_IRC=m
CONFIG_IP_NF_NAT_FTP=m
CONFIG_IP_NF_NAT_TFTP=m
CONFIG_IP_NF_NAT_AMANDA=m
CONFIG_IP_NF_MANGLE=m
CONFIG_IP_NF_TARGET_TOS=m
CONFIG_IP_NF_TARGET_ECN=m
CONFIG_IP_NF_TARGET_DSCP=m
CONFIG_IP_NF_TARGET_MARK=m
CONFIG_IP_NF_TARGET_CLASSIFY=m
CONFIG_IP_NF_TARGET_LOG=m
CONFIG_IP_NF_TARGET_ULOG=m
CONFIG_IP_NF_TARGET_TCPMSS=m
CONFIG_IP_NF_ARPTABLES=m
CONFIG_IP_NF_ARPFILTER=m
CONFIG_IP_NF_ARP_MANGLE=m
# CONFIG_IP_NF_RAW is not set
# CONFIG_IP_NF_MATCH_ADDRTYPE is not set
# CONFIG_IP_NF_MATCH_REALM is not set
CONFIG_XFRM=y
CONFIG_XFRM_USER=m

#
# SCTP Configuration (EXPERIMENTAL)
#
CONFIG_IP_SCTP=m
CONFIG_SCTP_DBG_MSG=y
CONFIG_SCTP_DBG_OBJCNT=y
CONFIG_SCTP_HMAC_NONE=y
# CONFIG_SCTP_HMAC_SHA1 is not set
# CONFIG_SCTP_HMAC_MD5 is not set
# CONFIG_ATM is not set
# CONFIG_BRIDGE is not set
# CONFIG_VLAN_8021Q is not set
# CONFIG_DECNET is not set
# CONFIG_LLC2 is not set
# CONFIG_IPX is not set
# CONFIG_ATALK is not set
# CONFIG_X25 is not set
# CONFIG_LAPB is not set
# CONFIG_NET_DIVERT is not set
# CONFIG_ECONET is not set
# CONFIG_WAN_ROUTER is not set
# CONFIG_NET_HW_FLOWCONTROL is not set

#
# QoS and/or fair queueing
#
# CONFIG_NET_SCHED is not set
# CONFIG_NET_CLS_ROUTE is not set

#
# Network testing
#
CONFIG_NET_PKTGEN=m
# CONFIG_NETPOLL is not set
# CONFIG_NET_POLL_CONTROLLER is not set
# CONFIG_HAMRADIO is not set
# CONFIG_IRDA is not set
# CONFIG_BT is not set
CONFIG_NETDEVICES=y
# CONFIG_DUMMY is not set
# CONFIG_BONDING is not set
# CONFIG_EQUALIZER is not set
# CONFIG_TUN is not set

#
# ARCnet devices
#
# CONFIG_ARCNET is not set

#
# Ethernet (10 or 100Mbit)
#
CONFIG_NET_ETHERNET=y
CONFIG_MII=m
# CONFIG_HAPPYMEAL is not set
# CONFIG_SUNGEM is not set
CONFIG_NET_VENDOR_3COM=y
CONFIG_VORTEX=m
# CONFIG_TYPHOON is not set

#
# Tulip family network device support
#
# CONFIG_NET_TULIP is not set
# CONFIG_HP100 is not set
CONFIG_NET_PCI=y
# CONFIG_PCNET32 is not set
CONFIG_AMD8111_ETH=m
# CONFIG_AMD8111E_NAPI is not set
# CONFIG_ADAPTEC_STARFIRE is not set
CONFIG_B44=m
# CONFIG_FORCEDETH is not set
# CONFIG_DGRS is not set
# CONFIG_EEPRO100 is not set
CONFIG_E100=m
# CONFIG_E100_NAPI is not set
# CONFIG_FEALNX is not set
# CONFIG_NATSEMI is not set
# CONFIG_NE2K_PCI is not set
# CONFIG_8139CP is not set
CONFIG_8139TOO=m
# CONFIG_8139TOO_PIO is not set
# CONFIG_8139TOO_TUNE_TWISTER is not set
# CONFIG_8139TOO_8129 is not set
# CONFIG_8139_OLD_RX_RESET is not set
# CONFIG_SIS900 is not set
# CONFIG_EPIC100 is not set
# CONFIG_SUNDANCE is not set
# CONFIG_VIA_RHINE is not set
# CONFIG_VIA_VELOCITY is not set

#
# Ethernet (1000 Mbit)
#
# CONFIG_ACENIC is not set
# CONFIG_DL2K is not set
# CONFIG_E1000 is not set
# CONFIG_NS83820 is not set
# CONFIG_HAMACHI is not set
# CONFIG_YELLOWFIN is not set
# CONFIG_R8169 is not set
# CONFIG_SK98LIN is not set
CONFIG_TIGON3=m

#
# Ethernet (10000 Mbit)
#
# CONFIG_IXGB is not set
# CONFIG_S2IO is not set

#
# Token Ring devices
#
# CONFIG_TR is not set

#
# Wireless LAN (non-hamradio)
#
# CONFIG_NET_RADIO is not set

#
# Wan interfaces
#
# CONFIG_WAN is not set
# CONFIG_FDDI is not set
# CONFIG_HIPPI is not set
# CONFIG_PLIP is not set
CONFIG_PPP=m
# CONFIG_PPP_MULTILINK is not set
CONFIG_PPP_FILTER=y
CONFIG_PPP_ASYNC=m
CONFIG_PPP_SYNC_TTY=m
CONFIG_PPP_DEFLATE=m
CONFIG_PPP_BSDCOMP=m
CONFIG_PPPOE=m
# CONFIG_SLIP is not set
# CONFIG_NET_FC is not set
CONFIG_SHAPER=m
# CONFIG_NETCONSOLE is not set

#
# ISDN subsystem
#
# CONFIG_ISDN is not set

#
# Telephony Support
#
# CONFIG_PHONE is not set

#
# Input device support
#
CONFIG_INPUT=y

#
# Userland interfaces
#
CONFIG_INPUT_MOUSEDEV=y
CONFIG_INPUT_MOUSEDEV_PSAUX=y
CONFIG_INPUT_MOUSEDEV_SCREEN_X=1280
CONFIG_INPUT_MOUSEDEV_SCREEN_Y=1024
CONFIG_INPUT_JOYDEV=m
# CONFIG_INPUT_TSDEV is not set
# CONFIG_INPUT_EVDEV is not set
# CONFIG_INPUT_EVBUG is not set

#
# Input I/O drivers
#
CONFIG_GAMEPORT=m
CONFIG_SOUND_GAMEPORT=m
# CONFIG_GAMEPORT_NS558 is not set
# CONFIG_GAMEPORT_L4 is not set
# CONFIG_GAMEPORT_EMU10K1 is not set
# CONFIG_GAMEPORT_VORTEX is not set
# CONFIG_GAMEPORT_FM801 is not set
# CONFIG_GAMEPORT_CS461x is not set
CONFIG_SERIO=y
CONFIG_SERIO_I8042=y
CONFIG_SERIO_SERPORT=y
# CONFIG_SERIO_CT82C710 is not set
# CONFIG_SERIO_PARKBD is not set
CONFIG_SERIO_PCIPS2=m

#
# Input Device Drivers
#
CONFIG_INPUT_KEYBOARD=y
CONFIG_KEYBOARD_ATKBD=y
# CONFIG_KEYBOARD_SUNKBD is not set
# CONFIG_KEYBOARD_LKKBD is not set
# CONFIG_KEYBOARD_XTKBD is not set
# CONFIG_KEYBOARD_NEWTON is not set
CONFIG_INPUT_MOUSE=y
CONFIG_MOUSE_PS2=y
# CONFIG_MOUSE_SERIAL is not set
# CONFIG_MOUSE_VSXXXAA is not set
# CONFIG_INPUT_JOYSTICK is not set
# CONFIG_INPUT_TOUCHSCREEN is not set
# CONFIG_INPUT_MISC is not set

#
# Character devices
#
CONFIG_VT=y
CONFIG_VT_CONSOLE=y
CONFIG_HW_CONSOLE=y
# CONFIG_SERIAL_NONSTANDARD is not set

#
# Serial drivers
#
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_SERIAL_8250_ACPI=y
CONFIG_SERIAL_8250_NR_UARTS=4
CONFIG_SERIAL_8250_EXTENDED=y
# CONFIG_SERIAL_8250_MANY_PORTS is not set
CONFIG_SERIAL_8250_SHARE_IRQ=y
# CONFIG_SERIAL_8250_DETECT_IRQ is not set
# CONFIG_SERIAL_8250_MULTIPORT is not set
# CONFIG_SERIAL_8250_RSA is not set

#
# Non-8250 serial port support
#
CONFIG_SERIAL_CORE=y
CONFIG_SERIAL_CORE_CONSOLE=y
CONFIG_UNIX98_PTYS=y
CONFIG_LEGACY_PTYS=y
CONFIG_LEGACY_PTY_COUNT=256
CONFIG_PRINTER=m
# CONFIG_LP_CONSOLE is not set
# CONFIG_PPDEV is not set
# CONFIG_TIPAR is not set
# CONFIG_QIC02_TAPE is not set

#
# IPMI
#
CONFIG_IPMI_HANDLER=m
CONFIG_IPMI_PANIC_EVENT=y
# CONFIG_IPMI_PANIC_STRING is not set
CONFIG_IPMI_DEVICE_INTERFACE=m
# CONFIG_IPMI_SI is not set
CONFIG_IPMI_WATCHDOG=m

#
# Watchdog Cards
#
# CONFIG_WATCHDOG is not set
CONFIG_HW_RANDOM=y
CONFIG_NVRAM=m
CONFIG_RTC=y
CONFIG_DTLK=m
# CONFIG_R3964 is not set
# CONFIG_APPLICOM is not set

#
# Ftape, the floppy tape device driver
#
CONFIG_AGP=y
CONFIG_AGP_AMD64=y
# CONFIG_AGP_INTEL_MCH is not set
CONFIG_DRM=y
# CONFIG_DRM_TDFX is not set
# CONFIG_DRM_GAMMA is not set
# CONFIG_DRM_R128 is not set
# CONFIG_DRM_RADEON is not set
# CONFIG_DRM_SIS is not set
# CONFIG_MWAVE is not set
# CONFIG_RAW_DRIVER is not set
# CONFIG_HPET is not set
CONFIG_HANGCHECK_TIMER=m

#
# I2C support
#
# CONFIG_I2C is not set

#
# Dallas's 1-wire bus
#
# CONFIG_W1 is not set

#
# Misc devices
#
# CONFIG_IBM_ASM is not set

#
# Multimedia devices
#
# CONFIG_VIDEO_DEV is not set

#
# Digital Video Broadcasting Devices
#
# CONFIG_DVB is not set

#
# Graphics support
#
CONFIG_FB=y
# CONFIG_FB_CIRRUS is not set
# CONFIG_FB_PM2 is not set
# CONFIG_FB_CYBER2000 is not set
# CONFIG_FB_ASILIANT is not set
# CONFIG_FB_IMSTT is not set
# CONFIG_FB_VGA16 is not set
CONFIG_FB_VESA=y
CONFIG_VIDEO_SELECT=y
# CONFIG_FB_HGA is not set
# CONFIG_FB_RIVA is not set
# CONFIG_FB_MATROX is not set
# CONFIG_FB_RADEON_OLD is not set
# CONFIG_FB_RADEON is not set
# CONFIG_FB_ATY128 is not set
# CONFIG_FB_ATY is not set
# CONFIG_FB_SIS is not set
# CONFIG_FB_NEOMAGIC is not set
# CONFIG_FB_KYRO is not set
# CONFIG_FB_3DFX is not set
# CONFIG_FB_VOODOO1 is not set
# CONFIG_FB_TRIDENT is not set
# CONFIG_FB_VIRTUAL is not set

#
# Console display driver support
#
CONFIG_VGA_CONSOLE=y
# CONFIG_MDA_CONSOLE is not set
CONFIG_DUMMY_CONSOLE=y
CONFIG_FRAMEBUFFER_CONSOLE=y
CONFIG_FONTS=y
CONFIG_FONT_8x8=y
CONFIG_FONT_8x16=y
# CONFIG_FONT_6x11 is not set
# CONFIG_FONT_PEARL_8x8 is not set
# CONFIG_FONT_ACORN_8x8 is not set
# CONFIG_FONT_MINI_4x6 is not set
# CONFIG_FONT_SUN8x16 is not set
# CONFIG_FONT_SUN12x22 is not set

#
# Logo configuration
#
CONFIG_LOGO=y
# CONFIG_LOGO_LINUX_MONO is not set
# CONFIG_LOGO_LINUX_VGA16 is not set
CONFIG_LOGO_LINUX_CLUT224=y

#
# Sound
#
CONFIG_SOUND=m

#
# Advanced Linux Sound Architecture
#
CONFIG_SND=m
CONFIG_SND_TIMER=m
CONFIG_SND_PCM=m
CONFIG_SND_HWDEP=m
CONFIG_SND_RAWMIDI=m
CONFIG_SND_SEQUENCER=m
CONFIG_SND_SEQ_DUMMY=m
CONFIG_SND_OSSEMUL=y
CONFIG_SND_MIXER_OSS=m
CONFIG_SND_PCM_OSS=m
CONFIG_SND_SEQUENCER_OSS=y
CONFIG_SND_BIT32_EMUL=m
CONFIG_SND_RTCTIMER=m
CONFIG_SND_VERBOSE_PRINTK=y
CONFIG_SND_DEBUG=y
CONFIG_SND_DEBUG_MEMORY=y
CONFIG_SND_DEBUG_DETECT=y

#
# Generic devices
#
CONFIG_SND_MPU401_UART=m
CONFIG_SND_DUMMY=m
CONFIG_SND_VIRMIDI=m
CONFIG_SND_MTPAV=m
CONFIG_SND_SERIAL_U16550=m
CONFIG_SND_MPU401=m

#
# PCI devices
#
CONFIG_SND_AC97_CODEC=m
# CONFIG_SND_ALI5451 is not set
# CONFIG_SND_ATIIXP is not set
# CONFIG_SND_AU8810 is not set
# CONFIG_SND_AU8820 is not set
# CONFIG_SND_AU8830 is not set
# CONFIG_SND_AZT3328 is not set
# CONFIG_SND_BT87X is not set
# CONFIG_SND_CS46XX is not set
# CONFIG_SND_CS4281 is not set
# CONFIG_SND_EMU10K1 is not set
# CONFIG_SND_KORG1212 is not set
# CONFIG_SND_MIXART is not set
# CONFIG_SND_NM256 is not set
# CONFIG_SND_RME32 is not set
# CONFIG_SND_RME96 is not set
# CONFIG_SND_RME9652 is not set
# CONFIG_SND_HDSP is not set
# CONFIG_SND_TRIDENT is not set
# CONFIG_SND_YMFPCI is not set
# CONFIG_SND_ALS4000 is not set
# CONFIG_SND_CMIPCI is not set
# CONFIG_SND_ENS1370 is not set
# CONFIG_SND_ENS1371 is not set
# CONFIG_SND_ES1938 is not set
# CONFIG_SND_ES1968 is not set
# CONFIG_SND_MAESTRO3 is not set
# CONFIG_SND_FM801 is not set
# CONFIG_SND_ICE1712 is not set
# CONFIG_SND_ICE1724 is not set
CONFIG_SND_INTEL8X0=m
# CONFIG_SND_INTEL8X0M is not set
# CONFIG_SND_SONICVIBES is not set
# CONFIG_SND_VIA82XX is not set
# CONFIG_SND_VX222 is not set

#
# ALSA USB devices
#
# CONFIG_SND_USB_AUDIO is not set

#
# Open Sound System
#
CONFIG_SOUND_PRIME=m
# CONFIG_SOUND_BT878 is not set
# CONFIG_SOUND_CMPCI is not set
# CONFIG_SOUND_EMU10K1 is not set
# CONFIG_SOUND_FUSION is not set
# CONFIG_SOUND_CS4281 is not set
# CONFIG_SOUND_ES1370 is not set
# CONFIG_SOUND_ES1371 is not set
# CONFIG_SOUND_ESSSOLO1 is not set
# CONFIG_SOUND_MAESTRO is not set
# CONFIG_SOUND_MAESTRO3 is not set
CONFIG_SOUND_ICH=m
# CONFIG_SOUND_SONICVIBES is not set
# CONFIG_SOUND_TRIDENT is not set
# CONFIG_SOUND_MSNDCLAS is not set
# CONFIG_SOUND_MSNDPIN is not set
# CONFIG_SOUND_VIA82CXXX is not set
# CONFIG_SOUND_OSS is not set
# CONFIG_SOUND_ALI5455 is not set
# CONFIG_SOUND_FORTE is not set
# CONFIG_SOUND_RME96XX is not set
# CONFIG_SOUND_AD1980 is not set

#
# USB support
#
CONFIG_USB=m
# CONFIG_USB_DEBUG is not set

#
# Miscellaneous USB options
#
CONFIG_USB_DEVICEFS=y
CONFIG_USB_BANDWIDTH=y
CONFIG_USB_DYNAMIC_MINORS=y

#
# USB Host Controller Drivers
#
CONFIG_USB_EHCI_HCD=m
# CONFIG_USB_EHCI_SPLIT_ISO is not set
# CONFIG_USB_EHCI_ROOT_HUB_TT is not set
CONFIG_USB_OHCI_HCD=m
# CONFIG_USB_UHCI_HCD is not set

#
# USB Device Class drivers
#
CONFIG_USB_AUDIO=m
# CONFIG_USB_BLUETOOTH_TTY is not set
# CONFIG_USB_MIDI is not set
# CONFIG_USB_ACM is not set
CONFIG_USB_PRINTER=m
CONFIG_USB_STORAGE=m
# CONFIG_USB_STORAGE_DEBUG is not set
# CONFIG_USB_STORAGE_RW_DETECT is not set
CONFIG_USB_STORAGE_DATAFAB=y
CONFIG_USB_STORAGE_FREECOM=y
CONFIG_USB_STORAGE_ISD200=y
CONFIG_USB_STORAGE_DPCM=y
CONFIG_USB_STORAGE_HP8200e=y
CONFIG_USB_STORAGE_SDDR09=y
CONFIG_USB_STORAGE_SDDR55=y
CONFIG_USB_STORAGE_JUMPSHOT=y

#
# USB Human Interface Devices (HID)
#
CONFIG_USB_HID=m
CONFIG_USB_HIDINPUT=y
# CONFIG_HID_FF is not set
CONFIG_USB_HIDDEV=y

#
# USB HID Boot Protocol drivers
#
# CONFIG_USB_KBD is not set
# CONFIG_USB_MOUSE is not set
# CONFIG_USB_AIPTEK is not set
# CONFIG_USB_WACOM is not set
# CONFIG_USB_KBTAB is not set
# CONFIG_USB_POWERMATE is not set
# CONFIG_USB_MTOUCH is not set
# CONFIG_USB_EGALAX is not set
# CONFIG_USB_XPAD is not set
# CONFIG_USB_ATI_REMOTE is not set

#
# USB Imaging devices
#
# CONFIG_USB_MDC800 is not set
# CONFIG_USB_MICROTEK is not set
CONFIG_USB_HPUSBSCSI=m

#
# USB Multimedia devices
#
# CONFIG_USB_DABUSB is not set

#
# Video4Linux support is needed for USB Multimedia device support
#

#
# USB Network adaptors
#
# CONFIG_USB_CATC is not set
# CONFIG_USB_KAWETH is not set
# CONFIG_USB_PEGASUS is not set
# CONFIG_USB_RTL8150 is not set
# CONFIG_USB_USBNET is not set

#
# USB port drivers
#
# CONFIG_USB_USS720 is not set

#
# USB Serial Converter support
#
CONFIG_USB_SERIAL=m
CONFIG_USB_SERIAL_GENERIC=y
# CONFIG_USB_SERIAL_BELKIN is not set
# CONFIG_USB_SERIAL_DIGI_ACCELEPORT is not set
# CONFIG_USB_SERIAL_EMPEG is not set
# CONFIG_USB_SERIAL_FTDI_SIO is not set
# CONFIG_USB_SERIAL_VISOR is not set
# CONFIG_USB_SERIAL_IPAQ is not set
# CONFIG_USB_SERIAL_IR is not set
# CONFIG_USB_SERIAL_EDGEPORT is not set
# CONFIG_USB_SERIAL_EDGEPORT_TI is not set
# CONFIG_USB_SERIAL_KEYSPAN_PDA is not set
# CONFIG_USB_SERIAL_KEYSPAN is not set
# CONFIG_USB_SERIAL_KLSI is not set
# CONFIG_USB_SERIAL_KOBIL_SCT is not set
# CONFIG_USB_SERIAL_MCT_U232 is not set
CONFIG_USB_SERIAL_PL2303=m
# CONFIG_USB_SERIAL_SAFE is not set
# CONFIG_USB_SERIAL_CYBERJACK is not set
# CONFIG_USB_SERIAL_XIRCOM is not set
# CONFIG_USB_SERIAL_OMNINET is not set

#
# USB Miscellaneous drivers
#
# CONFIG_USB_EMI62 is not set
# CONFIG_USB_EMI26 is not set
# CONFIG_USB_TIGL is not set
# CONFIG_USB_AUERSWALD is not set
# CONFIG_USB_RIO500 is not set
# CONFIG_USB_LEGOTOWER is not set
# CONFIG_USB_LCD is not set
# CONFIG_USB_LED is not set
# CONFIG_USB_CYTHERM is not set
# CONFIG_USB_PHIDGETSERVO is not set
CONFIG_USB_TEST=m

#
# USB Gadget Support
#
# CONFIG_USB_GADGET is not set

#
# Firmware Drivers
#
# CONFIG_EDD is not set

#
# File systems
#
CONFIG_EXT2_FS=y
CONFIG_EXT2_FS_XATTR=y
CONFIG_EXT2_FS_POSIX_ACL=y
CONFIG_EXT2_FS_SECURITY=y
CONFIG_EXT3_FS=y
CONFIG_EXT3_FS_XATTR=y
CONFIG_EXT3_FS_POSIX_ACL=y
CONFIG_EXT3_FS_SECURITY=y
CONFIG_JBD=y
# CONFIG_JBD_DEBUG is not set
CONFIG_FS_MBCACHE=y
CONFIG_REISERFS_FS=y
# CONFIG_REISERFS_CHECK is not set
CONFIG_REISERFS_PROC_INFO=y
CONFIG_REISERFS_FS_XATTR=y
CONFIG_REISERFS_FS_POSIX_ACL=y
CONFIG_REISERFS_FS_SECURITY=y
# CONFIG_JFS_FS is not set
CONFIG_FS_POSIX_ACL=y
# CONFIG_XFS_FS is not set
# CONFIG_MINIX_FS is not set
# CONFIG_ROMFS_FS is not set
CONFIG_QUOTA=y
CONFIG_QFMT_V1=y
CONFIG_QFMT_V2=y
CONFIG_QUOTACTL=y
CONFIG_AUTOFS_FS=m
CONFIG_AUTOFS4_FS=m

#
# CD-ROM/DVD Filesystems
#
CONFIG_ISO9660_FS=m
CONFIG_JOLIET=y
CONFIG_ZISOFS=y
CONFIG_ZISOFS_FS=m
CONFIG_UDF_FS=m
CONFIG_UDF_NLS=y

#
# DOS/FAT/NT Filesystems
#
CONFIG_FAT_FS=m
CONFIG_MSDOS_FS=m
CONFIG_VFAT_FS=m
CONFIG_FAT_DEFAULT_CODEPAGE=437
CONFIG_FAT_DEFAULT_IOCHARSET="iso8859-1"
# CONFIG_NTFS_FS is not set

#
# Pseudo filesystems
#
CONFIG_PROC_FS=y
CONFIG_PROC_KCORE=y
CONFIG_SYSFS=y
# CONFIG_DEVFS_FS is not set
CONFIG_DEVPTS_FS_XATTR=y
CONFIG_DEVPTS_FS_SECURITY=y
CONFIG_TMPFS=y
CONFIG_HUGETLBFS=y
CONFIG_HUGETLB_PAGE=y
CONFIG_RAMFS=y

#
# Miscellaneous filesystems
#
# CONFIG_ADFS_FS is not set
# CONFIG_AFFS_FS is not set
# CONFIG_HFS_FS is not set
# CONFIG_HFSPLUS_FS is not set
# CONFIG_BEFS_FS is not set
# CONFIG_BFS_FS is not set
# CONFIG_EFS_FS is not set
# CONFIG_CRAMFS is not set
# CONFIG_VXFS_FS is not set
# CONFIG_HPFS_FS is not set
# CONFIG_QNX4FS_FS is not set
# CONFIG_SYSV_FS is not set
# CONFIG_UFS_FS is not set

#
# Network File Systems
#
CONFIG_NFS_FS=m
CONFIG_NFS_V3=y
CONFIG_NFS_V4=y
CONFIG_NFS_DIRECTIO=y
CONFIG_NFSD=m
CONFIG_NFSD_V3=y
CONFIG_NFSD_V4=y
CONFIG_NFSD_TCP=y
CONFIG_LOCKD=m
CONFIG_LOCKD_V4=y
CONFIG_EXPORTFS=m
CONFIG_SUNRPC=m
CONFIG_SUNRPC_GSS=m
CONFIG_RPCSEC_GSS_KRB5=m
CONFIG_SMB_FS=m
CONFIG_SMB_NLS_DEFAULT=y
CONFIG_SMB_NLS_REMOTE="cp852"
CONFIG_CIFS=m
CONFIG_CIFS_STATS=y
# CONFIG_CIFS_XATTR is not set
CONFIG_CIFS_POSIX=y
# CONFIG_NCP_FS is not set
# CONFIG_CODA_FS is not set
# CONFIG_AFS_FS is not set

#
# Partition Types
#
# CONFIG_PARTITION_ADVANCED is not set
CONFIG_MSDOS_PARTITION=y

#
# Native Language Support
#
CONFIG_NLS=y
CONFIG_NLS_DEFAULT="iso8859-2"
CONFIG_NLS_CODEPAGE_437=m
# CONFIG_NLS_CODEPAGE_737 is not set
# CONFIG_NLS_CODEPAGE_775 is not set
# CONFIG_NLS_CODEPAGE_850 is not set
CONFIG_NLS_CODEPAGE_852=m
# CONFIG_NLS_CODEPAGE_855 is not set
# CONFIG_NLS_CODEPAGE_857 is not set
# CONFIG_NLS_CODEPAGE_860 is not set
# CONFIG_NLS_CODEPAGE_861 is not set
# CONFIG_NLS_CODEPAGE_862 is not set
# CONFIG_NLS_CODEPAGE_863 is not set
# CONFIG_NLS_CODEPAGE_864 is not set
# CONFIG_NLS_CODEPAGE_865 is not set
# CONFIG_NLS_CODEPAGE_866 is not set
# CONFIG_NLS_CODEPAGE_869 is not set
# CONFIG_NLS_CODEPAGE_936 is not set
# CONFIG_NLS_CODEPAGE_950 is not set
# CONFIG_NLS_CODEPAGE_932 is not set
# CONFIG_NLS_CODEPAGE_949 is not set
# CONFIG_NLS_CODEPAGE_874 is not set
# CONFIG_NLS_ISO8859_8 is not set
CONFIG_NLS_CODEPAGE_1250=m
CONFIG_NLS_CODEPAGE_1251=m
# CONFIG_NLS_ASCII is not set
CONFIG_NLS_ISO8859_1=m
CONFIG_NLS_ISO8859_2=y
# CONFIG_NLS_ISO8859_3 is not set
# CONFIG_NLS_ISO8859_4 is not set
# CONFIG_NLS_ISO8859_5 is not set
# CONFIG_NLS_ISO8859_6 is not set
# CONFIG_NLS_ISO8859_7 is not set
# CONFIG_NLS_ISO8859_9 is not set
# CONFIG_NLS_ISO8859_13 is not set
# CONFIG_NLS_ISO8859_14 is not set
CONFIG_NLS_ISO8859_15=m
# CONFIG_NLS_KOI8_R is not set
# CONFIG_NLS_KOI8_U is not set
CONFIG_NLS_UTF8=y

#
# Profiling support
#
CONFIG_PROFILING=y
CONFIG_OPROFILE=m

#
# Kernel hacking
#
CONFIG_DEBUG_KERNEL=y
CONFIG_DEBUG_SLAB=y
CONFIG_MAGIC_SYSRQ=y
# CONFIG_DEBUG_SPINLOCK is not set
# CONFIG_INIT_DEBUG is not set
# CONFIG_DEBUG_INFO is not set
# CONFIG_FRAME_POINTER is not set
# CONFIG_IOMMU_DEBUG is not set

#
# Security options
#
CONFIG_SECURITY=y
CONFIG_SECURITY_NETWORK=y
CONFIG_SECURITY_CAPABILITIES=y
CONFIG_SECURITY_ROOTPLUG=m
CONFIG_SECURITY_SELINUX=y
CONFIG_SECURITY_SELINUX_BOOTPARAM=y
# CONFIG_SECURITY_SELINUX_DISABLE is not set
CONFIG_SECURITY_SELINUX_DEVELOP=y
CONFIG_SECURITY_SELINUX_MLS=y

#
# Cryptographic options
#
CONFIG_CRYPTO=y
CONFIG_CRYPTO_HMAC=y
CONFIG_CRYPTO_NULL=m
CONFIG_CRYPTO_MD4=m
CONFIG_CRYPTO_MD5=y
CONFIG_CRYPTO_SHA1=y
CONFIG_CRYPTO_SHA256=m
CONFIG_CRYPTO_SHA512=m
CONFIG_CRYPTO_DES=y
CONFIG_CRYPTO_BLOWFISH=m
CONFIG_CRYPTO_TWOFISH=m
CONFIG_CRYPTO_SERPENT=m
CONFIG_CRYPTO_AES=m
CONFIG_CRYPTO_CAST5=m
CONFIG_CRYPTO_CAST6=m
CONFIG_CRYPTO_TEA=m
CONFIG_CRYPTO_ARC4=m
CONFIG_CRYPTO_KHAZAD=m
CONFIG_CRYPTO_DEFLATE=y
CONFIG_CRYPTO_MICHAEL_MIC=m
CONFIG_CRYPTO_CRC32C=m
CONFIG_CRYPTO_TEST=m

#
# Library routines
#
CONFIG_CRC_CCITT=m
CONFIG_CRC32=m
CONFIG_LIBCRC32C=m
CONFIG_ZLIB_INFLATE=y
CONFIG_ZLIB_DEFLATE=y

Re: [patch] voluntary-preempt-2.6.8.1-P7

[K.R. Foley]

Ingo Molnar wrote:
 > i've uploaded the -P7 patch:
 >
 >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P7
 >

Is this one for real, or another false positive? A ~4119 usec latency in
the scheduler?

http://www.cybsft.com/testresults/2.6.8.1-P7/2.6.8.1-P7-5.txt

kr

Re: [patch] voluntary-preempt-2.6.8.1-P7

[Lee Revell]

On Sun, 2004-08-22 at 03:49, Lee Revell wrote:
> On Sat, 2004-08-21 at 10:05, Ingo Molnar wrote:
> > i've uploaded the -P7 patch:
> > 
> >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P7
> 
> Here's a pretty bad one, over 500 usecs in rt_garbage_collect:
> 

Grr, apt-get upgrade (debian unstable) broke php on my web server. 
subsequent apt-get upgrades have not fixed it, and I have not had time
to file a bug report.  Traces can be found here:

http://krustophenia.net/2.6.8.1-P7

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P7

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> to file a bug report.  Traces can be found here:
> 
> http://krustophenia.net/2.6.8.1-P7

re the skb latencies: Mark H Johnson managed to reduce net-input
latencies by decreasing /proc/sys/net/core/netdev_max_backlog to 8 -
could you try this, does it help? (you could try an even more extreme
setting like 4.)

	Ingo

[patch] voluntary-preempt-2.6.8.1-P8

[Ingo Molnar]

i've uploaded the -P8 patch:

  http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P8

Changes since -P8:

 - fixes the DRI/DRM latency in radeon (and other drivers). The concept 
   was investigated/tested by Dave Airlie.

 - reduce netdev_max_backlog to 8 (Mark H Johnson)

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P7

[K.R. Foley]

Ingo Molnar wrote:
> i've uploaded the -P7 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P7
> 

Here are a couple more traces in the ~200 - ~300 usec range:

http://www.cybsft.com/testresults/2.6.8.1-P7/2.6.8.1-P7-4.txt

http://www.cybsft.com/testresults/2.6.8.1-P7/2.6.8.1-P7-10.txt

Also, can someone tell me why I often see traces in the 4 ms range with 
  3 - 4 ms being reported by a single routine?

kr

Re: [patch] voluntary-preempt-2.6.8.1-P7

[Lee Revell]

On Sun, 2004-08-22 at 03:49, Lee Revell wrote:
> On Sat, 2004-08-21 at 10:05, Ingo Molnar wrote:
> > i've uploaded the -P7 patch:
> > 
> >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P7
> 
> Here's a pretty bad one, over 500 usecs in rt_garbage_collect:
> 
> http://krustophenia.net/testresults.php?dataset=2.6.8.1-P7#/var/www/2.6.8.1-P7/trace7.txt
> 

Thanks to everyone who pointed out that PHP had broken on my web
server.  It's fixed, all the links in these threads should work now.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Lee Revell]

On Mon, 2004-08-23 at 17:01, Ingo Molnar wrote:
> i've uploaded the -P8 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P8
> 
> Changes since -P8:
> 
>  - fixes the DRI/DRM latency in radeon (and other drivers). The concept 
>    was investigated/tested by Dave Airlie.
> 
>  - reduce netdev_max_backlog to 8 (Mark H Johnson)
> 

Should this fix the 500+ usec latency I saw in rt_garbage_collect?  This
one took a while to occur (overnight).

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> On Mon, 2004-08-23 at 17:01, Ingo Molnar wrote:
> > i've uploaded the -P8 patch:
> > 
> >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P8
> > 
> > Changes since -P8:
> > 
> >  - fixes the DRI/DRM latency in radeon (and other drivers). The concept 
> >    was investigated/tested by Dave Airlie.
> > 
> >  - reduce netdev_max_backlog to 8 (Mark H Johnson)
> > 
> 
> Should this fix the 500+ usec latency I saw in rt_garbage_collect? 
> This one took a while to occur (overnight).

i dont think it will. Does the patch below help?

	Ingo

--- net/ipv4/route.c.orig
+++ net/ipv4/route.c
@@ -738,7 +738,7 @@ static int rt_garbage_collect(void)
 
 		if (atomic_read(&ipv4_dst_ops.entries) < ip_rt_max_size)
 			goto out;
-	} while (!in_softirq() && time_before_eq(jiffies, now));
+	} while (!in_softirq() && time_before_eq(jiffies, now) && !need_resched());
 
 	if (atomic_read(&ipv4_dst_ops.entries) < ip_rt_max_size)
 		goto out;

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Lee Revell]

On Mon, 2004-08-23 at 18:52, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> > On Mon, 2004-08-23 at 17:01, Ingo Molnar wrote:
> > > 
> > >   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P8
> > > 
> > Should this fix the 500+ usec latency I saw in rt_garbage_collect? 
> > This one took a while to occur (overnight).
> 
> i dont think it will. Does the patch below help?
> 

Probably.  This one is pretty rare, it might take a day to recur, and
it's pretty clear the patch will fix the problem, so I recommend adding
it to P9.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Lee Revell]

On Mon, 2004-08-23 at 17:01, Ingo Molnar wrote:
> i've uploaded the -P8 patch:
> 
>   http://redhat.com/~mingo/voluntary-preempt/voluntary-preempt-2.6.8.1-P8
> 
> Changes since -P8:
> 
>  - fixes the DRI/DRM latency in radeon (and other drivers). The concept 
>    was investigated/tested by Dave Airlie.
> 
>  - reduce netdev_max_backlog to 8 (Mark H Johnson)


With -P8 I have not seen a latency over 100 usecs yet.  The most common
latencies are those involving zap_pte_range, journal_get_undo_access and
netdev_process_backlog (even with netdev_max_backlog set to 4).

Traces:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P8

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P7

[Lee Revell]

On Mon, 2004-08-23 at 15:43, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > to file a bug report.  Traces can be found here:
> > 
> > http://krustophenia.net/2.6.8.1-P7
> 
> re the skb latencies: Mark H Johnson managed to reduce net-input
> latencies by decreasing /proc/sys/net/core/netdev_max_backlog to 8 -
> could you try this, does it help? (you could try an even more extreme
> setting like 4.)
> 

Spoke too soon with my last report.  I am actually still seeing
latencies close to 200 usecs in netdev_process_backlog.  Reducing
netdev_max_backlog to 4, then 2 did not seem to help.

netdev_process_backlog = 4:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P8#/var/www/2.6.8.1-P8/trace9.txt

netdev_process_backlog = 2:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P8#/var/www/2.6.8.1-P8/trace12.txt

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Lee Revell]

On Mon, 2004-08-23 at 17:01, Ingo Molnar wrote:

>  - reduce netdev_max_backlog to 8 (Mark H Johnson)
> 

On my system this setting has absolutely no effect on the skb related
latencies.  I tested setting netdev_max_backlog to every power of two 
between 1 and 128, and regardless of this setting, I can produce a
450-600 usec latency with:

ping -s 65507 -f $DEFAULT_GATEWAY

Looks like skb_checksum is the problem.  Here is one of the traces:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P8#/var/www/2.6.8.1-P8/trace14.txt

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P8

[K.R. Foley]

Lee Revell wrote:
> On Mon, 2004-08-23 at 17:01, Ingo Molnar wrote:
> 
> 
>> - reduce netdev_max_backlog to 8 (Mark H Johnson)
>>
> 
> 
> On my system this setting has absolutely no effect on the skb related
> latencies.  I tested setting netdev_max_backlog to every power of two 
> between 1 and 128, and regardless of this setting, I can produce a
> 450-600 usec latency with:
> 
> ping -s 65507 -f $DEFAULT_GATEWAY
> 
> Looks like skb_checksum is the problem.  Here is one of the traces:
> 
> http://krustophenia.net/testresults.php?dataset=2.6.8.1-P8#/var/www/2.6.8.1-P8/trace14.txt
> 
> Lee
> 
> -

Setting netdev_max_backlog doesn't seem to have any effect for me either.

I get a similar latency when I try to reproduce your result above, ~612 
usec. However, my trace is very different than yours:

http://www.cybsft.com/testresults/2.6.8.1-P8/latency_trace3.txt

In fact most any network activity, with the POSSIBLE exception of 
already open connections, seem to be able to trigger higher latencies. 
As you can see here:

http://www.cybsft.com/testresults/2.6.8.1-P8/


kr

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Lee Revell]

On Mon, 2004-08-23 at 23:56, K.R. Foley wrote:
> Lee Revell wrote:
> > On Mon, 2004-08-23 at 17:01, Ingo Molnar wrote:
> > 
> > 
> >> - reduce netdev_max_backlog to 8 (Mark H Johnson)
> >>
> > 
> > 
> > On my system this setting has absolutely no effect on the skb related
> > latencies.  I tested setting netdev_max_backlog to every power of two 
> > between 1 and 128, and regardless of this setting, I can produce a
> > 450-600 usec latency with:
> > 
> > ping -s 65507 -f $DEFAULT_GATEWAY
> > 
> > Looks like skb_checksum is the problem.  Here is one of the traces:
> > 
> > http://krustophenia.net/testresults.php?dataset=2.6.8.1-P8#/var/www/2.6.8.1-P8/trace14.txt
> > 
> > Lee
> > 
> > -
> 
> Setting netdev_max_backlog doesn't seem to have any effect for me either.
> 
> I get a similar latency when I try to reproduce your result above, ~612 
> usec. However, my trace is very different than yours:
> 
> http://www.cybsft.com/testresults/2.6.8.1-P8/latency_trace3.txt
>

I have some that are similar to this.  Looks like many of the
differences are due to different network drivers (e100 vs via-rhine).

> In fact most any network activity, with the POSSIBLE exception of 
> already open connections, seem to be able to trigger higher latencies. 
> As you can see here:
> 
> http://www.cybsft.com/testresults/2.6.8.1-P8/
> 

Looks like except for the one above, all of these involve the
extract_entropy issue.  I am using a hack to disable the random driver
to avoid these latencies.

There are a few other threads going on discussing a real solution to the
extract_entropy issue.  However, none of the solutions that has been
posted yet fix the problem.

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> >  - reduce netdev_max_backlog to 8 (Mark H Johnson)
> 
> On my system this setting has absolutely no effect on the skb related
> latencies. [...]

it has an effect on input queue length. Output queue lengths can be
reduced via 'ifconfig eth0 txqueuelen 8'.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Lee Revell]

On Tue, 2004-08-24 at 01:41, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > >  - reduce netdev_max_backlog to 8 (Mark H Johnson)
> > 
> > On my system this setting has absolutely no effect on the skb related
> > latencies. [...]
> 
> it has an effect on input queue length. Output queue lengths can be
> reduced via 'ifconfig eth0 txqueuelen 8'.
> 

OK.  With both of these set to 4, the largest latency I was able to
generate by ping flooding was 253 usecs.

Is there any particular reason one of these parameters is set via a
sysctl/proc entry, and one by ifconfig?

Lee

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > it has an effect on input queue length. Output queue lengths can be
> > reduced via 'ifconfig eth0 txqueuelen 8'.
> 
> OK.  With both of these set to 4, the largest latency I was able to
> generate by ping flooding was 253 usecs.
> 
> Is there any particular reason one of these parameters is set via a
> sysctl/proc entry, and one by ifconfig?

well, the input packet backlog is global, the output one is local. No
good reason otherwise.

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Ingo Molnar]

* K.R. Foley <kr@cybsft.com> wrote:

> http://www.cybsft.com/testresults/2.6.8.1-P8/latency_trace3.txt

this trace shows a TX processing latency of 45 frames - please try
'ifconfig eth0 txqueuelen 8', does it help?

	Ingo

Re: [patch] voluntary-preempt-2.6.8.1-P8

[K.R. Foley]

Ingo Molnar wrote:
> * K.R. Foley <kr@cybsft.com> wrote:
> 
> 
>>http://www.cybsft.com/testresults/2.6.8.1-P8/latency_trace3.txt
> 
> 
> this trace shows a TX processing latency of 45 frames - please try
> 'ifconfig eth0 txqueuelen 8', does it help?
> 
> 	Ingo
> 
There's no difference that jumps out at me. Here are a couple more with
txqueuelen=8

http://www.cybsft.com/testresults/2.6.8.1-P8/latency_trace7.txt
http://www.cybsft.com/testresults/2.6.8.1-P8/latency_trace8.txt

kr

Re: [patch] voluntary-preempt-2.6.8.1-P8

[K.R. Foley]

Lee Revell wrote:
> On Tue, 2004-08-24 at 01:41, Ingo Molnar wrote:
> 
>>* Lee Revell <rlrevell@joe-job.com> wrote:
>>
>>
>>>> - reduce netdev_max_backlog to 8 (Mark H Johnson)
>>>
>>>On my system this setting has absolutely no effect on the skb related
>>>latencies. [...]
>>
>>it has an effect on input queue length. Output queue lengths can be
>>reduced via 'ifconfig eth0 txqueuelen 8'.
>>
> 
> 
> OK.  With both of these set to 4, the largest latency I was able to
> generate by ping flooding was 253 usecs.
> 

Even with both of these set to 4, I still get similar results ~647 usec :(

<snip>

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Lee Revell]

On Tue, 2004-08-24 at 09:48, K.R. Foley wrote:
> Lee Revell wrote:
> > On Tue, 2004-08-24 at 01:41, Ingo Molnar wrote:
> > 
> >>* Lee Revell <rlrevell@joe-job.com> wrote:
> >>
> >>
> >>>> - reduce netdev_max_backlog to 8 (Mark H Johnson)
> >>>
> >>>On my system this setting has absolutely no effect on the skb related
> >>>latencies. [...]
> >>
> >>it has an effect on input queue length. Output queue lengths can be
> >>reduced via 'ifconfig eth0 txqueuelen 8'.
> >>
> > 
> > 
> > OK.  With both of these set to 4, the largest latency I was able to
> > generate by ping flooding was 253 usecs.
> > 
> 
> Even with both of these set to 4, I still get similar results ~647 usec :(
> 

I am able to generate a 1012 usec latency by flood pinging the broadcast
address.  These are pretty pathological cases, but if we are going for
bounded latency it seems like they should be addressed.

Lee

Re: [patch] intel8x0 latency fix

[Takashi Iwai]

At Fri, 20 Aug 2004 18:19:12 +0200,
karl.vogel@seagha.com wrote:
> 
> karl.vogel@seagha.com writes:
> 
> > # lspci -s 00:06.0
> > 00:06.0 Multimedia audio controller: nVidia Corporation nForce3 Audio (rev a2)
> >
> >  latency: 50752 us, entries: 267 (267)
> 
> 
> Following patch fixes it for 2 channel devices (like my notebook). Although I'm
> not sure if this is all that useful... if there are workloads other than audio
> work that need low latency, then this might be useful.
> 

In the recent version (linux-sound tree), we can use msleep() (that is
more latency friendely) in that callback.  The following patch was
applied to the current version.


Takashi

--- linux/sound/pci/intel8x0.c	17 Aug 2004 17:09:19 -0000	1.158
+++ linux/sound/pci/intel8x0.c	23 Aug 2004 13:57:52 -0000
@@ -1024,8 +1024,10 @@
 			/* reset to 2ch once to keep the 6 channel data in alignment,
 			 * to start from Front Left always
 			 */
-			iputdword(chip, ICHREG(GLOB_CNT), (cnt & 0xcfffff));
-			mdelay(50); /* grrr... */
+			if (cnt & ICH_PCM_246_MASK) {
+				iputdword(chip, ICHREG(GLOB_CNT), cnt & ~ICH_PCM_246_MASK);
+				msleep(50);
+			}
 		} else if (chip->device_type == DEVICE_INTEL_ICH4) {
 			if (sample_bits > 16)
 				cnt |= ICH_PCM_20BIT;

Re: [patch] voluntary-preempt-2.6.8.1-P8

[K.R. Foley]

Lee Revell wrote:
<snip>
> 
> I am able to generate a 1012 usec latency by flood pinging the broadcast
> address.  These are pretty pathological cases, but if we are going for
> bounded latency it seems like they should be addressed.
> 
> Lee
> 

I agree, if that is possible. Whether it's possible in all cases is not 
something I am really qualified to answer.

kr

Re: [patch] voluntary-preempt-2.6.8.1-P8

[Lee Revell]

On Mon, 2004-08-23 at 18:52, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> > Should this fix the 500+ usec latency I saw in rt_garbage_collect? 
> > This one took a while to occur (overnight).
> 
> i dont think it will. Does the patch below help?
> 
> 	Ingo
> 
> --- net/ipv4/route.c.orig
> +++ net/ipv4/route.c
> @@ -738,7 +738,7 @@ static int rt_garbage_collect(void)
>  
>  		if (atomic_read(&ipv4_dst_ops.entries) < ip_rt_max_size)
>  			goto out;
> -	} while (!in_softirq() && time_before_eq(jiffies, now));
> +	} while (!in_softirq() && time_before_eq(jiffies, now) && !need_resched());
>  
>  	if (atomic_read(&ipv4_dst_ops.entries) < ip_rt_max_size)
>  		goto out;
> 

Nope, the above does not actually fix it.  I got a 716 usec latency in
rt_garbage_collect:

http://krustophenia.net/testresults.php?dataset=2.6.8.1-P9#/var/www/2.6.8.1-P9/trace12.txt

I believe this is associated with heavy route cache activity, I did not
see this one again until I left a gnutella client running overnight.

Lee