Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Karsten Wiese]

[-- Attachment #1: Type: text/plain, Size: 1006 bytes --]

Hi Ingo

I've refined io_apic.c a little more:
- merged all ioapic data into a struct ioapic_data_struct
- Allocate the ioapic cache memory from bootmem. Allocate only,
  what is needed ((nr of registers + 5)*sizeod(u64)).
- Removed "cache overrun by reg out of bounds" check.
  this should never happen, as reg is calculated out of irq_2_pin[irq].pin.
- Do the index lookup 
	struct ioapic_data_struct *ioapic = ioapic_data[apic];
  as early and as rarely as possible.
  In the consequence many functions parameter changed from
	io_apic_*(unsigned int apic,....
  to
	io_apic_*(struct ioapic_data_struct *ioapic,.... 
- New function setup_IO_APIC_early(int _ioapic).
  It does the setup of the ioapic's struct ioapic_data_struct.
  if IOAPIC_CACHE is defined, it calls the also new function
  ioapic_cache_init().
  ioapic_cache_init() initializes the cache completely.
  Thereby we may safely omit to query a caches elements validity before
  reading it.

Works fine here applied against -51-27

Karsten

[-- Attachment #2: io_apic-RT-51-23 --]
[-- Type: text/x-diff, Size: 22712 bytes --]

diff -ur ../linux-2.6.12-RT-51-23/arch/i386/kernel/io_apic.c ./arch/i386/kernel/io_apic.c
--- ../linux-2.6.12-RT-51-23/arch/i386/kernel/io_apic.c	2005-07-09 23:49:21.000000000 +0200
+++ ./arch/i386/kernel/io_apic.c	2005-07-12 01:08:34.000000000 +0200
@@ -31,6 +31,7 @@
 #include <linux/mc146818rtc.h>
 #include <linux/compiler.h>
 #include <linux/acpi.h>
+#include <linux/bootmem.h>
 
 #include <linux/sysdev.h>
 #include <asm/io.h>
@@ -55,11 +56,6 @@
 int sis_apic_bug = -1;
 
 /*
- * # of IRQ routing registers
- */
-int nr_ioapic_registers[MAX_IO_APICS];
-
-/*
  * Rough estimation of how many shared IRQs there are, can
  * be changed anytime.
  */
@@ -132,88 +128,74 @@
 # define IOAPIC_CACHE
 #endif
 
-#ifdef IOAPIC_CACHE
-# define MAX_IOAPIC_CACHE 512
 
-/*
- * Cache register values:
- */
-static struct {
-	unsigned int reg;
-	unsigned int val[MAX_IOAPIC_CACHE];
-} io_apic_cache[MAX_IO_APICS]
-		____cacheline_aligned_in_smp;
+
+struct ioapic_data_struct {
+	struct sys_device dev;
+	int nr_registers;	//  # of IRQ routing registers
+	volatile unsigned int *base;
+	struct IO_APIC_route_entry *entry;
+#ifdef IOAPIC_CACHE
+	unsigned int reg_set;
+	u32 cached_val[0];
 #endif
+};
 
-volatile unsigned int *io_apic_base[MAX_IO_APICS];
+static struct ioapic_data_struct *ioapic_data[MAX_IO_APICS];
 
-static inline unsigned int __raw_io_apic_read(unsigned int apic, unsigned int reg)
+
+static inline unsigned int __raw_io_apic_read(struct ioapic_data_struct *ioapic, unsigned int reg)
 {
-	volatile unsigned int *io_apic;
-#ifdef IOAPIC_CACHE
-	io_apic_cache[apic].reg = reg;
-#endif
-	io_apic = io_apic_base[apic];
-	io_apic[0] = reg;
-	return io_apic[4];
+# ifdef IOAPIC_CACHE
+	ioapic->reg_set = reg;
+# endif
+	ioapic->base[0] = reg;
+	return ioapic->base[4];
 }
 
-unsigned int raw_io_apic_read(unsigned int apic, unsigned int reg)
+
+# ifdef IOAPIC_CACHE
+static void __init ioapic_cache_init(struct ioapic_data_struct *ioapic)
 {
-	unsigned int val = __raw_io_apic_read(apic, reg);
+	int reg;
+	for (reg = 0; reg < (ioapic->nr_registers + 10); reg++)
+		ioapic->cached_val[reg] = __raw_io_apic_read(ioapic, reg);
+}
+# endif
 
-#ifdef IOAPIC_CACHE
-	io_apic_cache[apic].val[reg] = val;
-#endif
+
+static unsigned int raw_io_apic_read(struct ioapic_data_struct *ioapic, unsigned int reg)
+{
+	unsigned int val = __raw_io_apic_read(ioapic, reg);
+
+# ifdef IOAPIC_CACHE
+	ioapic->cached_val[reg] = val;
+# endif
 	return val;
 }
 
-unsigned int io_apic_read(unsigned int apic, unsigned int reg)
+static unsigned int io_apic_read(struct ioapic_data_struct *ioapic, unsigned int reg)
 {
-#ifdef IOAPIC_CACHE
-	if (unlikely(reg >= MAX_IOAPIC_CACHE)) {
-		static int once = 1;
-
-		if (once) {
-			once = 0;
-			printk("WARNING: ioapic register cache overflow: %d.\n",
-				reg);
-			dump_stack();
-		}
-		return __raw_io_apic_read(apic, reg);
-	}
-	if (io_apic_cache[apic].val[reg] && !sis_apic_bug) {
-		io_apic_cache[apic].reg = -1;
-		return io_apic_cache[apic].val[reg];
+# ifdef IOAPIC_CACHE
+	if (likely(!sis_apic_bug)) {
+		ioapic->reg_set = -1;
+		return ioapic->cached_val[reg];
 	}
-#endif
-	return raw_io_apic_read(apic, reg);
+# endif
+	return raw_io_apic_read(ioapic, reg);
 }
 
-void io_apic_write(unsigned int apic, unsigned int reg, unsigned int val)
+static void io_apic_write(struct ioapic_data_struct *ioapic, unsigned int reg, unsigned int val)
 {
-	volatile unsigned int *io_apic;
-#ifdef IOAPIC_CACHE
-	if (unlikely(reg >= MAX_IOAPIC_CACHE)) {
-		static int once = 1;
-
-		if (once) {
-			once = 0;
-			printk("WARNING: ioapic register cache overflow: %d.\n",
-				reg);
-			dump_stack();
-		}
-	} else
-		io_apic_cache[apic].val[reg] = val;
-#endif
-	io_apic = io_apic_base[apic];
-#ifdef IOAPIC_CACHE
-	io_apic_cache[apic].reg = reg;
-#endif
-	io_apic[0] = reg;
-	io_apic[4] = val;
+# ifdef IOAPIC_CACHE
+	ioapic->cached_val[reg] = val;
+	ioapic->reg_set = reg;
+# endif
+	ioapic->base[0] = reg;
+	ioapic->base[4] = val;
 }
 
+
 /*
  * Some systems need a POST flush or else level-triggered interrupts
  * generate lots of spurious interrupts due to the POST-ed write not
@@ -231,44 +213,42 @@
  *
  * Older SiS APIC requires we rewrite the index regiser
  */
-void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int val)
+static void io_apic_modify(struct ioapic_data_struct *ioapic, unsigned int reg, unsigned int val)
 {
-	volatile unsigned int *io_apic;
-#ifdef IOAPIC_CACHE
-	io_apic_cache[apic].val[reg] = val;
-#endif
-	io_apic = io_apic_base[apic];
 #ifdef IOAPIC_CACHE
-	if (io_apic_cache[apic].reg != reg || sis_apic_bug) {
-		io_apic_cache[apic].reg = reg;
+	ioapic->cached_val[reg] = val;
+	if (ioapic->reg_set != reg || sis_apic_bug) {
+		ioapic->reg_set = reg;
 #else
 	if (unlikely(sis_apic_bug)) {
 #endif
-		io_apic[0] = reg;
+		ioapic->base[0] = reg;
 	}
-	io_apic[4] = val;
+	ioapic->base[4] = val;
 #ifndef IOAPIC_POSTFLUSH
 	if (unlikely(sis_apic_bug))
 #endif
 		/*
 		 * Force POST flush by reading:
  		 */
-		val = io_apic[4];
+		val = ioapic->base[4];
 }
 
 static void __modify_IO_APIC_irq (unsigned int irq, unsigned long enable, unsigned long disable)
 {
 	struct irq_pin_list *entry = irq_2_pin + irq;
 	unsigned int pin, val;
+	struct ioapic_data_struct *ioapic;
 
 	for (;;) {
 		pin = entry->pin;
 		if (pin == -1)
 			break;
-		val = io_apic_read(entry->apic, 0x10 + pin*2);
+		ioapic = ioapic_data[entry->apic];
+		val = io_apic_read(ioapic, 0x10 + pin*2);
 		val &= ~disable;
 		val |= enable;
-		io_apic_modify(entry->apic, 0x10 + pin*2, val);
+		io_apic_modify(ioapic, 0x10 + pin*2, val);
 		if (!entry->next)
 			break;
 		entry = irq_2_pin + entry->next;
@@ -305,15 +285,15 @@
 	spin_unlock_irqrestore(&ioapic_lock, flags);
 }
 
-static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
+static void clear_IO_APIC_pin(struct ioapic_data_struct *ioapic, unsigned int pin)
 {
 	struct IO_APIC_route_entry entry;
 	unsigned long flags;
 	
 	/* Check delivery_mode to be sure we're not clearing an SMI pin */
 	spin_lock_irqsave(&ioapic_lock, flags);
-	*(((int*)&entry) + 0) = io_apic_read(apic, 0x10 + 2 * pin);
-	*(((int*)&entry) + 1) = io_apic_read(apic, 0x11 + 2 * pin);
+	*(((int*)&entry) + 0) = io_apic_read(ioapic, 0x10 + 2 * pin);
+	*(((int*)&entry) + 1) = io_apic_read(ioapic, 0x11 + 2 * pin);
 	spin_unlock_irqrestore(&ioapic_lock, flags);
 	if (entry.delivery_mode == dest_SMI)
 		return;
@@ -324,8 +304,8 @@
 	memset(&entry, 0, sizeof(entry));
 	entry.mask = 1;
 	spin_lock_irqsave(&ioapic_lock, flags);
-	io_apic_write(apic, 0x10 + 2 * pin, *(((int *)&entry) + 0));
-	io_apic_write(apic, 0x11 + 2 * pin, *(((int *)&entry) + 1));
+	io_apic_write(ioapic, 0x10 + 2 * pin, *(((int *)&entry) + 0));
+	io_apic_write(ioapic, 0x11 + 2 * pin, *(((int *)&entry) + 1));
 	spin_unlock_irqrestore(&ioapic_lock, flags);
 }
 
@@ -334,11 +314,12 @@
 	int apic, pin;
 
 	for (apic = 0; apic < nr_ioapics; apic++) {
+		struct ioapic_data_struct *ioapic = ioapic_data[apic];
 #ifdef IOAPIC_CACHE
-		io_apic_cache[apic].reg = -1;
+		ioapic->reg_set = -1;
 #endif
-		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
-			clear_IO_APIC_pin(apic, pin);
+		for (pin = 0; pin < ioapic->nr_registers; pin++)
+			clear_IO_APIC_pin(ioapic, pin);
 	}
 }
 
@@ -357,7 +338,7 @@
 		pin = entry->pin;
 		if (pin == -1)
 			break;
-		io_apic_write(entry->apic, 0x10 + 1 + pin*2, apicid_value);
+		io_apic_write(ioapic_data[entry->apic], 0x10 + 1 + pin*2, apicid_value);
 		if (!entry->next)
 			break;
 		entry = irq_2_pin + entry->next;
@@ -947,7 +928,7 @@
 		return;
 
 	for (ioapic = 0; ioapic < nr_ioapics; ioapic++) {
-		for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
+		for (pin = 0; pin < ioapic_data[ioapic]->nr_registers; pin++) {
 			irq_entry = find_irq_entry(ioapic, pin, mp_INT);
 			if (irq_entry == -1)
 				continue;
@@ -1190,7 +1171,7 @@
 			 */
 			i = irq = 0;
 			while (i < apic)
-				irq += nr_ioapic_registers[i++];
+				irq += ioapic_data[i++]->nr_registers;
 			irq += pin;
 
 			/*
@@ -1233,7 +1214,7 @@
 	int apic, idx, pin;
 
 	for (apic = 0; apic < nr_ioapics; apic++) {
-		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
+		for (pin = 0; pin < ioapic_data[apic]->nr_registers; pin++) {
 			idx = find_irq_entry(apic,pin,mp_INT);
 			if ((idx != -1) && (irq == pin_2_irq(idx,apic,pin)))
 				return irq_trigger(idx);
@@ -1305,11 +1286,13 @@
 	struct IO_APIC_route_entry entry;
 	int apic, pin, idx, irq, first_notcon = 1, vector;
 	unsigned long flags;
+	struct ioapic_data_struct *ioapic;
 
 	apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
 
 	for (apic = 0; apic < nr_ioapics; apic++) {
-	for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
+		ioapic = ioapic_data[apic];
+	for (pin = 0; pin < ioapic->nr_registers; pin++) {
 
 		/*
 		 * add it to the IO-APIC irq-routing table:
@@ -1366,8 +1349,8 @@
 				disable_8259A_irq(irq);
 		}
 		spin_lock_irqsave(&ioapic_lock, flags);
-		io_apic_write(apic, 0x11+2*pin, *(((int *)&entry)+1));
-		io_apic_write(apic, 0x10+2*pin, *(((int *)&entry)+0));
+		io_apic_write(ioapic, 0x11+2*pin, *(((int *)&entry)+1));
+		io_apic_write(ioapic, 0x10+2*pin, *(((int *)&entry)+0));
 		spin_unlock_irqrestore(&ioapic_lock, flags);
 	}
 	}
@@ -1432,6 +1415,7 @@
 	union IO_APIC_reg_02 reg_02;
 	union IO_APIC_reg_03 reg_03;
 	unsigned long flags;
+	struct ioapic_data_struct *ioapic;
 
 	if (apic_verbosity == APIC_QUIET)
 		return;
@@ -1439,7 +1423,7 @@
  	printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
 	for (i = 0; i < nr_ioapics; i++)
 		printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
-		       mp_ioapics[i].mpc_apicid, nr_ioapic_registers[i]);
+		       mp_ioapics[i].mpc_apicid, ioapic_data[i]->nr_registers);
 
 	/*
 	 * We are a bit conservative about what we expect.  We have to
@@ -1448,14 +1432,14 @@
 	printk(KERN_INFO "testing the IO APIC.......................\n");
 
 	for (apic = 0; apic < nr_ioapics; apic++) {
-
+	ioapic = ioapic_data[apic];
 	spin_lock_irqsave(&ioapic_lock, flags);
-	reg_00.raw = io_apic_read(apic, 0);
-	reg_01.raw = io_apic_read(apic, 1);
+	reg_00.raw = io_apic_read(ioapic, 0);
+	reg_01.raw = io_apic_read(ioapic, 1);
 	if (reg_01.bits.version >= 0x10)
-		reg_02.raw = io_apic_read(apic, 2);
+		reg_02.raw = io_apic_read(ioapic, 2);
 	if (reg_01.bits.version >= 0x20)
-		reg_03.raw = io_apic_read(apic, 3);
+		reg_03.raw = io_apic_read(ioapic, 3);
 	spin_unlock_irqrestore(&ioapic_lock, flags);
 
 	printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mpc_apicid);
@@ -1526,8 +1510,8 @@
 		struct IO_APIC_route_entry entry;
 
 		spin_lock_irqsave(&ioapic_lock, flags);
-		*(((int *)&entry)+0) = raw_io_apic_read(apic, 0x10+i*2);
-		*(((int *)&entry)+1) = raw_io_apic_read(apic, 0x11+i*2);
+		*(((int *)&entry)+0) = raw_io_apic_read(ioapic, 0x10+i*2);
+		*(((int *)&entry)+1) = raw_io_apic_read(ioapic, 0x11+i*2);
 		spin_unlock_irqrestore(&ioapic_lock, flags);
 
 		printk(KERN_DEBUG " %02x %03X %02X  ",
@@ -1720,9 +1704,7 @@
 
 static void __init enable_IO_APIC(void)
 {
-	union IO_APIC_reg_01 reg_01;
 	int i;
-	unsigned long flags;
 
 	for (i = 0; i < PIN_MAP_SIZE; i++) {
 		irq_2_pin[i].pin = -1;
@@ -1733,16 +1715,6 @@
 			pirq_entries[i] = -1;
 
 	/*
-	 * The number of IO-APIC IRQ registers (== #pins):
-	 */
-	for (i = 0; i < nr_ioapics; i++) {
-		spin_lock_irqsave(&ioapic_lock, flags);
-		reg_01.raw = io_apic_read(i, 1);
-		spin_unlock_irqrestore(&ioapic_lock, flags);
-		nr_ioapic_registers[i] = reg_01.bits.entries+1;
-	}
-
-	/*
 	 * Do not trust the IO-APIC being empty at bootup
 	 */
 	clear_IO_APIC();
@@ -1777,6 +1749,7 @@
 	int i;
 	unsigned char old_id;
 	unsigned long flags;
+	struct ioapic_data_struct *ioapic;
 
 	/*
 	 * This is broken; anything with a real cpu count has to
@@ -1788,10 +1761,10 @@
 	 * Set the IOAPIC ID to the value stored in the MPC table.
 	 */
 	for (apic = 0; apic < nr_ioapics; apic++) {
-
+		ioapic = ioapic_data[apic];
 		/* Read the register 0 value */
 		spin_lock_irqsave(&ioapic_lock, flags);
-		reg_00.raw = io_apic_read(apic, 0);
+		reg_00.raw = io_apic_read(ioapic, 0);
 		spin_unlock_irqrestore(&ioapic_lock, flags);
 		
 		old_id = mp_ioapics[apic].mpc_apicid;
@@ -1856,14 +1829,14 @@
 
 		reg_00.bits.ID = mp_ioapics[apic].mpc_apicid;
 		spin_lock_irqsave(&ioapic_lock, flags);
-		io_apic_write(apic, 0, reg_00.raw);
+		io_apic_write(ioapic, 0, reg_00.raw);
 		spin_unlock_irqrestore(&ioapic_lock, flags);
 
 		/*
 		 * Sanity check
 		 */
 		spin_lock_irqsave(&ioapic_lock, flags);
-		reg_00.raw = io_apic_read(apic, 0);
+		reg_00.raw = io_apic_read(ioapic, 0);
 		spin_unlock_irqrestore(&ioapic_lock, flags);
 		if (reg_00.bits.ID != mp_ioapics[apic].mpc_apicid)
 			printk("could not set ID!\n");
@@ -2230,7 +2203,7 @@
  * cycles as some i82489DX-based boards have glue logic that keeps the
  * 8259A interrupt line asserted until INTA.  --macro
  */
-static inline void unlock_ExtINT_logic(void)
+static void __init unlock_ExtINT_logic(void)
 {
 	int pin, i;
 	struct IO_APIC_route_entry entry0, entry1;
@@ -2291,7 +2264,7 @@
  * is so screwy.  Thanks to Brian Perkins for testing/hacking this beast
  * fanatically on his truly buggy board.
  */
-static inline void check_timer(void)
+static void __init check_timer(void)
 {
 	int pin1, pin2;
 	int vector;
@@ -2402,6 +2375,43 @@
 		"report.  Then try booting with the 'noapic' option");
 }
 
+void __init setup_IO_APIC_early(int _ioapic)
+{
+	union IO_APIC_reg_01 reg_01;
+	unsigned long flags;
+	int size, nr_ioapic_registers;
+	volatile int *ioapic;
+	if (ioapic_data[_ioapic]) {
+		printk("been in %s before !!!!!\n", __FUNCTION__);
+		return;
+	}
+
+	/*
+	 * The number of IO-APIC IRQ registers (== #pins):
+	 */
+	ioapic = IO_APIC_BASE(_ioapic);
+	spin_lock_irqsave(&ioapic_lock, flags);
+	ioapic[0] = 1;
+	reg_01.raw = ioapic[4];
+	spin_unlock_irqrestore(&ioapic_lock, flags);
+	nr_ioapic_registers = reg_01.bits.entries+1;
+
+	/*
+	 * Initialsize ioapic_data struct:
+	 */
+	size = sizeof(struct ioapic_data_struct);
+#ifdef IOAPIC_CACHE
+	size += 0x10 * sizeof(u32) + nr_ioapic_registers * sizeof(struct IO_APIC_route_entry);
+#endif
+	ioapic_data[_ioapic] = alloc_bootmem(size);
+	memset(ioapic_data[_ioapic], 0, size);
+	ioapic_data[_ioapic]->nr_registers = nr_ioapic_registers;
+	ioapic_data[_ioapic]->base = ioapic;
+#ifdef IOAPIC_CACHE
+	ioapic_cache_init(ioapic_data[_ioapic]);
+#endif
+}
+
 /*
  *
  * IRQ's that are handled by the PIC in the MPS IOAPIC case.
@@ -2449,25 +2459,22 @@
 
 late_initcall(io_apic_bug_finalize);
 
-struct sysfs_ioapic_data {
-	struct sys_device dev;
-	struct IO_APIC_route_entry entry[0];
-};
-static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS];
-
 static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
 {
 	struct IO_APIC_route_entry *entry;
-	struct sysfs_ioapic_data *data;
+	struct ioapic_data_struct *data;
 	unsigned long flags;
 	int i;
+	struct ioapic_data_struct *ioapic;
 	
-	data = container_of(dev, struct sysfs_ioapic_data, dev);
+	data = container_of(dev, struct ioapic_data_struct, dev);
 	entry = data->entry;
+
+	ioapic = ioapic_data[dev->id];
 	spin_lock_irqsave(&ioapic_lock, flags);
-	for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ ) {
-		*(((int *)entry) + 1) = io_apic_read(dev->id, 0x11 + 2 * i);
-		*(((int *)entry) + 0) = io_apic_read(dev->id, 0x10 + 2 * i);
+	for (i = 0; i < ioapic_data[dev->id]->nr_registers; i ++, entry ++) {
+		*(((int *)entry) + 1) = io_apic_read(ioapic, 0x11 + 2 * i);
+		*(((int *)entry) + 0) = io_apic_read(ioapic, 0x10 + 2 * i);
 	}
 	spin_unlock_irqrestore(&ioapic_lock, flags);
 
@@ -2477,23 +2484,25 @@
 static int ioapic_resume(struct sys_device *dev)
 {
 	struct IO_APIC_route_entry *entry;
-	struct sysfs_ioapic_data *data;
+	struct ioapic_data_struct *data;
 	unsigned long flags;
 	union IO_APIC_reg_00 reg_00;
 	int i;
-	
-	data = container_of(dev, struct sysfs_ioapic_data, dev);
+	struct ioapic_data_struct *ioapic;
+
+	data = container_of(dev, struct ioapic_data_struct, dev);
 	entry = data->entry;
 
+	ioapic = ioapic_data[dev->id];
 	spin_lock_irqsave(&ioapic_lock, flags);
-	reg_00.raw = io_apic_read(dev->id, 0);
+	reg_00.raw = io_apic_read(ioapic, 0);
 	if (reg_00.bits.ID != mp_ioapics[dev->id].mpc_apicid) {
 		reg_00.bits.ID = mp_ioapics[dev->id].mpc_apicid;
-		io_apic_write(dev->id, 0, reg_00.raw);
+		io_apic_write(ioapic, 0, reg_00.raw);
 	}
-	for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ ) {
-		io_apic_write(dev->id, 0x11+2*i, *(((int *)entry)+1));
-		io_apic_write(dev->id, 0x10+2*i, *(((int *)entry)+0));
+	for (i = 0; i < ioapic_data[dev->id]->nr_registers; i ++, entry ++) {
+		io_apic_write(ioapic, 0x11+2*i, *(((int *)entry)+1));
+		io_apic_write(ioapic, 0x10+2*i, *(((int *)entry)+0));
 	}
 	spin_unlock_irqrestore(&ioapic_lock, flags);
 
@@ -2516,21 +2525,20 @@
 		return error;
 
 	for (i = 0; i < nr_ioapics; i++ ) {
-		size = sizeof(struct sys_device) + nr_ioapic_registers[i] 
-			* sizeof(struct IO_APIC_route_entry);
-		mp_ioapic_data[i] = kmalloc(size, GFP_KERNEL);
-		if (!mp_ioapic_data[i]) {
+		size = ioapic_data[i]->nr_registers * sizeof(struct IO_APIC_route_entry);
+		ioapic_data[i]->entry = kmalloc(size, GFP_KERNEL);
+		if (!ioapic_data[i]->entry) {
 			printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
 			continue;
 		}
-		memset(mp_ioapic_data[i], 0, size);
-		dev = &mp_ioapic_data[i]->dev;
+		memset(ioapic_data[i]->entry, 0, size);
+		dev = &ioapic_data[i]->dev;
 		dev->id = i; 
 		dev->cls = &ioapic_sysdev_class;
 		error = sysdev_register(dev);
 		if (error) {
-			kfree(mp_ioapic_data[i]);
-			mp_ioapic_data[i] = NULL;
+			kfree(ioapic_data[i]->entry);
+			ioapic_data[i]->entry = NULL;
 			printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
 			continue;
 		}
@@ -2547,13 +2555,14 @@
 
 #ifdef CONFIG_ACPI_BOOT
 
-int __init io_apic_get_unique_id (int ioapic, int apic_id)
+int __init io_apic_get_unique_id (int apic, int apic_id)
 {
 	union IO_APIC_reg_00 reg_00;
 	static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
 	physid_mask_t tmp;
 	unsigned long flags;
 	int i = 0;
+	struct ioapic_data_struct *ioapic = ioapic_data[apic];
 
 	/*
 	 * The P4 platform supports up to 256 APIC IDs on two separate APIC 
@@ -2573,7 +2582,7 @@
 
 	if (apic_id >= get_physical_broadcast()) {
 		printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
-			"%d\n", ioapic, apic_id, reg_00.bits.ID);
+			"%d\n", apic, apic_id, reg_00.bits.ID);
 		apic_id = reg_00.bits.ID;
 	}
 
@@ -2592,7 +2601,7 @@
 			panic("Max apic_id exceeded!\n");
 
 		printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
-			"trying %d\n", ioapic, apic_id, i);
+			"trying %d\n", apic, apic_id, i);
 
 		apic_id = i;
 	} 
@@ -2610,50 +2619,50 @@
 
 		/* Sanity check */
 		if (reg_00.bits.ID != apic_id)
-			panic("IOAPIC[%d]: Unable change apic_id!\n", ioapic);
+			panic("IOAPIC[%d]: Unable change apic_id!\n", apic);
 	}
 
 	apic_printk(APIC_VERBOSE, KERN_INFO
-			"IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
+			"IOAPIC[%d]: Assigned apic_id %d\n", apic, apic_id);
 
 	return apic_id;
 }
 
 
-int __init io_apic_get_version (int ioapic)
+int __init io_apic_get_version (int apic)
 {
 	union IO_APIC_reg_01	reg_01;
 	unsigned long flags;
 
 	spin_lock_irqsave(&ioapic_lock, flags);
-	reg_01.raw = io_apic_read(ioapic, 1);
+	reg_01.raw = io_apic_read(ioapic_data[apic], 1);
 	spin_unlock_irqrestore(&ioapic_lock, flags);
 
 	return reg_01.bits.version;
 }
 
 
-int __init io_apic_get_redir_entries (int ioapic)
+int __init io_apic_get_redir_entries (int apic)
 {
 	union IO_APIC_reg_01	reg_01;
 	unsigned long flags;
 
 	spin_lock_irqsave(&ioapic_lock, flags);
-	reg_01.raw = io_apic_read(ioapic, 1);
+	reg_01.raw = io_apic_read(ioapic_data[apic], 1);
 	spin_unlock_irqrestore(&ioapic_lock, flags);
 
 	return reg_01.bits.entries;
 }
 
 
-int io_apic_set_pci_routing (int ioapic, int pin, int irq, int edge_level, int active_high_low)
+int io_apic_set_pci_routing (int apic, int pin, int irq, int edge_level, int active_high_low)
 {
 	struct IO_APIC_route_entry entry;
 	unsigned long flags;
-
+	struct ioapic_data_struct *ioapic = ioapic_data[apic];
 	if (!IO_APIC_IRQ(irq)) {
 		printk(KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
-			ioapic);
+			apic);
 		return -EINVAL;
 	}
 
@@ -2676,18 +2685,18 @@
 	 * IRQs < 16 are already in the irq_2_pin[] map
 	 */
 	if (irq >= 16)
-		add_pin_to_irq(irq, ioapic, pin);
+		add_pin_to_irq(irq, apic, pin);
 
 	entry.vector = assign_irq_vector(irq);
 
 	apic_printk(APIC_DEBUG, KERN_DEBUG "IOAPIC[%d]: Set PCI routing entry "
-		"(%d-%d -> 0x%x -> IRQ %d Mode:%i Active:%i)\n", ioapic,
-		mp_ioapics[ioapic].mpc_apicid, pin, entry.vector, irq,
+		"(%d-%d -> 0x%x -> IRQ %d Mode:%i Active:%i)\n", apic,
+		mp_ioapics[apic].mpc_apicid, pin, entry.vector, irq,
 		edge_level, active_high_low);
 
 	ioapic_register_intr(irq, entry.vector, edge_level);
 
-	if (!ioapic && (irq < 16))
+	if (!apic && (irq < 16))
 		disable_8259A_irq(irq);
 
 	spin_lock_irqsave(&ioapic_lock, flags);
diff -ur ../linux-2.6.12-RT-51-23/arch/i386/kernel/mpparse.c ./arch/i386/kernel/mpparse.c
--- ../linux-2.6.12-RT-51-23/arch/i386/kernel/mpparse.c	2005-07-09 23:53:19.000000000 +0200
+++ ./arch/i386/kernel/mpparse.c	2005-07-11 20:30:55.000000000 +0200
@@ -263,7 +263,7 @@
 		return;
 	}
 	mp_ioapics[nr_ioapics] = *m;
-	io_apic_base[nr_ioapics] = IO_APIC_BASE(nr_ioapics);
+	setup_IO_APIC_early(nr_ioapics);
 	nr_ioapics++;
 }
 
@@ -915,7 +915,7 @@
 	mp_ioapics[idx].mpc_apicaddr = address;
 
 	set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
-	io_apic_base[idx] = IO_APIC_BASE(idx);
+	setup_IO_APIC_early(idx);
 	mp_ioapics[idx].mpc_apicid = io_apic_get_unique_id(idx, id);
 	mp_ioapics[idx].mpc_apicver = io_apic_get_version(idx);
 	
diff -ur ../linux-2.6.12-RT-51-23/include/asm-i386/io_apic.h ./include/asm-i386/io_apic.h
--- ../linux-2.6.12-RT-51-23/include/asm-i386/io_apic.h	2005-07-09 23:55:50.000000000 +0200
+++ ./include/asm-i386/io_apic.h	2005-07-11 20:28:42.000000000 +0200
@@ -155,15 +155,14 @@
 /* MP IRQ source entries */
 extern struct mpc_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
 
-extern volatile unsigned int *io_apic_base[MAX_IO_APICS];
-
 /* non-0 if default (table-less) MP configuration */
 extern int mpc_default_type;
 
-extern unsigned int raw_io_apic_read(unsigned int apic, unsigned int reg);
-extern unsigned int io_apic_read(unsigned int apic, unsigned int reg);
-extern void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value);
-extern void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value);
+extern void setup_IO_APIC_early(int ioapic);
 extern int sis_apic_bug;
 
 /* 1 if "noapic" boot option passed */

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Karsten Wiese <annabellesgarden@yahoo.de> wrote:

> Hi Ingo
> 
> I've refined io_apic.c a little more:

great. I've applied these changes and have released the -28 patch. (note 
that the last chunk of your patch was malformed, have applied it by 
hand.)

i'm wondering what your thoughts are about IOAPIC_POSTFLUSH - i had to 
turn it on unconditionally again, to get rid of spurious interrupts and 
outright interrupt storms (and resulting lockups) on some systems.  
IOAPIC_POSTFLUSH is now causing much of the IO-APIC related IRQ handling 
overhead.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Daniel Walker]

On Tue, 2005-07-12 at 16:02 +0200, Ingo Molnar wrote:
> * Karsten Wiese <annabellesgarden@yahoo.de> wrote:
> 
> > Hi Ingo
> > 
> > I've refined io_apic.c a little more:
> 
> great. I've applied these changes and have released the -28 patch. (note 
> that the last chunk of your patch was malformed, have applied it by 
> hand.)
> 
> i'm wondering what your thoughts are about IOAPIC_POSTFLUSH - i had to 
> turn it on unconditionally again, to get rid of spurious interrupts and 
> outright interrupt storms (and resulting lockups) on some systems.  
> IOAPIC_POSTFLUSH is now causing much of the IO-APIC related IRQ handling 
> overhead.

I observed a situation on a dual xeon where IOAPIC_POSTFLUSH , if on,
would actually cause spurious interrupts. It was odd cause it's suppose
to stop them .. If there was a lot of interrupt traffic on one IRQ , it
would cause interrupt traffic on another IRQ. This would result in
"nobody cared" messages , and the storming IRQ line would get shutdown.

This would only happen in PREEMPT_RT .

Daniel

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Daniel Walker <dwalker@mvista.com> wrote:

> I observed a situation on a dual xeon where IOAPIC_POSTFLUSH , if on, 
> would actually cause spurious interrupts. It was odd cause it's 
> suppose to stop them .. If there was a lot of interrupt traffic on one 
> IRQ , it would cause interrupt traffic on another IRQ. This would 
> result in "nobody cared" messages , and the storming IRQ line would 
> get shutdown.
> 
> This would only happen in PREEMPT_RT .

does it happen with the latest kernel too? There were a couple of things 
broken in the IOAPIC code in various earlier versions.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Daniel Walker]

On Tue, 2005-07-12 at 16:28 +0200, Ingo Molnar wrote:
> * Daniel Walker <dwalker@mvista.com> wrote:
> 
> > I observed a situation on a dual xeon where IOAPIC_POSTFLUSH , if on, 
> > would actually cause spurious interrupts. It was odd cause it's 
> > suppose to stop them .. If there was a lot of interrupt traffic on one 
> > IRQ , it would cause interrupt traffic on another IRQ. This would 
> > result in "nobody cared" messages , and the storming IRQ line would 
> > get shutdown.
> > 
> > This would only happen in PREEMPT_RT .
> 
> does it happen with the latest kernel too? There were a couple of things 
> broken in the IOAPIC code in various earlier versions.


I haven't tested it recently . This was on an older version of RT
though . I could try it if it's interesting ? Or do you think it's
already fixed?

Daniel

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Daniel Walker <dwalker@mvista.com> wrote:

> I haven't tested it recently . This was on an older version of RT 
> though . I could try it if it's interesting ? Or do you think it's 
> already fixed?

it would be definitely interesting to see how robust the latest IO-APIC 
code is.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[K.R. Foley]

Ingo Molnar wrote:
> * Daniel Walker <dwalker@mvista.com> wrote:
> 
> 
>>I observed a situation on a dual xeon where IOAPIC_POSTFLUSH , if on, 
>>would actually cause spurious interrupts. It was odd cause it's 
>>suppose to stop them .. If there was a lot of interrupt traffic on one 
>>IRQ , it would cause interrupt traffic on another IRQ. This would 
>>result in "nobody cared" messages , and the storming IRQ line would 
>>get shutdown.
>>
>>This would only happen in PREEMPT_RT .
> 
> 
> does it happen with the latest kernel too? There were a couple of things 
> broken in the IOAPIC code in various earlier versions.
> 
> 	Ingo

Is this why I have been able to boot the latest versions without the 
noapic option (and without noticeable keyboard repeat problems) or has 
it just been dumb luck?

-- 
    kr

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Daniel Walker]

On Tue, 2005-07-12 at 16:46 +0200, Ingo Molnar wrote:
> * Daniel Walker <dwalker@mvista.com> wrote:
> 
> > I haven't tested it recently . This was on an older version of RT 
> > though . I could try it if it's interesting ? Or do you think it's 
> > already fixed?
> 
> it would be definitely interesting to see how robust the latest IO-APIC 
> code is.

I'm sure you test that all the time, but I'll try it ..

Daniel

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* K.R. Foley <kr@cybsft.com> wrote:

> Ingo Molnar wrote:
> >* Daniel Walker <dwalker@mvista.com> wrote:
> >
> >
> >>I observed a situation on a dual xeon where IOAPIC_POSTFLUSH , if on, 
> >>would actually cause spurious interrupts. It was odd cause it's 
> >>suppose to stop them .. If there was a lot of interrupt traffic on one 
> >>IRQ , it would cause interrupt traffic on another IRQ. This would 
> >>result in "nobody cared" messages , and the storming IRQ line would 
> >>get shutdown.
> >>
> >>This would only happen in PREEMPT_RT .
> >
> >
> >does it happen with the latest kernel too? There were a couple of things 
> >broken in the IOAPIC code in various earlier versions.
> >
> >	Ingo
> 
> Is this why I have been able to boot the latest versions without the 
> noapic option (and without noticeable keyboard repeat problems) or has 
> it just been dumb luck?

yes, i think it's related - the IO-APIC code is now more robust than 
ever, and that's why any known-broken system would be important to 
re-check.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[K.R. Foley]

Ingo Molnar wrote:
> * K.R. Foley <kr@cybsft.com> wrote:
> 
> 
>>Ingo Molnar wrote:
>>
>>>* Daniel Walker <dwalker@mvista.com> wrote:
>>>
>>>
>>>
>>>>I observed a situation on a dual xeon where IOAPIC_POSTFLUSH , if on, 
>>>>would actually cause spurious interrupts. It was odd cause it's 
>>>>suppose to stop them .. If there was a lot of interrupt traffic on one 
>>>>IRQ , it would cause interrupt traffic on another IRQ. This would 
>>>>result in "nobody cared" messages , and the storming IRQ line would 
>>>>get shutdown.
>>>>
>>>>This would only happen in PREEMPT_RT .
>>>
>>>
>>>does it happen with the latest kernel too? There were a couple of things 
>>>broken in the IOAPIC code in various earlier versions.
>>>
>>>	Ingo
>>
>>Is this why I have been able to boot the latest versions without the 
>>noapic option (and without noticeable keyboard repeat problems) or has 
>>it just been dumb luck?
> 
> 
> yes, i think it's related - the IO-APIC code is now more robust than 
> ever, and that's why any known-broken system would be important to 
> re-check.
> 
> 	Ingo
> 

Well I have booted -27 a couple of times and -28 once now without 
supplying the noapic boot option and I haven't seen any of the keyboard 
repeat problems that I reported late last week. This was on my dual 2.6 
Xeon w/HT. I have never seen this behavior on any of my older systems. 
Because of the fact that the problem showed up sporadically, I can't say 
for sure that it is gone. However, so far so good. I will report any 
changes that I see.

-- 
    kr

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[William Weston]

On Tue, 12 Jul 2005, Ingo Molnar wrote:

> * K.R. Foley <kr@cybsft.com> wrote:
> 
> > Is this why I have been able to boot the latest versions without the 
> > noapic option (and without noticeable keyboard repeat problems) or has 
> > it just been dumb luck?
> 
> yes, i think it's related - the IO-APIC code is now more robust than 
> ever, and that's why any known-broken system would be important to 
> re-check.

Just brought the SIS based Xeon box up on -51-28 with CONFIG_IOAPIC_FAST
and without the noapic option, and all is well.  On -50-43, noapic was 
needed to avoid the spurious interrupt and APIC error on CPU0 messages.

Are there any non-SIS chipsets out there that need the sis_apic_bug 
checks?  What about IOAPIC_POSTFLUSH?

--ww

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Karsten Wiese]

Am Dienstag, 12. Juli 2005 16:25 schrieb Daniel Walker:
> On Tue, 2005-07-12 at 16:02 +0200, Ingo Molnar wrote:
> > * Karsten Wiese <annabellesgarden@yahoo.de> wrote:
> > 
> > > Hi Ingo
> > > 
> > > I've refined io_apic.c a little more:
> > 
> > great. I've applied these changes and have released the -28 patch. (note 
> > that the last chunk of your patch was malformed, have applied it by 
> > hand.)
> > 
> > i'm wondering what your thoughts are about IOAPIC_POSTFLUSH - i had to 
> > turn it on unconditionally again, to get rid of spurious interrupts and 
> > outright interrupt storms (and resulting lockups) on some systems.  
> > IOAPIC_POSTFLUSH is now causing much of the IO-APIC related IRQ handling 
> > overhead.
> 
> I observed a situation on a dual xeon where IOAPIC_POSTFLUSH , if on,
> would actually cause spurious interrupts. It was odd cause it's suppose
> to stop them .. If there was a lot of interrupt traffic on one IRQ , it
> would cause interrupt traffic on another IRQ. This would result in
> "nobody cared" messages , and the storming IRQ line would get shutdown.
> 
> This would only happen in PREEMPT_RT .
> 
i've only tested on 2005ish Ahlon64_UP@k8T800: it doesn't need any of the quirks
  IOAPIC_POSTFLUSH, sis_bug, level-edge cleanup.
IOAPIC_POSTFLUSH caused no negative influence neither.
i've an io_apic_one.c here, that doesn't have any of the quirks and is stripped down
to handle just one ioapic. It runs just fine on PREEMPT_RT.
Could be used as a "Do i crash/suffer without the quirks?" test for 1 ioapic equipped machines.
Should i post it?

On vanilla level-interrupts behave "nervously" showing the "doubled rate" effect:
For level interrupts vanilla just sends an EOI to the apic. Nothing else should be necessary (io)apic wise.
Only here, when the edge-level triggered hardware-handler acknowledges its client
(== resets the interrupt line) the doubled level interrupt is triggered on the same pin.
Thus the same interrupt routine runs again as soon as EOI has been sent and IFlag is open again.
hardware-handler receives it, does nothing and returns with "uninterested" <<insert correct macro here<.
everything works correctly though. its just some cycles wasted.
Odd, no? I found that the correct interrupt handling can be achieve by obeying the PREEMPT_RT method:
	level-interrupt triggers
		ioapic pin is masked
			harware-handler deasserts pin/line
		apic gets eoi
		ioapic pin is unmasked
this could be optimized by ioapic caching , but there must be some configuration tweak......
that i don't know of. there are no via docs for me. erm.. anybody sends me a link _offlist_ ?-)
Or even _knows that tweak_ ?
And again IOAPIC_POSTFLUSH has no influence here on the doubled interrupt (VIA K8T800 only ?)
bug.

    Karsten

	

	
		
___________________________________________________________ 
Gesendet von Yahoo! Mail - Jetzt mit 1GB Speicher kostenlos - Hier anmelden: http://mail.yahoo.de

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[William Weston]

On Wed, 13 Jul 2005, Karsten Wiese wrote:

> i've only tested on 2005ish Ahlon64_UP@k8T800: it doesn't need any of the quirks
>   IOAPIC_POSTFLUSH, sis_bug, level-edge cleanup.
> IOAPIC_POSTFLUSH caused no negative influence neither.
> i've an io_apic_one.c here, that doesn't have any of the quirks and is stripped down
> to handle just one ioapic. It runs just fine on PREEMPT_RT.
> Could be used as a "Do i crash/suffer without the quirks?" test for 1 ioapic equipped machines.
> Should i post it?

Please do.  I'll give it a spin on several machines and let you know how 
it goes ;-}

--ww

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Karsten Wiese]

[-- Attachment #1: Type: text/plain, Size: 1087 bytes --]

Am Mittwoch, 13. Juli 2005 02:08 schrieb William Weston:
> On Wed, 13 Jul 2005, Karsten Wiese wrote:
> 
> > i've only tested on 2005ish Ahlon64_UP@k8T800: it doesn't need any of the quirks
> >   IOAPIC_POSTFLUSH, sis_bug, level-edge cleanup.
> > IOAPIC_POSTFLUSH caused no negative influence neither.
> > i've an io_apic_one.c here, that doesn't have any of the quirks and is stripped down
> > to handle just one ioapic. It runs just fine on PREEMPT_RT.
> > Could be used as a "Do i crash/suffer without the quirks?" test for 1 ioapic equipped machines.
> > Should i post it?
> 
> Please do.  I'll give it a spin on several machines and let you know how 
> it goes ;-}
> 
This is it. Apply it on top of -51-30 and 'make oldconfig'.
You'll be asked about the new CONFIG_ONE_IOAPIC. Answering yes, 
io_apic_one.c will be used, which is
((io_apic.c minus quirks) minus multi_io_apic_capability)
.
It'll propably crash some machines, as the quirks are there for some reason...
With CONFIG_ONE_IOAPIC unset, you'll get the same kernel as with my
previously sent patch.
have fun :-)

	Karsten

[-- Attachment #2: io_apic-RT-51-30+io_apic_one.bz2 --]
[-- Type: application/x-bzip2, Size: 18261 bytes --]

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Daniel Walker]

On Tue, 2005-07-12 at 07:56 -0700, Daniel Walker wrote:
> On Tue, 2005-07-12 at 16:46 +0200, Ingo Molnar wrote:
> > * Daniel Walker <dwalker@mvista.com> wrote:
> > 
> > > I haven't tested it recently . This was on an older version of RT 
> > > though . I could try it if it's interesting ? Or do you think it's 
> > > already fixed?
> > 
> > it would be definitely interesting to see how robust the latest IO-APIC 
> > code is.
> 
> I'm sure you test that all the time, but I'll try it ..

I tried it, and it appears to be fixed in current RT ..

Daniel

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Daniel Walker <dwalker@mvista.com> wrote:

> On Tue, 2005-07-12 at 07:56 -0700, Daniel Walker wrote:
> > On Tue, 2005-07-12 at 16:46 +0200, Ingo Molnar wrote:
> > > * Daniel Walker <dwalker@mvista.com> wrote:
> > > 
> > > > I haven't tested it recently . This was on an older version of RT 
> > > > though . I could try it if it's interesting ? Or do you think it's 
> > > > already fixed?
> > > 
> > > it would be definitely interesting to see how robust the latest IO-APIC 
> > > code is.
> > 
> > I'm sure you test that all the time, but I'll try it ..
> 
> I tried it, and it appears to be fixed in current RT ..

cool, thanks.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Karsten Wiese <annabellesgarden@yahoo.de> wrote:

> This is it. Apply it on top of -51-30 and 'make oldconfig'. You'll be 
> asked about the new CONFIG_ONE_IOAPIC. Answering yes, io_apic_one.c 
> will be used, which is ((io_apic.c minus quirks) minus 
> multi_io_apic_capability) . It'll propably crash some machines, as the 
> quirks are there for some reason... With CONFIG_ONE_IOAPIC unset, 
> you'll get the same kernel as with my previously sent patch. have fun 
> :-)

this could only be offered if it's merged into the existing io_apic.c.  
The current IOAPIC_FAST option will go away eventually, but ONE_IOAPIC 
would/could be intentionally incompatible.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Lee Revell]

On Wed, 2005-07-13 at 08:35 +0200, Ingo Molnar wrote:
> * Lee Revell <rlrevell@joe-job.com> wrote:
> 
> > > I have found that heavy network traffic really kills the interactive
> > > performance.  In the top excerpt below, gtk-gnutella is generating about
> > > 320KB/sec of traffic.
> > > 
> > > These priorities do not look right:
> > 
> > Never mind, I just rediscovered the RT priority leakage bug.
> 
> ok. And is it fixed by recent kernels?
> 

Yes, I have not seen the problem yet with V0.7.51-28.

Lee

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Lee Revell <rlrevell@joe-job.com> wrote:

> > I have found that heavy network traffic really kills the interactive
> > performance.  In the top excerpt below, gtk-gnutella is generating about
> > 320KB/sec of traffic.
> > 
> > These priorities do not look right:
> 
> Never mind, I just rediscovered the RT priority leakage bug.

ok. And is it fixed by recent kernels?

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Lee Revell]

On Tue, 2005-07-12 at 21:18 -0400, Lee Revell wrote:
> On Mon, 2005-07-04 at 13:16 +0200, Ingo Molnar wrote:
> > i'd first suggest to try the latest kernel, before changing your X 
> > config - i think the bug might have been fixed meanwhile.
> 
> I have found that heavy network traffic really kills the interactive
> performance.  In the top excerpt below, gtk-gnutella is generating about
> 320KB/sec of traffic.
> 
> These priorities do not look right:

Never mind, I just rediscovered the RT priority leakage bug.

Lee

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Lee Revell]

On Mon, 2005-07-04 at 13:16 +0200, Ingo Molnar wrote:
> i'd first suggest to try the latest kernel, before changing your X 
> config - i think the bug might have been fixed meanwhile.

I have found that heavy network traffic really kills the interactive
performance.  In the top excerpt below, gtk-gnutella is generating about
320KB/sec of traffic.

These priorities do not look right:

   14 root     -50  -5     0    0 S  0.0  0.0   0:00.00    0    0    0    0 [IRQ 9]                                        irqd     
  686 root     -49  -5     0    0 S  0.0  0.0  87:58.18    0    0    0    0 [IRQ 8]                                        irqd     
  689 root     -48  -5     0    0 S  0.0  0.0   0:00.00    0    0    0    0 [IRQ 7]                                        irqd     
  694 root     -47  -5     0    0 S  0.0  0.0   0:00.00    0    0    0    0 [IRQ 12]                                       irqd     
  714 root     -46  -5     0    0 S  2.5  0.0   5:25.33    0    0    0    0 [IRQ 15]                                       irqd     
  731 root     -45  -5     0    0 S  0.0  0.0   0:04.74    0    0    0    0 [IRQ 1]                                        irqd     
    3 root     -44 -10     0    0 S  0.0  0.0   1:42.55    0    0    0    0 [softirq-timer/0]                              ksoftirqd
    5 root     -44 -10     0    0 S  2.5  0.0   5:06.49    0    0    0    0 [softirq-net-rx/]                              ksoftirqd
 1541 root     -44  -5     0    0 S  0.0  0.0   9:39.03    0    0    0    0 [IRQ 10]                                       irqd     
 5650 rlrevell -44   0  2804 1616 S 14.8  0.4   5:52.83 1188   52  784    1 /usr/lib/gamin/gam_server                      stext    
30313 rlrevell -44   0 24572  19m S 99.9  4.4  68:15.42 4520 1800  12m   31 gtk-gnutella                                   stext    
 2285 root     -43  -5     0    0 S  0.0  0.0   3:47.34    0    0    0    0 [IRQ 11]                                       irqd     
 2807 root     -42  -5     0    0 S  0.0  0.0   0:00.00    0    0    0    0 [IRQ 14]                                       irqd     
    9 root      -2  -5     0    0 S  0.0  0.0   0:16.05    0    0    0    0 [events/0]                                     worker_th
    4 root       5 -10     0    0 S  0.0  0.0   0:25.95    0    0    0    0 [softirq-net-tx/]                              ksoftirqd
    7 root       5 -10     0    0 S  0.0  0.0   0:01.22    0    0    0    0 [softirq-tasklet]                              ksoftirqd
    8 root       5 -10     0    0 S  0.0  0.0   0:01.30    0    0    0    0 [desched/0]                                    desched_t
28988 root       5 -10  106m  30m S  4.9  6.9 242:59.04  76m 1556  32m    3 /usr/X11R6/bin/X :0 -audit 0 -auth /var/lib/gd stext    
    2 root      10 -10     0    0 S  0.0  0.0   0:00.00    0    0    0    0 [softirq-high/0]                               ksoftirqd
    6 root      10 -10     0    0 S  0.0  0.0   0:00.00    0    0    0    0 [softirq-scsi/0]

Why do gtk-gnutella and gam_server get a higher priority than the disk
(14) and network (11) IRQs?  And shouldn't the softirq threads run at
lower priority than all the hardirq threads?

Looking at the top output I'd expect the machine to be crawling, and
that's what it does.

Lee

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* William Weston <weston@sysex.net> wrote:

> Still looking into this issue on -51-06.  Found something really odd: 
> SCHED_NORMAL tasks will start to inherit the priority value of some 
> other SCHED_FIFO task.  If JACK is started at a given SCHED_FIFO 
> priority, X and all of its children will inherit the same priority 
> value after login.  Other random processes will inherit this, too -- 
> sometimes init...
> 
> SCHED_NORMAL tasks suddenly inheriting priority values in the range 
> normally reserved for SCHED_FIFO could explain at least part of the 
> meltdown I've been seeing.  Any thoughts?

is this inheritance perpetual? It is normal for some tasks to be boosted 
momentarily, but if the condition remains even after jackd has exited, 
it's clearly an anomaly. (lets call it "RT priority leakage".) Priority 
leakage on SMP was fixed recently, but there could be other bugs 
remaining.

There's priority-leakage debugging code in the -RT kernel, which is 
activated if you enable CONFIG_DEBUG_PREEMPT. This debugging code, if 
triggered, should produce 'BUG: comm/123 leaked RT prio 123 (123)?" type 
of messages. But ... due to a bug it would not print out anything but 
would lock up hard if it detects the condition (and if 
RT_DEADLOCK_DETECT is enabled). I have fixed this reporting bug in the 
-51-08 kernel.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[William Weston]

On Sun, 3 Jul 2005, Ingo Molnar wrote:

> ok, found a bug that could explain the situation: mutex sleeps+wakeups 
> were incorrectly credited as 'interactive sleep' periods, causing the dd 
> processes to be boosted incorrectly. The dd processes created a workload 
> in which they blocked each other in such a pattern that they got boosted 
> periodically, starving pretty much every other task.
> 
> the fix is significant and affects alot of workloads, and should further 
> improve interactivity in noticeable ways. I'm not 100% sure it solves 
> all the starvation problems (e.g. how could normal-prio dd tasks starve 
> the SCHED_FIFO irq threads that drove SysRq?), but the results so far 
> look promising.
> 
> i've uploaded the -50-45 patch, can you under this kernel trigger a 
> 'meltdown' on your SMT box?

Still looking into this issue on -51-06.  Found something really odd:  
SCHED_NORMAL tasks will start to inherit the priority value of some other
SCHED_FIFO task.  If JACK is started at a given SCHED_FIFO priority, X and
all of its children will inherit the same priority value after login.  
Other random processes will inherit this, too -- sometimes init...

SCHED_NORMAL tasks suddenly inheriting priority values in the range
normally reserved for SCHED_FIFO could explain at least part of the
meltdown I've been seeing.  Any thoughts?


Cheers,
--ww

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* William Weston <weston@sysex.net> wrote:

> > Which video driver is X using?  What nice value is the X server running
> > at?
> 
> Hardware is Intel 82865G (integrated) with DRM i915 1.1.0 20040405 and 
> xorg-3.8.2 i810 driver, running at nice 0, priority 15.  Should I bump 
> the priority up?  To realtime?

i'd first suggest to try the latest kernel, before changing your X 
config - i think the bug might have been fixed meanwhile.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[William Weston]

On Fri, 1 Jul 2005, Lee Revell wrote:

> On Fri, 2005-07-01 at 18:46 -0700, William Weston wrote:
> > FWIW, I'm still seeing the SMT scheduling? meltdown issues with
> > -50-42.  
> > Running two instances of 'dd if=/dev/zero of=/dev/null bs=65536'
> > instead 
> > of 'burnP6' results in the same behavior.  Here's a quick recap:
> > 
> > - Start (or login to ) X.
> > - Start an X app that constantly updates the screen, like wmcube, or
> > vlc. 
> 
> Which video driver is X using?  What nice value is the X server running
> at?

Hardware is Intel 82865G (integrated) with DRM i915 1.1.0 20040405 and 
xorg-3.8.2 i810 driver, running at nice 0, priority 15.  Should I bump the 
priority up?  To realtime?

> Does adding:
> 
> Option "NoAccel"
> 
> to the Device section of your X config file make any difference?

Disabling the dri and drm modules didn't help.  I'll turn on NoAccel when 
I'm back in the office on Tuesday.

> (on most systems X is the only thing besides the kernel that can access
> hardware directly, which can cause problems)

So would running X through the framebuffer device be the way to go for 
stability under realtime?  It's been a couple years since I've used fb.

--ww

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> > A few things are left working (but not enough to get the system back):
> > 
> > - Mouse pointer (movements are chunky) and window focus.
> > - Mouse scroll wheel can still scroll xterms and switch workspaces.
> > - SysRq-B
> 
> hm, i can reproduce a variant of this, by starting enough 'dd' tasks.  
> (it needed more than two on a 2-way/4-way HT testbox though) Indeed 
> everything seems to be starved, but SysRq still worked so i was able 
> to SysRq-kIll all tasks and thus the system recovered.
> 
> i'm debugging this now.

ok, found a bug that could explain the situation: mutex sleeps+wakeups 
were incorrectly credited as 'interactive sleep' periods, causing the dd 
processes to be boosted incorrectly. The dd processes created a workload 
in which they blocked each other in such a pattern that they got boosted 
periodically, starving pretty much every other task.

the fix is significant and affects alot of workloads, and should further 
improve interactivity in noticeable ways. I'm not 100% sure it solves 
all the starvation problems (e.g. how could normal-prio dd tasks starve 
the SCHED_FIFO irq threads that drove SysRq?), but the results so far 
look promising.

i've uploaded the -50-45 patch, can you under this kernel trigger a 
'meltdown' on your SMT box?

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* William Weston <weston@sysex.net> wrote:

> FWIW, I'm still seeing the SMT scheduling? meltdown issues with 
> -50-42.  Running two instances of 'dd if=/dev/zero of=/dev/null 
> bs=65536' instead of 'burnP6' results in the same behavior.  Here's a 
> quick recap:
> 
> - Start (or login to ) X.
> - Start an X app that constantly updates the screen, like wmcube, or vlc.
> - Run 'burnP6' or 'dd if=/dev/zero of=/dev/null bs=65536'.
> - Run trace-it.  Trace completes without any troubles.
> - Run another 'burnP6' or 'dd if=/dev/zero of=/dev/null bs=65536'.
> 
> At this point, most of the system is unresponsive:
> 
> - Most X apps are frozen (even top in its own xterm).
> - Mouse lost synchro serio warnings show up on serial console.
> - Serial console is otherwise unresponsive (no SysRq).
> - X server quits responding to keyboard input.
> - Kbd input makes mouse temporarily unresponsive (for .1 to >5 secs).
> - Mouse immediately after kbd triggers more 'mouse lost synchro' messages.
> - Networking is lost (box won't respond to pings).
> - Any script automating starting burnP6 or dd and then trace-it hangs.
> 
> A few things are left working (but not enough to get the system back):
> 
> - Mouse pointer (movements are chunky) and window focus.
> - Mouse scroll wheel can still scroll xterms and switch workspaces.
> - SysRq-B

hm, i can reproduce a variant of this, by starting enough 'dd' tasks.  
(it needed more than two on a 2-way/4-way HT testbox though) Indeed 
everything seems to be starved, but SysRq still worked so i was able to 
SysRq-kIll all tasks and thus the system recovered.

i'm debugging this now.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Lee Revell]

On Fri, 2005-07-01 at 18:46 -0700, William Weston wrote:
> FWIW, I'm still seeing the SMT scheduling? meltdown issues with
> -50-42.  
> Running two instances of 'dd if=/dev/zero of=/dev/null bs=65536'
> instead 
> of 'burnP6' results in the same behavior.  Here's a quick recap:
> 
> - Start (or login to ) X.
> - Start an X app that constantly updates the screen, like wmcube, or
> vlc. 

Which video driver is X using?  What nice value is the X server running
at?

Does adding:

Option "NoAccel"

to the Device section of your X config file make any difference?

(on most systems X is the only thing besides the kernel that can access
hardware directly, which can cause problems)

Lee

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[William Weston]

On Fri, 1 Jul 2005, Ingo Molnar wrote:

> oops - i had it in my tree (so all my tests passed), but it escaped the 
> -39 patch. I've uploaded -50-40 with this included too.

FWIW, I'm still seeing the SMT scheduling? meltdown issues with -50-42.  
Running two instances of 'dd if=/dev/zero of=/dev/null bs=65536' instead 
of 'burnP6' results in the same behavior.  Here's a quick recap:

- Start (or login to ) X.
- Start an X app that constantly updates the screen, like wmcube, or vlc.
- Run 'burnP6' or 'dd if=/dev/zero of=/dev/null bs=65536'.
- Run trace-it.  Trace completes without any troubles.
- Run another 'burnP6' or 'dd if=/dev/zero of=/dev/null bs=65536'.

At this point, most of the system is unresponsive:

- Most X apps are frozen (even top in its own xterm).
- Mouse lost synchro serio warnings show up on serial console.
- Serial console is otherwise unresponsive (no SysRq).
- X server quits responding to keyboard input.
- Kbd input makes mouse temporarily unresponsive (for .1 to >5 secs).
- Mouse immediately after kbd triggers more 'mouse lost synchro' messages.
- Networking is lost (box won't respond to pings).
- Any script automating starting burnP6 or dd and then trace-it hangs.

A few things are left working (but not enough to get the system back):

- Mouse pointer (movements are chunky) and window focus.
- Mouse scroll wheel can still scroll xterms and switch workspaces.
- SysRq-B

All of this behavior is 100% reproduceable on recent realtime SMP/SMT
kernels, and is _not_ seen with vanilla SMP/SMT, Fedora SMP, or realtime
UP kernels.  I'll be running realtime UP kernels on this Xeon/HT box for a
while unless someone gives me reason to enable SMT again.

You can tell me to shut up now.... at least until I decide to hunt down
another corner case ;-}


Best Regards,
--William Weston

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Chuck Harding]

On Fri, 1 Jul 2005, Ingo Molnar wrote:

>
> * Karsten Wiese <annabellesgarden@yahoo.de> wrote:
>
>> Ooops, you didn't apply this part of the patch:
>
> oops - i had it in my tree (so all my tests passed), but it escaped the
> -39 patch. I've uploaded -50-40 with this included too.
>
> 	Ingo
> -
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
>

Well, the -50-42 still has the problem with sox locking up too tightly to
kill. Just saw that the sox processes are all in state 'D', FWIW.

-- 
Charles D. (Chuck) Harding <charding@llnl.gov>  Voice: 925-423-8879
Senior Computer Associate         ICCD            Fax: 925-423-6961
Lawrence Livermore National Laboratory      Computation Directorate
Livermore, CA USA  http://www.llnl.gov  GPG Public Key ID: B9EB6601
------------------ http://tinyurl.com/5w5ey -----------------------
-- Bad breath is better than no breath. --

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Karsten Wiese <annabellesgarden@yahoo.de> wrote:

> Ooops, you didn't apply this part of the patch:

oops - i had it in my tree (so all my tests passed), but it escaped the 
-39 patch. I've uploaded -50-40 with this included too.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[William Weston]

On Thu, 30 Jun 2005, William Weston wrote:
> On Fri, 1 Jul 2005, Karsten Wiese wrote:
> > --- linux-2.6.12-RT-50-35/arch/i386/kernel/mpparse.c	2005-06-30 16:38:00.000000000 +0200
> > +++ linux-2.6.12-RT/arch/i386/kernel/mpparse.c	2005-06-29 20:30:50.000000000 +0200
> > @@ -263,6 +263,7 @@
> >  		return;
> >  	}
> >  	mp_ioapics[nr_ioapics] = *m;
> > +	io_apic_base[nr_ioapics] = IO_APIC_BASE(nr_ioapics);
> >  	nr_ioapics++;
> >  }
> >  
> > @@ -914,6 +915,7 @@
> >  	mp_ioapics[idx].mpc_apicaddr = address;
> >  
> >  	set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
> > +	io_apic_base[idx] = IO_APIC_BASE(idx);
> >  	mp_ioapics[idx].mpc_apicid = io_apic_get_unique_id(idx, id);
> >  	mp_ioapics[idx].mpc_apicver = io_apic_get_version(idx);
> 
> Thanks, Karsten.  This patch was needed to get my Xeon/HT (i865) to boot 
> -50-39.  Next I'll see if IOAPIC_FAST will work.  I'll keep you posted.

After applying this patch to -50-39, IOAPIC_FAST now works on both the 
Xeon/HT and the non-HT Xeon boxes (i865 and i845 respectively).  Wakeup 
latencies are looking better, especially on the Xeon/HT (max of 53us so 
far, as compared to 183us with -50-37 without IOAPIC_FAST).

I'll test this out on the Athlon box (which hasn't done well with 
IOAPIC_FAST previously) when I get home.

--ww

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Karsten Wiese]

Am Freitag, 1. Juli 2005 00:27 schrieb Karsten Wiese:
> Am Donnerstag, 30. Juni 2005 22:50 schrieb Ingo Molnar:
> > 
> > * Karsten Wiese <annabellesgarden@yahoo.de> wrote:
> > 
> > > Here come some numbers to back up the usefullness of 
> > > CONFIG_X86_UP_IOAPIC_FAST. (and to show that my patch actually works 
> > > ;-)) All measurement where taken on an UP Athlon64 2Ghz running 32bit 
> > > 2.6.12-RT-50-35 PREEMPT_RT on a K8T800 mobo.
> > 
> > thanks - the numbers are pretty convincing. I've applied most of your 
> > patch (except the instrumentation bits), and it seems to work quite well 
> > - one of my testsystems that had interrupt storms before can now run 
> > IOAPIC_FAST. (i also enabled the option to be selectable for SMP kernels 
> > too. If things work out fine we can make it default-on.) I've uploaded 
> > the -50-39 patch with these changes included.
> > 
> Ooops, you didn't apply this part of the patch:
> 
> <snip>
> --- linux-2.6.12-RT-50-35/arch/i386/kernel/mpparse.c	2005-06-30 16:38:00.000000000 +0200
> +++ linux-2.6.12-RT/arch/i386/kernel/mpparse.c	2005-06-29 20:30:50.000000000 +0200
> @@ -263,6 +263,7 @@
>  		return;
>  	}
>  	mp_ioapics[nr_ioapics] = *m;
> +	io_apic_base[nr_ioapics] = IO_APIC_BASE(nr_ioapics);
>  	nr_ioapics++;
>  }
>  
> @@ -914,6 +915,7 @@
>  	mp_ioapics[idx].mpc_apicaddr = address;
>  
>  	set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
> +	io_apic_base[idx] = IO_APIC_BASE(idx);
>  	mp_ioapics[idx].mpc_apicid = io_apic_get_unique_id(idx, id);
>  	mp_ioapics[idx].mpc_apicver = io_apic_get_version(idx);
> </snip>
> 
> It provides the precalculation of the ioapics's virtual address.
> Or does IO_APIC_BASE() compile to an indexed lookup?
>  then io_apic_base[] wouldn't be needed...
> 
It is needed (only for optimization):
There are ioapics that have baseaddresses not on a page boundary.
Another idea: to merge the virtual ioapic base addresses into the io_apic_cache struct to use 
the cpu-cache more efficiently ;-) 

Karsten

	

	
		
___________________________________________________________ 
Gesendet von Yahoo! Mail - Jetzt mit 1GB Speicher kostenlos - Hier anmelden: http://mail.yahoo.de

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[William Weston]

On Fri, 1 Jul 2005, Karsten Wiese wrote:
> 
> <snip>
> --- linux-2.6.12-RT-50-35/arch/i386/kernel/mpparse.c	2005-06-30 16:38:00.000000000 +0200
> +++ linux-2.6.12-RT/arch/i386/kernel/mpparse.c	2005-06-29 20:30:50.000000000 +0200
> @@ -263,6 +263,7 @@
>  		return;
>  	}
>  	mp_ioapics[nr_ioapics] = *m;
> +	io_apic_base[nr_ioapics] = IO_APIC_BASE(nr_ioapics);
>  	nr_ioapics++;
>  }
>  
> @@ -914,6 +915,7 @@
>  	mp_ioapics[idx].mpc_apicaddr = address;
>  
>  	set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
> +	io_apic_base[idx] = IO_APIC_BASE(idx);
>  	mp_ioapics[idx].mpc_apicid = io_apic_get_unique_id(idx, id);
>  	mp_ioapics[idx].mpc_apicver = io_apic_get_version(idx);
> </snip>
> 
> It provides the precalculation of the ioapics's virtual address.
> Or does IO_APIC_BASE() compile to an indexed lookup?
>  then io_apic_base[] wouldn't be needed...

Thanks, Karsten.  This patch was needed to get my Xeon/HT (i865) to boot 
-50-39.  Next I'll see if IOAPIC_FAST will work.  I'll keep you posted.

--ww

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Karsten Wiese]

Am Donnerstag, 30. Juni 2005 22:50 schrieb Ingo Molnar:
> 
> * Karsten Wiese <annabellesgarden@yahoo.de> wrote:
> 
> > Here come some numbers to back up the usefullness of 
> > CONFIG_X86_UP_IOAPIC_FAST. (and to show that my patch actually works 
> > ;-)) All measurement where taken on an UP Athlon64 2Ghz running 32bit 
> > 2.6.12-RT-50-35 PREEMPT_RT on a K8T800 mobo.
> 
> thanks - the numbers are pretty convincing. I've applied most of your 
> patch (except the instrumentation bits), and it seems to work quite well 
> - one of my testsystems that had interrupt storms before can now run 
> IOAPIC_FAST. (i also enabled the option to be selectable for SMP kernels 
> too. If things work out fine we can make it default-on.) I've uploaded 
> the -50-39 patch with these changes included.
> 
Ooops, you didn't apply this part of the patch:

<snip>
--- linux-2.6.12-RT-50-35/arch/i386/kernel/mpparse.c	2005-06-30 16:38:00.000000000 +0200
+++ linux-2.6.12-RT/arch/i386/kernel/mpparse.c	2005-06-29 20:30:50.000000000 +0200
@@ -263,6 +263,7 @@
 		return;
 	}
 	mp_ioapics[nr_ioapics] = *m;
+	io_apic_base[nr_ioapics] = IO_APIC_BASE(nr_ioapics);
 	nr_ioapics++;
 }
 
@@ -914,6 +915,7 @@
 	mp_ioapics[idx].mpc_apicaddr = address;
 
 	set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
+	io_apic_base[idx] = IO_APIC_BASE(idx);
 	mp_ioapics[idx].mpc_apicid = io_apic_get_unique_id(idx, id);
 	mp_ioapics[idx].mpc_apicver = io_apic_get_version(idx);
</snip>

It provides the precalculation of the ioapics's virtual address.
Or does IO_APIC_BASE() compile to an indexed lookup?
 then io_apic_base[] wouldn't be needed...

Karsten

	

	
		
___________________________________________________________ 
Gesendet von Yahoo! Mail - Jetzt mit 1GB Speicher kostenlos - Hier anmelden: http://mail.yahoo.de

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Karsten Wiese <annabellesgarden@yahoo.de> wrote:

> Here come some numbers to back up the usefullness of 
> CONFIG_X86_UP_IOAPIC_FAST. (and to show that my patch actually works 
> ;-)) All measurement where taken on an UP Athlon64 2Ghz running 32bit 
> 2.6.12-RT-50-35 PREEMPT_RT on a K8T800 mobo.

thanks - the numbers are pretty convincing. I've applied most of your 
patch (except the instrumentation bits), and it seems to work quite well 
- one of my testsystems that had interrupt storms before can now run 
IOAPIC_FAST. (i also enabled the option to be selectable for SMP kernels 
too. If things work out fine we can make it default-on.) I've uploaded 
the -50-39 patch with these changes included.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Karsten Wiese]

[-- Attachment #1: Type: text/plain, Size: 3389 bytes --]

Am Donnerstag, 30. Juni 2005 01:36 schrieb Karsten Wiese:
> Am Mittwoch, 29. Juni 2005 21:38 schrieb Ingo Molnar:
> > 
> > * Karsten Wiese <annabellesgarden@yahoo.de> wrote:
> > 
> > > attached patch for io_apic.c lets
> > > 1. gcc 3.4.3 optimize io_apic access a little better.
> > > 2. CONFIG_X86_UP_IOAPIC_FAST work here.
> > >    Didn't check, if it really speeds up things.
> > 
> > which change made CONFIG_X86_UP_IOAPIC_FAST work on your box? It seems 
> > you've changed the per-register frontside read-cache to something else - 
> > was that on purpose?
> > 
> CONFIG_X86_UP_IOAPIC_FAST started working here, when I made io_apic_modify()
> look like that:
> 
> void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int val)
> {
> #ifdef IOAPIC_CACHE
> 	io_apic_cache[apic][reg] = val;
> #endif
> //	if (unlikely(sis_apic_bug))
> commented this ^^ out 
> 
> 		*IO_APIC_BASE(apic) = reg;
> 	*(IO_APIC_BASE(apic)+4) = val;
> #ifndef IOAPIC_POSTFLUSH
> 	if (unlikely(sis_apic_bug))
> #endif
> 		/*
> 		 * Force POST flush by reading:
>  		 */
> 		val = *(IO_APIC_BASE(apic)+4);
> }
> 
> This change does it, 'cause when we read a cached value instead of from the ioapic,
> the ioapic's address register isn't set and thus the following write in io_apic_modify
> might not scratch the right ioapic register.
> 
> On top of the above the patch adds
> 	unsigned int reg;
> to io_apic_cache.
> with that "reg" struct member a mark -1 can be set, when we read from cache,
> or the reg-number, when we read from the ioapic.
> Then by comparing parameter reg with io_apic_cache[apic].reg
> the patched io_apic_modify() knows, if it has to set the address register or not.
> The register caching in the patch should be the same as before,
> only the cache changed from a 2 dimensional array
> to an array holding structs, which contain an array:
> io_apic_cache[apic][reg] should be equivalent to io_apic_cache[apic].val[reg], no?
> 
Here come some numbers to back up the usefullness of CONFIG_X86_UP_IOAPIC_FAST.
(and to show that my patch actually works ;-))
All measurement where taken on an UP Athlon64 2Ghz running 32bit 2.6.12-RT-50-35 PREEMPT_RT
on a K8T800 mobo.

I measured the tsc-cycles spent in __do_IRQ() plus those used for interruptcontroller clean-up
after the irq-thread finished the handler.

Did that with 3 Setups:
 XT-PIC
 IO-APIC uncached
 IO-APIC cached
 IO-APIC cached + cpu-cache thrashing application hogging the cpu 

Characteristic values found:
 XT-PIC
  irq#       | cycles/irq
  -----------|------------
   0 timer   | 18733
   5 VIA8237 | 19089
  11 eth0    | 22676
  14 ide0    | 23057

 IO-APIC uncached
  irq#       | cycles/irq
  -----------|------------
   0 timer   |  4711
  22 VIA8237 | 32107
  16 eth0    | 32680
  14 ide0    | 13404

 IO-APIC cached
  irq#       | cycles/irq
  -----------|------------
   0 timer   |  5004
  22 VIA8237 |  1484
  16 eth0    |  1598
  14 ide0    |   905

 IO-APIC cached + cpu-cache thrashing application hogging the cpu 
  irq#       | cycles/irq
  -----------|------------
   0 timer   |  6344
  22 VIA8237 |  2512
  16 eth0    |  2774
  14 ide0    |  1723

Clear winner is IO-APIC cached.

The patch used + full /proc/interrupt logs are attached.
The patch differs from the previously sent one by precalculation 
of the ioapic's virtual address and the cycle-measurement code.

Cheers,
Karsten









[-- Attachment #2: ints_io_apic_cached_fullmemload --]
[-- Type: text/plain, Size: 810 bytes --]

           CPU0       
  0:    7390558    IO-APIC-edge      0/  558  6344 3540429, timer
  1:      14651    IO-APIC-edge      0/ 4651  1583 7364358, i8042
  8:         12    IO-APIC-edge      0/   12  1462 17550, rtc
  9:          0   IO-APIC-level      0/    0     0 0, acpi
 12:     120985    IO-APIC-edge      0/  985  1251 1232698, i8042
 14:     117323    IO-APIC-edge      0/ 7321  1723 12621315, ide0
 15:     328390    IO-APIC-edge      0/ 8388   619 5194067, ide1
 16:     458342   IO-APIC-level      0/ 8342  2774 23146219, eth0, nvidia
 21:          0   IO-APIC-level      0/    0     0 0, ehci_hcd:usb1, uhci_hcd:usb2, uhci_hcd:usb3, uhci_hcd:usb4, uhci_hcd:usb5
 22:       7826   IO-APIC-level      0/ 7826  2512 19665441, VIA8237
NMI:          0 
LOC:    7390775 
ERR:          0
MIS:          0

[-- Attachment #3: ints_xt-pic_noload --]
[-- Type: text/plain, Size: 869 bytes --]

           CPU0       
  0:     622269          XT-PIC      0/ 2269 18733 42507219, timer
  1:        886          XT-PIC      0/  886 19145 16962916, i8042
  2:          0          XT-PIC      0/    0     0 0, cascade
  3:          0          XT-PIC      0/    0     0 0, uhci_hcd:usb4, uhci_hcd:usb5
  5:       6381          XT-PIC      0/ 6381 19089 121807578, VIA8237, ehci_hcd:usb1
  8:          1          XT-PIC      0/    1 23697 23697, rtc
  9:          0          XT-PIC      0/    0     0 0, acpi
 11:       1086          XT-PIC      0/ 1086 22676 24626747, uhci_hcd:usb2, uhci_hcd:usb3, eth0
 12:         93          XT-PIC      0/   93 22848 2124875, i8042
 14:       3393          XT-PIC      0/ 3392 23057 78211017, ide0
 15:       5410          XT-PIC      0/ 5409 22818 123423742, ide1
NMI:          0 
LOC:     622240 
ERR:          0
MIS:          0

[-- Attachment #4: ints_io_apic_cached_noload --]
[-- Type: text/plain, Size: 806 bytes --]

           CPU0       
  0:    7520343    IO-APIC-edge      0/  343  5004 1716578, timer
  1:      15028    IO-APIC-edge      0/ 5028  1569 7890363, i8042
  8:         12    IO-APIC-edge      0/   12  1462 17550, rtc
  9:          0   IO-APIC-level      0/    0     0 0, acpi
 12:     122185    IO-APIC-edge      0/ 2185   998 2181980, i8042
 14:     133059    IO-APIC-edge      0/ 3057   905 2769148, ide0
 15:     334204    IO-APIC-edge      0/ 4202   515 2166602, ide1
 16:     466139   IO-APIC-level      0/ 6139  1598 9813868, eth0, nvidia
 21:          0   IO-APIC-level      0/    0     0 0, ehci_hcd:usb1, uhci_hcd:usb2, uhci_hcd:usb3, uhci_hcd:usb4, uhci_hcd:usb5
 22:      10619   IO-APIC-level      0/  619  1484 919075, VIA8237
NMI:          0 
LOC:    7520565 
ERR:          0
MIS:          0

[-- Attachment #5: ints_io_apic_noload --]
[-- Type: text/plain, Size: 801 bytes --]

           CPU0       
  0:     260074    IO-APIC-edge      0/   74  4711 348679, timer
  1:        596    IO-APIC-edge      0/  596 11293 6731173, i8042
  8:          1    IO-APIC-edge      0/    1 22686 22686, rtc
  9:          0   IO-APIC-level      0/    0     0 0, acpi
 12:         93    IO-APIC-edge      0/   93 11070 1029547, i8042
 14:       3305    IO-APIC-edge      0/ 3303 13404 44275707, ide0
 15:       2135    IO-APIC-edge      0/ 2133 10937 23329953, ide1
 16:        298   IO-APIC-level      0/  298 32680 9738917, eth0
 21:          0   IO-APIC-level      0/    0     0 0, ehci_hcd:usb1, uhci_hcd:usb2, uhci_hcd:usb3, uhci_hcd:usb4, uhci_hcd:usb5
 22:       1294   IO-APIC-level      0/ 1294 32107 41547375, VIA8237
NMI:          0 
LOC:     260001 
ERR:          0
MIS:          0

[-- Attachment #6: io_apic-RT-50-35.2.diff --]
[-- Type: text/x-diff, Size: 10320 bytes --]

diff -ru linux-2.6.12-RT-50-35/arch/i386/kernel/io_apic.c linux-2.6.12-RT/arch/i386/kernel/io_apic.c
--- linux-2.6.12-RT-50-35/arch/i386/kernel/io_apic.c	2005-06-30 16:38:19.000000000 +0200
+++ linux-2.6.12-RT/arch/i386/kernel/io_apic.c	2005-06-29 20:01:10.000000000 +0200
@@ -138,14 +138,24 @@
 /*
  * Cache register values:
  */
-static unsigned int io_apic_cache[MAX_IO_APICS][MAX_IOAPIC_CACHE]
+static struct {
+	unsigned int reg;
+	unsigned int val[MAX_IOAPIC_CACHE];
+} io_apic_cache[MAX_IO_APICS]
 		____cacheline_aligned_in_smp;
 #endif
 
-inline unsigned int __raw_io_apic_read(unsigned int apic, unsigned int reg)
+volatile unsigned int *io_apic_base[MAX_IO_APICS];
+
+static inline unsigned int __raw_io_apic_read(unsigned int apic, unsigned int reg)
 {
-	*IO_APIC_BASE(apic) = reg;
-	return *(IO_APIC_BASE(apic)+4);
+	volatile unsigned int *io_apic;
+#ifdef IOAPIC_CACHE
+	io_apic_cache[apic].reg = reg;
+#endif
+	io_apic = io_apic_base[apic];
+	io_apic[0] = reg;
+	return io_apic[4];
 }
 
 unsigned int raw_io_apic_read(unsigned int apic, unsigned int reg)
@@ -153,7 +163,7 @@
 	unsigned int val = __raw_io_apic_read(apic, reg);
 
 #ifdef IOAPIC_CACHE
-	io_apic_cache[apic][reg] = val;
+	io_apic_cache[apic].val[reg] = val;
 #endif
 	return val;
 }
@@ -172,14 +182,17 @@
 		}
 		return __raw_io_apic_read(apic, reg);
 	}
-	if (io_apic_cache[apic][reg] && !sis_apic_bug)
-		return io_apic_cache[apic][reg];
+	if (io_apic_cache[apic].val[reg] && !sis_apic_bug) {
+		io_apic_cache[apic].reg = -1;
+		return io_apic_cache[apic].val[reg];
+	}
 #endif
 	return raw_io_apic_read(apic, reg);
 }
 
 void io_apic_write(unsigned int apic, unsigned int reg, unsigned int val)
 {
+	volatile unsigned int *io_apic;
 #ifdef IOAPIC_CACHE
 	if (unlikely(reg >= MAX_IOAPIC_CACHE)) {
 		static int once = 1;
@@ -191,10 +204,14 @@
 			dump_stack();
 		}
 	} else
-		io_apic_cache[apic][reg] = val;
+		io_apic_cache[apic].val[reg] = val;
 #endif
-	*IO_APIC_BASE(apic) = reg;
-	*(IO_APIC_BASE(apic)+4) = val;
+	io_apic = io_apic_base[apic];
+#ifdef IOAPIC_CACHE
+	io_apic_cache[apic].reg = reg;
+#endif
+	io_apic[0] = reg;
+	io_apic[4] = val;
 }
 
 /*
@@ -214,34 +231,42 @@
  */
 void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int val)
 {
+	volatile unsigned int *io_apic;
 #ifdef IOAPIC_CACHE
-	io_apic_cache[apic][reg] = val;
+	io_apic_cache[apic].val[reg] = val;
 #endif
-	if (unlikely(sis_apic_bug))
-		*IO_APIC_BASE(apic) = reg;
-	*(IO_APIC_BASE(apic)+4) = val;
+	io_apic = io_apic_base[apic];
+#ifdef IOAPIC_CACHE
+	if (io_apic_cache[apic].reg != reg || sis_apic_bug) {
+		io_apic_cache[apic].reg = reg;
+#else
+	if (unlikely(sis_apic_bug)) {
+#endif
+		io_apic[0] = reg;
+	}
+	io_apic[4] = val;
 #ifndef IOAPIC_POSTFLUSH
 	if (unlikely(sis_apic_bug))
 #endif
 		/*
 		 * Force POST flush by reading:
  		 */
-		val = *(IO_APIC_BASE(apic)+4);
+		val = io_apic[4];
 }
 
 static void __modify_IO_APIC_irq (unsigned int irq, unsigned long enable, unsigned long disable)
 {
 	struct irq_pin_list *entry = irq_2_pin + irq;
-	unsigned int pin, reg;
+	unsigned int pin, val;
 
 	for (;;) {
 		pin = entry->pin;
 		if (pin == -1)
 			break;
-		reg = io_apic_read(entry->apic, 0x10 + pin*2);
-		reg &= ~disable;
-		reg |= enable;
-		io_apic_modify(entry->apic, 0x10 + pin*2, reg);
+		val = io_apic_read(entry->apic, 0x10 + pin*2);
+		val &= ~disable;
+		val |= enable;
+		io_apic_modify(entry->apic, 0x10 + pin*2, val);
 		if (!entry->next)
 			break;
 		entry = irq_2_pin + entry->next;
@@ -249,13 +274,13 @@
 }
 
 /* mask = 1 */
-static void __mask_IO_APIC_irq (unsigned int irq)
+static inline void __mask_IO_APIC_irq (unsigned int irq)
 {
 	__modify_IO_APIC_irq(irq, 0x00010000, 0);
 }
 
 /* mask = 0 */
-static void __unmask_IO_APIC_irq (unsigned int irq)
+static inline void __unmask_IO_APIC_irq (unsigned int irq)
 {
 	__modify_IO_APIC_irq(irq, 0, 0x00010000);
 }
@@ -306,9 +331,13 @@
 {
 	int apic, pin;
 
-	for (apic = 0; apic < nr_ioapics; apic++)
+	for (apic = 0; apic < nr_ioapics; apic++) {
+#ifdef IOAPIC_CACHE
+		io_apic_cache[apic].reg = -1;
+#endif
 		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
 			clear_IO_APIC_pin(apic, pin);
+	}
 }
 
 static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t cpumask)
Nur in linux-2.6.12-RT/arch/i386/kernel: io_apic.c~.
Nur in linux-2.6.12-RT/arch/i386/kernel: io_apic.c-50-33.
diff -ru linux-2.6.12-RT-50-35/arch/i386/kernel/irq.c linux-2.6.12-RT/arch/i386/kernel/irq.c
--- linux-2.6.12-RT-50-35/arch/i386/kernel/irq.c	2005-06-30 16:38:19.000000000 +0200
+++ linux-2.6.12-RT/arch/i386/kernel/irq.c	2005-06-30 16:11:18.000000000 +0200
@@ -217,6 +217,8 @@
 	}
 
 	if (i < NR_IRQS) {
+		unsigned int irq_count;
+		cycles_t cycles;
 		spin_lock_irqsave(&irq_desc[i].lock, flags);
 		action = irq_desc[i].action;
 		if (!action)
@@ -230,12 +232,16 @@
 				seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]);
 #endif
 		seq_printf(p, " %14s", irq_desc[i].handler->typename);
-		seq_printf(p, "  %s", action->name);
 
-		for (action=action->next; action; action = action->next)
-			seq_printf(p, ", %s", action->name);
-		seq_printf(p, "  %d/%d", irq_desc[i].irqs_unhandled, irq_desc[i].irq_count);
+		irq_count = irq_desc[i].irq_count;
+		cycles =  irq_desc[i].cycles;
+		if (irq_count) {
+			do_div(cycles, irq_count);
+		}
+		seq_printf(p, "  %5d/%5d %5lld %lld", irq_desc[i].irqs_unhandled, irq_count, cycles, irq_desc[i].cycles);
 
+		for (; action; action = action->next)
+			seq_printf(p, ", %s", action->name);
 		seq_putc(p, '\n');
 skip:
 		spin_unlock_irqrestore(&irq_desc[i].lock, flags);
Nur in linux-2.6.12-RT/arch/i386/kernel: irq.c~.
diff -ru linux-2.6.12-RT-50-35/arch/i386/kernel/mpparse.c linux-2.6.12-RT/arch/i386/kernel/mpparse.c
--- linux-2.6.12-RT-50-35/arch/i386/kernel/mpparse.c	2005-06-30 16:38:00.000000000 +0200
+++ linux-2.6.12-RT/arch/i386/kernel/mpparse.c	2005-06-29 20:30:50.000000000 +0200
@@ -263,6 +263,7 @@
 		return;
 	}
 	mp_ioapics[nr_ioapics] = *m;
+	io_apic_base[nr_ioapics] = IO_APIC_BASE(nr_ioapics);
 	nr_ioapics++;
 }
 
@@ -914,6 +915,7 @@
 	mp_ioapics[idx].mpc_apicaddr = address;
 
 	set_fixmap_nocache(FIX_IO_APIC_BASE_0 + idx, address);
+	io_apic_base[idx] = IO_APIC_BASE(idx);
 	mp_ioapics[idx].mpc_apicid = io_apic_get_unique_id(idx, id);
 	mp_ioapics[idx].mpc_apicver = io_apic_get_version(idx);
 	
Nur in linux-2.6.12-RT/arch/i386/kernel: mpparse.c~.
Nur in linux-2.6.12-RT/arch/i386/kernel: mpparse.c-RT-50-35.
Nur in linux-2.6.12-RT: cscope.files.
Nur in linux-2.6.12-RT: cscope.files~.
Nur in linux-2.6.12-RT: cscope.out.
Nur in linux-2.6.12-RT/Documentation: realtime-lsm.txt.
diff -ru linux-2.6.12-RT-50-35/include/asm-i386/io_apic.h linux-2.6.12-RT/include/asm-i386/io_apic.h
--- linux-2.6.12-RT-50-35/include/asm-i386/io_apic.h	2005-06-30 16:38:20.000000000 +0200
+++ linux-2.6.12-RT/include/asm-i386/io_apic.h	2005-06-29 19:59:17.000000000 +0200
@@ -155,6 +155,8 @@
 /* MP IRQ source entries */
 extern struct mpc_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
 
+extern volatile unsigned int *io_apic_base[MAX_IO_APICS];
+
 /* non-0 if default (table-less) MP configuration */
 extern int mpc_default_type;
 
Nur in linux-2.6.12-RT/include/asm-i386: io_apic.h~.
diff -ru linux-2.6.12-RT-50-35/include/linux/irq.h linux-2.6.12-RT/include/linux/irq.h
--- linux-2.6.12-RT-50-35/include/linux/irq.h	2005-06-30 16:38:20.000000000 +0200
+++ linux-2.6.12-RT/include/linux/irq.h	2005-06-30 13:24:46.000000000 +0200
@@ -79,7 +79,7 @@
 	unsigned int irqs_unhandled;
 	struct task_struct *thread;
 	wait_queue_head_t wait_for_handler;
-	cycles_t timestamp;
+	cycles_t cycles;
 	raw_spinlock_t lock;
 } ____cacheline_aligned irq_desc_t;
 
Nur in linux-2.6.12-RT/include/linux: irq.h~.
Nur in linux-2.6.12-RT/include: sound.
Nur in linux-2.6.12-RT/kernel: cscope.files.
Nur in linux-2.6.12-RT/kernel: cscope.out.
diff -ru linux-2.6.12-RT-50-35/kernel/irq/handle.c linux-2.6.12-RT/kernel/irq/handle.c
--- linux-2.6.12-RT-50-35/kernel/irq/handle.c	2005-06-30 16:38:20.000000000 +0200
+++ linux-2.6.12-RT/kernel/irq/handle.c	2005-06-30 13:36:11.000000000 +0200
@@ -143,10 +143,10 @@
 	return retval;
 }
 
-cycles_t irq_timestamp(unsigned int irq)
-{
-	return irq_desc[irq].timestamp;
-}
+/* cycles_t irq_timestamp(unsigned int irq) */
+/* { */
+/* 	return irq_desc[irq].timestamp; */
+/* } */
 
 /*
  * do_IRQ handles all normal device IRQ's (the special
@@ -160,7 +160,7 @@
 	unsigned int status;
 #ifdef CONFIG_PREEMPT_RT
 	unsigned long flags;
-
+	cycles_t timestamp;
 	/*
 	 * Disable the soft-irq-flag:
 	 */
@@ -178,7 +178,8 @@
 		desc->handler->end(irq);
 		return 1;
 	}
-	desc->timestamp = get_cycles();
+
+	timestamp = get_cycles();
 
 	spin_lock(&desc->lock);
 	desc->handler->ack(irq);
@@ -256,6 +257,7 @@
 	local_irq_restore(flags);
 	/* re-disable interrupts because callers expect irqs off: */
 //	raw_local_irq_disable();
+	desc->cycles += get_cycles() - timestamp;
 #endif
 	return 1;
 }
Nur in linux-2.6.12-RT/kernel/irq: handle.c~.
diff -ru linux-2.6.12-RT-50-35/kernel/irq/manage.c linux-2.6.12-RT/kernel/irq/manage.c
--- linux-2.6.12-RT-50-35/kernel/irq/manage.c	2005-06-30 16:38:20.000000000 +0200
+++ linux-2.6.12-RT/kernel/irq/manage.c	2005-06-30 13:39:35.000000000 +0200
@@ -412,6 +412,7 @@
 	spin_lock_irq(&desc->lock);
 
 	if (desc->status & IRQ_INPROGRESS) {
+		cycles_t timestamp;
 		action = desc->action;
 		for (;;) {
 			irqreturn_t action_ret = 0;
@@ -434,9 +435,11 @@
 		 * The ->end() handler has to deal with interrupts which got
 		 * disabled while the handler was running.
 		 */
+		timestamp = get_cycles();
 		desc->handler->end(irq);
 		if (!(desc->status & IRQ_DISABLED))
 			desc->handler->enable(irq);
+		desc->cycles += get_cycles() - timestamp;
 	}
 	spin_unlock_irq(&desc->lock);
 
Nur in linux-2.6.12-RT/kernel/irq: manage.c~.
diff -ru linux-2.6.12-RT-50-35/kernel/irq/spurious.c linux-2.6.12-RT/kernel/irq/spurious.c
--- linux-2.6.12-RT-50-35/kernel/irq/spurious.c	2005-06-30 16:38:20.000000000 +0200
+++ linux-2.6.12-RT/kernel/irq/spurious.c	2005-06-30 15:41:54.000000000 +0200
@@ -74,11 +74,12 @@
 	}
 
 	desc->irq_count++;
-	if (desc->irq_count < 100000)
+	if (desc->irq_count < 10000)
 		return;
 
 	desc->irq_count = 0;
-	if (desc->irqs_unhandled > 99900) {
+	desc->cycles = 0;
+	if (desc->irqs_unhandled > 9990) {
 		/*
 		 * The interrupt is stuck
 		 */

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[William Weston]

On Wed, 29 Jun 2005, Ingo Molnar wrote:

> great! The SMP box running BurnP6 is another system, right? Could you 
> sum up the remaining regressions you are seeing under -RT? (the 
> latency.c warning is one, what others are remaining?)

Right.  I'm doing the bulk of my RT testing on three machines:

Home:  UP  Athlon  2GHz    VIA KT400   desktop, audio & MIDI dev
Work:  UP  Xeon    1.8GHz  Intel 845   video encoding & mcast streaming
Work:  SMT Xeon/HT 3.2GHz  Intel 865   desktop, video streaming testing

Here's the max wakeup latencies I'm seeing on each machine with -50-35:

Athlon   15us 
Xeon     22us
Xeon/HT  290us

In all fairness, the Xeon/HT machine is still running with the debug and 
trace options enabled, while the other two have all debug except wakeup 
latency timing disabled.

On UP, everything looks good.  No JACK xruns, even while running burnK7.  
Streaming with VLC from a Hauppauge card to UDP/multicast is almost
flawless (usually takes at least a day before any UDP packets are sent out
too late).  I only saw one warning in my logs on the Athlon box for the
whole -50 series, and that was in -50-15.  The non-HT Xeon box hasn't seen
any bug warnings since plist_init() on -48-33.

On SMT, everything works well under normal desktop load (X, wmaker, 10
dockapps, 10-20 xterms, and firefox).  VLC runs very smoothly when I'm not
stress-testing.  System responsiveness is much better than the 
vanilla kernels.  Wakeup latencies have decreased since the -48 series
(generally <100us, spiking to <300us now instead of generally <200us
spiking to <1000us).

When it comes to stress testing, I'm still finding conditions where CPU
hungry processes (like burnP6) can make the box completely unresponsive.  
So far, I've narrowed it down to the following conditions:  X is running
(xorg-6.8.2), along with two burnP6 instances, and X apps that are
actively updating the screen (like VLC or a collection of wmaker
dockapps).  If one of these conditions is absent, there's no meltdown.

I've been toying with a script to fire up two instances of burnP6, grab
some traces with your trace-it code, and kill off those burnP6 processes.  
The script generally takes the form:

#!/bin/bash
trace-it > trace1.out
sync
burn &
usleep 100000
trace-it > trace2.out
sync
burn &
usleep 100000
trace-it > trace2.out
sync
killall burn

I copied 'burnP6' to 'burn' so that the process name shows up in the 
traces.  Every time system response disappears, I never get any traces 
after the second burnP6 fires up.  Most of the time, I can still move the 
mouse pointer for a few minutes (with serio lost synchro warnings on the 
serial console) before it goes dead, too.  Keyboard is almost always lost.

The results of two tests (one resulting in a meltdown) are located at:

http://sysex.net/testing/2.6.12-RT-V0.7.50-35

The 'test' directory has some traces with X and burnP6 (no VLC or
dockapps)  running.  These traces have chunks of dead time ranging from
roughly 500 to 900 us.

The 'crash' directory has a trace with X, burnP6, and VLC running, and a
bug warning.  As soon as the second burnP6 instance started up, the system
showed no signs of being alive other than the mouse synchro warnings on
the serial console.

Overall, I would have to say that RT_PREEMPT is nearly ready for
primetime, especially for audio, MIDI, and video.  SMT needs some help
when it comes to doing anything that requires X.  I'm not sure about true
SMP, since I can't find a true SMP system to play with right now.

Great Work, Ingo!


Best Regards,
--ww

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Karsten Wiese]

Am Mittwoch, 29. Juni 2005 21:38 schrieb Ingo Molnar:
> 
> * Karsten Wiese <annabellesgarden@yahoo.de> wrote:
> 
> > attached patch for io_apic.c lets
> > 1. gcc 3.4.3 optimize io_apic access a little better.
> > 2. CONFIG_X86_UP_IOAPIC_FAST work here.
> >    Didn't check, if it really speeds up things.
> 
> which change made CONFIG_X86_UP_IOAPIC_FAST work on your box? It seems 
> you've changed the per-register frontside read-cache to something else - 
> was that on purpose?
> 
CONFIG_X86_UP_IOAPIC_FAST started working here, when I made io_apic_modify()
look like that:

void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int val)
{
#ifdef IOAPIC_CACHE
	io_apic_cache[apic][reg] = val;
#endif
//	if (unlikely(sis_apic_bug))
commented this ^^ out 

		*IO_APIC_BASE(apic) = reg;
	*(IO_APIC_BASE(apic)+4) = val;
#ifndef IOAPIC_POSTFLUSH
	if (unlikely(sis_apic_bug))
#endif
		/*
		 * Force POST flush by reading:
 		 */
		val = *(IO_APIC_BASE(apic)+4);
}

This change does it, 'cause when we read a cached value instead of from the ioapic,
the ioapic's address register isn't set and thus the following write in io_apic_modify
might not scratch the right ioapic register.

On top of the above the patch adds
	unsigned int reg;
to io_apic_cache.
with that "reg" struct member a mark -1 can be set, when we read from cache,
or the reg-number, when we read from the ioapic.
Then by comparing parameter reg with io_apic_cache[apic].reg
the patched io_apic_modify() knows, if it has to set the address register or not.
The register caching in the patch should be the same as before,
only the cache changed from a 2 dimensional array
to an array holding structs, which contain an array:
io_apic_cache[apic][reg] should be equivalent to io_apic_cache[apic].val[reg], no?

Karsten




	

	
		
___________________________________________________________ 
Gesendet von Yahoo! Mail - Jetzt mit 1GB Speicher kostenlos - Hier anmelden: http://mail.yahoo.de

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Karsten Wiese <annabellesgarden@yahoo.de> wrote:

> attached patch for io_apic.c lets
> 1. gcc 3.4.3 optimize io_apic access a little better.
> 2. CONFIG_X86_UP_IOAPIC_FAST work here.
>    Didn't check, if it really speeds up things.

which change made CONFIG_X86_UP_IOAPIC_FAST work on your box? It seems 
you've changed the per-register frontside read-cache to something else - 
was that on purpose?

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Karsten Wiese]

[-- Attachment #1: Type: text/plain, Size: 937 bytes --]

Am Mittwoch, 29. Juni 2005 11:15 schrieb William Weston:
> On Wed, 29 Jun 2005, Ingo Molnar wrote:
> 
> > > [...] but i think i'm going to revert that, it's causing too many 
> > > problems all around.
> > 
> > reverted it and this enabled the removal of the extra ->disable() you 
> > noticed - this should further speed up the IOAPIC code. These changes 
> > are in the -50-34 kernel i just uploaded.
> 
> -50-34 fixed the wakeup latency regression I was seeing on my Athlon box
> with -50-33, and seems a bit more responsive than -50-25.  Max wakeup
> latency is back down to 14us (from 39us), even while running JACK (xrun
> free) and two instances of burnK7.  Overall system response is probably
> the best I've seen with the RT kernels ;-}
> 
attached patch for io_apic.c lets
1. gcc 3.4.3 optimize io_apic access a little better.
2. CONFIG_X86_UP_IOAPIC_FAST work here.
   Didn't check, if it really speeds up things.

Karsten




[-- Attachment #2: io_apic-RT-50-35.diff --]
[-- Type: text/x-diff, Size: 3593 bytes --]

--- arch/i386/kernel/io_apic.c-50-33	2005-06-29 16:27:40.000000000 +0200
+++ arch/i386/kernel/io_apic.c	2005-06-29 16:27:40.000000000 +0200
@@ -138,14 +138,22 @@
 /*
  * Cache register values:
  */
-static unsigned int io_apic_cache[MAX_IO_APICS][MAX_IOAPIC_CACHE]
+static struct {
+	unsigned int reg;
+	unsigned int val[MAX_IOAPIC_CACHE];
+} io_apic_cache[MAX_IO_APICS]
 		____cacheline_aligned_in_smp;
 #endif
 
 inline unsigned int __raw_io_apic_read(unsigned int apic, unsigned int reg)
 {
-	*IO_APIC_BASE(apic) = reg;
-	return *(IO_APIC_BASE(apic)+4);
+	volatile unsigned int *io_apic;
+	io_apic = IO_APIC_BASE(apic);
+#ifdef IOAPIC_CACHE
+	io_apic_cache[apic].reg = reg;
+#endif
+	io_apic[0] = reg;
+	return io_apic[4];
 }
 
 unsigned int raw_io_apic_read(unsigned int apic, unsigned int reg)
@@ -153,7 +161,7 @@
 	unsigned int val = __raw_io_apic_read(apic, reg);
 
 #ifdef IOAPIC_CACHE
-	io_apic_cache[apic][reg] = val;
+	io_apic_cache[apic].val[reg] = val;
 #endif
 	return val;
 }
@@ -172,14 +180,17 @@
 		}
 		return __raw_io_apic_read(apic, reg);
 	}
-	if (io_apic_cache[apic][reg] && !sis_apic_bug)
-		return io_apic_cache[apic][reg];
+	if (io_apic_cache[apic].val[reg] && !sis_apic_bug) {
+		io_apic_cache[apic].reg = -1;
+		return io_apic_cache[apic].val[reg];
+	}
 #endif
 	return raw_io_apic_read(apic, reg);
 }
 
 void io_apic_write(unsigned int apic, unsigned int reg, unsigned int val)
 {
+	volatile unsigned int *io_apic;
 #ifdef IOAPIC_CACHE
 	if (unlikely(reg >= MAX_IOAPIC_CACHE)) {
 		static int once = 1;
@@ -191,10 +202,14 @@
 			dump_stack();
 		}
 	} else
-		io_apic_cache[apic][reg] = val;
+		io_apic_cache[apic].val[reg] = val;
 #endif
-	*IO_APIC_BASE(apic) = reg;
-	*(IO_APIC_BASE(apic)+4) = val;
+	io_apic = IO_APIC_BASE(apic);
+#ifdef IOAPIC_CACHE
+	io_apic_cache[apic].reg = reg;
+#endif
+	io_apic[0] = reg;
+	io_apic[4] = val;
 }
 
 /*
@@ -214,34 +229,42 @@
  */
 void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int val)
 {
+	volatile unsigned int *io_apic;
 #ifdef IOAPIC_CACHE
-	io_apic_cache[apic][reg] = val;
+	io_apic_cache[apic].val[reg] = val;
 #endif
-	if (unlikely(sis_apic_bug))
-		*IO_APIC_BASE(apic) = reg;
-	*(IO_APIC_BASE(apic)+4) = val;
+	io_apic = IO_APIC_BASE(apic);
+#ifdef IOAPIC_CACHE
+	if (io_apic_cache[apic].reg != reg || sis_apic_bug) {
+		io_apic_cache[apic].reg = reg;
+#else
+	if (unlikely(sis_apic_bug)) {
+#endif
+		io_apic[0] = reg;
+	}
+	io_apic[4] = val;
 #ifndef IOAPIC_POSTFLUSH
 	if (unlikely(sis_apic_bug))
 #endif
 		/*
 		 * Force POST flush by reading:
  		 */
-		val = *(IO_APIC_BASE(apic)+4);
+		val = io_apic[4];
 }
 
 static void __modify_IO_APIC_irq (unsigned int irq, unsigned long enable, unsigned long disable)
 {
 	struct irq_pin_list *entry = irq_2_pin + irq;
-	unsigned int pin, reg;
+	unsigned int pin, val;
 
 	for (;;) {
 		pin = entry->pin;
 		if (pin == -1)
 			break;
-		reg = io_apic_read(entry->apic, 0x10 + pin*2);
-		reg &= ~disable;
-		reg |= enable;
-		io_apic_modify(entry->apic, 0x10 + pin*2, reg);
+		val = io_apic_read(entry->apic, 0x10 + pin*2);
+		val &= ~disable;
+		val |= enable;
+		io_apic_modify(entry->apic, 0x10 + pin*2, val);
 		if (!entry->next)
 			break;
 		entry = irq_2_pin + entry->next;
@@ -306,9 +329,13 @@
 {
 	int apic, pin;
 
-	for (apic = 0; apic < nr_ioapics; apic++)
+	for (apic = 0; apic < nr_ioapics; apic++) {
+#ifdef IOAPIC_CACHE
+		io_apic_cache[apic].reg = -1;
+#endif
 		for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
 			clear_IO_APIC_pin(apic, pin);
+	}
 }
 
 static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t cpumask)

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* William Weston <weston@sysex.net> wrote:

> On Wed, 29 Jun 2005, Ingo Molnar wrote:
> 
> > > [...] but i think i'm going to revert that, it's causing too many 
> > > problems all around.
> > 
> > reverted it and this enabled the removal of the extra ->disable() you 
> > noticed - this should further speed up the IOAPIC code. These changes 
> > are in the -50-34 kernel i just uploaded.
> 
> -50-34 fixed the wakeup latency regression I was seeing on my Athlon 
> box with -50-33, and seems a bit more responsive than -50-25.  Max 
> wakeup latency is back down to 14us (from 39us), even while running 
> JACK (xrun free) and two instances of burnK7.  Overall system response 
> is probably the best I've seen with the RT kernels ;-}

great! The SMP box running BurnP6 is another system, right? Could you 
sum up the remaining regressions you are seeing under -RT? (the 
latency.c warning is one, what others are remaining?)

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[William Weston]

On Wed, 29 Jun 2005, Ingo Molnar wrote:

> > [...] but i think i'm going to revert that, it's causing too many 
> > problems all around.
> 
> reverted it and this enabled the removal of the extra ->disable() you 
> noticed - this should further speed up the IOAPIC code. These changes 
> are in the -50-34 kernel i just uploaded.

-50-34 fixed the wakeup latency regression I was seeing on my Athlon box
with -50-33, and seems a bit more responsive than -50-25.  Max wakeup
latency is back down to 14us (from 39us), even while running JACK (xrun
free) and two instances of burnK7.  Overall system response is probably
the best I've seen with the RT kernels ;-}

--ww

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> [...] but i think i'm going to revert that, it's causing too many 
> problems all around.

reverted it and this enabled the removal of the extra ->disable() you 
noticed - this should further speed up the IOAPIC code. These changes 
are in the -50-34 kernel i just uploaded.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Karsten Wiese <annabellesgarden@yahoo.de> wrote:

> looked at -50-33 now and wonder why is mask_IO_APIC_irq() called twice 
> from __do_IRQ()? given a threaded interrupt: __do_IRQ() calls 
> desc->handler->ack(irq). ack points to 
> mask_and_ack_level_ioapic_irq(), which calls mask_IO_APIC_irq(irq).  
> some lines later in __do_IRQ() desc->handler->disable(irq) is called.  
> disable points to mask_IO_APIC_irq(), now being called a 2nd time. I 
> think this 2nd call isn't necessary. Is there a difference between 
> masking an interrupt line and disabling it? What am I missing?

you are not missing anything - but i found no easy way for the time 
being to get rid of the second masking.

> Back at 2.6.12-rc5-RT-48-16 mask_and_ack_level_ioapic_irq() also 
> contained the mask_IO_APIC_irq(irq) call and level interrupt-rates 
> where fine. Some versions later it vanished there. Why was that?

i reorganized how redirection is being done, and i've implemented 
auto-ACK for the i8259A, to reduce IRQ handling costs. One goal was to 
avoid the masking of the interrupt line for the timer interrupt on 
i8259A - but i think i'm going to revert that, it's causing too many 
problems all around.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Chuck Harding]

On Tue, 28 Jun 2005, Chuck Harding wrote:

> On Tue, 28 Jun 2005, Ingo Molnar wrote:
>
>> 
>> * Ingo Molnar <mingo@elte.hu> wrote:
>> 
>>> * Steven Rostedt <rostedt@goodmis.org> wrote:
>>> 
>>>> Although turning off apm works, this is a fix to the symptom and not a
>>>> cure.  Has someone already taken a look at this code? Since
>>>> apm_bios_call_simple calls local_save_flags and afterwards
>>>> raw_lock_irq_restore is then called.  Shouldn't that have been
>>>> raw_local_save_flags?
>>> 
>>> ah, indeed. I fixed this bug and have uploaded the -50-26 patch.
>>> Chuck, does this fix the APM problems for you?
>> 
>> i've also uploaded -50-27 in which i've improved the irq-flags debugging
>> code. They are activated if CONFIG_DEBUG_PREEMPT is enabled, and can
>> come in two variants of kernel messages:
>> 
>> BUG: bad raw irq-flag value 80000000, test/3810!
>> BUG: bad soft irq-flag value 00000202, test/3810!
>> 
>> so we should now be able to detect mismatches of irq flags right where
>> they occur.
>>
>> 	Ingo
>
> Ack!! I didn't have that enabled so I am rebuilding again. One thing I've
> noticed is that sox seems to be hanging when it is trying to play a .wav
> file (for system beeps) and there aren't any error messages about what
> might be going on.
>
>

More info - sox is locked up so tight that no kill can stop it. I'm not
seeing any of the BUG messages either even though I enabled 
CONFIG_DEBUG_PREEMPT.

-- 
Charles D. (Chuck) Harding <charding@llnl.gov>  Voice: 925-423-8879
Senior Computer Associate         ICCD            Fax: 925-423-6961
Lawrence Livermore National Laboratory      Computation Directorate
Livermore, CA USA  http://www.llnl.gov  GPG Public Key ID: B9EB6601
------------------ http://tinyurl.com/5w5ey -----------------------
-- LISP: To call a spade a thpade. --

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Karsten Wiese]

Am Dienstag, 28. Juni 2005 22:30 schrieb Ingo Molnar:
> 
> * Ingo Molnar <mingo@elte.hu> wrote:
> 
> > 
> > * Karsten Wiese <annabellesgarden@yahoo.de> wrote:
> > 
> > > Hi Ingo,
> > > 
> > > suffering (not really ;-) double-rated IO-APIC level-interrupts I 
> > > found the following patch as a solution:
> > 
> > thanks. I've applied your patch but also tweaked this area a bit, to 
> > make the i8259A PIC work too. I've uploaded the -31 patch with these 
> > fixes included.
> 
> make that -50-32, had a leftover hack in io_apic.c.
> 
looked at -50-33 now and wonder why is mask_IO_APIC_irq() called twice
from  __do_IRQ()?
given a threaded interrupt:
__do_IRQ() calls desc->handler->ack(irq).
ack points to mask_and_ack_level_ioapic_irq(), which calls mask_IO_APIC_irq(irq).
some lines later in __do_IRQ() desc->handler->disable(irq) is called.
disable points to  mask_IO_APIC_irq(), now being called a 2nd time.
I think this 2nd call isn't necessary.
Is there a difference between masking an interrupt line and disabling it?
What am I missing?

Back at 2.6.12-rc5-RT-48-16 mask_and_ack_level_ioapic_irq() also contained the mask_IO_APIC_irq(irq)
call and level interrupt-rates where fine.
Some versions later it vanished there. Why was that?

Karsten



  

	

	
		
___________________________________________________________ 
Gesendet von Yahoo! Mail - Jetzt mit 1GB Speicher kostenlos - Hier anmelden: http://mail.yahoo.de

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Chuck Harding]

On Tue, 28 Jun 2005, Ingo Molnar wrote:

>
> * Ingo Molnar <mingo@elte.hu> wrote:
>
>> * Steven Rostedt <rostedt@goodmis.org> wrote:
>>
>>> Although turning off apm works, this is a fix to the symptom and not a
>>> cure.  Has someone already taken a look at this code? Since
>>> apm_bios_call_simple calls local_save_flags and afterwards
>>> raw_lock_irq_restore is then called.  Shouldn't that have been
>>> raw_local_save_flags?
>>
>> ah, indeed. I fixed this bug and have uploaded the -50-26 patch.
>> Chuck, does this fix the APM problems for you?
>
> i've also uploaded -50-27 in which i've improved the irq-flags debugging
> code. They are activated if CONFIG_DEBUG_PREEMPT is enabled, and can
> come in two variants of kernel messages:
>
> BUG: bad raw irq-flag value 80000000, test/3810!
> BUG: bad soft irq-flag value 00000202, test/3810!
>
> so we should now be able to detect mismatches of irq flags right where
> they occur.
>
> 	Ingo

Ack!! I didn't have that enabled so I am rebuilding again. One thing I've
noticed is that sox seems to be hanging when it is trying to play a .wav
file (for system beeps) and there aren't any error messages about what
might be going on.

-- 
Charles D. (Chuck) Harding <charding@llnl.gov>  Voice: 925-423-8879
Senior Computer Associate         ICCD            Fax: 925-423-6961
Lawrence Livermore National Laboratory      Computation Directorate
Livermore, CA USA  http://www.llnl.gov  GPG Public Key ID: B9EB6601
------------------ http://tinyurl.com/5w5ey -----------------------
-- If I knew what I was doing...I'd be dangerous... --

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> 
> * Karsten Wiese <annabellesgarden@yahoo.de> wrote:
> 
> > Hi Ingo,
> > 
> > suffering (not really ;-) double-rated IO-APIC level-interrupts I 
> > found the following patch as a solution:
> 
> thanks. I've applied your patch but also tweaked this area a bit, to 
> make the i8259A PIC work too. I've uploaded the -31 patch with these 
> fixes included.

make that -50-32, had a leftover hack in io_apic.c.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Karsten Wiese <annabellesgarden@yahoo.de> wrote:

> Hi Ingo,
> 
> suffering (not really ;-) double-rated IO-APIC level-interrupts I 
> found the following patch as a solution:

thanks. I've applied your patch but also tweaked this area a bit, to 
make the i8259A PIC work too. I've uploaded the -31 patch with these 
fixes included.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Chuck Harding]

On Tue, 28 Jun 2005, Ingo Molnar wrote:

>
> * Michal Schmidt <xschmi00@stud.feec.vutbr.cz> wrote:
>
>> Ingo Molnar wrote:
>>> i've also uploaded -50-27 in which i've improved the irq-flags debugging
>>> code.
>>
>> Hi Ingo,
>> check_raw_flags needs to be exported. The attached one-line patch is
>> against -V0.7.50-29.
>
> thanks, i've uploaded the -30 patch with your fix included.
>
> 	Ingo

It's working fine now - switching back and forth between graphical mode
and text console (which works with or with out the shift key B-)  and no
more flooding of BUG: scheduling with irqsw disabled in dmesg. I did
notice that 50-24 kernal was hard locked up when I came in this morning.
SysRq-B would not reboot it and pinging from another machine said it
was down. I went ahead and booted up another kernel and saw the message
about 50-30 so I built that and that is what I am running on now. I really
like using it because X works much better at rendering the screen and
screen animations are really smooth. (this is on a 2GHz P4 with 1Gb RAM)

-- 
Charles D. (Chuck) Harding <charding@llnl.gov>  Voice: 925-423-8879
Senior Computer Associate         ICCD            Fax: 925-423-6961
Lawrence Livermore National Laboratory      Computation Directorate
Livermore, CA USA  http://www.llnl.gov  GPG Public Key ID: B9EB6601
------------------ http://tinyurl.com/5w5ey -----------------------
-- Oxymoron: Spending Cuts. --

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Karsten Wiese]

Hi Ingo,

suffering (not really ;-) double-rated IO-APIC level-interrupts I found the following patch as a solution:

--- arch/i386/kernel/io_apic.c~	2005-06-28 19:07:49.000000000 +0200
+++ arch/i386/kernel/io_apic.c	2005-06-28 19:07:49.000000000 +0200
@@ -1959,6 +1959,7 @@
 static void mask_and_ack_level_ioapic_irq(unsigned int irq)
 {
 	move_irq(irq);
+	mask_IO_APIC_irq(irq);
 	ack_APIC_irq();
 }
 
--- kernel/irq/handle.c~	2005-06-28 19:19:32.000000000 +0200
+++ kernel/irq/handle.c	2005-06-28 19:19:32.000000000 +0200
@@ -214,7 +214,6 @@
 	 * hardirq redirection to the irqd process context:
 	 */
 	if (redirect_hardirq(desc)) {
-		desc->handler->disable(irq);
 		goto out_no_end;
 	}

it works here on a PREEMPT_RT 2.6.12-RT-V0.7.50-29 base.
Level-interrupts are sanely rated again.
Didn't check, if the patch breaks XT-PIC mode, which works ok without the patch.
Mainline shows the same effect here (VIA K8T800 UP), didn't dig there yet.

thanks,
Karsten

	

	
		
___________________________________________________________ 
Gesendet von Yahoo! Mail - Jetzt mit 1GB Speicher kostenlos - Hier anmelden: http://mail.yahoo.de

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Daniel Walker]

On Tue, 28 Jun 2005, Steven Rostedt wrote:

> 
> [Please CC Ingo Molnar on all RT kernel issues]
> 
> On Mon, 27 Jun 2005, Daniel Walker wrote:
> 
> > On Mon, 2005-06-27 at 12:01 -0700, Chuck Harding wrote:
> > > What can be causing the following message to appear in dmesg and
> > > how can I fix it?
> > >
> > > BUG: scheduling with irqs disabled: kapmd/0x00000000/46
> > > caller is schedule_timeout+0x51/0x9e
> > >   [<c02b3bc9>] schedule+0x96/0xf6 (8)
> > >   [<c02b43f7>] schedule_timeout+0x51/0x9e (28)
> > >   [<c01222ed>] process_timeout+0x0/0x5 (32)
> > >   [<c0112063>] apm_mainloop+0x7a/0x96 (24)
> > >   [<c0115e45>] default_wake_function+0x0/0x16 (12)
> > >   [<c0115e45>] default_wake_function+0x0/0x16 (32)
> > >   [<c0111485>] apm_driver_version+0x1c/0x38 (16)
> > >   [<c01126f7>] apm+0x0/0x289 (8)
> > >   [<c01127a6>] apm+0xaf/0x289 (8)
> > >   [<c010133c>] kernel_thread_helper+0x0/0xb (20)
> > >   [<c0101341>] kernel_thread_helper+0x5/0xb (4)
> > >
> > > This was also present in earlier final-V0.7.50 version I've tried
> > > (since -00) I don't get hangs but that doesn't look like it should
> > > be happening. Thanks.
> >
> > If you have PREEMPT_RT enabled, it looks like interrupts are hard
> > disabled then there is a schedule_timeout() requested. You could try
> > turning off power management and see if you still have problems.
> >
> 
> Although turning off apm works, this is a fix to the symptom and not a
> cure.  Has someone already taken a look at this code? Since
> apm_bios_call_simple calls local_save_flags and afterwards
> raw_lock_irq_restore is then called.  Shouldn't that have been
> raw_local_save_flags?
> 
> That apm_bios_call_simple_asm also looks pretty scary!  I haven't yet
> figured out how APM_FUNC_VERSION becomes a normal function.


I looked at them briefly.. It looks like there is some raw and non-raw
mixed usage .

Daniel

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Michal Schmidt <xschmi00@stud.feec.vutbr.cz> wrote:

> Ingo Molnar wrote:
> >i've also uploaded -50-27 in which i've improved the irq-flags debugging 
> >code.
> 
> Hi Ingo,
> check_raw_flags needs to be exported. The attached one-line patch is 
> against -V0.7.50-29.

thanks, i've uploaded the -30 patch with your fix included.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Michal Schmidt]

[-- Attachment #1: Type: text/plain, Size: 212 bytes --]

Ingo Molnar wrote:
> i've also uploaded -50-27 in which i've improved the irq-flags debugging 
> code.

Hi Ingo,
check_raw_flags needs to be exported. The attached one-line patch is 
against -V0.7.50-29.

Michal

[-- Attachment #2: rt-check_raw_flags-export.diff --]
[-- Type: text/plain, Size: 301 bytes --]

--- linux-RT.mich/kernel/rt.c.orig	2005-06-28 17:20:18.000000000 +0200
+++ linux-RT.mich/kernel/rt.c	2005-06-28 17:14:19.000000000 +0200
@@ -2088,6 +2088,7 @@ void check_raw_flags(unsigned long flags
 		}
 	}
 }
+EXPORT_SYMBOL(check_raw_flags);
 
 static void check_soft_flags(unsigned long flags)
 {

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> * Steven Rostedt <rostedt@goodmis.org> wrote:
> 
> > Although turning off apm works, this is a fix to the symptom and not a 
> > cure.  Has someone already taken a look at this code? Since 
> > apm_bios_call_simple calls local_save_flags and afterwards 
> > raw_lock_irq_restore is then called.  Shouldn't that have been 
> > raw_local_save_flags?
> 
> ah, indeed. I fixed this bug and have uploaded the -50-26 patch. 
> Chuck, does this fix the APM problems for you?

i've also uploaded -50-27 in which i've improved the irq-flags debugging 
code. They are activated if CONFIG_DEBUG_PREEMPT is enabled, and can 
come in two variants of kernel messages:

 BUG: bad raw irq-flag value 80000000, test/3810!
 BUG: bad soft irq-flag value 00000202, test/3810!

so we should now be able to detect mismatches of irq flags right where 
they occur.

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Steven Rostedt]

On Tue, 28 Jun 2005, Ingo Molnar wrote:

> ah, indeed. I fixed this bug and have uploaded the -50-26 patch. Chuck,
> does this fix the APM problems for you?

The question is where the call to schedule happened. If it was between
the local_save_flags and irq_restore, then I believe the interrupts are
off and the schedule can take place.  But if it was afterwards, then this
is probably the fix since the irqrestore probably didn't restore it.

I guess we need to wait for Chuck to answer this.

-- Steve

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Ingo Molnar]

* Steven Rostedt <rostedt@goodmis.org> wrote:

> Although turning off apm works, this is a fix to the symptom and not a 
> cure.  Has someone already taken a look at this code? Since 
> apm_bios_call_simple calls local_save_flags and afterwards 
> raw_lock_irq_restore is then called.  Shouldn't that have been 
> raw_local_save_flags?

ah, indeed. I fixed this bug and have uploaded the -50-26 patch. Chuck, 
does this fix the APM problems for you?

> That apm_bios_call_simple_asm also looks pretty scary!  I haven't yet 
> figured out how APM_FUNC_VERSION becomes a normal function.

it's all black magic, so when hacking that code i always close my eyes.  
(that's how the bug got introduced)

	Ingo

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Steven Rostedt]

[Please CC Ingo Molnar on all RT kernel issues]

On Mon, 27 Jun 2005, Daniel Walker wrote:

> On Mon, 2005-06-27 at 12:01 -0700, Chuck Harding wrote:
> > What can be causing the following message to appear in dmesg and
> > how can I fix it?
> >
> > BUG: scheduling with irqs disabled: kapmd/0x00000000/46
> > caller is schedule_timeout+0x51/0x9e
> >   [<c02b3bc9>] schedule+0x96/0xf6 (8)
> >   [<c02b43f7>] schedule_timeout+0x51/0x9e (28)
> >   [<c01222ed>] process_timeout+0x0/0x5 (32)
> >   [<c0112063>] apm_mainloop+0x7a/0x96 (24)
> >   [<c0115e45>] default_wake_function+0x0/0x16 (12)
> >   [<c0115e45>] default_wake_function+0x0/0x16 (32)
> >   [<c0111485>] apm_driver_version+0x1c/0x38 (16)
> >   [<c01126f7>] apm+0x0/0x289 (8)
> >   [<c01127a6>] apm+0xaf/0x289 (8)
> >   [<c010133c>] kernel_thread_helper+0x0/0xb (20)
> >   [<c0101341>] kernel_thread_helper+0x5/0xb (4)
> >
> > This was also present in earlier final-V0.7.50 version I've tried
> > (since -00) I don't get hangs but that doesn't look like it should
> > be happening. Thanks.
>
> If you have PREEMPT_RT enabled, it looks like interrupts are hard
> disabled then there is a schedule_timeout() requested. You could try
> turning off power management and see if you still have problems.
>

Although turning off apm works, this is a fix to the symptom and not a
cure.  Has someone already taken a look at this code? Since
apm_bios_call_simple calls local_save_flags and afterwards
raw_lock_irq_restore is then called.  Shouldn't that have been
raw_local_save_flags?

That apm_bios_call_simple_asm also looks pretty scary!  I haven't yet
figured out how APM_FUNC_VERSION becomes a normal function.

-- Steve

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Daniel Walker]

On Mon, 2005-06-27 at 13:28 -0700, Chuck Harding wrote:
> On Mon, 27 Jun 2005, Daniel Walker wrote:
> 
> > If you have PREEMPT_RT enabled, it looks like interrupts are hard
> > disabled then there is a schedule_timeout() requested. You could try
> > turning off power management and see if you still have problems.
> >
> > Daniel
> >
> 
> Well, putting apm=off in the kernel command line did the trick. I am
> using a desktop system so apm really isn't needed. Time to change my
> standard config file..... Thanks.

Did it solve everything , including the virtual terminal switching, and
the scheduling with irqs disabled ?

Daniel

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Gene Heskett]

On Monday 27 June 2005 15:42, Chuck Harding wrote:
>On Mon, 27 Jun 2005, Chuck Harding wrote:
>> What can be causing the following message to appear in dmesg and
>> how can I fix it?
>>
>> BUG: scheduling with irqs disabled: kapmd/0x00000000/46
>> caller is schedule_timeout+0x51/0x9e
>> [<c02b3bc9>] schedule+0x96/0xf6 (8)
>> [<c02b43f7>] schedule_timeout+0x51/0x9e (28)
>> [<c01222ed>] process_timeout+0x0/0x5 (32)
>> [<c0112063>] apm_mainloop+0x7a/0x96 (24)
>> [<c0115e45>] default_wake_function+0x0/0x16 (12)
>> [<c0115e45>] default_wake_function+0x0/0x16 (32)
>> [<c0111485>] apm_driver_version+0x1c/0x38 (16)
>> [<c01126f7>] apm+0x0/0x289 (8)
>> [<c01127a6>] apm+0xaf/0x289 (8)
>> [<c010133c>] kernel_thread_helper+0x0/0xb (20)
>> [<c0101341>] kernel_thread_helper+0x5/0xb (4)
>>
>> This was also present in earlier final-V0.7.50 version I've tried
>> (since -00) I don't get hangs but that doesn't look like it should
>> be happening. Thanks.
>
>another symptom (don't know if it's actually related) is that if I
>try to switch to a virtual consol (ctl-alt-chift-F[1..6] the screen
>won't change out of graphics mode - it goes black like it's trying
>to switch but comes back with the graphical screen which isn't
> responsive and hitting alt-F7 restores it to operation. I just
> rebooted to a kernel that doesn't have the RT-preempt patch but
> uses the same .config and everything for switching between X and
> virtual terminals works just fine.

Its working ok here, but not with those keystrokes. I just did it 
several times.  Its ctl+alt+Fn here, and always has been.

What I did see on VT#1, where the boinc stuff was being spit out 
before I logged in and did a startx, is that I have less then a 
screenfull of history available, and unlike previous kernels, I'm 
getting a very verbose running trail, almost like an strace output, 
of all the kde/kmail activities going on being output to VT#1.  VT#2 
seems normal and I was able to log into it just fine before hitting 
ctl+alt+F7 to come back to X.  Weird to say the least.

But this brings up the question re how do I enlarge the scrollback 
memory of an init-run-level=3 VT?

-- 
Cheers, Gene
"There are four boxes to be used in defense of liberty:
 soap, ballot, jury, and ammo. Please use in that order."
-Ed Howdershelt (Author)
99.35% setiathome rank, not too shabby for a WV hillbilly
Yahoo.com and AOL/TW attorneys please note, additions to the above
message by Gene Heskett are:
Copyright 2005 by Maurice Eugene Heskett, all rights reserved.

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Chuck Harding]

On Mon, 27 Jun 2005, Daniel Walker wrote:

> If you have PREEMPT_RT enabled, it looks like interrupts are hard
> disabled then there is a schedule_timeout() requested. You could try
> turning off power management and see if you still have problems.
>
> Daniel
>

Well, putting apm=off in the kernel command line did the trick. I am
using a desktop system so apm really isn't needed. Time to change my
standard config file..... Thanks.

-- 
Charles D. (Chuck) Harding <charding@llnl.gov>  Voice: 925-423-8879
Senior Computer Associate         ICCD            Fax: 925-423-6961
Lawrence Livermore National Laboratory      Computation Directorate
Livermore, CA USA  http://www.llnl.gov  GPG Public Key ID: B9EB6601
------------------ http://tinyurl.com/5w5ey -----------------------
-- Useless Invention: Rollerblade skates for peglegs. --

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Daniel Walker]

On Mon, 2005-06-27 at 12:01 -0700, Chuck Harding wrote:
> What can be causing the following message to appear in dmesg and
> how can I fix it?
> 
> BUG: scheduling with irqs disabled: kapmd/0x00000000/46
> caller is schedule_timeout+0x51/0x9e
>   [<c02b3bc9>] schedule+0x96/0xf6 (8)
>   [<c02b43f7>] schedule_timeout+0x51/0x9e (28)
>   [<c01222ed>] process_timeout+0x0/0x5 (32)
>   [<c0112063>] apm_mainloop+0x7a/0x96 (24)
>   [<c0115e45>] default_wake_function+0x0/0x16 (12)
>   [<c0115e45>] default_wake_function+0x0/0x16 (32)
>   [<c0111485>] apm_driver_version+0x1c/0x38 (16)
>   [<c01126f7>] apm+0x0/0x289 (8)
>   [<c01127a6>] apm+0xaf/0x289 (8)
>   [<c010133c>] kernel_thread_helper+0x0/0xb (20)
>   [<c0101341>] kernel_thread_helper+0x5/0xb (4)
> 
> This was also present in earlier final-V0.7.50 version I've tried
> (since -00) I don't get hangs but that doesn't look like it should
> be happening. Thanks.

If you have PREEMPT_RT enabled, it looks like interrupts are hard
disabled then there is a schedule_timeout() requested. You could try
turning off power management and see if you still have problems.

Daniel 

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Chuck Harding]

On Mon, 27 Jun 2005, Chuck Harding wrote:

> What can be causing the following message to appear in dmesg and
> how can I fix it?
>
> BUG: scheduling with irqs disabled: kapmd/0x00000000/46
> caller is schedule_timeout+0x51/0x9e
> [<c02b3bc9>] schedule+0x96/0xf6 (8)
> [<c02b43f7>] schedule_timeout+0x51/0x9e (28)
> [<c01222ed>] process_timeout+0x0/0x5 (32)
> [<c0112063>] apm_mainloop+0x7a/0x96 (24)
> [<c0115e45>] default_wake_function+0x0/0x16 (12)
> [<c0115e45>] default_wake_function+0x0/0x16 (32)
> [<c0111485>] apm_driver_version+0x1c/0x38 (16)
> [<c01126f7>] apm+0x0/0x289 (8)
> [<c01127a6>] apm+0xaf/0x289 (8)
> [<c010133c>] kernel_thread_helper+0x0/0xb (20)
> [<c0101341>] kernel_thread_helper+0x5/0xb (4)
>
> This was also present in earlier final-V0.7.50 version I've tried
> (since -00) I don't get hangs but that doesn't look like it should
> be happening. Thanks.
>
another symptom (don't know if it's actually related) is that if I
try to switch to a virtual consol (ctl-alt-chift-F[1..6] the screen
won't change out of graphics mode - it goes black like it's trying
to switch but comes back with the graphical screen which isn't responsive
and hitting alt-F7 restores it to operation. I just rebooted to a kernel
that doesn't have the RT-preempt patch but uses the same .config and
everything for switching between X and virtual terminals works just fine.

-- 
Charles D. (Chuck) Harding <charding@llnl.gov>  Voice: 925-423-8879
Senior Computer Associate         ICCD            Fax: 925-423-6961
Lawrence Livermore National Laboratory      Computation Directorate
Livermore, CA USA  http://www.llnl.gov  GPG Public Key ID: B9EB6601
------------------ http://tinyurl.com/5w5ey -----------------------
-- I'm not fat, just horizontally disproportionate. --

Re: Real-Time Preemption, -RT-2.6.12-final-V0.7.50-24

[Chuck Harding]

What can be causing the following message to appear in dmesg and
how can I fix it?

BUG: scheduling with irqs disabled: kapmd/0x00000000/46
caller is schedule_timeout+0x51/0x9e
  [<c02b3bc9>] schedule+0x96/0xf6 (8)
  [<c02b43f7>] schedule_timeout+0x51/0x9e (28)
  [<c01222ed>] process_timeout+0x0/0x5 (32)
  [<c0112063>] apm_mainloop+0x7a/0x96 (24)
  [<c0115e45>] default_wake_function+0x0/0x16 (12)
  [<c0115e45>] default_wake_function+0x0/0x16 (32)
  [<c0111485>] apm_driver_version+0x1c/0x38 (16)
  [<c01126f7>] apm+0x0/0x289 (8)
  [<c01127a6>] apm+0xaf/0x289 (8)
  [<c010133c>] kernel_thread_helper+0x0/0xb (20)
  [<c0101341>] kernel_thread_helper+0x5/0xb (4)

This was also present in earlier final-V0.7.50 version I've tried
(since -00) I don't get hangs but that doesn't look like it should
be happening. Thanks.

-- 
Charles D. (Chuck) Harding <charding@llnl.gov>  Voice: 925-423-8879
Senior Computer Associate         ICCD            Fax: 925-423-6961
Lawrence Livermore National Laboratory      Computation Directorate
Livermore, CA USA  http://www.llnl.gov  GPG Public Key ID: B9EB6601
------------------ http://tinyurl.com/5w5ey -----------------------
-- Unburdened by the rigors of coherent thought. --