diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/Makefile ipipe-2.6.19-arm/Makefile --- linux-2.6.19/Makefile 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/Makefile 2007-01-15 21:33:00.000000000 +0100 @@ -491,6 +491,10 @@ endif include $(srctree)/arch/$(ARCH)/Makefile +ifdef CONFIG_IPIPE_TRACE_MCOUNT +CFLAGS += -pg +endif + ifdef CONFIG_FRAME_POINTER CFLAGS += -fno-omit-frame-pointer $(call cc-option,-fno-optimize-sibling-calls,) else diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/Kconfig ipipe-2.6.19-arm/arch/arm/Kconfig --- linux-2.6.19/arch/arm/Kconfig 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/Kconfig 2007-01-15 21:38:05.000000000 +0100 @@ -487,6 +487,8 @@ config LOCAL_TIMERS accounting to be spread across the timer interval, preventing a "thundering herd" at every timer tick. +source "kernel/ipipe/Kconfig" + config PREEMPT bool "Preemptible Kernel (EXPERIMENTAL)" depends on EXPERIMENTAL diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/boot/compressed/head.S ipipe-2.6.19-arm/arch/arm/boot/compressed/head.S --- linux-2.6.19/arch/arm/boot/compressed/head.S 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/boot/compressed/head.S 2007-01-15 21:38:05.000000000 +0100 @@ -833,6 +833,16 @@ memdump: mov r12, r0 mov pc, r10 #endif +#ifdef CONFIG_IPIPE_TRACE_MCOUNT + .text + .align 0 + .type mcount %function + .global mcount +mcount: + mov pc, lr @ just return +#endif + + reloc_end: .align diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/kernel/Makefile ipipe-2.6.19-arm/arch/arm/kernel/Makefile --- linux-2.6.19/arch/arm/kernel/Makefile 2007-01-04 22:01:14.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/kernel/Makefile 2007-01-16 23:40:09.000000000 +0100 @@ -20,6 +20,8 @@ obj-$(CONFIG_ISA_DMA) += dma-isa.o obj-$(CONFIG_PCI) += bios32.o isa.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_OABI_COMPAT) += sys_oabi-compat.o +obj-$(CONFIG_IPIPE) += ipipe.o +obj-$(CONFIG_IPIPE_TRACE_MCOUNT) += ipipe-mcount.o obj-$(CONFIG_CRUNCH) += crunch.o crunch-bits.o AFLAGS_crunch-bits.o := -Wa,-mcpu=ep9312 diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/kernel/entry-armv.S ipipe-2.6.19-arm/arch/arm/kernel/entry-armv.S --- linux-2.6.19/arch/arm/kernel/entry-armv.S 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/kernel/entry-armv.S 2007-01-15 21:46:43.000000000 +0100 @@ -4,6 +4,7 @@ * Copyright (C) 1996,1997,1998 Russell King. * ARM700 fix by Matthew Godbolt (linux-user@domain.hid) * nommu support by Hyok S. Choi (hyok.choi@domain.hid) + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -33,7 +34,11 @@ @ routine called with r0 = irq number, r1 = struct pt_regs * @ adrne lr, 1b +#ifdef CONFIG_IPIPE + bne __ipipe_grab_irq +#else bne asm_do_IRQ +#endif #ifdef CONFIG_SMP /* @@ -202,6 +207,11 @@ __irq_svc: #endif irq_handler +#ifdef CONFIG_IPIPE + cmp r0, #0 + beq __ipipe_fast_svc_irq_exit +#endif + #ifdef CONFIG_PREEMPT ldr r0, [tsk, #TI_FLAGS] @ get flags tst r0, #_TIF_NEED_RESCHED @@ -212,6 +222,9 @@ preempt_return: teq r0, r7 strne r0, [r0, -r0] @ bug() #endif +#ifdef CONFIG_IPIPE +__ipipe_fast_svc_irq_exit: +#endif ldr r0, [sp, #S_PSR] @ irqs are already disabled msr spsr_cxsf, r0 #ifdef CONFIG_TRACE_IRQFLAGS @@ -231,19 +244,34 @@ svc_preempt: ldr r1, [r6, #8] @ local_bh_count adds r0, r0, r1 movne pc, lr +#ifdef CONFIG_IPIPE + bl __ipipe_stall_root +#endif mov r7, #0 @ preempt_schedule_irq str r7, [tsk, #TI_PREEMPT] @ expects preempt_count == 0 1: bl preempt_schedule_irq @ irq en/disable is done inside ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS tst r0, #_TIF_NEED_RESCHED +#ifdef CONFIG_IPIPE + bne 1b + bl __ipipe_unstall_root + b preempt_return @ go again +#else beq preempt_return @ go again b 1b #endif +#endif .align 5 __und_svc: svc_entry +#ifdef CONFIG_IPIPE + mov r1, sp @ r0 = trapno, r1 = ®s + bl __ipipe_dispatch_event @ branch to trap handler + cmp r0, #0 + bne 1f +#endif /* CONFIG_IPIPE */ @ @ call emulation code, which returns using r9 if it has emulated @ the instruction, or the more conventional lr if we are to treat @@ -416,6 +444,12 @@ __irq_usr: #endif irq_handler +#ifdef CONFIG_IPIPE + cmp r0, #0 + bne __ipipe_usr_irq_continue + slow_restore_user_regs @ Fast exit path over non-root domains +__ipipe_usr_irq_continue: +#endif #ifdef CONFIG_PREEMPT ldr r0, [tsk, #TI_PREEMPT] str r8, [tsk, #TI_PREEMPT] @@ -517,8 +551,8 @@ call_fpe: mov pc, lr @ CP#8 mov pc, lr @ CP#9 #ifdef CONFIG_VFP - b do_vfp @ CP#10 (VFP) - b do_vfp @ CP#11 (VFP) + b _do_vfp @ CP#10 (VFP) + b _do_vfp @ CP#11 (VFP) #else mov pc, lr @ CP#10 (VFP) mov pc, lr @ CP#11 (VFP) @@ -529,11 +563,35 @@ call_fpe: mov pc, lr @ CP#15 (Control) do_fpe: +#ifdef CONFIG_IPIPE + mov r4, r0 + mov r0, #5 @ == IPIPE_TRAP_FPU + mov r1, sp @ r0 = trapno, r1 = ®s + bl __ipipe_dispatch_event @ branch to trap handler + cmp r0, #0 + ldrne pc, [r9] + mov r0, r4 +#endif enable_irq ldr r4, .LCfp add r10, r10, #TI_FPSTATE @ r10 = workspace ldr pc, [r4] @ Call FP module USR entry point +#ifdef CONFIG_VFP +_do_vfp: +#ifdef CONFIG_IPIPE + mov r4, r0 + mov r0, #6 @ == IPIPE_TRAP_VFP + mov r1, sp @ r0 = trapno, r1 = ®s + bl __ipipe_dispatch_event @ branch to trap handler + cmp r0, #0 + ldrne pc, [r9] + mov r0, r4 +#endif + b do_vfp +#endif + + /* * The FP module is called with these registers set: * r0 = instruction diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/kernel/entry-common.S ipipe-2.6.19-arm/arch/arm/kernel/entry-common.S --- linux-2.6.19/arch/arm/kernel/entry-common.S 2007-01-04 22:01:14.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/kernel/entry-common.S 2007-01-19 21:00:59.000000000 +0100 @@ -2,6 +2,7 @@ * linux/arch/arm/kernel/entry-common.S * * Copyright (C) 2000 Russell King + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -19,13 +20,18 @@ * possible here, and this includes saving r0 back into the SVC * stack. */ +#ifdef CONFIG_IPIPE +__ipipe_ret_fast_syscall: + ldr r0, [sp, #S_R0+S_OFF] @ returned r0 + /* fall through */ +#endif ret_fast_syscall: disable_irq @ disable interrupts ldr r1, [tsk, #TI_FLAGS] tst r1, #_TIF_WORK_MASK bne fast_work_pending - @ fast_restore_user_regs +fast_restore_user_regs: ldr r1, [sp, #S_OFF + S_PSR] @ get calling cpsr ldr lr, [sp, #S_OFF + S_PC]! @ get pc msr spsr_cxsf, r1 @ save in spsr_svc @@ -34,6 +40,13 @@ ret_fast_syscall: add sp, sp, #S_FRAME_SIZE - S_PC movs pc, lr @ return & move spsr_svc into cpsr +#ifdef CONFIG_IPIPE +__ipipe_fast_exit_syscall: + ldr r0, [sp, #S_R0+S_OFF] @ returned r0 + disable_irq @ disable interrupts + b fast_restore_user_regs +#endif /* CONFIG_IPIPE */ + /* * Ok, we need to do extra processing, enter the slow path. */ @@ -61,19 +74,15 @@ ret_slow_syscall: tst r1, #_TIF_WORK_MASK bne work_pending no_work_pending: - @ slow_restore_user_regs - ldr r1, [sp, #S_PSR] @ get calling cpsr - ldr lr, [sp, #S_PC]! @ get pc - msr spsr_cxsf, r1 @ save in spsr_svc - ldmdb sp, {r0 - lr}^ @ get calling r1 - lr - mov r0, r0 - add sp, sp, #S_FRAME_SIZE - S_PC - movs pc, lr @ return & move spsr_svc into cpsr + slow_restore_user_regs /* * This is how we return from a fork. */ ENTRY(ret_from_fork) +#ifdef CONFIG_IPIPE + enable_irq +#endif /* CONFIG_IPIPE */ bl schedule_tail get_thread_info tsk ldr r1, [tsk, #TI_FLAGS] @ check for syscall tracing @@ -198,6 +207,17 @@ ENTRY(vector_swi) #endif stmdb sp!, {r4, r5} @ push fifth and sixth args +#ifdef CONFIG_IPIPE + stmfd sp!, {r0-r3, ip} + add r1, sp, #S_OFF + add r1, r1, #20 + mov r0, scno + bl __ipipe_syscall_root + cmp r0, #0 + ldmfd sp!, {r0-r3, ip} + blt __ipipe_ret_fast_syscall + bgt __ipipe_fast_exit_syscall +#endif /* CONFIG_IPIPE */ tst ip, #_TIF_SYSCALL_TRACE @ are we tracing syscalls? bne __sys_trace @@ -244,6 +264,9 @@ __sys_trace_return: __cr_alignment: .word cr_alignment #endif +#ifdef CONFIG_IPIPE + .word __ipipe_syscall_root +#endif .ltorg /* @@ -390,3 +413,28 @@ ENTRY(sys_oabi_call_table) #endif + +#ifdef CONFIG_FRAME_POINTER + + .text + .align 0 + .type arm_return_addr %function + .global arm_return_addr + +arm_return_addr: + mov ip, r0 + mov r0, fp +3: + cmp r0, #0 + beq 1f @ frame list hit end, bail + cmp ip, #0 + beq 2f @ reached desired frame + ldr r0, [r0, #-12] @ else continue, get next fp + sub ip, ip, #1 + b 3b +2: + ldr r0, [r0, #-4] @ get target return address +1: + mov pc, lr + +#endif diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/kernel/entry-header.S ipipe-2.6.19-arm/arch/arm/kernel/entry-header.S --- linux-2.6.19/arch/arm/kernel/entry-header.S 2007-01-04 22:01:14.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/kernel/entry-header.S 2007-01-15 21:38:05.000000000 +0100 @@ -49,6 +49,15 @@ #endif .endm + .macro slow_restore_user_regs + ldr r1, [sp, #S_PSR] @ get calling cpsr + ldr lr, [sp, #S_PC]! @ get pc + msr spsr_cxsf, r1 @ save in spsr_svc + ldmdb sp, {r0 - lr}^ @ get calling r1 - lr + mov r0, r0 + add sp, sp, #S_FRAME_SIZE - S_PC + movs pc, lr @ return & move spsr_svc into cpsr + .endm /* * These are the registers used in the syscall handler, and allow us to diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/kernel/ipipe-mcount.S ipipe-2.6.19-arm/arch/arm/kernel/ipipe-mcount.S --- linux-2.6.19/arch/arm/kernel/ipipe-mcount.S 1970-01-01 01:00:00.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/kernel/ipipe-mcount.S 2007-01-15 21:38:05.000000000 +0100 @@ -0,0 +1,40 @@ +/* + * linux/arch/arm/kernel/ipipe-mcount.S + * + * Copyright (C) 2006 Sebastian Smolorz , emlix GmbH + */ + +#include + +#ifdef CONFIG_FRAME_POINTER + + .text + .align 0 + .type mcount %function + .global mcount + +mcount: + + ldr ip, =ipipe_trace_enable @ leave early, if disabled + ldr ip, [ip] + cmp ip, #0 + moveq pc,lr + + mov ip, sp + stmdb sp!, {r0 - r3, fp, ip, lr, pc} @ create stack frame + + mov r3, #0 @ no additional value (v) + ldr r2, [fp, #-4] @ get lr (the return address + @ of the caller of the + @ instrumented function) + mov r1, lr @ get lr - (the return address + @ of the instrumented function) + mov r0, #0 @ IPIPE_TRACE_FN + + sub fp, ip, #4 @ point fp at this frame + + bl __ipipe_trace + + ldmdb fp, {r0 - r3, fp, sp, pc} @ pop entry frame and return + +#endif diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/kernel/ipipe.c ipipe-2.6.19-arm/arch/arm/kernel/ipipe.c --- linux-2.6.19/arch/arm/kernel/ipipe.c 1970-01-01 01:00:00.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/kernel/ipipe.c 2007-01-19 22:34:33.000000000 +0100 @@ -0,0 +1,507 @@ +/* -*- linux-c -*- + * linux/arch/arm/kernel/ipipe.c + * + * Copyright (C) 2002-2005 Philippe Gerum. + * Copyright (C) 2004 Wolfgang Grandegger (Adeos/arm port over 2.4). + * Copyright (C) 2005 Heikki Lindholm (PowerPC 970 fixes). + * Copyright (C) 2005 Stelian Pop. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Architecture-dependent I-PIPE support for ARM. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Current reload value for the decrementer. */ +unsigned long __ipipe_decr_ticks; + +/* Next tick date (timebase value). */ +unsigned long long __ipipe_decr_next[IPIPE_NR_CPUS]; + +struct pt_regs __ipipe_tick_regs[IPIPE_NR_CPUS]; + +extern struct irqdesc irq_desc[]; +extern asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs); + +#ifdef CONFIG_SMP + +static cpumask_t __ipipe_cpu_sync_map; + +static cpumask_t __ipipe_cpu_lock_map; + +static ipipe_spinlock_t __ipipe_cpu_barrier = IPIPE_SPIN_LOCK_UNLOCKED; + +static atomic_t __ipipe_critical_count = ATOMIC_INIT(0); + +static void (*__ipipe_cpu_sync) (void); + +/* Always called with hw interrupts off. */ + +void __ipipe_do_critical_sync(unsigned irq) +{ + ipipe_declare_cpuid; + + ipipe_load_cpuid(); + + cpu_set(cpuid, __ipipe_cpu_sync_map); + + /* + * Now we are in sync with the lock requestor running on another + * CPU. Enter a spinning wait until he releases the global + * lock. + */ + spin_lock_hw(&__ipipe_cpu_barrier); + + /* Got it. Now get out. */ + + if (__ipipe_cpu_sync) + /* Call the sync routine if any. */ + __ipipe_cpu_sync(); + + spin_unlock_hw(&__ipipe_cpu_barrier); + + cpu_clear(cpuid, __ipipe_cpu_sync_map); +} + +#endif /* CONFIG_SMP */ + +/* + * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline + * just like if it has been actually received from a hw source. Also + * works for virtual interrupts. + */ +int ipipe_trigger_irq(unsigned irq) +{ + unsigned long flags; + + if (irq >= IPIPE_NR_IRQS || + (ipipe_virtual_irq_p(irq) + && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))) + return -EINVAL; + + local_irq_save_hw(flags); + + __ipipe_handle_irq(irq, NULL); + + local_irq_restore_hw(flags); + + return 1; +} + +void __ipipe_init_platform(void) +{ + __ipipe_decr_ticks = __ipipe_mach_ticks_per_jiffy; +} + +int ipipe_get_sysinfo(struct ipipe_sysinfo *info) +{ + info->ncpus = num_online_cpus(); + info->cpufreq = ipipe_cpu_freq(); + info->archdep.tmirq = __ipipe_mach_timerint; + info->archdep.tmfreq = info->cpufreq; + + return 0; +} + +static void __ipipe_set_decr(void) +{ + ipipe_declare_cpuid; + + ipipe_load_cpuid(); + + __ipipe_decr_next[cpuid] = __ipipe_read_timebase() + __ipipe_decr_ticks; + __ipipe_mach_set_dec(__ipipe_decr_ticks); +} + +int ipipe_tune_timer(unsigned long ns, int flags) +{ + unsigned long x, ticks; + + if (flags & IPIPE_RESET_TIMER) + ticks = __ipipe_mach_ticks_per_jiffy; + else { + ticks = (ns / 1000) * (__ipipe_mach_ticks_per_jiffy) / (1000000 / HZ); + + if (ticks > __ipipe_mach_ticks_per_jiffy) + return -EINVAL; + } + + x = ipipe_critical_enter(&__ipipe_set_decr); /* Sync with all CPUs */ + __ipipe_decr_ticks = ticks; + __ipipe_set_decr(); + ipipe_critical_exit(x); + + return 0; +} + +static int __ipipe_ack_irq(unsigned irq) +{ + irq_desc_t *desc = irq_desc + irq; + desc->ipipe_ack(irq, desc); + return 1; +} + +static int __ipipe_ack_timerirq(unsigned irq) +{ + irq_desc_t *desc = irq_desc + irq; + desc->ipipe_ack(irq, desc); + __ipipe_mach_acktimer(); +#ifdef irq_finish + irq_finish(irq); +#endif /* irq_finish */ + desc->ipipe_end(irq, desc); + return 1; +} + +void __ipipe_enable_irqdesc(unsigned irq) +{ + irq_desc[irq].status &= ~IRQ_DISABLED; +} + +static void __ipipe_enable_sync(void) +{ + __ipipe_decr_next[ipipe_processor_id()] = + __ipipe_read_timebase() + __ipipe_mach_get_dec(); +} + +/* + * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw + * interrupts are off, and secondary CPUs are still lost in space. + */ +void __ipipe_enable_pipeline(void) +{ + unsigned long flags; + unsigned irq; + + flags = ipipe_critical_enter(&__ipipe_enable_sync); + + /* First, virtualize all interrupts from the root domain. */ + + for (irq = 0; irq < NR_IRQS; irq++) + ipipe_virtualize_irq(ipipe_root_domain, + irq, + (ipipe_irq_handler_t)&asm_do_IRQ, NULL, + ((irq == __ipipe_mach_timerint) + ? &__ipipe_ack_timerirq + : &__ipipe_ack_irq), + IPIPE_HANDLE_MASK | IPIPE_PASS_MASK); + + __ipipe_decr_next[ipipe_processor_id()] = + __ipipe_read_timebase() + __ipipe_mach_get_dec(); + + ipipe_critical_exit(flags); +} + +/* + * ipipe_critical_enter() -- Grab the superlock excluding all CPUs + * but the current one from a critical section. This lock is used when + * we must enforce a global critical section for a single CPU in a + * possibly SMP system whichever context the CPUs are running. + */ +unsigned long ipipe_critical_enter(void (*syncfn) (void)) +{ + unsigned long flags; + + local_irq_save_hw(flags); + +#ifdef CONFIG_SMP + if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a UP box... */ + ipipe_declare_cpuid; + cpumask_t lock_map; + + ipipe_load_cpuid(); + + if (!cpu_test_and_set(cpuid, __ipipe_cpu_lock_map)) { + while (cpu_test_and_set(BITS_PER_LONG - 1, + __ipipe_cpu_lock_map)) { + int n = 0; + do { + cpu_relax(); + } while (++n < cpuid); + } + + spin_lock_hw(&__ipipe_cpu_barrier); + + __ipipe_cpu_sync = syncfn; + + /* Send the sync IPI to all processors but the current one. */ + send_IPI_allbutself(IPIPE_CRITICAL_VECTOR); + + cpus_andnot(lock_map, cpu_online_map, + __ipipe_cpu_lock_map); + + while (!cpus_equal(__ipipe_cpu_sync_map, lock_map)) + cpu_relax(); + } + + atomic_inc(&__ipipe_critical_count); + } +#endif /* CONFIG_SMP */ + + return flags; +} + +/* ipipe_critical_exit() -- Release the superlock. */ + +void ipipe_critical_exit(unsigned long flags) +{ +#ifdef CONFIG_SMP + if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a UP box... */ + ipipe_declare_cpuid; + + ipipe_load_cpuid(); + + if (atomic_dec_and_test(&__ipipe_critical_count)) { + spin_unlock_hw(&__ipipe_cpu_barrier); + + while (!cpus_empty(__ipipe_cpu_sync_map)) + cpu_relax(); + + cpu_clear(cpuid, __ipipe_cpu_lock_map); + cpu_clear(BITS_PER_LONG - 1, __ipipe_cpu_lock_map); + } + } +#endif /* CONFIG_SMP */ + + local_irq_restore_hw(flags); +} + +asmlinkage int __ipipe_check_root(struct pt_regs *regs) +{ + ipipe_declare_cpuid; + /* + * This routine is called with hw interrupts off, so no migration + * can occur while checking the identity of the current domain. + */ + ipipe_load_cpuid(); + return (per_cpu(ipipe_percpu_domain, cpuid) == ipipe_root_domain && + !test_bit(IPIPE_STALL_FLAG, + &ipipe_root_domain->cpudata[cpuid].status)); +} + +asmlinkage int __ipipe_syscall_root(unsigned long scno, struct pt_regs *regs) +{ + ipipe_declare_cpuid; + unsigned long flags, origr7; + + /* We use r7 to pass the syscall number to the other domains */ + origr7 = regs->ARM_r7; + regs->ARM_r7 = __NR_SYSCALL_BASE + scno; + /* + * This routine either returns: + * 0 -- if the syscall is to be passed to Linux; + * >0 -- if the syscall should not be passed to Linux, and no + * tail work should be performed; + * <0 -- if the syscall should not be passed to Linux but the + * tail work has to be performed (for handling signals etc). + */ + + if (__ipipe_syscall_watched_p(current, regs->ARM_r7) && + __ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL) && + __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL,regs) > 0) { + if (ipipe_current_domain == ipipe_root_domain && !in_atomic()) { + /* + * Sync pending VIRQs before _TIF_NEED_RESCHED + * is tested. + */ + ipipe_lock_cpu(flags); + if ((ipipe_root_domain->cpudata[cpuid].irq_pending_hi & IPIPE_IRQMASK_VIRT) != 0) + __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT); + ipipe_unlock_cpu(flags); + regs->ARM_r7 = origr7; + return -1; + } + regs->ARM_r7 = origr7; + return 1; + } + + regs->ARM_r7 = origr7; + return 0; +} + +/* + * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic + * interrupt protection log is maintained here for each domain. Hw + * interrupts are off on entry. + */ +void __ipipe_handle_irq(int irq, struct pt_regs *regs) +{ + struct ipipe_domain *this_domain, *next_domain; + struct list_head *head, *pos; + ipipe_declare_cpuid; + int m_ack; + + m_ack = (regs == NULL); + + if (irq >= IPIPE_NR_IRQS) { + printk(KERN_ERR "I-pipe: spurious interrupt %d\n", irq); + return; + } + + ipipe_load_cpuid(); + + this_domain = per_cpu(ipipe_percpu_domain, cpuid); + + if (test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control)) + head = &this_domain->p_link; + else { + head = __ipipe_pipeline.next; + next_domain = list_entry(head, struct ipipe_domain, p_link); + if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) { + if (!m_ack && next_domain->irqs[irq].acknowledge != NULL) + next_domain->irqs[irq].acknowledge(irq); + if (likely(__ipipe_dispatch_wired(next_domain, irq))) + goto finalize; + return; + } + } + + /* Ack the interrupt. */ + + pos = head; + + while (pos != &__ipipe_pipeline) { + next_domain = list_entry(pos, struct ipipe_domain, p_link); + + /* + * For each domain handling the incoming IRQ, mark it as + * pending in its log. + */ + if (test_bit(IPIPE_HANDLE_FLAG, + &next_domain->irqs[irq].control)) { + /* + * Domains that handle this IRQ are polled for + * acknowledging it by decreasing priority order. The + * interrupt must be made pending _first_ in the + * domain's status flags before the PIC is unlocked. + */ + + next_domain->cpudata[cpuid].irq_counters[irq].total_hits++; + next_domain->cpudata[cpuid].irq_counters[irq].pending_hits++; + __ipipe_set_irq_bit(next_domain, cpuid, irq); + + /* + * Always get the first master acknowledge available. + * Once we've got it, allow slave acknowledge + * handlers to run (until one of them stops us). + */ + if (!m_ack && next_domain->irqs[irq].acknowledge != NULL) + m_ack = next_domain->irqs[irq].acknowledge(irq); + } + + /* + * If the domain does not want the IRQ to be passed down the + * interrupt pipe, exit the loop now. + */ + + if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control)) + break; + + pos = next_domain->p_link.next; + } + +finalize: + /* + * Now walk the pipeline, yielding control to the highest + * priority domain that has pending interrupt(s) or + * immediately to the current domain if the interrupt has been + * marked as 'sticky'. This search does not go beyond the + * current domain in the pipeline. + */ + + __ipipe_walk_pipeline(head, cpuid); +} + +asmlinkage int __ipipe_grab_irq(int irq, struct pt_regs *regs) +{ + ipipe_declare_cpuid; + + if (irq == __ipipe_mach_timerint) { + + __ipipe_tick_regs[cpuid].ARM_cpsr = regs->ARM_cpsr; + __ipipe_tick_regs[cpuid].ARM_pc = regs->ARM_pc; + + if (__ipipe_decr_ticks != __ipipe_mach_ticks_per_jiffy) { + unsigned long long next_date, now; + + next_date = __ipipe_decr_next[cpuid]; + + while ((now = __ipipe_read_timebase()) >= next_date) + next_date += __ipipe_decr_ticks; + + __ipipe_mach_set_dec(next_date - now); + + __ipipe_decr_next[cpuid] = next_date; + } + } + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + ipipe_trace_begin(regs->ARM_ORIG_r0); +#endif + + if (__ipipe_mach_irq_mux_p(irq)) + __ipipe_mach_demux_irq(irq, regs); + else + __ipipe_handle_irq(irq, regs); + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + ipipe_trace_end(regs->ARM_ORIG_r0); +#endif + +#ifdef irq_finish + if (irq != __ipipe_mach_timerint) + irq_finish(irq); +#endif /* irq_finish */ + ipipe_load_cpuid(); + + return (per_cpu(ipipe_percpu_domain, cpuid) == ipipe_root_domain && + !test_bit(IPIPE_STALL_FLAG, + &ipipe_root_domain->cpudata[cpuid].status)); +} + +EXPORT_SYMBOL(__ipipe_decr_ticks); +EXPORT_SYMBOL(__ipipe_decr_next); +EXPORT_SYMBOL(ipipe_critical_enter); +EXPORT_SYMBOL(ipipe_critical_exit); +EXPORT_SYMBOL(ipipe_trigger_irq); +EXPORT_SYMBOL(ipipe_get_sysinfo); +EXPORT_SYMBOL(ipipe_tune_timer); + +EXPORT_SYMBOL_GPL(show_stack); +#ifndef MULTI_CPU +EXPORT_SYMBOL_GPL(cpu_do_switch_mm); +#endif + +#ifdef CONFIG_IPIPE_TRACE_MCOUNT +void notrace mcount(void); +EXPORT_SYMBOL(mcount); +#endif /* CONFIG_IPIPE_TRACE_MCOUNT */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/kernel/irq.c ipipe-2.6.19-arm/arch/arm/kernel/irq.c --- linux-2.6.19/arch/arm/kernel/irq.c 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/kernel/irq.c 2007-01-16 00:31:15.000000000 +0100 @@ -125,8 +125,10 @@ asmlinkage void asm_do_IRQ(unsigned int desc_handle_irq(irq, desc); +#ifndef CONFIG_IPIPE /* AT91 specific workaround */ irq_finish(irq); +#endif /* !CONFIG_IPIPE */ irq_exit(); set_irq_regs(old_regs); diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/kernel/process.c ipipe-2.6.19-arm/arch/arm/kernel/process.c --- linux-2.6.19/arch/arm/kernel/process.c 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/kernel/process.c 2007-01-15 21:38:05.000000000 +0100 @@ -125,6 +125,12 @@ static void default_idle(void) local_irq_disable(); if (!need_resched()) { timer_dyn_reprogram(); +#ifdef CONFIG_IPIPE + __ipipe_unstall_root(); +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + ipipe_trace_end(0x8000000E); +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ +#endif /* CONFIG_IPIPE */ arch_idle(); } local_irq_enable(); @@ -153,6 +159,7 @@ void cpu_idle(void) if (!idle) idle = default_idle; + ipipe_suspend_domain(); leds_event(led_idle_start); while (!need_resched()) idle(); diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/kernel/ptrace.c ipipe-2.6.19-arm/arch/arm/kernel/ptrace.c --- linux-2.6.19/arch/arm/kernel/ptrace.c 2007-01-04 22:01:15.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/kernel/ptrace.c 2007-01-15 21:38:05.000000000 +0100 @@ -485,6 +485,10 @@ void ptrace_break(struct task_struct *ts static int break_trap(struct pt_regs *regs, unsigned int instr) { + + if (ipipe_trap_notify(IPIPE_TRAP_BREAK,regs)) + return 0; + ptrace_break(current, regs); return 0; } diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/kernel/traps.c ipipe-2.6.19-arm/arch/arm/kernel/traps.c --- linux-2.6.19/arch/arm/kernel/traps.c 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/kernel/traps.c 2007-01-15 21:38:05.000000000 +0100 @@ -306,6 +306,9 @@ asmlinkage void do_undefinstr(struct pt_ } spin_unlock_irq(&undef_lock); + if (ipipe_trap_notify(IPIPE_TRAP_UNDEFINSTR,regs)) + return; + #ifdef CONFIG_DEBUG_USER if (user_debug & UDBG_UNDEFINED) { printk(KERN_INFO "%s (%d): undefined instruction: pc=%p\n", diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-at91rm9200/Kconfig ipipe-2.6.19-arm/arch/arm/mach-at91rm9200/Kconfig --- linux-2.6.19/arch/arm/mach-at91rm9200/Kconfig 2007-01-04 22:01:15.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-at91rm9200/Kconfig 2007-01-30 00:48:57.090002907 +0100 @@ -82,6 +82,24 @@ config MACH_KAFA help Select this if you are using Sperry-Sun's KAFA board. +config IPIPE_AT91_TC + depends on IPIPE + int "AT91 TC used as time base" + default 0 + help + When Adeos interrupt pipeline is enabled, TC0 is used by default + as time base, but you can use TC1 or TC2 by setting this variable to 1 + or 2. This should only be needed to avoid conflicts with other drivers. + +config IPIPE_AT91_MCK + depends on IPIPE + int "AT91 Master clock Frequency" + default 46080000 + help + When Adeos interrupt pipeline is enabled, AT91 timer is based on + the AT91 master clock, whose frequency need hence to be known at + compilation time. + endif # ---------------------------------------------------------- diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-at91rm9200/at91rm9200.c ipipe-2.6.19-arm/arch/arm/mach-at91rm9200/at91rm9200.c --- linux-2.6.19/arch/arm/mach-at91rm9200/at91rm9200.c 2007-01-18 00:05:24.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-at91rm9200/at91rm9200.c 2007-01-18 00:08:41.000000000 +0100 @@ -51,6 +51,13 @@ static struct map_desc at91rm9200_io_des .length = SZ_16K, .type = MT_DEVICE, }, { +#ifdef CONFIG_IPIPE + .virtual = AT91_VA_BASE_TCB0, + .pfn = __phys_to_pfn(AT91RM9200_BASE_TCB0), + .length = SZ_16K, + .type = MT_DEVICE, + }, { +#endif /* CONFIG_IPIPE */ .virtual = AT91_SRAM_VIRT_BASE, .pfn = __phys_to_pfn(AT91RM9200_SRAM_BASE), .length = AT91RM9200_SRAM_SIZE, diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-at91rm9200/at91rm9200_time.c ipipe-2.6.19-arm/arch/arm/mach-at91rm9200/at91rm9200_time.c --- linux-2.6.19/arch/arm/mach-at91rm9200/at91rm9200_time.c 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-at91rm9200/at91rm9200_time.c 2007-01-30 00:42:50.365627627 +0100 @@ -32,6 +32,62 @@ static unsigned long last_crtr; +#ifdef CONFIG_IPIPE +#include +#include +#include +#include +#include "clock.h" + +#ifdef CONFIG_NO_IDLE_HZ +#error "dynamic tick timer not yet supported with IPIPE" +#endif /* CONFIG_NO_IDLE_HZ */ + +#define TCNXCNS(timer,v) ((v) << ((timer)<<1)) +#define AT91_TC_REG_MASK (0xffff) + +#if (CONFIG_IPIPE_AT91_TC==0) +# define KERNEL_TIMER_IRQ_NUM AT91RM9200_ID_TC0 +#elif (CONFIG_IPIPE_AT91_TC==1) +# define KERNEL_TIMER_IRQ_NUM AT91RM9200_ID_TC1 +#elif (CONFIG_IPIPE_AT91_TC==2) +# define KERNEL_TIMER_IRQ_NUM AT91RM9200_ID_TC2 +#else +#error IPIPE_AT91_TC must be 0, 1 or 2. +#endif + +static inline unsigned int at91_tc_read(unsigned int reg_offset) +{ + unsigned long addr = + (AT91_VA_BASE_TCB0 + 0x40 * CONFIG_IPIPE_AT91_TC); + + return readl((void __iomem *)(addr + reg_offset)); +} + +static inline void at91_tc_write(unsigned int reg_offset, unsigned long value) +{ + unsigned long addr = + (AT91_VA_BASE_TCB0 + 0x40 * CONFIG_IPIPE_AT91_TC); + + writel(value, (void __iomem *)(addr + reg_offset)); +} + +#define read_CV() at91_tc_read(AT91_TC_CV) +#define read_RC() at91_tc_read(AT91_TC_RC) +#define write_RC(value) at91_tc_write(AT91_TC_RC, value) + +int __ipipe_mach_timerint = KERNEL_TIMER_IRQ_NUM; +EXPORT_SYMBOL(__ipipe_mach_timerint); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); + +unsigned int __ipipe_mach_ticks_per_jiffy = LATCH; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +static int at91_timer_initialized; +#endif /* CONFIG_IPIPE */ + /* * The ST_CRTR is updated asynchronously to the master clock. It is therefore * necessary to read it twice (with the same value) to ensure accuracy. @@ -39,14 +95,16 @@ static unsigned long last_crtr; static inline unsigned long read_CRTR(void) { unsigned long x1, x2; + x2 = at91_sys_read(AT91_ST_CRTR); do { - x1 = at91_sys_read(AT91_ST_CRTR); + x1 = x2; x2 = at91_sys_read(AT91_ST_CRTR); } while (x1 != x2); return x1; } +#ifndef CONFIG_IPIPE /* * Returns number of microseconds since last timer interrupt. Note that interrupts * will have been disabled by do_gettimeofday() @@ -139,3 +197,209 @@ struct sys_timer at91rm9200_timer = { .resume = at91rm9200_timer_reset, }; +#else /* CONFIG_IPIPE */ + +/* + * Returns number of microseconds since last timer interrupt. Note that interrupts + * will have been disabled by do_gettimeofday() + * 'LATCH' is hwclock ticks (see CLOCK_TICK_RATE in timex.h) per jiffy. + * 'tick' is usecs per jiffy (linux/timex.h). + */ +static unsigned long at91rm9200_gettimeoffset(void) +{ + unsigned long elapsed; + + elapsed = (read_CV() - last_crtr) & AT91_TC_REG_MASK; + + return (unsigned long) (elapsed * (tick_nsec / 1000)) / LATCH; +} + +void __ipipe_mach_acktimer(void) +{ + at91_tc_read(AT91_TC_SR); +} + +/* + * IRQ handler for the timer. + */ +static irqreturn_t at91rm9200_timer_interrupt(int irq, void *dev_id) +{ + /* + * - if Linux is running under ipipe, but it still has the control over + * the timer (no Xenomai for example), then reprogram the timer (ipipe + * has already acked it) + * - if some other domain has taken over the timer, then do nothing + * (ipipe has acked it, and the other domain has reprogramed it) + */ + + write_seqlock(&xtime_lock); + + if (__ipipe_mach_timerstolen) { + timer_tick(); + last_crtr = (last_crtr + LATCH) & AT91_TC_REG_MASK; + } else { + while (((read_CV() - last_crtr) & AT91_TC_REG_MASK) >= LATCH) { + timer_tick(); + last_crtr = (last_crtr + LATCH) & AT91_TC_REG_MASK; + } + write_RC((last_crtr + LATCH) & AT91_TC_REG_MASK); + } + + write_sequnlock(&xtime_lock); + + return IRQ_HANDLED; +} + +static irqreturn_t at91rm9200_bad_freq(int irq, void *dev_id) +{ + static int ticks = 0; + + if (++ticks != HZ * 120) { + if (!console_drivers || try_acquire_console_sem()) + return at91rm9200_timer_interrupt(irq, dev_id); + + release_console_sem(); + } + + panic("AT91 clock rate incorrectly set.\n" + "Please recompile with IPIPE_AT91_MCK set to %lu Hz.", + clk_get_rate(clk_get(NULL, "mck"))); +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + if (likely(at91_timer_initialized)) { + static union { +#ifdef __BIG_ENDIAN + struct { + unsigned long high; + unsigned short mid; + unsigned short low; + }; +#else /* __LITTLE_ENDIAN */ + struct { + unsigned short low; + unsigned short mid; + unsigned long high; + }; +#endif /* __LITTLE_ENDIAN */ + unsigned long long full; + } tsc[NR_CPUS], *local_tsc; + unsigned long long result; + unsigned short stamp; + unsigned long flags; + + local_irq_save_hw(flags); + local_tsc = &tsc[ipipe_processor_id()]; + stamp = read_CV(); + if (unlikely(stamp < local_tsc->low)) { + if (unlikely(!++local_tsc->mid)) + /* 32 bit counter wrapped, increment high word. */ + local_tsc->high++; + } + local_tsc->low = stamp; + result = local_tsc->full; + local_irq_restore_hw(flags); + + return result; + } + + return 0; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +/* + * Reprogram the timer + */ + +void __ipipe_mach_set_dec(unsigned long delay) +{ + unsigned long flags; + + if (delay > 2) { + local_irq_save_hw(flags); + write_RC((read_CV() + delay) & AT91_TC_REG_MASK); + local_irq_restore_hw(flags); + } else + ipipe_trigger_irq(KERNEL_TIMER_IRQ_NUM); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + __ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy); +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return (read_RC() - read_CV()) & AT91_TC_REG_MASK; +} + +static struct irqaction at91rm9200_timer_irq = { + .name = "at91_tick", + .flags = IRQF_DISABLED | IRQF_TIMER, + .handler = &at91rm9200_timer_interrupt +}; + +static char clk_name [] = "tc%"; + +static struct clk tc = { + .name = (const char *) clk_name, + .users = 0, + .type = CLK_TYPE_PERIPHERAL, + .pmc_mask = 1 << (KERNEL_TIMER_IRQ_NUM), +}; + +void __init at91rm9200_timer_init(void) +{ + unsigned long v; + + if (clk_get_rate(clk_get(NULL, "mck")) != CONFIG_IPIPE_AT91_MCK) + at91rm9200_timer_irq.handler = &at91rm9200_bad_freq; + + snprintf(clk_name, sizeof(clk_name), "tc%d", CONFIG_IPIPE_AT91_TC); + clk_register(&tc); + clk_enable(&tc); + + /* No Sync. */ + at91_tc_write(AT91_TC_BCR, 0); + + /* program NO signal on XCN */ + v = readl((void __iomem *) (AT91_VA_BASE_TCB0 + AT91_TC_BMR)); + v &= ~TCNXCNS(CONFIG_IPIPE_AT91_TC, 3); + v |= TCNXCNS(CONFIG_IPIPE_AT91_TC, 1); /* AT91_TC_TCNXCNS_NONE */ + writel(v, (void __iomem *) (AT91_VA_BASE_TCB0 + AT91_TC_BMR)); + + /* Disable the channel */ + at91_tc_write(AT91_TC_CCR, AT91_TC_CLKDIS); + + /* Select TIMER_CLOCK3 (MCLK/32) as input frequency for TC. */ + at91_tc_write(AT91_TC_CMR, AT91_TC_TIMER_CLOCK3); + + /* Disable all interrupts. */ + at91_tc_write(AT91_TC_IDR, ~0ul); + + /* Load the TC register C. */ + last_crtr = 0; + write_RC(LATCH); + + /* Enable CPCS interrupt. */ + at91_tc_write(AT91_TC_IER, AT91_TC_CPCS); + + /* Set up the interrupt. */ + setup_irq(KERNEL_TIMER_IRQ_NUM, &at91rm9200_timer_irq); + + /* Enable the channel. */ + at91_tc_write(AT91_TC_CCR, AT91_TC_CLKEN | AT91_TC_SWTRG); + + at91_timer_initialized = 1; +} + +struct sys_timer at91rm9200_timer = { + .init = at91rm9200_timer_init, + .offset = at91rm9200_gettimeoffset, + .suspend = NULL, + .resume = NULL, +}; +#endif /* CONFIG_IPIPE */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-at91rm9200/gpio.c ipipe-2.6.19-arm/arch/arm/mach-at91rm9200/gpio.c --- linux-2.6.19/arch/arm/mach-at91rm9200/gpio.c 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-at91rm9200/gpio.c 2007-01-20 02:35:03.000000000 +0100 @@ -20,6 +20,11 @@ #include #include #include +#ifdef CONFIG_IPIPE +#include + +unsigned __ipipe_at91_gpio_banks = 0; +#endif /* CONFIG_IPIPE */ #include "generic.h" @@ -326,6 +331,9 @@ static int gpio_irq_type(unsigned pin, u static struct irq_chip gpio_irqchip = { .name = "GPIO", +#ifdef CONFIG_IPIPE + .ack = gpio_irq_mask, +#endif .mask = gpio_irq_mask, .unmask = gpio_irq_unmask, .set_type = gpio_irq_type, @@ -374,6 +382,50 @@ static void gpio_irq_handler(unsigned ir /* now it may re-trigger */ } +#ifdef CONFIG_IPIPE +void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs) +{ + struct irqdesc *desc = irq_desc + irq; + unsigned pin; + struct irqdesc *gpio; + void __iomem *pio; + u32 isr; + + pio = get_irq_chip_data(irq); + + /* temporarily mask (level sensitive) parent IRQ */ + desc->chip->ack(irq); + for (;;) { + isr = __raw_readl(pio + PIO_ISR) & __raw_readl(pio + PIO_IMR); + if (!isr) + break; + + pin = (unsigned) get_irq_data(irq); + gpio = &irq_desc[pin]; + + while (isr) { + if (isr & 1) { + if (unlikely(gpio->depth)) { + /* + * The core ARM interrupt handler lazily disables IRQs so + * another IRQ must be generated before it actually gets + * here to be disabled on the GPIO controller. + */ + gpio_irq_mask(pin); + } + else + __ipipe_handle_irq(pin, regs); + } + pin++; + gpio++; + isr >>= 1; + } + } + desc->chip->unmask(irq); + /* now it may re-trigger */ +} +#endif /* CONFIG_IPIPE */ + /*--------------------------------------------------------------------------*/ /* @@ -422,4 +474,7 @@ void __init at91_gpio_init(struct at91_g gpio = data; gpio_banks = nr_banks; +#ifdef CONFIG_IPIPE + __ipipe_at91_gpio_banks = nr_banks; +#endif /* CONFIG_IPIPE */ } diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-integrator/core.c ipipe-2.6.19-arm/arch/arm/mach-integrator/core.c --- linux-2.6.19/arch/arm/mach-integrator/core.c 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-integrator/core.c 2007-01-20 02:10:32.000000000 +0100 @@ -2,6 +2,7 @@ * linux/arch/arm/mach-integrator/core.c * * Copyright (C) 2000-2003 Deep Blue Solutions Ltd + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2, as @@ -195,53 +196,57 @@ EXPORT_SYMBOL(cm_control); /* * How long is the timer interval? */ -#define TIMER_INTERVAL (TICKS_PER_uSEC * mSEC_10) -#if TIMER_INTERVAL >= 0x100000 -#define TICKS2USECS(x) (256 * (x) / TICKS_PER_uSEC) -#elif TIMER_INTERVAL >= 0x10000 -#define TICKS2USECS(x) (16 * (x) / TICKS_PER_uSEC) -#else #define TICKS2USECS(x) ((x) / TICKS_PER_uSEC) -#endif static unsigned long timer_reload; +static unsigned long timer_interval; +static unsigned long timer_lxlost; +static int tscok; + +#ifdef CONFIG_IPIPE +int __ipipe_mach_timerint = IRQ_TIMERINT1; +static unsigned long long __ipipe_mach_tsc; +static IPIPE_DEFINE_SPINLOCK(timer_lock); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); +#endif /* - * Returns number of ms since last clock interrupt. Note that interrupts - * will have been disabled by do_gettimeoffset() + * Called with IRQ disabled from do_gettimeofday(). */ -unsigned long integrator_gettimeoffset(void) +static inline unsigned long integrator_getticksoffset(void) { - unsigned long ticks1, ticks2, status; + unsigned long ticks; - /* - * Get the current number of ticks. Note that there is a race - * condition between us reading the timer and checking for - * an interrupt. We get around this by ensuring that the - * counter has not reloaded between our two reads. - */ - ticks2 = readl(TIMER1_VA_BASE + TIMER_VALUE) & 0xffff; - do { - ticks1 = ticks2; - status = __raw_readl(VA_IC_BASE + IRQ_RAW_STATUS); - ticks2 = readl(TIMER1_VA_BASE + TIMER_VALUE) & 0xffff; - } while (ticks2 > ticks1); + if (!tscok) + return 0; - /* - * Number of ticks since last interrupt. - */ - ticks1 = timer_reload - ticks2; + ticks = readl(TIMER1_VA_BASE + TIMER_VALUE) & 0xffff; - /* - * Interrupt pending? If so, we've reloaded once already. - */ - if (status & (1 << IRQ_TIMERINT1)) - ticks1 += timer_reload; + if (ticks > timer_reload) + ticks = 0xffff + timer_reload - ticks; + else + ticks = timer_reload - ticks; + if (timer_interval < 0x10000) + return ticks; + else if (timer_interval < 0x100000) + return ticks * 16; + else + return ticks * 256; +} + +/* + * Returns number of ms since last clock interrupt. Note that interrupts + * will have been disabled by do_gettimeoffset() + */ +unsigned long integrator_gettimeoffset(void) +{ /* * Convert the ticks to usecs */ - return TICKS2USECS(ticks1); + return TICKS2USECS(timer_lxlost + integrator_getticksoffset()); } /* @@ -252,10 +257,22 @@ integrator_timer_interrupt(int irq, void { write_seqlock(&xtime_lock); + timer_lxlost = 0; + +#ifdef CONFIG_IPIPE /* - * clear the interrupt + * If Linux is the only domain, ack the timer and reprogram it */ - writel(1, TIMER1_VA_BASE + TIMER_INTCLR); + if (!__ipipe_mach_timerstolen) { + __ipipe_mach_tsc += integrator_getticksoffset(); +#else + writel(1, TIMER1_VA_BASE + TIMER_INTCLR); +#endif + + writel(timer_reload, TIMER1_VA_BASE + TIMER_LOAD); +#ifdef CONFIG_IPIPE + } +#endif /* * the clock tick routines are only processed on the @@ -286,24 +303,30 @@ static struct irqaction integrator_timer .handler = integrator_timer_interrupt, }; -/* - * Set up timer interrupt, and return the current time in seconds. - */ -void __init integrator_time_init(unsigned long reload, unsigned int ctrl) +static inline void set_dec(unsigned long reload) { - unsigned int timer_ctrl = TIMER_CTRL_ENABLE | TIMER_CTRL_PERIODIC; + unsigned int ctrl = TIMER_CTRL_ENABLE | TIMER_CTRL_IE; timer_reload = reload; - timer_ctrl |= ctrl; + timer_interval = reload; - if (timer_reload > 0x100000) { + if (timer_reload >= 0x100000) { timer_reload >>= 8; - timer_ctrl |= TIMER_CTRL_DIV256; - } else if (timer_reload > 0x010000) { + ctrl |= TIMER_CTRL_DIV256; + } else if (timer_reload >= 0x010000) { timer_reload >>= 4; - timer_ctrl |= TIMER_CTRL_DIV16; + ctrl |= TIMER_CTRL_DIV16; } + writel(ctrl, TIMER1_VA_BASE + TIMER_CTRL); + writel(timer_reload, TIMER1_VA_BASE + TIMER_LOAD); +} + +/* + * Set up timer interrupt, and return the current time in seconds. + */ +void __init integrator_time_init(unsigned long reload, unsigned int ctrl) +{ /* * Initialise to a known state (all timers off) */ @@ -311,12 +334,57 @@ void __init integrator_time_init(unsigne writel(0, TIMER1_VA_BASE + TIMER_CTRL); writel(0, TIMER2_VA_BASE + TIMER_CTRL); - writel(timer_reload, TIMER1_VA_BASE + TIMER_LOAD); - writel(timer_reload, TIMER1_VA_BASE + TIMER_VALUE); - writel(timer_ctrl, TIMER1_VA_BASE + TIMER_CTRL); + set_dec(reload); /* * Make irqs happen for the system timer */ setup_irq(IRQ_TIMERINT1, &integrator_timer_irq); + + tscok = 1; +} + +#ifdef CONFIG_IPIPE +void __ipipe_mach_acktimer(void) +{ + writel(1, TIMER1_VA_BASE + TIMER_INTCLR); } + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + unsigned long long result; + unsigned long flags; + + spin_lock_irqsave(&timer_lock, flags); + result = __ipipe_mach_tsc + integrator_getticksoffset(); + spin_unlock_irqrestore(&timer_lock, flags); + return result; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +void __ipipe_mach_set_dec(unsigned long reload) +{ + unsigned long ticks; + unsigned long flags; + + spin_lock_irqsave(&timer_lock, flags); + ticks = integrator_getticksoffset(); + __ipipe_mach_tsc += ticks; + timer_lxlost += ticks; + + set_dec(reload); + spin_unlock_irqrestore(&timer_lock, flags); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + __ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy); +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return readl(TIMER1_VA_BASE + TIMER_VALUE); +} +#endif /* CONFIG_IPIPE */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-integrator/integrator_cp.c ipipe-2.6.19-arm/arch/arm/mach-integrator/integrator_cp.c --- linux-2.6.19/arch/arm/mach-integrator/integrator_cp.c 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-integrator/integrator_cp.c 2007-01-20 02:11:41.000000000 +0100 @@ -2,6 +2,7 @@ * linux/arch/arm/mach-integrator/integrator_cp.c * * Copyright (C) 2003 Deep Blue Solutions Ltd + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,6 +20,7 @@ #include #include #include +#include #include #include @@ -222,6 +224,31 @@ sic_handle_irq(unsigned int irq, struct } while (status); } +#ifdef CONFIG_IPIPE +void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs) +{ + unsigned long status = sic_readl(INTCP_VA_SIC_BASE + IRQ_STATUS); + struct irqdesc *desc_unused = irq_desc + irq; + unsigned irq_unused = irq; + + if (status == 0) { + do_bad_IRQ(irq, desc_unused); + return; + } + + do { + irq = ffs(status) - 1; + status &= ~(1 << irq); + + irq += IRQ_SIC_START; + + __ipipe_handle_irq(irq, regs); + } while (status); + + desc_unused->chip->unmask(irq_unused); +} +#endif /* CONFIG_IPIPE */ + static void __init intcp_init_irq(void) { unsigned int i; @@ -568,9 +595,14 @@ static void __init intcp_init(void) #define TIMER_CTRL_IE (1 << 5) /* Interrupt Enable */ +#ifdef CONFIG_IPIPE +unsigned int __ipipe_mach_ticks_per_jiffy = 1000000 * TICKS_PER_uSEC / HZ; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); +#endif + static void __init intcp_timer_init(void) { - integrator_time_init(1000000 / HZ, TIMER_CTRL_IE); + integrator_time_init(1000000 * TICKS_PER_uSEC / HZ, TIMER_CTRL_IE); } static struct sys_timer cp_timer = { diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-pxa/irq.c ipipe-2.6.19-arm/arch/arm/mach-pxa/irq.c --- linux-2.6.19/arch/arm/mach-pxa/irq.c 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-pxa/irq.c 2007-01-15 21:38:05.000000000 +0100 @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -216,6 +217,42 @@ static void pxa_gpio_demux_handler(unsig } while (loop); } +#ifdef CONFIG_IPIPE +void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs) +{ + struct irqdesc *desc_unused = irq_desc + irq; + unsigned irq_unused = irq; + unsigned int i, mask[4]; + int loop; + + do { + loop = 0; + + mask[0] = GEDR0 & ~3; + mask[1] = GEDR1; + mask[2] = GEDR2; +#if PXA_LAST_GPIO < 96 + i = 3; +#else /* PXA_LAST_GPIO >= 96 */ + mask[3] = GEDR3; + i = 4; +#endif /* PXA_LAST_GPIO >= 96 */ + for (; i; i--) { + loop |= mask[i - 1]; + while (mask[i - 1]) { + irq = fls(mask[i - 1]) - 1; + mask[i - 1] &= ~(1 << irq); + irq = IRQ_GPIO((i - 1) * 32 + irq); + + __ipipe_handle_irq(irq, regs); + } + } + } while (loop); + + desc_unused->chip->unmask(irq_unused); +} +#endif /* CONFIG_IPIPE */ + static void pxa_ack_muxed_gpio(unsigned int irq) { int gpio = irq - IRQ_GPIO(2) + 2; diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-pxa/time.c ipipe-2.6.19-arm/arch/arm/mach-pxa/time.c --- linux-2.6.19/arch/arm/mach-pxa/time.c 2007-01-04 22:04:12.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-pxa/time.c 2007-01-15 21:55:57.000000000 +0100 @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,23 @@ #include +#ifdef CONFIG_IPIPE +#ifdef CONFIG_NO_IDLE_HZ +#error "dynamic tick timer not yet supported with IPIPE" +#endif /* CONFIG_NO_IDLE_HZ */ +int __ipipe_mach_timerint = IRQ_OST0; +EXPORT_SYMBOL(__ipipe_mach_timerint); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); + +unsigned int __ipipe_mach_ticks_per_jiffy = LATCH; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +static int pxa_timer_initialized; +static unsigned long last_jiffy_time; +#endif /* CONFIG_IPIPE */ + static inline unsigned long pxa_get_rtc_time(void) { return RCNR; @@ -53,6 +71,9 @@ static unsigned long pxa_gettimeoffset ( { long ticks_to_match, elapsed, usec; +#ifdef CONFIG_IPIPE + if (!__ipipe_mach_timerstolen) { +#endif /* Get ticks before next timer match */ ticks_to_match = OSMR0 - OSCR; @@ -62,6 +83,10 @@ static unsigned long pxa_gettimeoffset ( /* don't get fooled by the workaround in pxa_timer_interrupt() */ if (elapsed <= 0) return 0; +#ifdef CONFIG_IPIPE + } else + elapsed = OSCR - last_jiffy_time; +#endif /* Now convert them to usec */ usec = (unsigned long)(elapsed * (tick_nsec / 1000))/LATCH; @@ -104,9 +129,27 @@ pxa_timer_interrupt(int irq, void *dev_i * affect things only when the timer IRQ has been delayed by nearly * exactly one tick period which should be a pretty rare event. */ +#ifdef CONFIG_IPIPE + /* + * - if Linux is running natively (no ipipe), ack and reprogram the timer + * - if Linux is running under ipipe, but it still has the control over + * the timer (no Xenomai for example), then reprogram the timer (ipipe + * has already acked it) + * - if some other domain has taken over the timer, then do nothing + * (ipipe has acked it, and the other domain has reprogramed it) + */ + if (__ipipe_mach_timerstolen) { + timer_tick(); + last_jiffy_time += LATCH; + } else +#endif /* CONFIG_IPIPE */ do { timer_tick(); +#ifdef CONFIG_IPIPE + last_jiffy_time += LATCH; +#else /* !CONFIG_IPIPE */ OSSR = OSSR_M0; /* Clear match on timer 0 */ +#endif /* !CONFIG_IPIPE */ next_match = (OSMR0 += LATCH); } while( (signed long)(next_match - OSCR) <= 8 ); @@ -138,6 +181,10 @@ static void __init pxa_timer_init(void) setup_irq(IRQ_OST0, &pxa_timer_irq); OIER = OIER_E0; /* enable match on timer 0 to cause interrupts */ OSCR = 0; /* initialize free-running timer */ + +#ifdef CONFIG_IPIPE + pxa_timer_initialized = 1; +#endif /* CONFIG_IPIPE */ } #ifdef CONFIG_NO_IDLE_HZ @@ -215,3 +262,75 @@ struct sys_timer pxa_timer = { .dyn_tick = &pxa_dyn_tick, #endif }; + +#ifdef CONFIG_IPIPE +void __ipipe_mach_acktimer(void) +{ + OSSR = OSSR_M0; /* Clear match on timer 0 */ +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + if (likely(pxa_timer_initialized)) { + static union { +#ifdef __BIG_ENDIAN + struct { + unsigned long high; + unsigned long low; + }; +#else /* __LITTLE_ENDIAN */ + struct { + unsigned long low; + unsigned long high; + }; +#endif /* __LITTLE_ENDIAN */ + unsigned long long full; + } tsc[NR_CPUS], *local_tsc; + unsigned long stamp, flags; + unsigned long long result; + + local_irq_save_hw(flags); + local_tsc = &tsc[ipipe_processor_id()]; + stamp = OSCR; + if (unlikely(stamp < local_tsc->low)) + /* 32 bit counter wrapped, increment high word. */ + local_tsc->high++; + local_tsc->low = stamp; + result = local_tsc->full; + local_irq_restore_hw(flags); + + return result; + } + + return 0; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +/* + * Reprogram the timer + */ + +void __ipipe_mach_set_dec(unsigned long delay) +{ + if (delay > 8) { + unsigned long flags; + + local_irq_save_hw(flags); + OSMR0 = delay + OSCR; + local_irq_restore_hw(flags); + } else + ipipe_trigger_irq(IRQ_OST0); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + __ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy); +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return OSMR0 - OSCR; +} +#endif /* CONFIG_IPIPE */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-s3c2410/irq.c ipipe-2.6.19-arm/arch/arm/mach-s3c2410/irq.c --- linux-2.6.19/arch/arm/mach-s3c2410/irq.c 2007-01-04 22:04:13.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-s3c2410/irq.c 2007-01-15 21:38:05.000000000 +0100 @@ -3,6 +3,8 @@ * Copyright (c) 2003,2004 Simtec Electronics * Ben Dooks * + * Copyright (C) 2006 Sebastian Smolorz , emlix GmbH + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -48,7 +50,10 @@ * * 25-Jul-2005 Ben Dooks * Split the S3C2440 IRQ code to seperate file -*/ + * + * 30-Oct-2006 Sebastian Smolorz + * Added Adeos/I-pipe support + */ #include #include @@ -56,6 +61,7 @@ #include #include #include +#include #include #include @@ -70,6 +76,14 @@ #include "pm.h" #include "irq.h" +#ifdef CONFIG_IPIPE +#ifdef CONFIG_CPU_S3C2440 +extern void __ipipe_s3c_irq_demux_wdtac97(unsigned int irq, + struct pt_regs *regs); +extern void __ipipe_s3c_irq_demux_cam(unsigned int irq, struct pt_regs *regs); +#endif /* CONFIG_CPU_S3C2440 */ +#endif /* CONFIG_IPIPE */ + /* wakeup irq control */ #ifdef CONFIG_PM @@ -668,6 +682,79 @@ int s3c24xx_irq_resume(struct sys_device #define s3c24xx_irq_resume NULL #endif +#ifdef CONFIG_IPIPE +static void __ipipe_s3c_irq_demux_uart(unsigned int start, + unsigned int subsrc, + struct pt_regs *regs) +{ + unsigned int offset = start - IRQ_S3CUART_RX0; + + subsrc >>= offset; + subsrc &= 7; + + if (subsrc != 0) { + if (subsrc & 1) + __ipipe_handle_irq(start, regs); + if (subsrc & 2) + __ipipe_handle_irq(start+1, regs); + if (subsrc & 4) + __ipipe_handle_irq(start+2, regs); + } +} + +static void __ipipe_s3c_irq_demux_adc(unsigned int subsrc, + struct pt_regs *regs) +{ + subsrc >>= 9; + subsrc &= 3; + + if (subsrc != 0) { + if (subsrc & 1) + __ipipe_handle_irq(IRQ_TC, regs); + if (subsrc & 2) + __ipipe_handle_irq(IRQ_ADC, regs); + } +} + +void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs) +{ + unsigned int subsrc, submsk; + struct irqdesc *desc_unused = irq_desc + irq; + + /* read the current pending interrupts, and the mask + * for what it is available */ + subsrc = __raw_readl(S3C2410_SUBSRCPND); + submsk = __raw_readl(S3C2410_INTSUBMSK); + + subsrc &= ~submsk; + + switch (irq) { + case IRQ_UART0: + __ipipe_s3c_irq_demux_uart(IRQ_S3CUART_RX0, subsrc, regs); + break; + case IRQ_UART1: + __ipipe_s3c_irq_demux_uart(IRQ_S3CUART_RX1, subsrc, regs); + break; + case IRQ_UART2: + __ipipe_s3c_irq_demux_uart(IRQ_S3CUART_RX2, subsrc, regs); + break; + case IRQ_ADCPARENT: + __ipipe_s3c_irq_demux_adc(subsrc, regs); + break; +#ifdef CONFIG_CPU_S3C2440 + case IRQ_WDT: + __ipipe_s3c_irq_demux_wdtac97(subsrc, regs); + break; + case IRQ_CAM: + __ipipe_s3c_irq_demux_cam(subsrc, regs); + break; +#endif /* CONFIG_CPU_S3C2440 */ + } + + desc_unused->chip->unmask(irq); +} +#endif /* CONFIG_IPIPE */ + /* s3c24xx_init_irq * * Initialise S3C2410 IRQ system diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-s3c2410/s3c2440-irq.c ipipe-2.6.19-arm/arch/arm/mach-s3c2410/s3c2440-irq.c --- linux-2.6.19/arch/arm/mach-s3c2410/s3c2440-irq.c 2007-01-04 22:04:13.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-s3c2410/s3c2440-irq.c 2007-01-15 21:38:05.000000000 +0100 @@ -3,6 +3,8 @@ * Copyright (c) 2003,2004 Simtec Electronics * Ben Dooks * + * Copyright (C) 2006 Sebastian Smolorz , emlix GmbH + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -25,6 +27,7 @@ #include #include #include +#include #include #include @@ -96,6 +99,34 @@ static struct irqchip s3c_irq_wdtac97 = .ack = s3c_irq_wdtac97_ack, }; +#ifdef CONFIG_IPIPE +void __ipipe_s3c_irq_demux_wdtac97(unsigned int subsrc, struct pt_regs *regs) +{ + subsrc >>= 13; + subsrc &= 3; + + if (subsrc != 0) { + if (subsrc & 1) + __ipipe_handle_irq(IRQ_S3C2440_WDT, regs); + if (subsrc & 2) + __ipipe_handle_irq(IRQ_S3C2440_AC97, regs); + } +} + +void __ipipe_s3c_irq_demux_cam(unsigned int subsrc, struct pt_regs *regs) +{ + subsrc >>= 11; + subsrc &= 3; + + if (subsrc != 0) { + if (subsrc & 1) + __ipipe_handle_irq(IRQ_S3C2440_CAM_C, regs); + if (subsrc & 2) + __ipipe_handle_irq(IRQ_S3C2440_CAM_P, regs); + } +} +#endif /* CONFIG_IPIPE */ + static int s3c2440_irq_add(struct sys_device *sysdev) { unsigned int irqno; diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-s3c2410/time.c ipipe-2.6.19-arm/arch/arm/mach-s3c2410/time.c --- linux-2.6.19/arch/arm/mach-s3c2410/time.c 2007-01-04 22:04:13.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-s3c2410/time.c 2007-01-20 02:09:36.000000000 +0100 @@ -3,6 +3,8 @@ * Copyright (C) 2003-2005 Simtec Electronics * Ben Dooks, * + * Copyright (C) 2006 Sebastian Smolorz , emlix GmbH + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -25,6 +27,7 @@ #include #include #include +#include #include #include @@ -40,7 +43,6 @@ #include "clock.h" #include "cpu.h" -static unsigned long timer_startval; static unsigned long timer_usec_ticks; #define TIMER_USEC_SHIFT 16 @@ -55,6 +57,24 @@ static unsigned long timer_usec_ticks; * Original patch by Dimitry Andric, updated by Ben Dooks */ +static unsigned long last_free_running_tcnt = 0; +static unsigned long free_running_tcon = 0; +static unsigned long timer_lxlost = 0; + +#ifdef CONFIG_IPIPE +unsigned int __ipipe_mach_ticks_per_jiffy; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +int __ipipe_mach_timerint = IRQ_TIMER4; +EXPORT_SYMBOL(__ipipe_mach_timerint); + +static unsigned long long __ipipe_mach_tsc = 0; +static unsigned long timer_ackval = 1UL << (IRQ_TIMER4 - IRQ_EINT0); +static IPIPE_DEFINE_SPINLOCK(timer_lock); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); +#endif /* CONFIG_IPIPE */ /* timer_mask_usec_ticks * @@ -85,44 +105,46 @@ static inline unsigned long timer_ticks_ return res >> TIMER_USEC_SHIFT; } -/*** - * Returns microsecond since last clock interrupt. Note that interrupts - * will have been disabled by do_gettimeoffset() - * IRQs are disabled before entering here from do_gettimeofday() - */ - -#define SRCPND_TIMER4 (1<<(IRQ_TIMER4 - IRQ_EINT0)) -static unsigned long s3c2410_gettimeoffset (void) +static inline unsigned long timer_freerunning_getvalue(void) { - unsigned long tdone; - unsigned long irqpend; - unsigned long tval; - - /* work out how many ticks have gone since last timer interrupt */ + return __raw_readl(S3C2410_TCNTO(3)); +} - tval = __raw_readl(S3C2410_TCNTO(4)); - tdone = timer_startval - tval; +static inline unsigned long timer_freerunning_getticksoffset(unsigned long tval) +{ + long tdone; - /* check to see if there is an interrupt pending */ + tdone = last_free_running_tcnt - tval; + if (tdone < 0) + tdone += 0x10000; - irqpend = __raw_readl(S3C2410_SRCPND); - if (irqpend & SRCPND_TIMER4) { - /* re-read the timer, and try and fix up for the missed - * interrupt. Note, the interrupt may go off before the - * timer has re-loaded from wrapping. - */ + return tdone; +} - tval = __raw_readl(S3C2410_TCNTO(4)); - tdone = timer_startval - tval; +static inline unsigned long getticksoffset(void) +{ + return timer_freerunning_getticksoffset(timer_freerunning_getvalue()); +} - if (tval != 0) - tdone += timer_startval; - } +#ifdef CONFIG_IPIPE +static inline unsigned long getticksoffset_tscupdate(void) +{ + unsigned long tval; + unsigned long ticks; - return timer_ticks_to_usec(tdone); + tval = timer_freerunning_getvalue(); + ticks = timer_freerunning_getticksoffset(tval); + last_free_running_tcnt = tval; + __ipipe_mach_tsc += ticks; + return ticks; } +#endif /* CONFIG_IPIPE */ +static unsigned long s3c2410_gettimeoffset (void) +{ + return timer_ticks_to_usec(timer_lxlost + getticksoffset()); +} /* * IRQ handler for the timer @@ -131,6 +153,14 @@ static irqreturn_t s3c2410_timer_interrupt(int irq, void *dev_id) { write_seqlock(&xtime_lock); + +#ifdef CONFIG_IPIPE + timer_lxlost = 0; + + if (!__ipipe_mach_timerstolen) + getticksoffset_tscupdate(); +#endif /* CONFIG_IPIPE */ + timer_tick(); write_sequnlock(&xtime_lock); return IRQ_HANDLED; @@ -149,10 +179,10 @@ static struct irqaction s3c2410_timer_ir machine_is_osiris() ) /* - * Set up timer interrupt, and return the current time in seconds. + * Set up timer interrupt. * - * Currently we only use timer4, as it is the only timer which has no - * other function that can be exploited externally + * Currently we use timer4 as event timer and timer3 as tick counter which + * permanently counts ticks without interrupt generation. */ static void s3c2410_timer_setup (void) { @@ -160,6 +190,7 @@ static void s3c2410_timer_setup (void) unsigned long tcnt; unsigned long tcfg1; unsigned long tcfg0; + unsigned long intmask; tcnt = 0xffff; /* default value for tcnt */ @@ -176,8 +207,8 @@ static void s3c2410_timer_setup (void) timer_usec_ticks = timer_mask_usec_ticks(1, 12000000); tcnt = 12000000 / HZ; - tcfg1 &= ~S3C2410_TCFG1_MUX4_MASK; - tcfg1 |= S3C2410_TCFG1_MUX4_TCLK1; + tcfg1 &= ~(S3C2410_TCFG1_MUX4_MASK | S3C2410_TCFG1_MUX3_MASK); + tcfg1 |= (S3C2410_TCFG1_MUX4_TCLK1 | S3C2410_TCFG1_MUX3_TCLK1); } else { unsigned long pclk; struct clk *clk; @@ -205,8 +236,8 @@ static void s3c2410_timer_setup (void) timer_usec_ticks = timer_mask_usec_ticks(6, pclk); - tcfg1 &= ~S3C2410_TCFG1_MUX4_MASK; - tcfg1 |= S3C2410_TCFG1_MUX4_DIV2; + tcfg1 &= ~(S3C2410_TCFG1_MUX4_MASK | S3C2410_TCFG1_MUX3_MASK); + tcfg1 |= (S3C2410_TCFG1_MUX4_DIV2 | S3C2410_TCFG1_MUX3_DIV2); tcfg0 &= ~S3C2410_TCFG_PRESCALER1_MASK; tcfg0 |= ((6 - 1) / 2) << S3C2410_TCFG_PRESCALER1_SHIFT; @@ -214,6 +245,10 @@ static void s3c2410_timer_setup (void) tcnt = (pclk / 6) / HZ; } +#ifdef CONFIG_IPIPE + __ipipe_mach_ticks_per_jiffy = tcnt; +#endif /* CONFIG_IPIPE */ + /* timers reload after counting zero, so reduce the count by 1 */ tcnt--; @@ -230,23 +265,37 @@ static void s3c2410_timer_setup (void) __raw_writel(tcfg1, S3C2410_TCFG1); __raw_writel(tcfg0, S3C2410_TCFG0); - timer_startval = tcnt; - __raw_writel(tcnt, S3C2410_TCNTB(4)); - - /* ensure timer is stopped... */ + /* ensure timers are stopped... */ + tcon &= ~(0x3f<<17); + __raw_writel(tcon, S3C2410_TCON); - tcon &= ~(7<<20); - tcon |= S3C2410_TCON_T4RELOAD; - tcon |= S3C2410_TCON_T4MANUALUPD; + /* Mask timer3 interrupt. */ + intmask = __raw_readl(S3C2410_INTMSK); + intmask |= 1UL << (IRQ_TIMER3 - IRQ_EINT0); + __raw_writel(intmask, S3C2410_INTMSK); - __raw_writel(tcon, S3C2410_TCON); + /* Set timer values */ __raw_writel(tcnt, S3C2410_TCNTB(4)); __raw_writel(tcnt, S3C2410_TCMPB(4)); + __raw_writel(0xffff, S3C2410_TCNTB(3)); + __raw_writel(0xffff, S3C2410_TCMPB(3)); - /* start the timer running */ - tcon |= S3C2410_TCON_T4START; - tcon &= ~S3C2410_TCON_T4MANUALUPD; + /* Set base tcon value for later programming of timer 4 by Xenomai. */ + free_running_tcon = tcon | S3C2410_TCON_T3RELOAD | S3C2410_TCON_T3START; + + /* Set auto reloads for both timers. */ + tcon |= S3C2410_TCON_T3RELOAD | S3C2410_TCON_T4RELOAD; + + /* Manual update */ + __raw_writel(tcon | S3C2410_TCON_T3MANUALUPD + | S3C2410_TCON_T4MANUALUPD, S3C2410_TCON); + + tcon |= S3C2410_TCON_T3START | S3C2410_TCON_T4START; + /* Start timers.*/ __raw_writel(tcon, S3C2410_TCON); + + /* Save start value of timer 3 as begining of first period. */ + last_free_running_tcnt = 0xffff; } static void __init s3c2410_timer_init (void) @@ -260,3 +309,56 @@ struct sys_timer s3c24xx_timer = { .offset = s3c2410_gettimeoffset, .resume = s3c2410_timer_setup }; + +#ifdef CONFIG_IPIPE +void __ipipe_mach_acktimer(void) +{ + __raw_writel(timer_ackval, S3C2410_SRCPND); + __raw_writel(timer_ackval, S3C2410_INTPND); +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + unsigned long long result; + unsigned long flags; + + spin_lock_irqsave(&timer_lock, flags); + result = __ipipe_mach_tsc + getticksoffset(); + spin_unlock_irqrestore(&timer_lock, flags); + return result; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +static inline void set_dec(unsigned long reload) +{ + __raw_writel(reload, S3C2410_TCNTB(4)); + /* Manual update */ + __raw_writel(free_running_tcon | S3C2410_TCON_T4MANUALUPD, S3C2410_TCON); + /* Start timer */ + __raw_writel(free_running_tcon | S3C2410_TCON_T4START, S3C2410_TCON); +} + +void __ipipe_mach_set_dec(unsigned long reload) +{ + unsigned long flags; + + spin_lock_irqsave(&timer_lock, flags); + timer_lxlost += getticksoffset_tscupdate(); + set_dec(reload); + spin_unlock_irqrestore(&timer_lock, flags); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + free_running_tcon |= S3C2410_TCON_T4RELOAD; + __ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy - 1); + free_running_tcon &= ~S3C2410_TCON_T4RELOAD; +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return __raw_readl(S3C2410_TCNTO(4)); +} +#endif /* CONFIG_IPIPE */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-sa1100/irq.c ipipe-2.6.19-arm/arch/arm/mach-sa1100/irq.c --- linux-2.6.19/arch/arm/mach-sa1100/irq.c 2007-01-04 22:04:13.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-sa1100/irq.c 2007-01-15 21:38:05.000000000 +0100 @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -137,6 +138,37 @@ sa1100_high_gpio_handler(unsigned int ir } while (mask); } +#ifdef CONFIG_IPIPE +void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs) +{ + struct irqdesc *desc_unused = irq_desc + irq; + unsigned irq_unused = irq; + unsigned int mask; + + mask = GEDR & 0xfffff800; + do { + /* + * clear down all currently active IRQ sources. + * We will be processing them all. + */ + GEDR = mask; + + irq = IRQ_GPIO11; + mask >>= 11; + do { + if (mask & 1) + __ipipe_handle_irq(irq, regs); + mask >>= 1; + irq++; + } while (mask); + + mask = GEDR & 0xfffff800; + } while (mask); + + desc_unused->chip->unmask(irq_unused); +} +#endif /* CONFIG_IPIPE */ + /* * Like GPIO0 to 10, GPIO11-27 IRQs need to be handled specially. * In addition, the IRQs are all collected up into one bit in the diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mach-sa1100/time.c ipipe-2.6.19-arm/arch/arm/mach-sa1100/time.c --- linux-2.6.19/arch/arm/mach-sa1100/time.c 2007-01-04 22:04:13.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mach-sa1100/time.c 2007-01-20 00:06:53.000000000 +0100 @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -21,6 +22,23 @@ #define RTC_DEF_DIVIDER (32768 - 1) #define RTC_DEF_TRIM 0 +#ifdef CONFIG_IPIPE +#ifdef CONFIG_NO_IDLE_HZ +#error "dynamic tick timer not yet supported with IPIPE" +#endif /* CONFIG_NO_IDLE_HZ */ +int __ipipe_mach_timerint = IRQ_OST0; +EXPORT_SYMBOL(__ipipe_mach_timerint); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); + +unsigned int __ipipe_mach_ticks_per_jiffy = LATCH; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +static int sa1100_timer_initialized; +static unsigned long last_jiffy_time; +#endif /* CONFIG_IPIPE */ + static unsigned long __init sa1100_get_rtc_time(void) { /* @@ -59,11 +77,18 @@ static unsigned long sa1100_gettimeoffse { unsigned long ticks_to_match, elapsed, usec; +#ifdef CONFIG_IPIPE + if (!__ipipe_mach_timerstolen) { +#endif /* Get ticks before next timer match */ ticks_to_match = OSMR0 - OSCR; /* We need elapsed ticks since last match */ elapsed = LATCH - ticks_to_match; +#ifdef CONFIG_IPIPE + } else + elapsed = OSCR - last_jiffy_time; +#endif /* Now convert them to usec */ usec = (unsigned long)(elapsed * (tick_nsec / 1000))/LATCH; @@ -98,9 +123,27 @@ sa1100_timer_interrupt(int irq, void *de * ensured, hence we can use do_gettimeofday() from interrupt * handlers. */ +#ifdef CONFIG_IPIPE + /* + * - if Linux is running natively (no ipipe), ack and reprogram the timer + * - if Linux is running under ipipe, but it still has the control over + * the timer (no Xenomai for example), then reprogram the timer (ipipe + * has already acked it) + * - if some other domain has taken over the timer, then do nothing + * (ipipe has acked it, and the other domain has reprogramed it) + */ + if (__ipipe_mach_timerstolen) { + timer_tick(); + last_jiffy_time += LATCH; + } else +#endif /* CONFIG_IPIPE */ do { timer_tick(); +#ifdef CONFIG_IPIPE + last_jiffy_time += LATCH; +#else /* !CONFIG_IPIPE */ OSSR = OSSR_M0; /* Clear match on timer 0 */ +#endif /* !CONFIG_IPIPE */ next_match = (OSMR0 += LATCH); } while ((signed long)(next_match - OSCR) <= 0); @@ -132,6 +175,10 @@ static void __init sa1100_timer_init(voi setup_irq(IRQ_OST0, &sa1100_timer_irq); OIER = OIER_E0; /* enable match on timer 0 to cause interrupts */ OSCR = 0; /* initialize free-running timer */ + +#ifdef CONFIG_IPIPE + sa1100_timer_initialized = 1; +#endif /* CONFIG_IPIPE */ } #ifdef CONFIG_NO_IDLE_HZ @@ -210,3 +257,72 @@ struct sys_timer sa1100_timer = { .dyn_tick = &sa1100_dyn_tick, #endif }; + +#ifdef CONFIG_IPIPE +void __ipipe_mach_acktimer(void) +{ + OSSR = OSSR_M0; /* Clear match on timer 0 */ +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + if (likely(sa1100_timer_initialized)) { + static union { +#ifdef __BIG_ENDIAN + struct { + unsigned long high; + unsigned long low; + }; +#else /* __LITTLE_ENDIAN */ + struct { + unsigned long low; + unsigned long high; + }; +#endif /* __LITTLE_ENDIAN */ + unsigned long long full; + } tsc[NR_CPUS], *local_tsc; + unsigned long stamp, flags; + unsigned long long result; + + local_irq_save_hw(flags); + local_tsc = &tsc[ipipe_processor_id()]; + stamp = OSCR; + if (unlikely(stamp < local_tsc->low)) + /* 32 bit counter wrapped, increment high word. */ + local_tsc->high++; + local_tsc->low = stamp; + result = local_tsc->full; + local_irq_restore_hw(flags); + + return result; + } + + return 0; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +/* + * Reprogram the timer + */ + +void __ipipe_mach_set_dec(unsigned long delay) +{ + unsigned long flags; + + local_irq_save_hw(flags); + OSMR0 = delay + OSCR; + local_irq_restore_hw(flags); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + __ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy); +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return OSMR0 - OSCR; +} +#endif /* CONFIG_IPIPE */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/arch/arm/mm/fault.c ipipe-2.6.19-arm/arch/arm/mm/fault.c --- linux-2.6.19/arch/arm/mm/fault.c 2007-01-04 22:04:13.000000000 +0100 +++ ipipe-2.6.19-arm/arch/arm/mm/fault.c 2007-01-15 22:10:26.000000000 +0100 @@ -223,6 +223,9 @@ do_page_fault(unsigned long addr, unsign struct mm_struct *mm; int fault, sig, code; + if (ipipe_trap_notify(IPIPE_TRAP_ACCESS,regs)) + return 0; + tsk = current; mm = tsk->mm; @@ -351,6 +354,9 @@ do_translation_fault(unsigned long addr, return 0; bad_area: + if (ipipe_trap_notify(IPIPE_TRAP_ACCESS,regs)) + return 0; + do_bad_area(addr, fsr, regs); return 0; } @@ -362,6 +368,10 @@ bad_area: static int do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { + + if (ipipe_trap_notify(IPIPE_TRAP_SECTION,regs)) + return 0; + do_bad_area(addr, fsr, regs); return 0; } @@ -372,6 +382,9 @@ do_sect_fault(unsigned long addr, unsign static int do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { + if (ipipe_trap_notify(IPIPE_TRAP_DABT,regs)) + return 0; + return 1; } @@ -447,6 +460,9 @@ do_DataAbort(unsigned long addr, unsigne if (!inf->fn(addr, fsr, regs)) return; + if (ipipe_trap_notify(IPIPE_TRAP_UNKNOWN,regs)) + return; + printk(KERN_ALERT "Unhandled fault: %s (0x%03x) at 0x%08lx\n", inf->name, fsr, addr); diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/drivers/pci/htirq.c ipipe-2.6.19-arm/drivers/pci/htirq.c --- linux-2.6.19/drivers/pci/htirq.c 2007-01-04 22:04:37.000000000 +0100 +++ ipipe-2.6.19-arm/drivers/pci/htirq.c 2007-01-15 21:33:00.000000000 +0100 @@ -21,7 +21,7 @@ * With multiple simultaneous hypertransport irq devices it might pay * to make this more fine grained. But start with simple, stupid, and correct. */ -static DEFINE_SPINLOCK(ht_irq_lock); +static IPIPE_DEFINE_SPINLOCK(ht_irq_lock); struct ht_irq_cfg { struct pci_dev *dev; diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/arch-at91rm9200/hardware.h ipipe-2.6.19-arm/include/asm-arm/arch-at91rm9200/hardware.h --- linux-2.6.19/include/asm-arm/arch-at91rm9200/hardware.h 2007-01-04 22:05:06.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/arch-at91rm9200/hardware.h 2007-01-18 00:13:54.000000000 +0100 @@ -39,6 +39,9 @@ #define AT91_VA_BASE_TWI AT91_IO_P2V(AT91RM9200_BASE_TWI) #define AT91_VA_BASE_MCI AT91_IO_P2V(AT91RM9200_BASE_MCI) #define AT91_VA_BASE_UDP AT91_IO_P2V(AT91RM9200_BASE_UDP) +#ifdef CONFIG_IPIPE +#define AT91_VA_BASE_TCB0 AT91_IO_P2V(AT91RM9200_BASE_TCB0) +#endif /* CONFIG_IPIPE */ /* Internal SRAM is mapped below the IO devices */ #define AT91_SRAM_VIRT_BASE (AT91_IO_VIRT_BASE - AT91RM9200_SRAM_SIZE) diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/arch-at91rm9200/irqs.h ipipe-2.6.19-arm/include/asm-arm/arch-at91rm9200/irqs.h --- linux-2.6.19/include/asm-arm/arch-at91rm9200/irqs.h 2007-01-04 22:05:06.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/arch-at91rm9200/irqs.h 2007-01-17 22:19:26.000000000 +0100 @@ -39,4 +39,12 @@ */ #define NR_IRQS (NR_AIC_IRQS + (4 * 32)) +#if defined(CONFIG_IPIPE) && !defined(__ASSEMBLY__) +extern unsigned __ipipe_at91_gpio_banks; + +#define __ipipe_mach_irq_mux_p(irq) \ + ((unsigned) (irq - AT91RM9200_ID_PIOA) < __ipipe_at91_gpio_banks) + +#endif /* CONFIG_IPIPE && !__ASSEMBLY__ */ + #endif diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/arch-at91rm9200/timex.h ipipe-2.6.19-arm/include/asm-arm/arch-at91rm9200/timex.h --- linux-2.6.19/include/asm-arm/arch-at91rm9200/timex.h 2007-01-04 22:02:39.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/arch-at91rm9200/timex.h 2007-01-15 23:53:40.000000000 +0100 @@ -23,6 +23,10 @@ #include +#ifndef CONFIG_IPIPE #define CLOCK_TICK_RATE (AT91_SLOW_CLOCK) +#else /* !CONFIG_IPIPE */ +#define CLOCK_TICK_RATE (CONFIG_IPIPE_AT91_MCK / 32) +#endif /* !CONFIG_IPIPE */ #endif diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/arch-integrator/entry-macro.S ipipe-2.6.19-arm/include/asm-arm/arch-integrator/entry-macro.S --- linux-2.6.19/include/asm-arm/arch-integrator/entry-macro.S 2006-05-07 16:42:04.000000000 +0200 +++ ipipe-2.6.19-arm/include/asm-arm/arch-integrator/entry-macro.S 2007-01-15 21:38:05.000000000 +0100 @@ -22,7 +22,11 @@ teq \irqstat, #0 ldreq \irqstat, [\base, #(INTEGRATOR_HDR_IC_OFFSET+IRQ_STATUS)] moveq \irqnr, #IRQ_CIC_START - +#ifdef CONFIG_IPIPE + tst \irqstat, #0x00000040 @ check IRQ_TIMERINT1 first + movne \irqnr, #6 + bne 1003f +#endif /* CONFIG_IPIPE */ 1001: tst \irqstat, #15 bne 1002f add \irqnr, \irqnr, #4 diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/arch-integrator/irqs.h ipipe-2.6.19-arm/include/asm-arm/arch-integrator/irqs.h --- linux-2.6.19/include/asm-arm/arch-integrator/irqs.h 2005-10-28 02:02:08.000000000 +0200 +++ ipipe-2.6.19-arm/include/asm-arm/arch-integrator/irqs.h 2007-01-15 21:38:05.000000000 +0100 @@ -80,3 +80,6 @@ #define NR_IRQS 47 +#ifdef CONFIG_IPIPE +#define __ipipe_mach_irq_mux_p(irq) ((irq) == IRQ_CP_CPPLDINT) +#endif /* CONFIG_IPIPE */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/arch-integrator/platform.h ipipe-2.6.19-arm/include/asm-arm/arch-integrator/platform.h --- linux-2.6.19/include/asm-arm/arch-integrator/platform.h 2005-10-28 02:02:08.000000000 +0200 +++ ipipe-2.6.19-arm/include/asm-arm/arch-integrator/platform.h 2007-01-20 00:52:56.000000000 +0100 @@ -26,7 +26,7 @@ * NOTE: This is a multi-hosted header file for use with uHAL and * supported debuggers. * - * $Id: platform.s,v 1.32 2000/02/18 10:51:39 asims Exp $ + * $Id: platform.h,v 1.2 2006/02/20 13:54:22 rpm Exp $ * * ***********************************************************************/ @@ -436,7 +436,7 @@ * Timer definitions * * Only use timer 1 & 2 - * (both run at 24MHz and will need the clock divider set to 16). + * (both run at 1MHZ on /CP and at 24MHz on /AP) * * Timer 0 runs at bus frequency and therefore could vary and currently * uHAL can't handle that. @@ -449,7 +449,12 @@ #define MAX_TIMER 2 #define MAX_PERIOD 699050 + +#ifdef CONFIG_ARCH_INTEGRATOR_CP +#define TICKS_PER_uSEC 1 +#else #define TICKS_PER_uSEC 24 +#endif /* * These are useconds NOT ticks. diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/arch-integrator/timex.h ipipe-2.6.19-arm/include/asm-arm/arch-integrator/timex.h --- linux-2.6.19/include/asm-arm/arch-integrator/timex.h 2005-10-28 02:02:08.000000000 +0200 +++ ipipe-2.6.19-arm/include/asm-arm/arch-integrator/timex.h 2007-01-15 21:38:06.000000000 +0100 @@ -21,6 +21,6 @@ */ /* - * ?? + * Timer rate */ -#define CLOCK_TICK_RATE (50000000 / 16) +#define CLOCK_TICK_RATE (1000000) diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/arch-pxa/irqs.h ipipe-2.6.19-arm/include/asm-arm/arch-pxa/irqs.h --- linux-2.6.19/include/asm-arm/arch-pxa/irqs.h 2007-01-04 22:05:06.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/arch-pxa/irqs.h 2007-01-15 21:38:06.000000000 +0100 @@ -72,6 +72,10 @@ ((i) - IRQ_GPIO(2) + 2) #define IRQ_TO_GPIO(i) (((i) < IRQ_GPIO(2)) ? ((i) - IRQ_GPIO0) : IRQ_TO_GPIO_2_x(i)) +#ifdef CONFIG_IPIPE +#define __ipipe_mach_irq_mux_p(irq) ((irq) == IRQ_GPIO_2_x) +#endif /* CONFIG_IPIPE */ + #if defined(CONFIG_PXA25x) #define PXA_LAST_GPIO 84 #elif defined(CONFIG_PXA27x) diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/arch-s3c2410/irqs.h ipipe-2.6.19-arm/include/asm-arm/arch-s3c2410/irqs.h --- linux-2.6.19/include/asm-arm/arch-s3c2410/irqs.h 2007-01-04 22:02:40.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/arch-s3c2410/irqs.h 2007-01-15 21:38:06.000000000 +0100 @@ -3,6 +3,8 @@ * Copyright (c) 2003-2005 Simtec Electronics * Ben Dooks * + * Copyright (C) 2006 Sebastian Smolorz , emlix GmbH + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. @@ -116,4 +118,26 @@ #define NR_IRQS (IRQ_S3C2440_AC97+1) +#ifdef CONFIG_IPIPE +#define __ipipe_irqbit(irq) (1 << ((irq) - S3C2410_CPUIRQ_OFFSET)) + +#ifdef CONFIG_CPU_S3C2440 +#define __ipipe_muxed_irqmask (__ipipe_irqbit(IRQ_UART0) | \ + __ipipe_irqbit(IRQ_UART1) | \ + __ipipe_irqbit(IRQ_UART2) | \ + __ipipe_irqbit(IRQ_ADCPARENT) | \ + __ipipe_irqbit(IRQ_WDT) | \ + __ipipe_irqbit(IRQ_CAM)) +#else /* !CONFIG_CPU_S3C2440 */ +#define __ipipe_muxed_irqmask (__ipipe_irqbit(IRQ_UART0) | \ + __ipipe_irqbit(IRQ_UART1) | \ + __ipipe_irqbit(IRQ_UART2) | \ + __ipipe_irqbit(IRQ_ADCPARENT)) +#endif /* CONFIG_CPU_S3C2440 */ + +#define __ipipe_mach_irq_mux_p(irq) ((irq) <= IRQ_ADCPARENT && \ + (__ipipe_irqbit(irq) & \ + __ipipe_muxed_irqmask)) +#endif /* CONFIG_IPIPE */ + #endif /* __ASM_ARCH_IRQ_H */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/arch-sa1100/irqs.h ipipe-2.6.19-arm/include/asm-arm/arch-sa1100/irqs.h --- linux-2.6.19/include/asm-arm/arch-sa1100/irqs.h 2007-01-04 22:02:40.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/arch-sa1100/irqs.h 2007-01-15 21:38:06.000000000 +0100 @@ -144,6 +144,10 @@ #define IRQ_LOCOMO_SPI_OVRN (IRQ_BOARD_END + 20) #define IRQ_LOCOMO_SPI_TEND (IRQ_BOARD_END + 21) +#ifdef CONFIG_IPIPE +#define __ipipe_mach_irq_mux_p(irq) ((irq) == IRQ_GPIO11_27) +#endif /* CONFIG_IPIPE */ + /* * Figure out the MAX IRQ number. * diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/atomic.h ipipe-2.6.19-arm/include/asm-arm/atomic.h --- linux-2.6.19/include/asm-arm/atomic.h 2007-01-04 22:05:06.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/atomic.h 2007-01-15 22:25:49.000000000 +0100 @@ -128,10 +128,10 @@ static inline int atomic_add_return(int unsigned long flags; int val; - raw_local_irq_save(flags); + local_irq_save_hw(flags); val = v->counter; v->counter = val += i; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return val; } @@ -141,10 +141,10 @@ static inline int atomic_sub_return(int unsigned long flags; int val; - raw_local_irq_save(flags); + local_irq_save_hw(flags); val = v->counter; v->counter = val -= i; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return val; } @@ -154,11 +154,11 @@ static inline int atomic_cmpxchg(atomic_ int ret; unsigned long flags; - raw_local_irq_save(flags); + local_irq_save_hw(flags); ret = v->counter; if (likely(ret == old)) v->counter = new; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return ret; } @@ -167,9 +167,9 @@ static inline void atomic_clear_mask(uns { unsigned long flags; - raw_local_irq_save(flags); + local_irq_save_hw(flags); *addr &= ~mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); } #endif /* __LINUX_ARM_ARCH__ */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/bitops.h ipipe-2.6.19-arm/include/asm-arm/bitops.h --- linux-2.6.19/include/asm-arm/bitops.h 2007-01-04 22:05:06.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/bitops.h 2007-01-15 22:27:33.000000000 +0100 @@ -37,9 +37,9 @@ static inline void ____atomic_set_bit(un p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); *p |= mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p) @@ -49,9 +49,9 @@ static inline void ____atomic_clear_bit( p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); *p &= ~mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p) @@ -61,9 +61,9 @@ static inline void ____atomic_change_bit p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); *p ^= mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline int @@ -75,10 +75,10 @@ ____atomic_test_and_set_bit(unsigned int p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); res = *p; *p = res | mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return res & mask; } @@ -92,10 +92,10 @@ ____atomic_test_and_clear_bit(unsigned i p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); res = *p; *p = res & ~mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return res & mask; } @@ -109,10 +109,10 @@ ____atomic_test_and_change_bit(unsigned p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); res = *p; *p = res ^ mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return res & mask; } diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/ipipe.h ipipe-2.6.19-arm/include/asm-arm/ipipe.h --- linux-2.6.19/include/asm-arm/ipipe.h 1970-01-01 01:00:00.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/ipipe.h 2007-01-20 02:04:17.000000000 +0100 @@ -0,0 +1,207 @@ +/* -*- linux-c -*- + * include/asm-arm/ipipe.h + * + * Copyright (C) 2002-2005 Philippe Gerum. + * Copyright (C) 2005 Stelian Pop. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __ARM_IPIPE_H +#define __ARM_IPIPE_H + +#ifdef CONFIG_IPIPE + +#include +#include + +#define IPIPE_ARCH_STRING "1.6-01" +#define IPIPE_MAJOR_NUMBER 1 +#define IPIPE_MINOR_NUMBER 6 +#define IPIPE_PATCH_NUMBER 1 + +#define IPIPE_NR_XIRQS NR_IRQS +#define IPIPE_IRQ_ISHIFT 5 /* 25 for 32bits arch. */ + +#ifdef CONFIG_SMP +#error "I-pipe/arm: SMP not yet implemented" +#define ipipe_processor_id() (current_thread_info()->cpu) +#else /* !CONFIG_SMP */ +#define ipipe_processor_id() 0 +#endif /* CONFIG_SMP */ + +/* Note that we disable the interrupts around context_switch, + * or we'll get into severe problems when scheduling Xenomai's + * user space real time threads. + * This can however cause high latencies, see for example: + * http://www.ussg.iu.edu/hypermail/linux/kernel/0405.2/1388.html + * This may need further optimization... + */ +#define prepare_arch_switch(next) \ +do { \ + ipipe_schedule_notify(current, next); \ + local_irq_disable_hw(); \ +} while(0) + +#define task_hijacked(p) \ + ({ \ + int __x__ = ipipe_current_domain != ipipe_root_domain; \ + /* We would need to clear the SYNC flag for the root domain */ \ + /* over the current processor in SMP mode. */ \ + local_irq_enable_hw(); __x__; \ + }) + +/* ARM traps */ +#define IPIPE_TRAP_ACCESS 0 /* Data or instruction access exception */ +#define IPIPE_TRAP_SECTION 1 /* Section fault */ +#define IPIPE_TRAP_DABT 2 /* Generic data abort */ +#define IPIPE_TRAP_UNKNOWN 3 /* Unknown exception */ +#define IPIPE_TRAP_BREAK 4 /* Instruction breakpoint */ +#define IPIPE_TRAP_FPU 5 /* Floating point exception */ +#define IPIPE_TRAP_VFP 6 /* VFP floating point exception */ +#define IPIPE_TRAP_UNDEFINSTR 7 /* Undefined instruction */ +#define IPIPE_NR_FAULTS 8 + +/* Pseudo-vectors used for kernel events */ +#define IPIPE_FIRST_EVENT IPIPE_NR_FAULTS +#define IPIPE_EVENT_SYSCALL (IPIPE_FIRST_EVENT) +#define IPIPE_EVENT_SCHEDULE (IPIPE_FIRST_EVENT + 1) +#define IPIPE_EVENT_SIGWAKE (IPIPE_FIRST_EVENT + 2) +#define IPIPE_EVENT_SETSCHED (IPIPE_FIRST_EVENT + 3) +#define IPIPE_EVENT_INIT (IPIPE_FIRST_EVENT + 4) +#define IPIPE_EVENT_EXIT (IPIPE_FIRST_EVENT + 5) +#define IPIPE_EVENT_CLEANUP (IPIPE_FIRST_EVENT + 6) +#define IPIPE_LAST_EVENT IPIPE_EVENT_CLEANUP +#define IPIPE_NR_EVENTS (IPIPE_LAST_EVENT + 1) + +extern unsigned long arm_return_addr(int level); + +#define BROKEN_BUILTIN_RETURN_ADDRESS +#define __BUILTIN_RETURN_ADDRESS0 arm_return_addr(0) +#define __BUILTIN_RETURN_ADDRESS1 arm_return_addr(1) + + +struct ipipe_domain; + +struct ipipe_sysinfo { + + int ncpus; /* Number of CPUs on board */ + u64 cpufreq; /* CPU frequency (in Hz) */ + + /* Arch-dependent block */ + + struct { + unsigned tmirq; /* Timer tick IRQ */ + u64 tmfreq; /* Timer frequency */ + } archdep; +}; + +/* arch specific stuff */ +extern int __ipipe_mach_timerint; +extern int __ipipe_mach_timerstolen; +extern unsigned int __ipipe_mach_ticks_per_jiffy; +extern void __ipipe_mach_acktimer(void); +extern unsigned long long __ipipe_mach_get_tsc(void); +extern void __ipipe_mach_set_dec(unsigned long); +extern void __ipipe_mach_release_timer(void); +extern unsigned long __ipipe_mach_get_dec(void); +extern void __ipipe_mach_demux_irq(unsigned irq, struct pt_regs *regs); + +#define ipipe_read_tsc(t) do { t = __ipipe_mach_get_tsc(); } while (0) +#define __ipipe_read_timebase() __ipipe_mach_get_tsc() + +#define ipipe_cpu_freq() (HZ * __ipipe_mach_ticks_per_jiffy) +#define ipipe_tsc2ns(t) \ +({ \ + unsigned long long delta = (t)*1000; \ + do_div(delta, ipipe_cpu_freq() / 1000000 + 1); \ + (unsigned long)delta; \ +}) +#define ipipe_tsc2us(t) \ +({ \ + unsigned long long delta = (t); \ + do_div(delta, ipipe_cpu_freq() / 1000000 + 1); \ + (unsigned long)delta; \ +}) + +/* Private interface -- Internal use only */ + +#define __ipipe_check_platform() do { } while(0) + +void __ipipe_init_platform(void); + +#define __ipipe_enable_irq(irq) irq_desc[irq].chip->enable(irq) + +#define __ipipe_disable_irq(irq) irq_desc[irq].chip->disable(irq) + +#define __ipipe_hook_critical_ipi(ipd) do { } while(0) + +void __ipipe_enable_irqdesc(unsigned irq); + +void __ipipe_enable_pipeline(void); + +void __ipipe_do_IRQ(int irq, + struct pt_regs *regs); + +void __ipipe_do_timer(int irq, + struct pt_regs *regs); + +void __ipipe_do_critical_sync(unsigned irq, + void *cookie); + +extern unsigned long __ipipe_decr_ticks; + +extern unsigned long long __ipipe_decr_next[]; + +extern struct pt_regs __ipipe_tick_regs[]; + +void __ipipe_handle_irq(int irq, + struct pt_regs *regs); + +#define __ipipe_tick_irq ipipe_timerint + +static inline unsigned long __ipipe_ffnz(unsigned long ul) +{ + return ffs(ul) - 1; +} + +/* When running handlers, enable hw interrupts for all domains but the + * one heading the pipeline, so that IRQs can never be significantly + * deferred for the latter. */ +#define __ipipe_run_isr(ipd, irq, cpuid) \ +do { \ + local_irq_enable_nohead(ipd); \ + if (ipd == ipipe_root_domain) { \ + ((void (*)(unsigned, struct pt_regs *)) \ + ipd->irqs[irq].handler) (irq, __ipipe_tick_regs + cpuid); \ + } else { \ + __clear_bit(IPIPE_SYNC_FLAG, &cpudata->status); \ + ipd->irqs[irq].handler(irq,ipd->irqs[irq].cookie); \ + __set_bit(IPIPE_SYNC_FLAG, &cpudata->status); \ + } \ + local_irq_disable_nohead(ipd); \ +} while(0) + +#define __ipipe_syscall_watched_p(p, sc) \ + (((p)->flags & PF_EVNOTIFY) || (unsigned long)sc >= __ARM_NR_BASE + 64) + +#else /* !CONFIG_IPIPE */ + +#define task_hijacked(p) 0 + +#endif /* CONFIG_IPIPE */ + +#endif /* !__ARM_IPIPE_H */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/irqflags.h ipipe-2.6.19-arm/include/asm-arm/irqflags.h --- linux-2.6.19/include/asm-arm/irqflags.h 2007-01-04 22:05:06.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/irqflags.h 2007-01-17 22:12:03.000000000 +0100 @@ -10,30 +10,30 @@ */ #if __LINUX_ARM_ARCH__ >= 6 -#define raw_local_irq_save(x) \ +#define local_irq_save_hw_notrace(x) \ ({ \ __asm__ __volatile__( \ - "mrs %0, cpsr @ local_irq_save\n" \ + "mrs %0, cpsr @ local_irq_save_hw\n" \ "cpsid i" \ : "=r" (x) : : "memory", "cc"); \ }) -#define raw_local_irq_enable() __asm__("cpsie i @ __sti" : : : "memory", "cc") -#define raw_local_irq_disable() __asm__("cpsid i @ __cli" : : : "memory", "cc") -#define local_fiq_enable() __asm__("cpsie f @ __stf" : : : "memory", "cc") -#define local_fiq_disable() __asm__("cpsid f @ __clf" : : : "memory", "cc") +#define local_irq_enable_hw() __asm__("cpsie i @ __sti" : : : "memory", "cc") +#define local_irq_disable_hw() __asm__("cpsid i @ __cli" : : : "memory", "cc") +#define local_fiq_enable_hw() __asm__("cpsie f @ __stf" : : : "memory", "cc") +#define local_fiq_disable_hw() __asm__("cpsid f @ __clf" : : : "memory", "cc") #else /* * Save the current interrupt enable state & disable IRQs */ -#define raw_local_irq_save(x) \ +#define local_irq_save_hw_notrace(x) \ ({ \ unsigned long temp; \ (void) (&temp == &x); \ __asm__ __volatile__( \ - "mrs %0, cpsr @ local_irq_save\n" \ + "mrs %0, cpsr @ local_irq_save_hw\n" \ " orr %1, %0, #128\n" \ " msr cpsr_c, %1" \ : "=r" (x), "=r" (temp) \ @@ -44,11 +44,11 @@ /* * Enable IRQs */ -#define raw_local_irq_enable() \ +#define local_irq_enable_hw_notrace() \ ({ \ unsigned long temp; \ __asm__ __volatile__( \ - "mrs %0, cpsr @ local_irq_enable\n" \ + "mrs %0, cpsr @ local_irq_enable_hw\n" \ " bic %0, %0, #128\n" \ " msr cpsr_c, %0" \ : "=r" (temp) \ @@ -59,11 +59,11 @@ /* * Disable IRQs */ -#define raw_local_irq_disable() \ +#define local_irq_disable_hw_notrace() \ ({ \ unsigned long temp; \ __asm__ __volatile__( \ - "mrs %0, cpsr @ local_irq_disable\n" \ + "mrs %0, cpsr @ local_irq_disable_hw\n" \ " orr %0, %0, #128\n" \ " msr cpsr_c, %0" \ : "=r" (temp) \ @@ -74,7 +74,7 @@ /* * Enable FIQs */ -#define local_fiq_enable() \ +#define local_fiq_enable_hw_notrace() \ ({ \ unsigned long temp; \ __asm__ __volatile__( \ @@ -89,7 +89,7 @@ /* * Disable FIQs */ -#define local_fiq_disable() \ +#define local_fiq_disable_hw_notrace() \ ({ \ unsigned long temp; \ __asm__ __volatile__( \ @@ -106,19 +106,19 @@ /* * Save the current interrupt enable state. */ -#define raw_local_save_flags(x) \ +#define local_save_flags_hw(x) \ ({ \ __asm__ __volatile__( \ - "mrs %0, cpsr @ local_save_flags" \ + "mrs %0, cpsr @ local_save_flags_hw" \ : "=r" (x) : : "memory", "cc"); \ }) /* * restore saved IRQ & FIQ state */ -#define raw_local_irq_restore(x) \ +#define local_irq_restore_hw_notrace(x) \ __asm__ __volatile__( \ - "msr cpsr_c, %0 @ local_irq_restore\n" \ + "msr cpsr_c, %0 @ local_irq_restore_hw\n" \ : \ : "r" (x) \ : "memory", "cc") @@ -128,5 +128,93 @@ (int)((flags) & PSR_I_BIT); \ }) +#define irqs_disabled_hw() \ +({ \ + unsigned long flags; \ + local_save_flags_hw(flags); \ + raw_irqs_disabled_flags(flags); \ +}) + + +#ifdef CONFIG_IPIPE + +void __ipipe_stall_root(void); +void __ipipe_unstall_root(void); +unsigned long __ipipe_test_root(void); +unsigned long __ipipe_test_and_stall_root(void); +void __ipipe_restore_root(unsigned long flags); + +/* PSR_I_BIT is bit no. 7 and is set if interrupts are _disabled_ */ +#define raw_local_irq_save(flags) ((flags) = __ipipe_test_and_stall_root() << 7) +#define raw_local_irq_enable() __ipipe_unstall_root() +#define raw_local_irq_disable() __ipipe_stall_root() +#define local_fiq_enable() __ipipe_unstall_root() +#define local_fiq_disable() __ipipe_stall_root() +#define raw_local_save_flags(flags) ((flags) = __ipipe_test_root() << 7) +#define raw_local_irq_restore(flags) __ipipe_restore_root(flags & (1 << 7)) + +#define raw_irqs_disabled() __ipipe_test_root() + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + +#include + +#define local_irq_disable_hw() do { \ + if (!irqs_disabled_hw()) { \ + local_irq_disable_hw_notrace(); \ + ipipe_trace_begin(0x80000000); \ + } \ +} while (0) +#define local_irq_enable_hw() do { \ + if (irqs_disabled_hw()) { \ + ipipe_trace_end(0x80000000); \ + local_irq_enable_hw_notrace(); \ + } \ +} while (0) +#define local_irq_save_hw(x) do { \ + local_save_flags_hw(x); \ + if (local_test_iflag_hw(x)) { \ + local_irq_disable_hw_notrace(); \ + ipipe_trace_begin(0x80000001); \ + } \ +} while (0) +#define local_irq_restore_hw(x) do { \ + if (local_test_iflag_hw(x)) \ + ipipe_trace_end(0x80000001); \ + local_irq_restore_hw_notrace(x); \ +} while (0) + +#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */ + +#define local_irq_save_hw(flags) local_irq_save_hw_notrace(flags) +#define local_irq_enable_hw() local_irq_enable_hw_notrace() +#define local_irq_disable_hw() local_irq_disable_hw_notrace() +#define local_fiq_enable_hw() local_fiq_enable_hw_notrace() +#define local_fiq_disable_hw() local_fiq_disable_hw_notrace() +#define local_irq_restore_hw(flags) local_irq_restore_hw_notrace(flags) + +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ + +#else /* !CONFIG_IPIPE */ + +#define raw_local_irq_save(flags) local_irq_save_hw_notrace(flags) +#define raw_local_irq_enable() local_irq_enable_hw_notrace() +#define raw_local_irq_disable() local_irq_disable_hw_notrace() +#define local_fiq_enable() local_fiq_enable_hw_notrace() +#define local_fiq_disable() local_fiq_disable_hw_notrace() +#define raw_local_save_flags(flags) local_save_flags_hw(flags) +#define raw_local_irq_restore(flags) local_irq_restore_hw_notrace(flags) + +#define local_irq_save_hw(flags) local_irq_save_hw_notrace(flags) +#define local_irq_enable_hw() local_irq_enable_hw_notrace() +#define local_irq_disable_hw() local_irq_disable_hw_notrace() +#define local_fiq_enable_hw() local_fiq_enable_hw_notrace() +#define local_fiq_disable_hw() local_fiq_disable_hw_notrace() +#define local_irq_restore_hw(flags) local_irq_restore_hw_notrace(flags) + +#define irqs_disabled() irqs_disabled_hw() + +#endif /* CONFIG_IPIPE */ + #endif #endif diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/mmu_context.h ipipe-2.6.19-arm/include/asm-arm/mmu_context.h --- linux-2.6.19/include/asm-arm/mmu_context.h 2007-01-04 22:02:40.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/mmu_context.h 2007-01-15 22:28:42.000000000 +0100 @@ -93,14 +93,17 @@ switch_mm(struct mm_struct *prev, struct struct task_struct *tsk) { #ifdef CONFIG_MMU - unsigned int cpu = smp_processor_id(); + unsigned int cpu = smp_processor_id_hw(); if (prev != next) { + unsigned long flags; + local_irq_save_hw_cond(flags); cpu_set(cpu, next->cpu_vm_mask); check_context(next); cpu_switch_mm(next->pgd, next); if (cache_is_vivt()) cpu_clear(cpu, prev->cpu_vm_mask); + local_irq_restore_hw_cond(flags); } #endif } diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/pgalloc.h ipipe-2.6.19-arm/include/asm-arm/pgalloc.h --- linux-2.6.19/include/asm-arm/pgalloc.h 2007-01-04 22:02:40.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/pgalloc.h 2007-01-15 21:38:06.000000000 +0100 @@ -23,6 +23,11 @@ #define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER)) #define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL)) +static inline void set_pgdir(unsigned long address, pgd_t entry) +{ + /* nop */ +} + /* * Since we have only two-level page tables, these are trivial */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/asm-arm/system.h ipipe-2.6.19-arm/include/asm-arm/system.h --- linux-2.6.19/include/asm-arm/system.h 2007-01-04 22:05:06.000000000 +0100 +++ ipipe-2.6.19-arm/include/asm-arm/system.h 2007-01-15 22:39:51.000000000 +0100 @@ -286,17 +286,17 @@ static inline unsigned long __xchg(unsig #error SMP is not supported on this platform #endif case 1: - raw_local_irq_save(flags); + local_irq_save_hw(flags); ret = *(volatile unsigned char *)ptr; *(volatile unsigned char *)ptr = x; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); break; case 4: - raw_local_irq_save(flags); + local_irq_save_hw(flags); ret = *(volatile unsigned long *)ptr; *(volatile unsigned long *)ptr = x; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); break; #else case 1: diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/linux/hardirq.h ipipe-2.6.19-arm/include/linux/hardirq.h --- linux-2.6.19/include/linux/hardirq.h 2007-01-04 22:05:12.000000000 +0100 +++ ipipe-2.6.19-arm/include/linux/hardirq.h 2007-01-15 21:33:00.000000000 +0100 @@ -128,7 +128,7 @@ static inline void account_system_vtime( */ extern void irq_exit(void); -#define nmi_enter() do { lockdep_off(); irq_enter(); } while (0) -#define nmi_exit() do { __irq_exit(); lockdep_on(); } while (0) +#define nmi_enter() do { if (ipipe_root_domain_p) { lockdep_off(); irq_enter(); } } while (0) +#define nmi_exit() do { if (ipipe_root_domain_p) { __irq_exit(); lockdep_on(); } } while (0) #endif /* LINUX_HARDIRQ_H */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/linux/ipipe.h ipipe-2.6.19-arm/include/linux/ipipe.h --- linux-2.6.19/include/linux/ipipe.h 1970-01-01 01:00:00.000000000 +0100 +++ ipipe-2.6.19-arm/include/linux/ipipe.h 2007-01-15 21:33:00.000000000 +0100 @@ -0,0 +1,710 @@ +/* -*- linux-c -*- + * include/linux/ipipe.h + * + * Copyright (C) 2002-2005 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_H +#define __LINUX_IPIPE_H + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_IPIPE + +#define IPIPE_VERSION_STRING IPIPE_ARCH_STRING +#define IPIPE_RELEASE_NUMBER ((IPIPE_MAJOR_NUMBER << 16) | \ + (IPIPE_MINOR_NUMBER << 8) | \ + (IPIPE_PATCH_NUMBER)) + +#ifndef BROKEN_BUILTIN_RETURN_ADDRESS +#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0)) +#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_address(1)) +#endif /* !BUILTIN_RETURN_ADDRESS */ + +#define IPIPE_ROOT_PRIO 100 +#define IPIPE_ROOT_ID 0 +#define IPIPE_ROOT_NPTDKEYS 4 /* Must be <= BITS_PER_LONG */ + +#define IPIPE_RESET_TIMER 0x1 +#define IPIPE_GRAB_TIMER 0x2 + +/* Global domain flags */ +#define IPIPE_SPRINTK_FLAG 0 /* Synchronous printk() allowed */ +#define IPIPE_AHEAD_FLAG 1 /* Domain always heads the pipeline */ + +/* Per-cpu pipeline status */ +#define IPIPE_STALL_FLAG 0 /* Stalls a pipeline stage -- guaranteed at bit #0 */ +#define IPIPE_SYNC_FLAG 1 /* The interrupt syncer is running for the domain */ +#define IPIPE_NOSTACK_FLAG 2 /* Domain currently runs on a foreign stack */ + +#define IPIPE_SYNC_MASK (1 << IPIPE_SYNC_FLAG) + +/* Interrupt control bits */ +#define IPIPE_HANDLE_FLAG 0 +#define IPIPE_PASS_FLAG 1 +#define IPIPE_ENABLE_FLAG 2 +#define IPIPE_DYNAMIC_FLAG IPIPE_HANDLE_FLAG +#define IPIPE_STICKY_FLAG 3 +#define IPIPE_SYSTEM_FLAG 4 +#define IPIPE_LOCK_FLAG 5 +#define IPIPE_WIRED_FLAG 6 +#define IPIPE_EXCLUSIVE_FLAG 7 + +#define IPIPE_HANDLE_MASK (1 << IPIPE_HANDLE_FLAG) +#define IPIPE_PASS_MASK (1 << IPIPE_PASS_FLAG) +#define IPIPE_ENABLE_MASK (1 << IPIPE_ENABLE_FLAG) +#define IPIPE_DYNAMIC_MASK IPIPE_HANDLE_MASK +#define IPIPE_STICKY_MASK (1 << IPIPE_STICKY_FLAG) +#define IPIPE_SYSTEM_MASK (1 << IPIPE_SYSTEM_FLAG) +#define IPIPE_LOCK_MASK (1 << IPIPE_LOCK_FLAG) +#define IPIPE_WIRED_MASK (1 << IPIPE_WIRED_FLAG) +#define IPIPE_EXCLUSIVE_MASK (1 << IPIPE_EXCLUSIVE_FLAG) + +#define IPIPE_DEFAULT_MASK (IPIPE_HANDLE_MASK|IPIPE_PASS_MASK) +#define IPIPE_STDROOT_MASK (IPIPE_HANDLE_MASK|IPIPE_PASS_MASK|IPIPE_SYSTEM_MASK) + +#define IPIPE_EVENT_SELF 0x80000000 + +/* Number of virtual IRQs */ +#define IPIPE_NR_VIRQS BITS_PER_LONG +/* First virtual IRQ # */ +#define IPIPE_VIRQ_BASE (((IPIPE_NR_XIRQS + BITS_PER_LONG - 1) / BITS_PER_LONG) * BITS_PER_LONG) +/* Total number of IRQ slots */ +#define IPIPE_NR_IRQS (IPIPE_VIRQ_BASE + IPIPE_NR_VIRQS) +/* Number of indirect words needed to map the whole IRQ space. */ +#define IPIPE_IRQ_IWORDS ((IPIPE_NR_IRQS + BITS_PER_LONG - 1) / BITS_PER_LONG) +#define IPIPE_IRQ_IMASK (BITS_PER_LONG - 1) +#define IPIPE_IRQMASK_ANY (~0L) +#define IPIPE_IRQMASK_VIRT (IPIPE_IRQMASK_ANY << (IPIPE_VIRQ_BASE / BITS_PER_LONG)) + +#ifdef CONFIG_SMP + +#define IPIPE_NR_CPUS NR_CPUS +#define ipipe_declare_cpuid int cpuid +#define ipipe_load_cpuid() do { \ + cpuid = ipipe_processor_id(); \ + } while(0) +#define ipipe_lock_cpu(flags) do { \ + local_irq_save_hw(flags); \ + cpuid = ipipe_processor_id(); \ + } while(0) +#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags) +#define ipipe_get_cpu(flags) ipipe_lock_cpu(flags) +#define ipipe_put_cpu(flags) ipipe_unlock_cpu(flags) +#define ipipe_current_domain per_cpu(ipipe_percpu_domain, ipipe_processor_id()) + +#else /* !CONFIG_SMP */ + +#define IPIPE_NR_CPUS 1 +#define ipipe_declare_cpuid const int cpuid = 0 +#define ipipe_load_cpuid() do { } while(0) +#define ipipe_lock_cpu(flags) local_irq_save_hw(flags) +#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags) +#define ipipe_get_cpu(flags) do { (void)(flags); } while(0) +#define ipipe_put_cpu(flags) do { } while(0) +#define ipipe_current_domain per_cpu(ipipe_percpu_domain, 0) + +#endif /* CONFIG_SMP */ + +#define ipipe_virtual_irq_p(irq) ((irq) >= IPIPE_VIRQ_BASE && \ + (irq) < IPIPE_NR_IRQS) + +typedef void (*ipipe_irq_handler_t)(unsigned irq, + void *cookie); + +#define IPIPE_SAME_HANDLER ((ipipe_irq_handler_t)(-1)) + +typedef int (*ipipe_irq_ackfn_t)(unsigned irq); + +typedef int (*ipipe_event_handler_t)(unsigned event, + struct ipipe_domain *from, + void *data); +struct ipipe_domain { + + struct list_head p_link; /* Link in pipeline */ + + struct ipcpudata { + unsigned long status; + unsigned long irq_pending_hi; + unsigned long irq_pending_lo[IPIPE_IRQ_IWORDS]; + struct ipirqcnt { + unsigned long pending_hits; + unsigned long total_hits; + } irq_counters[IPIPE_NR_IRQS]; + unsigned long long evsync; + } ____cacheline_aligned_in_smp cpudata[IPIPE_NR_CPUS]; + + struct { + unsigned long control; + ipipe_irq_ackfn_t acknowledge; + ipipe_irq_handler_t handler; + void *cookie; + } ____cacheline_aligned irqs[IPIPE_NR_IRQS]; + + ipipe_event_handler_t evhand[IPIPE_NR_EVENTS]; /* Event handlers. */ + unsigned long long evself; /* Self-monitored event bits. */ + unsigned long flags; + unsigned domid; + const char *name; + int priority; + void *pdd; + struct mutex mutex; +}; + +#define IPIPE_HEAD_PRIORITY (-1) /* For domains always heading the pipeline */ + +struct ipipe_domain_attr { + + unsigned domid; /* Domain identifier -- Magic value set by caller */ + const char *name; /* Domain name -- Warning: won't be dup'ed! */ + int priority; /* Priority in interrupt pipeline */ + void (*entry) (void); /* Domain entry point */ + void *pdd; /* Per-domain (opaque) data pointer */ +}; + +/* The following macros must be used hw interrupts off. */ + +#define __ipipe_irq_cookie(ipd,irq) (ipd)->irqs[irq].cookie +#define __ipipe_irq_handler(ipd,irq) (ipd)->irqs[irq].handler + +#define __ipipe_cpudata_irq_hits(ipd,cpuid,irq) ((ipd)->cpudata[cpuid].irq_counters[irq].total_hits) + +#define __ipipe_set_irq_bit(ipd,cpuid,irq) \ +do { \ + if (!test_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) { \ + __set_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \ + __set_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[cpuid].irq_pending_hi); \ + } \ +} while(0) + +#define __ipipe_clear_pend(ipd,cpuid,irq) \ +do { \ + __clear_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \ + if ((ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT] == 0) \ + __clear_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[cpuid].irq_pending_hi); \ +} while(0) + +#define __ipipe_lock_irq(ipd,cpuid,irq) \ +do { \ + if (!test_and_set_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) \ + __ipipe_clear_pend(ipd,cpuid,irq); \ +} while(0) + +#define __ipipe_unlock_irq(ipd,irq) \ +do { \ + int __cpuid, __nr_cpus = num_online_cpus(); \ + if (test_and_clear_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) \ + for (__cpuid = 0; __cpuid < __nr_cpus; __cpuid++) \ + if ((ipd)->cpudata[__cpuid].irq_counters[irq].pending_hits > 0) { /* We need atomic ops next. */ \ + set_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[__cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \ + set_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[__cpuid].irq_pending_hi); \ + } \ +} while(0) + +#define __ipipe_clear_irq(ipd,irq) \ +do { \ + int __cpuid, __nr_cpus = num_online_cpus(); \ + clear_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control); \ + for (__cpuid = 0; __cpuid < __nr_cpus; __cpuid++) { \ + (ipd)->cpudata[__cpuid].irq_counters[irq].pending_hits = 0; \ + __ipipe_clear_pend(ipd,__cpuid,irq); \ + } \ +} while(0) + +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) +#define write_lock_hw(x) __raw_write_lock(&(x)->raw_lock) +#define write_trylock_hw(x) __raw_write_trylock(&(x)->raw_lock) +#define write_unlock_hw(x) __raw_write_unlock(&(x)->raw_lock) +#define read_lock_hw(x) __raw_read_lock(&(x)->raw_lock) +#define read_trylock_hw(x) __raw_read_trylock(&(x)->raw_lock) +#define read_unlock_hw(x) __raw_read_unlock(&(x)->raw_lock) +#else /* UP non-debug */ +#define write_lock_hw(lock) do { (void)(lock); } while (0) +#define write_trylock_hw(lock) ({ (void)(lock); 1; }) +#define write_unlock_hw(lock) do { (void)(lock); } while (0) +#define read_lock_hw(lock) do { (void)(lock); } while (0) +#define read_trylock_hw(lock) ({ (void)(lock); 1; }) +#define read_unlock_hw(lock) do { (void)(lock); } while (0) +#endif /* CONFIG_SMP || CONFIG_DEBUG_SPINLOCK */ + +typedef rwlock_t ipipe_rwlock_t; +#define IPIPE_RW_LOCK_UNLOCKED RW_LOCK_UNLOCKED + +#define read_lock_irqsave_hw(lock, flags) \ +do { \ + local_irq_save_hw(flags); \ + read_lock_hw(lock); \ +} while (0) + +#define read_unlock_irqrestore_hw(lock, flags) \ +do { \ + read_unlock_hw(lock); \ + local_irq_restore_hw(flags); \ +} while (0) + +#define write_lock_irqsave_hw(lock, flags) \ +do { \ + local_irq_save_hw(flags); \ + write_lock_hw(lock); \ +} while (0) + +#define write_unlock_irqrestore_hw(lock, flags) \ +do { \ + write_unlock_hw(lock); \ + local_irq_restore_hw(flags); \ +} while (0) + +DECLARE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain); + +extern struct ipipe_domain ipipe_root; + +#define ipipe_root_domain (&ipipe_root) + +extern unsigned __ipipe_printk_virq; + +extern unsigned long __ipipe_virtual_irq_map; + +extern struct list_head __ipipe_pipeline; + +extern ipipe_spinlock_t __ipipe_pipelock; + +extern int __ipipe_event_monitors[]; + +/* Private interface */ + +void ipipe_init(void); + +#ifdef CONFIG_PROC_FS +void ipipe_init_proc(void); + +#ifdef CONFIG_IPIPE_TRACE +void __ipipe_init_tracer(void); +#else /* !CONFIG_IPIPE_TRACE */ +#define __ipipe_init_tracer() do { } while(0) +#endif /* CONFIG_IPIPE_TRACE */ + +#else /* !CONFIG_PROC_FS */ +#define ipipe_init_proc() do { } while(0) +#endif /* CONFIG_PROC_FS */ + +void __ipipe_init_stage(struct ipipe_domain *ipd); + +void __ipipe_cleanup_domain(struct ipipe_domain *ipd); + +void __ipipe_add_domain_proc(struct ipipe_domain *ipd); + +void __ipipe_remove_domain_proc(struct ipipe_domain *ipd); + +void __ipipe_flush_printk(unsigned irq, void *cookie); + +void __ipipe_stall_root(void); + +void __ipipe_unstall_root(void); + +unsigned long __ipipe_test_root(void); + +unsigned long __ipipe_test_and_stall_root(void); + +void fastcall __ipipe_walk_pipeline(struct list_head *pos, int cpuid); + +void fastcall __ipipe_restore_root(unsigned long x); + +int fastcall __ipipe_schedule_irq(unsigned irq, struct list_head *head); + +int fastcall __ipipe_dispatch_event(unsigned event, void *data); + +int fastcall __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq); + +void fastcall __ipipe_sync_stage(unsigned long syncmask); + +#ifndef __ipipe_sync_pipeline +#define __ipipe_sync_pipeline(syncmask) __ipipe_sync_stage(syncmask) +#endif + +#ifndef __ipipe_run_irqtail +#define __ipipe_run_irqtail() do { } while(0) +#endif + +#define __ipipe_pipeline_head_p(ipd) (&(ipd)->p_link == __ipipe_pipeline.next) + +/* + * Keep the following as a macro, so that client code could check for + * the support of the invariant pipeline head optimization. + */ +#define __ipipe_pipeline_head() list_entry(__ipipe_pipeline.next,struct ipipe_domain,p_link) + +#define __ipipe_event_monitored_p(ev) \ + (__ipipe_event_monitors[ev] > 0 || (ipipe_current_domain->evself & (1LL << ev))) + +#ifdef CONFIG_SMP + +cpumask_t __ipipe_set_irq_affinity(unsigned irq, + cpumask_t cpumask); + +int fastcall __ipipe_send_ipi(unsigned ipi, + cpumask_t cpumask); + +#endif /* CONFIG_SMP */ + +/* Called with hw interrupts off. */ +static inline void __ipipe_switch_to(struct ipipe_domain *out, + struct ipipe_domain *in, int cpuid) +{ + void ipipe_suspend_domain(void); + + /* + * "in" is guaranteed to be closer than "out" from the head of the + * pipeline (and obviously different). + */ + + out->cpudata[cpuid].evsync = 0; + per_cpu(ipipe_percpu_domain, cpuid) = in; + + ipipe_suspend_domain(); /* Sync stage and propagate interrupts. */ + ipipe_load_cpuid(); /* Processor might have changed. */ + + if (per_cpu(ipipe_percpu_domain, cpuid) == in) + /* + * Otherwise, something has changed the current domain under + * our feet recycling the register set; do not override. + */ + per_cpu(ipipe_percpu_domain, cpuid) = out; +} + +#define ipipe_sigwake_notify(p) \ +do { \ + if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SIGWAKE)) \ + __ipipe_dispatch_event(IPIPE_EVENT_SIGWAKE,p); \ +} while(0) + +#define ipipe_exit_notify(p) \ +do { \ + if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_EXIT)) \ + __ipipe_dispatch_event(IPIPE_EVENT_EXIT,p); \ +} while(0) + +#define ipipe_setsched_notify(p) \ +do { \ + if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SETSCHED)) \ + __ipipe_dispatch_event(IPIPE_EVENT_SETSCHED,p); \ +} while(0) + +#define ipipe_schedule_notify(prev, next) \ +do { \ + if ((((prev)->flags|(next)->flags) & PF_EVNOTIFY) && \ + __ipipe_event_monitored_p(IPIPE_EVENT_SCHEDULE)) \ + __ipipe_dispatch_event(IPIPE_EVENT_SCHEDULE,next); \ +} while(0) + +#define ipipe_trap_notify(ex, regs) \ +({ \ + ipipe_declare_cpuid; \ + int ret = 0; \ + ipipe_load_cpuid(); \ + if ((test_bit(IPIPE_NOSTACK_FLAG, &ipipe_current_domain->cpudata[cpuid].status) || \ + ((current)->flags & PF_EVNOTIFY)) && \ + __ipipe_event_monitored_p(ex)) \ + ret = __ipipe_dispatch_event(ex, regs); \ + ret; \ +}) + +static inline void ipipe_init_notify(struct task_struct *p) +{ + if (__ipipe_event_monitored_p(IPIPE_EVENT_INIT)) + __ipipe_dispatch_event(IPIPE_EVENT_INIT,p); +} + +struct mm_struct; + +static inline void ipipe_cleanup_notify(struct mm_struct *mm) +{ + if (__ipipe_event_monitored_p(IPIPE_EVENT_CLEANUP)) + __ipipe_dispatch_event(IPIPE_EVENT_CLEANUP,mm); +} + +/* Public interface */ + +int ipipe_register_domain(struct ipipe_domain *ipd, + struct ipipe_domain_attr *attr); + +int ipipe_unregister_domain(struct ipipe_domain *ipd); + +void ipipe_suspend_domain(void); + +int ipipe_virtualize_irq(struct ipipe_domain *ipd, + unsigned irq, + ipipe_irq_handler_t handler, + void *cookie, + ipipe_irq_ackfn_t acknowledge, + unsigned modemask); + +int ipipe_control_irq(unsigned irq, + unsigned clrmask, + unsigned setmask); + +unsigned ipipe_alloc_virq(void); + +int ipipe_free_virq(unsigned virq); + +int fastcall ipipe_trigger_irq(unsigned irq); + +static inline int ipipe_propagate_irq(unsigned irq) +{ + return __ipipe_schedule_irq(irq, ipipe_current_domain->p_link.next); +} + +static inline int ipipe_schedule_irq(unsigned irq) +{ + return __ipipe_schedule_irq(irq, &ipipe_current_domain->p_link); +} + +void fastcall ipipe_stall_pipeline_from(struct ipipe_domain *ipd); + +unsigned long fastcall ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd); + +void fastcall ipipe_unstall_pipeline_from(struct ipipe_domain *ipd); + +unsigned long fastcall ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd); + +void fastcall ipipe_restore_pipeline_from(struct ipipe_domain *ipd, + unsigned long x); + +static inline unsigned long ipipe_test_pipeline_from(struct ipipe_domain *ipd) +{ + unsigned long flags, x; + ipipe_declare_cpuid; + + ipipe_get_cpu(flags); + x = test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + ipipe_put_cpu(flags); + + return x; +} + +static inline void ipipe_restore_pipeline_nosync(struct ipipe_domain *ipd, + unsigned long x, int cpuid) +{ + /* + * If cpuid is current, then it must be held on entry + * (ipipe_get_cpu/local_irq_save_hw/local_irq_disable_hw). + */ + + if (x) + __set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + else + __clear_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); +} + +static inline void ipipe_stall_pipeline_head(void) +{ + ipipe_declare_cpuid; + + local_irq_disable_hw(); + ipipe_load_cpuid(); + __set_bit(IPIPE_STALL_FLAG, &__ipipe_pipeline_head()->cpudata[cpuid].status); +} + +static inline unsigned long ipipe_test_and_stall_pipeline_head(void) +{ + ipipe_declare_cpuid; + + local_irq_disable_hw(); + ipipe_load_cpuid(); + return __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_pipeline_head()->cpudata[cpuid].status); +} + +void ipipe_unstall_pipeline_head(void); + +void fastcall __ipipe_restore_pipeline_head(struct ipipe_domain *head, + unsigned long x); + +static inline void ipipe_restore_pipeline_head(unsigned long x) +{ + struct ipipe_domain *head = __ipipe_pipeline_head(); + /* On some archs, __test_and_set_bit() might return different + * truth value than test_bit(), so we test the exclusive OR of + * both statuses, assuming that the lowest bit is always set in + * the truth value (if this is wrong, the failed optimization will + * be caught in __ipipe_restore_pipeline_head() if + * CONFIG_DEBUG_KERNEL is set). */ + if ((x ^ test_bit(IPIPE_STALL_FLAG, &head->cpudata[ipipe_processor_id()].status)) & 1) + __ipipe_restore_pipeline_head(head,x); +} + +#define ipipe_unstall_pipeline() \ + ipipe_unstall_pipeline_from(ipipe_current_domain) + +#define ipipe_test_and_unstall_pipeline() \ + ipipe_test_and_unstall_pipeline_from(ipipe_current_domain) + +#define ipipe_test_pipeline() \ + ipipe_test_pipeline_from(ipipe_current_domain) + +#define ipipe_test_and_stall_pipeline() \ + ipipe_test_and_stall_pipeline_from(ipipe_current_domain) + +#define ipipe_stall_pipeline() \ + ipipe_stall_pipeline_from(ipipe_current_domain) + +#define ipipe_restore_pipeline(x) \ + ipipe_restore_pipeline_from(ipipe_current_domain, (x)) + +void ipipe_init_attr(struct ipipe_domain_attr *attr); + +int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo); + +int ipipe_tune_timer(unsigned long ns, + int flags); + +unsigned long ipipe_critical_enter(void (*syncfn) (void)); + +void ipipe_critical_exit(unsigned long flags); + +static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd) +{ + set_bit(IPIPE_SPRINTK_FLAG, &ipd->flags); +} + +static inline void ipipe_set_printk_async(struct ipipe_domain *ipd) +{ + clear_bit(IPIPE_SPRINTK_FLAG, &ipd->flags); +} + +static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd) +{ + /* Must be called hw interrupts off. */ + ipipe_declare_cpuid; + ipipe_load_cpuid(); + __set_bit(IPIPE_NOSTACK_FLAG, &ipd->cpudata[cpuid].status); +} + +static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd) +{ + /* Must be called hw interrupts off. */ + ipipe_declare_cpuid; + ipipe_load_cpuid(); + __clear_bit(IPIPE_NOSTACK_FLAG, &ipd->cpudata[cpuid].status); +} + +#define ipipe_safe_current() \ +({ \ + ipipe_declare_cpuid; \ + struct task_struct *p; \ + ipipe_load_cpuid(); \ + p = test_bit(IPIPE_NOSTACK_FLAG, \ + &per_cpu(ipipe_percpu_domain, cpuid)->cpudata[cpuid].status) ? &init_task : current; \ + p; \ +}) + +ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd, + unsigned event, + ipipe_event_handler_t handler); + +cpumask_t ipipe_set_irq_affinity(unsigned irq, + cpumask_t cpumask); + +int fastcall ipipe_send_ipi(unsigned ipi, + cpumask_t cpumask); + +int ipipe_setscheduler_root(struct task_struct *p, + int policy, + int prio); + +int ipipe_reenter_root(struct task_struct *prev, + int policy, + int prio); + +int ipipe_alloc_ptdkey(void); + +int ipipe_free_ptdkey(int key); + +int fastcall ipipe_set_ptd(int key, + void *value); + +void fastcall *ipipe_get_ptd(int key); + +#define local_irq_enable_hw_cond() local_irq_enable_hw() +#define local_irq_disable_hw_cond() local_irq_disable_hw() +#define local_irq_save_hw_cond(flags) local_irq_save_hw(flags) +#define local_irq_restore_hw_cond(flags) local_irq_restore_hw(flags) +#define local_irq_disable_head() ipipe_stall_pipeline_head() + +#define local_irq_enable_nohead(ipd) \ + do { \ + if (!__ipipe_pipeline_head_p(ipd)) \ + local_irq_enable_hw(); \ + } while(0) + +#define local_irq_disable_nohead(ipd) \ + do { \ + if (!__ipipe_pipeline_head_p(ipd)) \ + local_irq_disable_hw(); \ + } while(0) + +#define smp_processor_id_hw() ipipe_processor_id() + +#define ipipe_irq_lock(irq) \ + do { \ + ipipe_declare_cpuid; \ + ipipe_load_cpuid(); \ + __ipipe_lock_irq(per_cpu(ipipe_percpu_domain, cpuid), cpuid, irq);\ + } while(0) + +#define ipipe_irq_unlock(irq) \ + do { \ + ipipe_declare_cpuid; \ + ipipe_load_cpuid(); \ + __ipipe_unlock_irq(per_cpu(ipipe_percpu_domain, cpuid), irq); \ + } while(0) + +#define ipipe_root_domain_p (ipipe_current_domain == ipipe_root_domain) + +#else /* !CONFIG_IPIPE */ + +#define ipipe_init() do { } while(0) +#define ipipe_suspend_domain() do { } while(0) +#define ipipe_sigwake_notify(p) do { } while(0) +#define ipipe_setsched_notify(p) do { } while(0) +#define ipipe_init_notify(p) do { } while(0) +#define ipipe_exit_notify(p) do { } while(0) +#define ipipe_cleanup_notify(mm) do { } while(0) +#define ipipe_trap_notify(t,r) 0 +#define ipipe_init_proc() do { } while(0) + +#define local_irq_enable_hw_cond() do { } while(0) +#define local_irq_disable_hw_cond() do { } while(0) +#define local_irq_save_hw_cond(flags) do { (void)(flags); } while(0) +#define local_irq_restore_hw_cond(flags) do { } while(0) +#define smp_processor_id_hw() smp_processor_id() + +#define ipipe_irq_lock(irq) do { } while(0) +#define ipipe_irq_unlock(irq) do { } while(0) + +#define ipipe_root_domain_p 1 +#define ipipe_safe_current current + +#define local_irq_disable_head() local_irq_disable() + +#endif /* CONFIG_IPIPE */ + +#endif /* !__LINUX_IPIPE_H */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/linux/ipipe_trace.h ipipe-2.6.19-arm/include/linux/ipipe_trace.h --- linux-2.6.19/include/linux/ipipe_trace.h 1970-01-01 01:00:00.000000000 +0100 +++ ipipe-2.6.19-arm/include/linux/ipipe_trace.h 2007-01-15 21:33:00.000000000 +0100 @@ -0,0 +1,44 @@ +/* -*- linux-c -*- + * include/linux/ipipe_trace.h + * + * Copyright (C) 2005 Luotao Fu. + * 2005, 2006 Jan Kiszka. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef _LINUX_IPIPE_TRACE_H +#define _LINUX_IPIPE_TRACE_H + +#ifdef CONFIG_IPIPE_TRACE + +#include + +void ipipe_trace_begin(unsigned long v); +void ipipe_trace_end(unsigned long v); +void ipipe_trace_freeze(unsigned long v); +void ipipe_trace_special(unsigned char special_id, unsigned long v); +void ipipe_trace_pid(pid_t pid, short prio); + +int ipipe_trace_max_reset(void); +int ipipe_trace_frozen_reset(void); + +void ipipe_trace_panic_freeze(void); +void ipipe_trace_panic_dump(void); + +#endif /* CONFIG_IPIPE_TRACE */ + +#endif /* !__LINUX_IPIPE_H */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/linux/irq.h ipipe-2.6.19-arm/include/linux/irq.h --- linux-2.6.19/include/linux/irq.h 2007-01-04 22:05:12.000000000 +0100 +++ ipipe-2.6.19-arm/include/linux/irq.h 2007-01-15 21:33:00.000000000 +0100 @@ -146,6 +146,12 @@ struct irq_chip { * Pad this out to 32 bytes for cache and indexing reasons. */ struct irq_desc { +#ifdef CONFIG_IPIPE + void fastcall (*ipipe_ack)(unsigned int irq, + struct irq_desc *desc); + void fastcall (*ipipe_end)(unsigned int irq, + struct irq_desc *desc); +#endif /* CONFIG_IPIPE */ irq_flow_handler_t handle_irq; struct irq_chip *chip; void *handler_data; diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/linux/linkage.h ipipe-2.6.19-arm/include/linux/linkage.h --- linux-2.6.19/include/linux/linkage.h 2007-01-04 22:05:12.000000000 +0100 +++ ipipe-2.6.19-arm/include/linux/linkage.h 2007-01-15 21:33:00.000000000 +0100 @@ -64,4 +64,8 @@ #define fastcall #endif +#ifndef notrace +#define notrace __attribute__((no_instrument_function)) +#endif + #endif diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/linux/preempt.h ipipe-2.6.19-arm/include/linux/preempt.h --- linux-2.6.19/include/linux/preempt.h 2007-01-04 22:02:52.000000000 +0100 +++ ipipe-2.6.19-arm/include/linux/preempt.h 2007-01-15 21:33:00.000000000 +0100 @@ -26,29 +26,44 @@ asmlinkage void preempt_schedule(void); -#define preempt_disable() \ -do { \ - inc_preempt_count(); \ - barrier(); \ +#ifdef CONFIG_IPIPE +#include +DECLARE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain); +extern struct ipipe_domain ipipe_root; +#define ipipe_preempt_guard() (per_cpu(ipipe_percpu_domain, ipipe_processor_id()) == &ipipe_root) +#else /* !CONFIG_IPIPE */ +#define ipipe_preempt_guard() 1 +#endif /* CONFIG_IPIPE */ + +#define preempt_disable() \ +do { \ + if (ipipe_preempt_guard()) { \ + inc_preempt_count(); \ + barrier(); \ + } \ } while (0) -#define preempt_enable_no_resched() \ -do { \ - barrier(); \ - dec_preempt_count(); \ +#define preempt_enable_no_resched() \ +do { \ + if (ipipe_preempt_guard()) { \ + barrier(); \ + dec_preempt_count(); \ + } \ } while (0) -#define preempt_check_resched() \ -do { \ - if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ - preempt_schedule(); \ +#define preempt_check_resched() \ +do { \ + if (ipipe_preempt_guard()) { \ + if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ + preempt_schedule(); \ + } \ } while (0) -#define preempt_enable() \ -do { \ - preempt_enable_no_resched(); \ +#define preempt_enable() \ +do { \ + preempt_enable_no_resched(); \ barrier(); \ - preempt_check_resched(); \ + preempt_check_resched(); \ } while (0) #else diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/linux/sched.h ipipe-2.6.19-arm/include/linux/sched.h --- linux-2.6.19/include/linux/sched.h 2007-01-04 22:05:14.000000000 +0100 +++ ipipe-2.6.19-arm/include/linux/sched.h 2007-01-15 21:33:00.000000000 +0100 @@ -54,6 +54,7 @@ struct sched_param { #include #include #include +#include #include #include @@ -150,6 +151,13 @@ extern unsigned long weighted_cpuload(co /* in tsk->state again */ #define TASK_NONINTERACTIVE 64 #define TASK_DEAD 128 +#ifdef CONFIG_IPIPE +#define TASK_ATOMICSWITCH 512 +#define TASK_NOWAKEUP 1024 +#else /* !CONFIG_IPIPE */ +#define TASK_ATOMICSWITCH 0 +#define TASK_NOWAKEUP 0 +#endif /* CONFIG_IPIPE */ #define __set_task_state(tsk, state_value) \ do { (tsk)->state = (state_value); } while (0) @@ -1015,6 +1023,9 @@ struct task_struct { atomic_t fs_excl; /* holding fs exclusive resources */ struct rcu_head rcu; +#ifdef CONFIG_IPIPE + void *ptd[IPIPE_ROOT_NPTDKEYS]; +#endif /* * cache last used pipe for splice @@ -1115,6 +1126,11 @@ static inline void put_task_struct(struc #define PF_SPREAD_SLAB 0x02000000 /* Spread some slab caches over cpuset */ #define PF_MEMPOLICY 0x10000000 /* Non-default NUMA mempolicy */ #define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */ +#ifdef CONFIG_IPIPE +#define PF_EVNOTIFY 0x40000000 /* Notify other domains about internal events */ +#else +#define PF_EVNOTIFY 0 +#endif /* CONFIG_IPIPE */ /* * Only the _current_ task can read/write to tsk->flags, but other diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/linux/spinlock.h ipipe-2.6.19-arm/include/linux/spinlock.h --- linux-2.6.19/include/linux/spinlock.h 2007-01-04 22:05:14.000000000 +0100 +++ ipipe-2.6.19-arm/include/linux/spinlock.h 2007-01-15 21:33:00.000000000 +0100 @@ -171,7 +171,97 @@ do { \ #define read_trylock(lock) __cond_lock(lock, _read_trylock(lock)) #define write_trylock(lock) __cond_lock(lock, _write_trylock(lock)) -#define spin_lock(lock) _spin_lock(lock) +#undef TYPE_EQUAL +#define TYPE_EQUAL(lock, type) \ + __builtin_types_compatible_p(typeof(lock), type *) + +#define PICK_SPINOP(op, lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) \ + __raw_spin##op(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin##op((spinlock_t *)(lock)); \ +} while (0) + +#define PICK_SPINOP_RAW(op, lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) \ + __raw_spin##op(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + else if (TYPE_EQUAL(lock, spinlock_t)) \ + __raw_spin##op(&((spinlock_t *)(lock))->raw_lock); \ +} while (0) + +#define PICK_SPINLOCK_IRQ(lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + local_irq_disable_hw(); \ + __raw_spin_lock(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin_lock_irq((spinlock_t *)(lock)); \ +} while (0) + +#define PICK_SPINUNLOCK_IRQ(lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + __raw_spin_unlock(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + local_irq_enable_hw(); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin_unlock_irq((spinlock_t *)(lock)); \ +} while (0) + +#define PICK_SPINLOCK_IRQ_RAW(lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + local_irq_disable_hw(); \ + __raw_spin_lock(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + local_irq_disable(); \ + __raw_spin_lock(&((spinlock_t *)(lock))->raw_lock); \ +} while (0) + +#define PICK_SPINUNLOCK_IRQ_RAW(lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + __raw_spin_unlock(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + local_irq_enable_hw(); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + __raw_spin_unlock(&((spinlock_t *)(lock))->raw_lock); \ + local_irq_enable(); \ +} while (0) + +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) +extern int __bad_spinlock_type(void); + +#define PICK_SPINLOCK_IRQSAVE(lock, flags) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + local_irq_save_hw(flags); \ + __raw_spin_lock(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + flags = _spin_lock_irqsave((spinlock_t *)(lock)); \ + else __bad_spinlock_type(); \ +} while (0) +#else +#define PICK_SPINLOCK_IRQSAVE(lock, flags) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + local_irq_save_hw(flags); \ + __raw_spin_lock(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin_lock_irqsave((spinlock_t *)(lock), flags); \ +} while (0) +#endif + +#define PICK_SPINUNLOCK_IRQRESTORE(lock, flags) \ + do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + __raw_spin_unlock(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + local_irq_restore_hw(flags); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin_unlock_irqrestore((spinlock_t *)(lock), flags); \ +} while (0) + +#define spin_lock(lock) PICK_SPINOP(_lock, lock) #ifdef CONFIG_DEBUG_LOCK_ALLOC # define spin_lock_nested(lock, subclass) _spin_lock_nested(lock, subclass) @@ -184,7 +274,7 @@ do { \ #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) -#define spin_lock_irqsave(lock, flags) flags = _spin_lock_irqsave(lock) +#define spin_lock_irqsave(lock, flags) PICK_SPINLOCK_IRQSAVE(lock, flags) #define read_lock_irqsave(lock, flags) flags = _read_lock_irqsave(lock) #define write_lock_irqsave(lock, flags) flags = _write_lock_irqsave(lock) @@ -198,7 +288,7 @@ do { \ #else -#define spin_lock_irqsave(lock, flags) _spin_lock_irqsave(lock, flags) +#define spin_lock_irqsave(lock, flags) PICK_SPINLOCK_IRQSAVE(lock, flags) #define read_lock_irqsave(lock, flags) _read_lock_irqsave(lock, flags) #define write_lock_irqsave(lock, flags) _write_lock_irqsave(lock, flags) #define spin_lock_irqsave_nested(lock, flags, subclass) \ @@ -206,7 +296,7 @@ do { \ #endif -#define spin_lock_irq(lock) _spin_lock_irq(lock) +#define spin_lock_irq(lock) PICK_SPINLOCK_IRQ(lock) #define spin_lock_bh(lock) _spin_lock_bh(lock) #define read_lock_irq(lock) _read_lock_irq(lock) @@ -220,26 +310,25 @@ do { \ */ #if defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT) || \ !defined(CONFIG_SMP) -# define spin_unlock(lock) _spin_unlock(lock) +#define spin_unlock(lock) PICK_SPINOP(_unlock, lock) # define read_unlock(lock) _read_unlock(lock) # define write_unlock(lock) _write_unlock(lock) -# define spin_unlock_irq(lock) _spin_unlock_irq(lock) -# define read_unlock_irq(lock) _read_unlock_irq(lock) -# define write_unlock_irq(lock) _write_unlock_irq(lock) +# define spin_unlock_irq(lock) PICK_SPINUNLOCK_IRQ(lock) +# define read_unlock_irq(lock) _read_unlock_irq(lock) +# define write_unlock_irq(lock) _write_unlock_irq(lock) #else -# define spin_unlock(lock) __raw_spin_unlock(&(lock)->raw_lock) +# define spin_unlock(lock) PICK_SPINOP_RAW(_unlock, lock) # define read_unlock(lock) __raw_read_unlock(&(lock)->raw_lock) # define write_unlock(lock) __raw_write_unlock(&(lock)->raw_lock) -# define spin_unlock_irq(lock) \ - do { __raw_spin_unlock(&(lock)->raw_lock); local_irq_enable(); } while (0) +# define spin_unlock_irq(lock) PICK_SPINUNLOCK_IRQ_RAW(lock) # define read_unlock_irq(lock) \ do { __raw_read_unlock(&(lock)->raw_lock); local_irq_enable(); } while (0) # define write_unlock_irq(lock) \ do { __raw_write_unlock(&(lock)->raw_lock); local_irq_enable(); } while (0) #endif -#define spin_unlock_irqrestore(lock, flags) \ - _spin_unlock_irqrestore(lock, flags) +#define spin_unlock_irqrestore(lock, flags) \ + PICK_SPINUNLOCK_IRQRESTORE(lock, flags) #define spin_unlock_bh(lock) _spin_unlock_bh(lock) #define read_unlock_irqrestore(lock, flags) \ @@ -286,4 +375,16 @@ extern int _atomic_dec_and_lock(atomic_t */ #define spin_can_lock(lock) (!spin_is_locked(lock)) +#ifdef CONFIG_IPIPE +#define spin_lock_irqsave_cond(lock, flags) \ + spin_lock_irqsave(lock,flags) +#define spin_unlock_irqrestore_cond(lock, flags) \ + spin_unlock_irqrestore(lock,flags) +#else +#define spin_lock_irqsave_cond(lock, flags) \ + do { (void)(flags); spin_lock(lock); } while(0) +#define spin_unlock_irqrestore_cond(lock, flags) \ + spin_unlock(lock) +#endif + #endif /* __LINUX_SPINLOCK_H */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/include/linux/spinlock_types.h ipipe-2.6.19-arm/include/linux/spinlock_types.h --- linux-2.6.19/include/linux/spinlock_types.h 2007-01-04 22:02:52.000000000 +0100 +++ ipipe-2.6.19-arm/include/linux/spinlock_types.h 2007-01-15 21:33:00.000000000 +0100 @@ -31,6 +31,10 @@ typedef struct { #endif } spinlock_t; +typedef struct { + raw_spinlock_t __raw_lock; +} __ipipe_spinlock_t; + #define SPINLOCK_MAGIC 0xdead4ead typedef struct { @@ -86,9 +90,19 @@ typedef struct { #endif #define SPIN_LOCK_UNLOCKED __SPIN_LOCK_UNLOCKED(old_style_spin_init) +#define IPIPE_SPIN_LOCK_UNLOCKED \ + (__ipipe_spinlock_t) { .__raw_lock = __RAW_SPIN_LOCK_UNLOCKED } #define RW_LOCK_UNLOCKED __RW_LOCK_UNLOCKED(old_style_rw_init) #define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) #define DEFINE_RWLOCK(x) rwlock_t x = __RW_LOCK_UNLOCKED(x) +#ifdef CONFIG_IPIPE +# define ipipe_spinlock_t __ipipe_spinlock_t +# define IPIPE_DEFINE_SPINLOCK(x) ipipe_spinlock_t x = IPIPE_SPIN_LOCK_UNLOCKED +#else +# define ipipe_spinlock_t spinlock_t +# define IPIPE_DEFINE_SPINLOCK(x) DEFINE_SPINLOCK(x) +#endif + #endif /* __LINUX_SPINLOCK_TYPES_H */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/init/Kconfig ipipe-2.6.19-arm/init/Kconfig --- linux-2.6.19/init/Kconfig 2007-01-04 22:05:14.000000000 +0100 +++ ipipe-2.6.19-arm/init/Kconfig 2007-01-15 21:33:00.000000000 +0100 @@ -67,6 +67,7 @@ menu "General setup" config LOCALVERSION string "Local version - append to kernel release" + default "-ipipe" help Append an extra string to the end of your kernel version. This will show up when you type uname, for example. diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/init/main.c ipipe-2.6.19-arm/init/main.c --- linux-2.6.19/init/main.c 2007-01-04 22:05:14.000000000 +0100 +++ ipipe-2.6.19-arm/init/main.c 2007-01-15 21:33:00.000000000 +0100 @@ -489,7 +489,7 @@ asmlinkage void __init start_kernel(void unwind_init(); lockdep_init(); - local_irq_disable(); + local_irq_disable_hw(); early_boot_irqs_off(); early_init_irq_lock_class(); @@ -535,6 +535,11 @@ asmlinkage void __init start_kernel(void softirq_init(); timekeeping_init(); time_init(); + /* + * We need to wait for the interrupt and time subsystems to be + * initialized before enabling the pipeline. + */ + ipipe_init(); profile_init(); if (!irqs_disabled()) printk("start_kernel(): bug: interrupts were enabled early\n"); @@ -683,6 +688,7 @@ static void __init do_basic_setup(void) #ifdef CONFIG_SYSCTL sysctl_init(); #endif + ipipe_init_proc(); do_initcalls(); } diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/Makefile ipipe-2.6.19-arm/kernel/Makefile --- linux-2.6.19/kernel/Makefile 2007-01-04 22:05:14.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/Makefile 2007-01-15 21:33:00.000000000 +0100 @@ -48,6 +48,7 @@ obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ obj-$(CONFIG_SECCOMP) += seccomp.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o obj-$(CONFIG_RELAY) += relay.o +obj-$(CONFIG_IPIPE) += ipipe/ obj-$(CONFIG_UTS_NS) += utsname.o obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/exit.c ipipe-2.6.19-arm/kernel/exit.c --- linux-2.6.19/kernel/exit.c 2007-01-04 22:05:14.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/exit.c 2007-01-15 21:33:01.000000000 +0100 @@ -918,6 +918,7 @@ fastcall NORET_TYPE void do_exit(long co if (group_dead) acct_process(); + ipipe_exit_notify(tsk); exit_sem(tsk); __exit_files(tsk); __exit_fs(tsk); diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/fork.c ipipe-2.6.19-arm/kernel/fork.c --- linux-2.6.19/kernel/fork.c 2007-01-04 22:05:14.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/fork.c 2007-01-15 21:33:01.000000000 +0100 @@ -384,6 +384,7 @@ void mmput(struct mm_struct *mm) might_sleep(); if (atomic_dec_and_test(&mm->mm_users)) { + ipipe_cleanup_notify(mm); exit_aio(mm); exit_mmap(mm); if (!list_empty(&mm->mmlist)) { @@ -914,7 +915,7 @@ static inline void copy_flags(unsigned l { unsigned long new_flags = p->flags; - new_flags &= ~(PF_SUPERPRIV | PF_NOFREEZE); + new_flags &= ~(PF_SUPERPRIV | PF_NOFREEZE | PF_EVNOTIFY); new_flags |= PF_FORKNOEXEC; if (!(clone_flags & CLONE_PTRACE)) p->ptrace = 0; @@ -1258,6 +1259,14 @@ static struct task_struct *copy_process( spin_unlock(¤t->sighand->siglock); write_unlock_irq(&tasklist_lock); proc_fork_connector(p); +#ifdef CONFIG_IPIPE + { + int k; + + for (k = 0; k < IPIPE_ROOT_NPTDKEYS; k++) + p->ptd[k] = NULL; + } +#endif /* CONFIG_IPIPE */ return p; bad_fork_cleanup_namespaces: diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/ipipe/Kconfig ipipe-2.6.19-arm/kernel/ipipe/Kconfig --- linux-2.6.19/kernel/ipipe/Kconfig 1970-01-01 01:00:00.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/ipipe/Kconfig 2007-01-15 21:33:01.000000000 +0100 @@ -0,0 +1,6 @@ +config IPIPE + bool "Interrupt pipeline" + default y + ---help--- + Activate this option if you want the interrupt pipeline to be + compiled in. diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/ipipe/Kconfig.debug ipipe-2.6.19-arm/kernel/ipipe/Kconfig.debug --- linux-2.6.19/kernel/ipipe/Kconfig.debug 1970-01-01 01:00:00.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/ipipe/Kconfig.debug 2007-01-15 21:33:01.000000000 +0100 @@ -0,0 +1,69 @@ +config IPIPE_DEBUG + bool "I-pipe debugging" + depends on IPIPE + +config IPIPE_TRACE + bool "Latency tracing" + depends on IPIPE_DEBUG + select FRAME_POINTER + select KALLSYMS + select PROC_FS + ---help--- + Activate this option if you want to use per-function tracing of + the kernel. The tracer will collect data via instrumentation + features like the one below or with the help of explicite calls + of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the + in-kernel tracing API. The collected data and runtime control + is available via /proc/ipipe/trace/*. + +config IPIPE_TRACE_ENABLE + bool "Enable tracing on boot" + depends on IPIPE_TRACE + default y + ---help--- + Disable this option if you want to arm the tracer after booting + manually ("echo 1 > /proc/ipipe/tracer/enable"). This can reduce + boot time on slow embedded devices due to the tracer overhead. + +config IPIPE_TRACE_MCOUNT + bool "Instrument function entries" + depends on IPIPE_TRACE + default y + ---help--- + When enabled, records every kernel function entry in the tracer + log. While this slows down the system noticeably, it provides + the highest level of information about the flow of events. + However, it can be switch off in order to record only explicit + I-pipe trace points. + +config IPIPE_TRACE_IRQSOFF + bool "Trace IRQs-off times" + depends on IPIPE_TRACE + default y + ---help--- + Activate this option if I-pipe shall trace the longest path + with hard-IRQs switched off. + +config IPIPE_TRACE_SHIFT + int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)" + range 10 18 + default 14 + depends on IPIPE_TRACE + ---help--- + The number of trace points to hold tracing data for each + trace path, as a power of 2. + +config IPIPE_TRACE_VMALLOC + bool "Use vmalloc'ed trace buffer" + depends on IPIPE_TRACE + ---help--- + Instead of reserving static kernel data, the required buffer + is allocated via vmalloc during boot-up when this option is + enabled. This can help to start systems that are low on memory, + but it slightly degrades overall performance. Try this option + when a traced kernel hangs unexpectedly at boot time. + +config IPIPE_TRACE_ENABLE_VALUE + int + default 0 if !IPIPE_TRACE_ENABLE + default 1 if IPIPE_TRACE_ENABLE diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/ipipe/Makefile ipipe-2.6.19-arm/kernel/ipipe/Makefile --- linux-2.6.19/kernel/ipipe/Makefile 1970-01-01 01:00:00.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/ipipe/Makefile 2007-01-15 21:33:01.000000000 +0100 @@ -0,0 +1,3 @@ + +obj-$(CONFIG_IPIPE) += core.o +obj-$(CONFIG_IPIPE_TRACE) += tracer.o diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/ipipe/core.c ipipe-2.6.19-arm/kernel/ipipe/core.c --- linux-2.6.19/kernel/ipipe/core.c 1970-01-01 01:00:00.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/ipipe/core.c 2007-01-20 01:24:55.000000000 +0100 @@ -0,0 +1,1425 @@ +/* -*- linux-c -*- + * linux/kernel/ipipe/core.c + * + * Copyright (C) 2002-2005 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Architecture-independent I-PIPE core support. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_PROC_FS +#include +#endif /* CONFIG_PROC_FS */ +#include + +MODULE_DESCRIPTION("I-pipe"); +MODULE_LICENSE("GPL"); + +static int __ipipe_ptd_key_count; + +static unsigned long __ipipe_ptd_key_map; + +struct ipipe_domain ipipe_root = + { .cpudata = {[0 ... IPIPE_NR_CPUS-1] = + { .status = (1<name = "Linux"; + ipd->domid = IPIPE_ROOT_ID; + ipd->priority = IPIPE_ROOT_PRIO; + + __ipipe_init_stage(ipd); + + INIT_LIST_HEAD(&ipd->p_link); + list_add_tail(&ipd->p_link, &__ipipe_pipeline); + + __ipipe_init_platform(); + +#ifdef CONFIG_PRINTK + __ipipe_printk_virq = ipipe_alloc_virq(); /* Cannot fail here. */ + ipd->irqs[__ipipe_printk_virq].handler = &__ipipe_flush_printk; + ipd->irqs[__ipipe_printk_virq].cookie = NULL; + ipd->irqs[__ipipe_printk_virq].acknowledge = NULL; + ipd->irqs[__ipipe_printk_virq].control = IPIPE_HANDLE_MASK; +#endif /* CONFIG_PRINTK */ + + __ipipe_enable_pipeline(); + + printk(KERN_INFO "I-pipe %s: pipeline enabled.\n", + IPIPE_VERSION_STRING); +} + +void __ipipe_init_stage(struct ipipe_domain *ipd) +{ + int cpuid, n; + + for (cpuid = 0; cpuid < IPIPE_NR_CPUS; cpuid++) { + ipd->cpudata[cpuid].irq_pending_hi = 0; + + for (n = 0; n < IPIPE_IRQ_IWORDS; n++) + ipd->cpudata[cpuid].irq_pending_lo[n] = 0; + + for (n = 0; n < IPIPE_NR_IRQS; n++) { + ipd->cpudata[cpuid].irq_counters[n].total_hits = 0; + ipd->cpudata[cpuid].irq_counters[n].pending_hits = 0; + } + + ipd->cpudata[cpuid].evsync = 0; + } + + for (n = 0; n < IPIPE_NR_IRQS; n++) { + ipd->irqs[n].acknowledge = NULL; + ipd->irqs[n].handler = NULL; + ipd->irqs[n].control = IPIPE_PASS_MASK; /* Pass but don't handle */ + } + + for (n = 0; n < IPIPE_NR_EVENTS; n++) + ipd->evhand[n] = NULL; + + ipd->evself = 0LL; + mutex_init(&ipd->mutex); + + __ipipe_hook_critical_ipi(ipd); +} + +void __ipipe_cleanup_domain(struct ipipe_domain *ipd) +{ + ipipe_unstall_pipeline_from(ipd); + +#ifdef CONFIG_SMP + { + int cpu; + + for_each_online_cpu(cpu) { + while (ipd->cpudata[cpu].irq_pending_hi != 0) + cpu_relax(); + } + } +#endif /* CONFIG_SMP */ +} + +void __ipipe_stall_root(void) +{ + ipipe_declare_cpuid; + unsigned long flags; + + ipipe_get_cpu(flags); /* Care for migration. */ + set_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status); + ipipe_put_cpu(flags); +} + +void __ipipe_unstall_root(void) +{ + ipipe_declare_cpuid; + + local_irq_disable_hw(); + + ipipe_load_cpuid(); + + __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status); + + if (unlikely(ipipe_root_domain->cpudata[cpuid].irq_pending_hi != 0)) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + + local_irq_enable_hw(); +} + +unsigned long __ipipe_test_root(void) +{ + unsigned long flags, x; + ipipe_declare_cpuid; + + ipipe_get_cpu(flags); /* Care for migration. */ + x = test_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status); + ipipe_put_cpu(flags); + + return x; +} + +unsigned long __ipipe_test_and_stall_root(void) +{ + unsigned long flags, x; + ipipe_declare_cpuid; + + ipipe_get_cpu(flags); /* Care for migration. */ + x = test_and_set_bit(IPIPE_STALL_FLAG, + &ipipe_root_domain->cpudata[cpuid].status); + ipipe_put_cpu(flags); + + return x; +} + +void fastcall __ipipe_restore_root(unsigned long x) +{ + if (x) + __ipipe_stall_root(); + else + __ipipe_unstall_root(); +} + +void fastcall ipipe_stall_pipeline_from(struct ipipe_domain *ipd) +{ + ipipe_declare_cpuid; +#ifdef CONFIG_SMP + unsigned long flags; + + ipipe_lock_cpu(flags); /* Care for migration. */ + + __set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + + if (!__ipipe_pipeline_head_p(ipd)) + ipipe_unlock_cpu(flags); +#else /* CONFIG_SMP */ + set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + + if (__ipipe_pipeline_head_p(ipd)) + local_irq_disable_hw(); +#endif /* CONFIG_SMP */ +} + +unsigned long fastcall ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd) +{ + ipipe_declare_cpuid; + unsigned long s; +#ifdef CONFIG_SMP + unsigned long flags; + + ipipe_lock_cpu(flags); /* Care for migration. */ + + s = __test_and_set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + + if (!__ipipe_pipeline_head_p(ipd)) + ipipe_unlock_cpu(flags); +#else /* CONFIG_SMP */ + s = test_and_set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + + if (__ipipe_pipeline_head_p(ipd)) + local_irq_disable_hw(); +#endif /* CONFIG_SMP */ + + return s; +} + +/* + * ipipe_unstall_pipeline_from() -- Unstall the pipeline and + * synchronize pending interrupts for a given domain. See + * __ipipe_walk_pipeline() for more information. + */ +void fastcall ipipe_unstall_pipeline_from(struct ipipe_domain *ipd) +{ + struct list_head *pos; + unsigned long flags; + ipipe_declare_cpuid; + + ipipe_lock_cpu(flags); + + __clear_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + + if (ipd == per_cpu(ipipe_percpu_domain, cpuid)) + pos = &ipd->p_link; + else + pos = __ipipe_pipeline.next; + + __ipipe_walk_pipeline(pos, cpuid); + + if (__ipipe_pipeline_head_p(ipd)) + local_irq_enable_hw(); + else + ipipe_unlock_cpu(flags); +} + +unsigned long fastcall ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd) +{ + unsigned long flags, x; + ipipe_declare_cpuid; + + ipipe_get_cpu(flags); + x = test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + ipipe_unstall_pipeline_from(ipd); + ipipe_put_cpu(flags); + + return x; +} + +void fastcall ipipe_restore_pipeline_from(struct ipipe_domain *ipd, + unsigned long x) +{ + if (x) + ipipe_stall_pipeline_from(ipd); + else + ipipe_unstall_pipeline_from(ipd); +} + +void ipipe_unstall_pipeline_head(void) +{ + struct ipipe_domain *head; + ipipe_declare_cpuid; + + local_irq_disable_hw(); + ipipe_load_cpuid(); + head = __ipipe_pipeline_head(); + __clear_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status); + + if (unlikely(head->cpudata[cpuid].irq_pending_hi != 0)) { + if (likely(head == per_cpu(ipipe_percpu_domain, cpuid))) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + else + __ipipe_walk_pipeline(&head->p_link, cpuid); + } + + local_irq_enable_hw(); +} + +void fastcall __ipipe_restore_pipeline_head(struct ipipe_domain *head, unsigned long x) +{ + ipipe_declare_cpuid; + + local_irq_disable_hw(); + ipipe_load_cpuid(); + + if (x) { +#ifdef CONFIG_DEBUG_KERNEL + static int warned; + if (!warned && test_and_set_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status)) { + /* + * Already stalled albeit ipipe_restore_pipeline_head() + * should have detected it? Send a warning once.\n"); + */ + warned = 1; + printk(KERN_WARNING + "I-pipe: ipipe_restore_pipeline_head() optimization failed.\n"); + dump_stack(); + } +#else /* !CONFIG_DEBUG_KERNEL */ + set_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status); +#endif /* CONFIG_DEBUG_KERNEL */ + } + else { + __clear_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status); + if (unlikely(head->cpudata[cpuid].irq_pending_hi != 0)) { + if (likely(head == per_cpu(ipipe_percpu_domain, cpuid))) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + else + __ipipe_walk_pipeline(&head->p_link, cpuid); + } + local_irq_enable_hw(); + } +} + +/* __ipipe_walk_pipeline(): Plays interrupts pending in the log. Must + be called with local hw interrupts disabled. */ + +void fastcall __ipipe_walk_pipeline(struct list_head *pos, int cpuid) +{ + struct ipipe_domain *this_domain = per_cpu(ipipe_percpu_domain, cpuid); + + while (pos != &__ipipe_pipeline) { + struct ipipe_domain *next_domain = + list_entry(pos, struct ipipe_domain, p_link); + + if (test_bit + (IPIPE_STALL_FLAG, &next_domain->cpudata[cpuid].status)) + break; /* Stalled stage -- do not go further. */ + + if (next_domain->cpudata[cpuid].irq_pending_hi != 0) { + + if (next_domain == this_domain) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + else { + __ipipe_switch_to(this_domain, next_domain, + cpuid); + + ipipe_load_cpuid(); /* Processor might have changed. */ + + if (this_domain->cpudata[cpuid]. + irq_pending_hi != 0 + && !test_bit(IPIPE_STALL_FLAG, + &this_domain->cpudata[cpuid].status)) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + } + + break; + } else if (next_domain == this_domain) + break; + + pos = next_domain->p_link.next; + } +} + +/* + * ipipe_suspend_domain() -- Suspend the current domain, switching to + * the next one which has pending work down the pipeline. + */ +void ipipe_suspend_domain(void) +{ + struct ipipe_domain *this_domain, *next_domain; + struct list_head *ln; + unsigned long flags; + ipipe_declare_cpuid; + + ipipe_lock_cpu(flags); + + this_domain = next_domain = per_cpu(ipipe_percpu_domain, cpuid); + + __clear_bit(IPIPE_STALL_FLAG, &this_domain->cpudata[cpuid].status); + + if (this_domain->cpudata[cpuid].irq_pending_hi != 0) + goto sync_stage; + + for (;;) { + ln = next_domain->p_link.next; + + if (ln == &__ipipe_pipeline) + break; + + next_domain = list_entry(ln, struct ipipe_domain, p_link); + + if (test_bit(IPIPE_STALL_FLAG, + &next_domain->cpudata[cpuid].status)) + break; + + if (next_domain->cpudata[cpuid].irq_pending_hi == 0) + continue; + + per_cpu(ipipe_percpu_domain, cpuid) = next_domain; + +sync_stage: + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + + ipipe_load_cpuid(); /* Processor might have changed. */ + + if (per_cpu(ipipe_percpu_domain, cpuid) != next_domain) + /* + * Something has changed the current domain under our + * feet, recycling the register set; take note. + */ + this_domain = per_cpu(ipipe_percpu_domain, cpuid); + } + + per_cpu(ipipe_percpu_domain, cpuid) = this_domain; + + ipipe_unlock_cpu(flags); +} + +/* ipipe_alloc_virq() -- Allocate a pipelined virtual/soft interrupt. + * Virtual interrupts are handled in exactly the same way than their + * hw-generated counterparts wrt pipelining. + */ +unsigned ipipe_alloc_virq(void) +{ + unsigned long flags, irq = 0; + int ipos; + + spin_lock_irqsave(&__ipipe_pipelock, flags); + + if (__ipipe_virtual_irq_map != ~0) { + ipos = ffz(__ipipe_virtual_irq_map); + set_bit(ipos, &__ipipe_virtual_irq_map); + irq = ipos + IPIPE_VIRQ_BASE; + } + + spin_unlock_irqrestore(&__ipipe_pipelock, flags); + + return irq; +} + +/* ipipe_virtualize_irq() -- Attach a handler (and optionally a hw + acknowledge routine) to an interrupt for a given domain. */ + +int ipipe_virtualize_irq(struct ipipe_domain *ipd, + unsigned irq, + ipipe_irq_handler_t handler, + void *cookie, + ipipe_irq_ackfn_t acknowledge, + unsigned modemask) +{ + unsigned long flags; + int err; + + if (irq >= IPIPE_NR_IRQS) + return -EINVAL; + + if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK) + return -EPERM; + + if (!test_bit(IPIPE_AHEAD_FLAG, &ipd->flags)) + /* Silently unwire interrupts for non-heading domains. */ + modemask &= ~IPIPE_WIRED_MASK; + + spin_lock_irqsave(&__ipipe_pipelock, flags); + + if (handler != NULL) { + + if (handler == IPIPE_SAME_HANDLER) { + handler = ipd->irqs[irq].handler; + cookie = ipd->irqs[irq].cookie; + + if (handler == NULL) { + err = -EINVAL; + goto unlock_and_exit; + } + } else if ((modemask & IPIPE_EXCLUSIVE_MASK) != 0 && + ipd->irqs[irq].handler != NULL) { + err = -EBUSY; + goto unlock_and_exit; + } + + /* Wired interrupts can only be delivered to domains + * always heading the pipeline, and using dynamic + * propagation. */ + + if ((modemask & IPIPE_WIRED_MASK) != 0) { + if ((modemask & (IPIPE_PASS_MASK | IPIPE_STICKY_MASK)) != 0) { + err = -EINVAL; + goto unlock_and_exit; + } + modemask |= (IPIPE_HANDLE_MASK); + } + + if ((modemask & IPIPE_STICKY_MASK) != 0) + modemask |= IPIPE_HANDLE_MASK; + } else + modemask &= + ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK | + IPIPE_EXCLUSIVE_MASK | IPIPE_WIRED_MASK); + + if (acknowledge == NULL && !ipipe_virtual_irq_p(irq)) + /* Acknowledge handler unspecified for a hw interrupt: + use the Linux-defined handler instead. */ + acknowledge = ipipe_root_domain->irqs[irq].acknowledge; + + ipd->irqs[irq].handler = handler; + ipd->irqs[irq].cookie = cookie; + ipd->irqs[irq].acknowledge = acknowledge; + ipd->irqs[irq].control = modemask; + + if (irq < NR_IRQS && handler != NULL && !ipipe_virtual_irq_p(irq)) { + __ipipe_enable_irqdesc(irq); + + if ((modemask & IPIPE_ENABLE_MASK) != 0) { + if (ipd != ipipe_current_domain) { + /* IRQ enable/disable state is domain-sensitive, so we may + not change it for another domain. What is allowed + however is forcing some domain to handle an interrupt + source, by passing the proper 'ipd' descriptor which + thus may be different from ipipe_current_domain. */ + err = -EPERM; + goto unlock_and_exit; + } + + __ipipe_enable_irq(irq); + } + } + + err = 0; + + unlock_and_exit: + + spin_unlock_irqrestore(&__ipipe_pipelock, flags); + + return err; +} + +/* ipipe_control_irq() -- Change modes of a pipelined interrupt for + * the current domain. */ + +int ipipe_control_irq(unsigned irq, unsigned clrmask, unsigned setmask) +{ + struct ipipe_domain *ipd; + unsigned long flags; + + if (irq >= IPIPE_NR_IRQS) + return -EINVAL; + + ipd = ipipe_current_domain; + + if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK) + return -EPERM; + + if (ipd->irqs[irq].handler == NULL) + setmask &= ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK); + + if ((setmask & IPIPE_STICKY_MASK) != 0) + setmask |= IPIPE_HANDLE_MASK; + + if ((clrmask & (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK)) != 0) /* If one goes, both go. */ + clrmask |= (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK); + + spin_lock_irqsave(&__ipipe_pipelock, flags); + + ipd->irqs[irq].control &= ~clrmask; + ipd->irqs[irq].control |= setmask; + + if ((setmask & IPIPE_ENABLE_MASK) != 0) + __ipipe_enable_irq(irq); + else if ((clrmask & IPIPE_ENABLE_MASK) != 0) + __ipipe_disable_irq(irq); + + spin_unlock_irqrestore(&__ipipe_pipelock, flags); + + return 0; +} + +/* __ipipe_dispatch_event() -- Low-level event dispatcher. */ + +int fastcall __ipipe_dispatch_event (unsigned event, void *data) +{ + struct ipipe_domain *start_domain, *this_domain, *next_domain; + ipipe_event_handler_t evhand; + struct list_head *pos, *npos; + unsigned long flags; + ipipe_declare_cpuid; + int propagate = 1; + + ipipe_lock_cpu(flags); + + start_domain = this_domain = per_cpu(ipipe_percpu_domain, cpuid); + + list_for_each_safe(pos,npos,&__ipipe_pipeline) { + + /* + * Note: Domain migration may occur while running + * event or interrupt handlers, in which case the + * current register set is going to be recycled for a + * different domain than the initiating one. We do + * care for that, always tracking the current domain + * descriptor upon return from those handlers. + */ + next_domain = list_entry(pos,struct ipipe_domain,p_link); + + /* + * Keep a cached copy of the handler's address since + * ipipe_catch_event() may clear it under our feet. + */ + + evhand = next_domain->evhand[event]; + + if (evhand != NULL) { + per_cpu(ipipe_percpu_domain, cpuid) = next_domain; + next_domain->cpudata[cpuid].evsync |= (1LL << event); + ipipe_unlock_cpu(flags); + propagate = !evhand(event,start_domain,data); + ipipe_lock_cpu(flags); + next_domain->cpudata[cpuid].evsync &= ~(1LL << event); + if (per_cpu(ipipe_percpu_domain, cpuid) != next_domain) + this_domain = per_cpu(ipipe_percpu_domain, cpuid); + } + + if (next_domain != ipipe_root_domain && /* NEVER sync the root stage here. */ + next_domain->cpudata[cpuid].irq_pending_hi != 0 && + !test_bit(IPIPE_STALL_FLAG,&next_domain->cpudata[cpuid].status)) { + per_cpu(ipipe_percpu_domain, cpuid) = next_domain; + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + ipipe_load_cpuid(); + if (per_cpu(ipipe_percpu_domain, cpuid) != next_domain) + this_domain = per_cpu(ipipe_percpu_domain, cpuid); + } + + per_cpu(ipipe_percpu_domain, cpuid) = this_domain; + + if (next_domain == this_domain || !propagate) + break; + } + + ipipe_unlock_cpu(flags); + + return !propagate; +} + +/* + * __ipipe_dispatch_wired -- Wired interrupt dispatcher. Wired + * interrupts are immediately and unconditionally delivered to the + * domain heading the pipeline upon receipt, and such domain must have + * been registered as an invariant head for the system (priority == + * IPIPE_HEAD_PRIORITY). The motivation for using wired interrupts is + * to get an extra-fast dispatching path for those IRQs, by relying on + * a straightforward logic based on assumptions that must always be + * true for invariant head domains. The following assumptions are + * made when dealing with such interrupts: + * + * 1- Wired interrupts are purely dynamic, i.e. the decision to + * propagate them down the pipeline must be done from the head domain + * ISR. + * 2- Wired interrupts cannot be shared or sticky. + * 3- The root domain cannot be an invariant pipeline head, in + * consequence of what the root domain cannot handle wired + * interrupts. + * 4- Wired interrupts must have a valid acknowledge handler for the + * head domain (if needed), and in any case, must not rely on handlers + * provided by lower priority domains during the acknowledge cycle + * (see __ipipe_handle_irq). + * + * Called with hw interrupts off. + */ + +int fastcall __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq) +{ + struct ipcpudata *cpudata; + struct ipipe_domain *old; + ipipe_declare_cpuid; + + ipipe_load_cpuid(); + cpudata = &head->cpudata[cpuid]; + cpudata->irq_counters[irq].total_hits++; + + if (test_bit(IPIPE_LOCK_FLAG, &head->irqs[irq].control)) { + /* If we can't process this IRQ right now, we must + * mark it as pending, so that it will get played + * during normal log sync when the corresponding + * interrupt source is eventually unlocked. */ + cpudata->irq_counters[irq].pending_hits++; + return 0; + } + + if (test_bit(IPIPE_STALL_FLAG, &cpudata->status)) { + cpudata->irq_counters[irq].pending_hits++; + __ipipe_set_irq_bit(head, cpuid, irq); + return 0; + } + + old = per_cpu(ipipe_percpu_domain, cpuid); + per_cpu(ipipe_percpu_domain, cpuid) = head; /* Switch to the head domain. */ + + __set_bit(IPIPE_STALL_FLAG, &cpudata->status); + head->irqs[irq].handler(irq,head->irqs[irq].cookie); /* Call the ISR. */ + __ipipe_run_irqtail(); + __clear_bit(IPIPE_STALL_FLAG, &cpudata->status); + + /* We expect the caller to start a complete pipeline walk upon + * return, so that propagated interrupts will get played. */ + + if (per_cpu(ipipe_percpu_domain, cpuid) == head) + per_cpu(ipipe_percpu_domain, cpuid) = old; /* Back to the preempted domain. */ + + return 1; +} + +/* + * __ipipe_sync_stage() -- Flush the pending IRQs for the current + * domain (and processor). This routine flushes the interrupt log + * (see "Optimistic interrupt protection" from D. Stodolsky et al. for + * more on the deferred interrupt scheme). Every interrupt that + * occurred while the pipeline was stalled gets played. WARNING: + * callers on SMP boxen should always check for CPU migration on + * return of this routine. One can control the kind of interrupts + * which are going to be sync'ed using the syncmask + * parameter. IPIPE_IRQMASK_ANY plays them all, IPIPE_IRQMASK_VIRT + * plays virtual interrupts only. + * + * This routine must be called with hw interrupts off. + */ +void fastcall __ipipe_sync_stage(unsigned long syncmask) +{ + unsigned long mask, submask; + struct ipcpudata *cpudata; + struct ipipe_domain *ipd; + ipipe_declare_cpuid; + int level, rank; + unsigned irq; + + ipipe_load_cpuid(); + ipd = per_cpu(ipipe_percpu_domain, cpuid); + cpudata = &ipd->cpudata[cpuid]; + + if (__test_and_set_bit(IPIPE_SYNC_FLAG, &cpudata->status)) + return; + + /* + * The policy here is to keep the dispatching code interrupt-free + * by stalling the current stage. If the upper domain handler + * (which we call) wants to re-enable interrupts while in a safe + * portion of the code (e.g. SA_INTERRUPT flag unset for Linux's + * sigaction()), it will have to unstall (then stall again before + * returning to us!) the stage when it sees fit. + */ + while ((mask = (cpudata->irq_pending_hi & syncmask)) != 0) { + level = __ipipe_ffnz(mask); + + while ((submask = cpudata->irq_pending_lo[level]) != 0) { + rank = __ipipe_ffnz(submask); + irq = (level << IPIPE_IRQ_ISHIFT) + rank; + + if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) { + __clear_bit(rank, &cpudata->irq_pending_lo[level]); + continue; + } + + if (--cpudata->irq_counters[irq].pending_hits == 0) { + __clear_bit(rank, &cpudata->irq_pending_lo[level]); + if (cpudata->irq_pending_lo[level] == 0) + __clear_bit(level, &cpudata->irq_pending_hi); + } + + __set_bit(IPIPE_STALL_FLAG, &cpudata->status); + + if (ipd == ipipe_root_domain) + trace_hardirqs_off(); + + __ipipe_run_isr(ipd, irq, cpuid); +#ifdef CONFIG_SMP + { + int _cpuid = ipipe_processor_id(); + + if (_cpuid != cpuid) { /* Handle CPU migration. */ + /* + * We expect any domain to clear the SYNC bit each + * time it switches in a new task, so that preemptions + * and/or CPU migrations (in the SMP case) over the + * ISR do not lock out the log syncer for some + * indefinite amount of time. In the Linux case, + * schedule() handles this (see kernel/sched.c). For + * this reason, we don't bother clearing it here for + * the source CPU in the migration handling case, + * since it must have scheduled another task in by + * now. + */ + cpuid = _cpuid; + cpudata = &ipd->cpudata[cpuid]; + __set_bit(IPIPE_SYNC_FLAG, &cpudata->status); + } + } +#endif /* CONFIG_SMP */ + if (ipd == ipipe_root_domain && + test_bit(IPIPE_STALL_FLAG, &cpudata->status)) + trace_hardirqs_on(); + + __clear_bit(IPIPE_STALL_FLAG, &cpudata->status); + } + } + + __clear_bit(IPIPE_SYNC_FLAG, &cpudata->status); +} + +/* ipipe_register_domain() -- Link a new domain to the pipeline. */ + +int ipipe_register_domain(struct ipipe_domain *ipd, + struct ipipe_domain_attr *attr) +{ + struct list_head *pos; + unsigned long flags; + + if (ipipe_current_domain != ipipe_root_domain) { + printk(KERN_WARNING + "I-pipe: Only the root domain may register a new domain.\n"); + return -EPERM; + } + + if (attr->priority == IPIPE_HEAD_PRIORITY && + test_bit(IPIPE_AHEAD_FLAG,&__ipipe_pipeline_head()->flags)) + return -EAGAIN; /* Cannot override current head. */ + + flags = ipipe_critical_enter(NULL); + + list_for_each(pos, &__ipipe_pipeline) { + struct ipipe_domain *_ipd = + list_entry(pos, struct ipipe_domain, p_link); + if (_ipd->domid == attr->domid) + break; + } + + ipipe_critical_exit(flags); + + if (pos != &__ipipe_pipeline) + /* A domain with the given id already exists -- fail. */ + return -EBUSY; + + ipd->name = attr->name; + ipd->domid = attr->domid; + ipd->pdd = attr->pdd; + ipd->flags = 0; + + if (attr->priority == IPIPE_HEAD_PRIORITY) { + ipd->priority = INT_MAX; + __set_bit(IPIPE_AHEAD_FLAG,&ipd->flags); + } + else + ipd->priority = attr->priority; + + __ipipe_init_stage(ipd); + + INIT_LIST_HEAD(&ipd->p_link); + +#ifdef CONFIG_PROC_FS + __ipipe_add_domain_proc(ipd); +#endif /* CONFIG_PROC_FS */ + + flags = ipipe_critical_enter(NULL); + + list_for_each(pos, &__ipipe_pipeline) { + struct ipipe_domain *_ipd = + list_entry(pos, struct ipipe_domain, p_link); + if (ipd->priority > _ipd->priority) + break; + } + + list_add_tail(&ipd->p_link, pos); + + ipipe_critical_exit(flags); + + printk(KERN_WARNING "I-pipe: Domain %s registered.\n", ipd->name); + + /* + * Finally, allow the new domain to perform its initialization + * chores. + */ + + if (attr->entry != NULL) { + ipipe_declare_cpuid; + + ipipe_lock_cpu(flags); + + per_cpu(ipipe_percpu_domain, cpuid) = ipd; + attr->entry(); + per_cpu(ipipe_percpu_domain, cpuid) = ipipe_root_domain; + + ipipe_load_cpuid(); /* Processor might have changed. */ + + if (ipipe_root_domain->cpudata[cpuid].irq_pending_hi != 0 && + !test_bit(IPIPE_STALL_FLAG, + &ipipe_root_domain->cpudata[cpuid].status)) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + + ipipe_unlock_cpu(flags); + } + + return 0; +} + +/* ipipe_unregister_domain() -- Remove a domain from the pipeline. */ + +int ipipe_unregister_domain(struct ipipe_domain *ipd) +{ + unsigned long flags; + + if (ipipe_current_domain != ipipe_root_domain) { + printk(KERN_WARNING + "I-pipe: Only the root domain may unregister a domain.\n"); + return -EPERM; + } + + if (ipd == ipipe_root_domain) { + printk(KERN_WARNING + "I-pipe: Cannot unregister the root domain.\n"); + return -EPERM; + } +#ifdef CONFIG_SMP + { + int nr_cpus = num_online_cpus(), _cpuid; + unsigned irq; + + /* + * In the SMP case, wait for the logged events to drain on + * other processors before eventually removing the domain + * from the pipeline. + */ + + ipipe_unstall_pipeline_from(ipd); + + flags = ipipe_critical_enter(NULL); + + for (irq = 0; irq < IPIPE_NR_IRQS; irq++) { + clear_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control); + clear_bit(IPIPE_STICKY_FLAG, &ipd->irqs[irq].control); + set_bit(IPIPE_PASS_FLAG, &ipd->irqs[irq].control); + } + + ipipe_critical_exit(flags); + + for (_cpuid = 0; _cpuid < nr_cpus; _cpuid++) + for (irq = 0; irq < IPIPE_NR_IRQS; irq++) + while (ipd->cpudata[_cpuid].irq_counters[irq].pending_hits > 0) + cpu_relax(); + } +#endif /* CONFIG_SMP */ + + mutex_lock(&ipd->mutex); + +#ifdef CONFIG_PROC_FS + __ipipe_remove_domain_proc(ipd); +#endif /* CONFIG_PROC_FS */ + + /* + * Simply remove the domain from the pipeline and we are almost done. + */ + + flags = ipipe_critical_enter(NULL); + list_del_init(&ipd->p_link); + ipipe_critical_exit(flags); + + __ipipe_cleanup_domain(ipd); + + mutex_unlock(&ipd->mutex); + + printk(KERN_WARNING "I-pipe: Domain %s unregistered.\n", ipd->name); + + return 0; +} + +/* + * ipipe_propagate_irq() -- Force a given IRQ propagation on behalf of + * a running interrupt handler to the next domain down the pipeline. + * ipipe_schedule_irq() -- Does almost the same as above, but attempts + * to pend the interrupt for the current domain first. + */ +int fastcall __ipipe_schedule_irq(unsigned irq, struct list_head *head) +{ + struct list_head *ln; + unsigned long flags; + ipipe_declare_cpuid; + + if (irq >= IPIPE_NR_IRQS || + (ipipe_virtual_irq_p(irq) + && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))) + return -EINVAL; + + ipipe_lock_cpu(flags); + + ln = head; + + while (ln != &__ipipe_pipeline) { + struct ipipe_domain *ipd = + list_entry(ln, struct ipipe_domain, p_link); + + if (test_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control)) { + ipd->cpudata[cpuid].irq_counters[irq].total_hits++; + ipd->cpudata[cpuid].irq_counters[irq].pending_hits++; + __ipipe_set_irq_bit(ipd, cpuid, irq); + ipipe_unlock_cpu(flags); + return 1; + } + + ln = ipd->p_link.next; + } + + ipipe_unlock_cpu(flags); + + return 0; +} + +/* ipipe_free_virq() -- Release a virtual/soft interrupt. */ + +int ipipe_free_virq(unsigned virq) +{ + if (!ipipe_virtual_irq_p(virq)) + return -EINVAL; + + clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map); + + return 0; +} + +void ipipe_init_attr(struct ipipe_domain_attr *attr) +{ + attr->name = "anon"; + attr->domid = 1; + attr->entry = NULL; + attr->priority = IPIPE_ROOT_PRIO; + attr->pdd = NULL; +} + +/* + * ipipe_catch_event() -- Interpose or remove an event handler for a + * given domain. + */ +ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd, + unsigned event, + ipipe_event_handler_t handler) +{ + ipipe_event_handler_t old_handler; + unsigned long flags; + int self = 0, cpuid; + + if (event & IPIPE_EVENT_SELF) { + event &= ~IPIPE_EVENT_SELF; + self = 1; + } + + if (event >= IPIPE_NR_EVENTS) + return NULL; + + flags = ipipe_critical_enter(NULL); + + if (!(old_handler = xchg(&ipd->evhand[event],handler))) { + if (handler) { + if (self) + ipd->evself |= (1LL << event); + else + __ipipe_event_monitors[event]++; + } + } + else if (!handler) { + if (ipd->evself & (1LL << event)) + ipd->evself &= ~(1LL << event); + else + __ipipe_event_monitors[event]--; + } else if ((ipd->evself & (1LL << event)) && !self) { + __ipipe_event_monitors[event]++; + ipd->evself &= ~(1LL << event); + } else if (!(ipd->evself & (1LL << event)) && self) { + __ipipe_event_monitors[event]--; + ipd->evself |= (1LL << event); + } + + ipipe_critical_exit(flags); + + if (!handler && ipipe_root_domain_p) { + /* + * If we cleared a handler on behalf of the root + * domain, we have to wait for any current invocation + * to drain, since our caller might subsequently unmap + * the target domain. To this aim, this code + * synchronizes with __ipipe_dispatch_event(), + * guaranteeing that either the dispatcher sees a null + * handler in which case it discards the invocation + * (which also prevents from entering a livelock), or + * finds a valid handler and calls it. Symmetrically, + * ipipe_catch_event() ensures that the called code + * won't be unmapped under our feet until the event + * synchronization flag is cleared for the given event + * on all CPUs. + */ + + for_each_online_cpu(cpuid) { + while (ipd->cpudata[cpuid].evsync & (1LL << event)) + schedule_timeout_interruptible(HZ / 50); + } + } + + return old_handler; +} + +cpumask_t ipipe_set_irq_affinity (unsigned irq, cpumask_t cpumask) +{ +#ifdef CONFIG_SMP + if (irq >= IPIPE_NR_XIRQS) + /* Allow changing affinity of external IRQs only. */ + return CPU_MASK_NONE; + + if (num_online_cpus() > 1) + return __ipipe_set_irq_affinity(irq,cpumask); +#endif /* CONFIG_SMP */ + + return CPU_MASK_NONE; +} + +int fastcall ipipe_send_ipi (unsigned ipi, cpumask_t cpumask) + +{ +#ifdef CONFIG_SMP + return __ipipe_send_ipi(ipi,cpumask); +#else /* !CONFIG_SMP */ + return -EINVAL; +#endif /* CONFIG_SMP */ +} + +int ipipe_alloc_ptdkey (void) +{ + unsigned long flags; + int key = -1; + + spin_lock_irqsave(&__ipipe_pipelock,flags); + + if (__ipipe_ptd_key_count < IPIPE_ROOT_NPTDKEYS) { + key = ffz(__ipipe_ptd_key_map); + set_bit(key,&__ipipe_ptd_key_map); + __ipipe_ptd_key_count++; + } + + spin_unlock_irqrestore(&__ipipe_pipelock,flags); + + return key; +} + +int ipipe_free_ptdkey (int key) +{ + unsigned long flags; + + if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS) + return -EINVAL; + + spin_lock_irqsave(&__ipipe_pipelock,flags); + + if (test_and_clear_bit(key,&__ipipe_ptd_key_map)) + __ipipe_ptd_key_count--; + + spin_unlock_irqrestore(&__ipipe_pipelock,flags); + + return 0; +} + +int fastcall ipipe_set_ptd (int key, void *value) + +{ + if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS) + return -EINVAL; + + current->ptd[key] = value; + + return 0; +} + +void fastcall *ipipe_get_ptd (int key) + +{ + if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS) + return NULL; + + return current->ptd[key]; +} + +#ifdef CONFIG_PROC_FS + +#include + +struct proc_dir_entry *ipipe_proc_root; + +static int __ipipe_version_info_proc(char *page, + char **start, + off_t off, int count, int *eof, void *data) +{ + int len = sprintf(page, "%s\n", IPIPE_VERSION_STRING); + + len -= off; + + if (len <= off + count) + *eof = 1; + + *start = page + off; + + if(len > count) + len = count; + + if(len < 0) + len = 0; + + return len; +} + +static int __ipipe_common_info_proc(char *page, + char **start, + off_t off, int count, int *eof, void *data) +{ + char *p = page, handling, stickiness, lockbit, exclusive, virtuality; + struct ipipe_domain *ipd = (struct ipipe_domain *)data; + unsigned long ctlbits; + unsigned irq; + int len; + + p += sprintf(p, " +----- Handling ([A]ccepted, [G]rabbed, [W]ired, [D]iscarded)\n"); + p += sprintf(p, " |+---- Sticky\n"); + p += sprintf(p, " ||+--- Locked\n"); + p += sprintf(p, " |||+-- Exclusive\n"); + p += sprintf(p, " ||||+- Virtual\n"); + p += sprintf(p, "[IRQ] |||||\n"); + + mutex_lock(&ipd->mutex); + + for (irq = 0; irq < IPIPE_NR_IRQS; irq++) { + /* Remember to protect against + * ipipe_virtual_irq/ipipe_control_irq if more fields + * get involved. */ + ctlbits = ipd->irqs[irq].control; + + if (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq)) + /* + * There might be a hole between the last external + * IRQ and the first virtual one; skip it. + */ + continue; + + if (ipipe_virtual_irq_p(irq) + && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)) + /* Non-allocated virtual IRQ; skip it. */ + continue; + + /* + * Statuses are as follows: + * o "accepted" means handled _and_ passed down the pipeline. + * o "grabbed" means handled, but the interrupt might be + * terminated _or_ passed down the pipeline depending on + * what the domain handler asks for to the I-pipe. + * o "wired" is basically the same as "grabbed", except that + * the interrupt is unconditionally delivered to an invariant + * pipeline head domain. + * o "passed" means unhandled by the domain but passed + * down the pipeline. + * o "discarded" means unhandled and _not_ passed down the + * pipeline. The interrupt merely disappears from the + * current domain down to the end of the pipeline. + */ + if (ctlbits & IPIPE_HANDLE_MASK) { + if (ctlbits & IPIPE_PASS_MASK) + handling = 'A'; + else if (ctlbits & IPIPE_WIRED_MASK) + handling = 'W'; + else + handling = 'G'; + } else if (ctlbits & IPIPE_PASS_MASK) + /* Do not output if no major action is taken. */ + continue; + else + handling = 'D'; + + if (ctlbits & IPIPE_STICKY_MASK) + stickiness = 'S'; + else + stickiness = '.'; + + if (ctlbits & IPIPE_LOCK_MASK) + lockbit = 'L'; + else + lockbit = '.'; + + if (ctlbits & IPIPE_EXCLUSIVE_MASK) + exclusive = 'X'; + else + exclusive = '.'; + + if (ipipe_virtual_irq_p(irq)) + virtuality = 'V'; + else + virtuality = '.'; + + p += sprintf(p, " %3u: %c%c%c%c%c\n", + irq, handling, stickiness, lockbit, exclusive, virtuality); + } + + p += sprintf(p, "[Domain info]\n"); + + p += sprintf(p, "id=0x%.8x\n", ipd->domid); + + if (test_bit(IPIPE_AHEAD_FLAG,&ipd->flags)) + p += sprintf(p, "priority=topmost\n"); + else + p += sprintf(p, "priority=%d\n", ipd->priority); + + mutex_unlock(&ipd->mutex); + + len = p - page; + + if (len <= off + count) + *eof = 1; + + *start = page + off; + + len -= off; + + if (len > count) + len = count; + + if (len < 0) + len = 0; + + return len; +} + +void __ipipe_add_domain_proc(struct ipipe_domain *ipd) +{ + create_proc_read_entry(ipd->name,0444,ipipe_proc_root,&__ipipe_common_info_proc,ipd); +} + +void __ipipe_remove_domain_proc(struct ipipe_domain *ipd) +{ + remove_proc_entry(ipd->name,ipipe_proc_root); +} + +void ipipe_init_proc(void) +{ + ipipe_proc_root = create_proc_entry("ipipe",S_IFDIR, 0); + create_proc_read_entry("version",0444,ipipe_proc_root,&__ipipe_version_info_proc,NULL); + __ipipe_add_domain_proc(ipipe_root_domain); + + __ipipe_init_tracer(); +} + +#endif /* CONFIG_PROC_FS */ + +EXPORT_SYMBOL(ipipe_virtualize_irq); +EXPORT_SYMBOL(ipipe_control_irq); +EXPORT_SYMBOL(ipipe_suspend_domain); +EXPORT_SYMBOL(ipipe_alloc_virq); +EXPORT_PER_CPU_SYMBOL(ipipe_percpu_domain); +EXPORT_SYMBOL(ipipe_root); +EXPORT_SYMBOL(ipipe_stall_pipeline_from); +EXPORT_SYMBOL(ipipe_test_and_stall_pipeline_from); +EXPORT_SYMBOL(ipipe_unstall_pipeline_from); +EXPORT_SYMBOL(ipipe_restore_pipeline_from); +EXPORT_SYMBOL(ipipe_test_and_unstall_pipeline_from); +EXPORT_SYMBOL(ipipe_unstall_pipeline_head); +EXPORT_SYMBOL(__ipipe_restore_pipeline_head); +EXPORT_SYMBOL(__ipipe_unstall_root); +EXPORT_SYMBOL(__ipipe_stall_root); +EXPORT_SYMBOL(__ipipe_restore_root); +EXPORT_SYMBOL(__ipipe_test_and_stall_root); +EXPORT_SYMBOL(__ipipe_test_root); +EXPORT_SYMBOL(__ipipe_dispatch_event); +EXPORT_SYMBOL(__ipipe_dispatch_wired); +EXPORT_SYMBOL(__ipipe_sync_stage); +EXPORT_SYMBOL(__ipipe_pipeline); +EXPORT_SYMBOL(__ipipe_pipelock); +EXPORT_SYMBOL(__ipipe_virtual_irq_map); + +EXPORT_SYMBOL(ipipe_register_domain); +EXPORT_SYMBOL(ipipe_unregister_domain); +EXPORT_SYMBOL(ipipe_free_virq); +EXPORT_SYMBOL(ipipe_init_attr); +EXPORT_SYMBOL(ipipe_catch_event); +EXPORT_SYMBOL(ipipe_alloc_ptdkey); +EXPORT_SYMBOL(ipipe_free_ptdkey); +EXPORT_SYMBOL(ipipe_set_ptd); +EXPORT_SYMBOL(ipipe_get_ptd); +EXPORT_SYMBOL(ipipe_set_irq_affinity); +EXPORT_SYMBOL(ipipe_send_ipi); +EXPORT_SYMBOL(__ipipe_schedule_irq); diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/ipipe/tracer.c ipipe-2.6.19-arm/kernel/ipipe/tracer.c --- linux-2.6.19/kernel/ipipe/tracer.c 1970-01-01 01:00:00.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/ipipe/tracer.c 2007-01-15 21:33:01.000000000 +0100 @@ -0,0 +1,1244 @@ +/* -*- linux-c -*- + * kernel/ipipe/tracer.c + * + * Copyright (C) 2005 Luotao Fu. + * 2005, 2006 Jan Kiszka. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IPIPE_TRACE_PATHS 4 /* Do not lower below 3 */ +#define IPIPE_DEFAULT_ACTIVE 0 +#define IPIPE_DEFAULT_MAX 1 +#define IPIPE_DEFAULT_FROZEN 2 + +#define IPIPE_TRACE_POINTS (1 << CONFIG_IPIPE_TRACE_SHIFT) +#define WRAP_POINT_NO(point) ((point) & (IPIPE_TRACE_POINTS-1)) + +#define IPIPE_DEFAULT_PRE_TRACE 10 +#define IPIPE_DEFAULT_POST_TRACE 10 +#define IPIPE_DEFAULT_BACK_TRACE 30 + +#define IPIPE_DELAY_NOTE 1000 /* in nanoseconds */ +#define IPIPE_DELAY_WARN 10000 /* in nanoseconds */ + +#define IPIPE_TFLG_NMI_LOCK 0x0001 +#define IPIPE_TFLG_NMI_HIT 0x0002 +#define IPIPE_TFLG_NMI_FREEZE_REQ 0x0004 + +#define IPIPE_TFLG_HWIRQ_OFF 0x0100 +#define IPIPE_TFLG_FREEZING 0x0200 +#define IPIPE_TFLG_CURRDOM_SHIFT 10 /* bits 10..11: current domain */ +#define IPIPE_TFLG_CURRDOM_MASK 0x0C00 +#define IPIPE_TFLG_DOMSTATE_SHIFT 12 /* bits 12..15: domain stalled? */ +#define IPIPE_TFLG_DOMSTATE_BITS 3 + +#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \ + (point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT))) +#define IPIPE_TFLG_CURRENT_DOMAIN(point) \ + ((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT) + + +struct ipipe_trace_point{ + short type; + short flags; + unsigned long eip; + unsigned long parent_eip; + unsigned long v; + unsigned long long timestamp; +}; + +struct ipipe_trace_path{ + volatile int flags; + int dump_lock; /* separated from flags due to cross-cpu access */ + int trace_pos; /* next point to fill */ + int begin, end; /* finalised path begin and end */ + int post_trace; /* non-zero when in post-trace phase */ + unsigned long long length; /* max path length in cycles */ + unsigned long nmi_saved_eip; /* for deferred requests from NMIs */ + unsigned long nmi_saved_parent_eip; + unsigned long nmi_saved_v; + struct ipipe_trace_point point[IPIPE_TRACE_POINTS]; +} ____cacheline_aligned_in_smp; + +enum ipipe_trace_type +{ + IPIPE_TRACE_FUNC = 0, + IPIPE_TRACE_BEGIN, + IPIPE_TRACE_END, + IPIPE_TRACE_FREEZE, + IPIPE_TRACE_SPECIAL, + IPIPE_TRACE_PID, +}; + +#define IPIPE_TYPE_MASK 0x0007 +#define IPIPE_TYPE_BITS 3 + + +#ifdef CONFIG_IPIPE_TRACE_VMALLOC +#define IPIPE_DEFAULT_TRACE_STATE 0 + +static struct ipipe_trace_path *trace_paths[NR_CPUS]; + +#else /* !CONFIG_IPIPE_TRACE_VMALLOC */ +#define IPIPE_DEFAULT_TRACE_STATE CONFIG_IPIPE_TRACE_ENABLE_VALUE + +static struct ipipe_trace_path trace_paths[NR_CPUS][IPIPE_TRACE_PATHS] = + { [0 ... NR_CPUS-1] = + { [0 ... IPIPE_TRACE_PATHS-1] = + { .begin = -1, .end = -1 } + } + }; +#endif /* CONFIG_IPIPE_TRACE_VMALLOC */ + +int ipipe_trace_enable = IPIPE_DEFAULT_TRACE_STATE; + +static int active_path[NR_CPUS] = + { [0 ... NR_CPUS-1] = IPIPE_DEFAULT_ACTIVE }; +static int max_path[NR_CPUS] = + { [0 ... NR_CPUS-1] = IPIPE_DEFAULT_MAX }; +static int frozen_path[NR_CPUS] = + { [0 ... NR_CPUS-1] = IPIPE_DEFAULT_FROZEN }; +static IPIPE_DEFINE_SPINLOCK(global_path_lock); +static int pre_trace = IPIPE_DEFAULT_PRE_TRACE; +static int post_trace = IPIPE_DEFAULT_POST_TRACE; +static int back_trace = IPIPE_DEFAULT_BACK_TRACE; +static int verbose_trace = 0; + +static DECLARE_MUTEX(out_mutex); +static struct ipipe_trace_path *print_path; +static struct ipipe_trace_path *panic_path; +static int print_pre_trace; +static int print_post_trace; + + +static long __ipipe_signed_tsc2us(long long tsc); +static void +__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point); +static void __ipipe_print_symname(struct seq_file *m, unsigned long eip); + + +static notrace void +__ipipe_store_domain_states(struct ipipe_trace_point *point, int cpu_id) +{ + struct list_head *pos; + int i = 0; + + list_for_each_prev(pos, &__ipipe_pipeline) { + struct ipipe_domain *ipd = + list_entry(pos, struct ipipe_domain, p_link); + + if (test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpu_id].status)) + point->flags |= 1 << (i + IPIPE_TFLG_DOMSTATE_SHIFT); + + if (ipd == per_cpu(ipipe_percpu_domain, cpu_id)) + point->flags |= i << IPIPE_TFLG_CURRDOM_SHIFT; + + if (++i > IPIPE_TFLG_DOMSTATE_BITS) + break; + } +} + +static notrace int __ipipe_get_free_trace_path(int old, int cpu_id) +{ + int new_active = old; + struct ipipe_trace_path *tp; + + do { + if (++new_active == IPIPE_TRACE_PATHS) + new_active = 0; + tp = &trace_paths[cpu_id][new_active]; + } while ((new_active == max_path[cpu_id]) || + (new_active == frozen_path[cpu_id]) || + tp->dump_lock); + + return new_active; +} + +static notrace void +__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp, + struct ipipe_trace_path *old_tp, int old_pos) +{ + int i; + + new_tp->trace_pos = pre_trace+1; + + for (i = new_tp->trace_pos; i > 0; i--) + memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)], + &old_tp->point[WRAP_POINT_NO(old_pos-i)], + sizeof(struct ipipe_trace_point)); + + /* mark the end (i.e. the point before point[0]) invalid */ + new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0; +} + +static notrace struct ipipe_trace_path * +__ipipe_trace_end(int cpu_id, struct ipipe_trace_path *tp, int pos) +{ + struct ipipe_trace_path *old_tp = tp; + long active = active_path[cpu_id]; + unsigned long long length; + + /* do we have a new worst case? */ + length = tp->point[tp->end].timestamp - + tp->point[tp->begin].timestamp; + if (length > (trace_paths[cpu_id][max_path[cpu_id]]).length) { + /* we need protection here against other cpus trying + to start a proc dump */ + spin_lock(&global_path_lock); + + /* active path holds new worst case */ + tp->length = length; + max_path[cpu_id] = active; + + /* find next unused trace path */ + active = __ipipe_get_free_trace_path(active, cpu_id); + + spin_unlock(&global_path_lock); + + tp = &trace_paths[cpu_id][active]; + + /* migrate last entries for pre-tracing */ + __ipipe_migrate_pre_trace(tp, old_tp, pos); + } + + return tp; +} + +static notrace struct ipipe_trace_path * +__ipipe_trace_freeze(int cpu_id, struct ipipe_trace_path *tp, int pos) +{ + struct ipipe_trace_path *old_tp = tp; + long active = active_path[cpu_id]; + int i; + + /* frozen paths have no core (begin=end) */ + tp->begin = tp->end; + + /* we need protection here against other cpus trying + * to set their frozen path or to start a proc dump */ + spin_lock(&global_path_lock); + + frozen_path[cpu_id] = active; + + /* find next unused trace path */ + active = __ipipe_get_free_trace_path(active, cpu_id); + + /* check if this is the first frozen path */ + for_each_online_cpu(i) { + if ((i != cpu_id) && + (trace_paths[i][frozen_path[i]].end >= 0)) + tp->end = -1; + } + + spin_unlock(&global_path_lock); + + tp = &trace_paths[cpu_id][active]; + + /* migrate last entries for pre-tracing */ + __ipipe_migrate_pre_trace(tp, old_tp, pos); + + return tp; +} + +void notrace +__ipipe_trace(enum ipipe_trace_type type, unsigned long eip, + unsigned long parent_eip, unsigned long v) +{ + struct ipipe_trace_path *tp, *old_tp; + int pos, next_pos, begin; + struct ipipe_trace_point *point; + unsigned long flags; + int cpu_id; + + local_irq_save_hw_notrace(flags); + + cpu_id = ipipe_processor_id(); + restart: + tp = old_tp = &trace_paths[cpu_id][active_path[cpu_id]]; + + /* here starts a race window with NMIs - catched below */ + + /* check for NMI recursion */ + if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) { + tp->flags |= IPIPE_TFLG_NMI_HIT; + + /* first freeze request from NMI context? */ + if ((type == IPIPE_TRACE_FREEZE) && + !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) { + /* save arguments and mark deferred freezing */ + tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ; + tp->nmi_saved_eip = eip; + tp->nmi_saved_parent_eip = parent_eip; + tp->nmi_saved_v = v; + } + return; /* no need for restoring flags inside IRQ */ + } + + /* clear NMI events and set lock (atomically per cpu) */ + tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT | + IPIPE_TFLG_NMI_FREEZE_REQ)) + | IPIPE_TFLG_NMI_LOCK; + + /* check active_path again - some nasty NMI may have switched + * it meanwhile */ + if (unlikely(tp != &trace_paths[cpu_id][active_path[cpu_id]])) { + /* release lock on wrong path and restart */ + tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + + /* there is no chance that the NMI got deferred + * => no need to check for pending freeze requests */ + goto restart; + } + + /* get the point buffer */ + pos = tp->trace_pos; + point = &tp->point[pos]; + + /* store all trace point data */ + point->type = type; + point->flags = raw_irqs_disabled_flags(flags) ? IPIPE_TFLG_HWIRQ_OFF : 0; + point->eip = eip; + point->parent_eip = parent_eip; + point->v = v; + ipipe_read_tsc(point->timestamp); + + __ipipe_store_domain_states(point, cpu_id); + + /* forward to next point buffer */ + next_pos = WRAP_POINT_NO(pos+1); + tp->trace_pos = next_pos; + + /* only mark beginning if we haven't started yet */ + begin = tp->begin; + if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0)) + tp->begin = pos; + + /* end of critical path, start post-trace if not already started */ + if (unlikely(type == IPIPE_TRACE_END) && + (begin >= 0) && !tp->post_trace) + tp->post_trace = post_trace + 1; + + /* freeze only if the slot is free and we are not already freezing */ + if (unlikely(type == IPIPE_TRACE_FREEZE) && + (trace_paths[cpu_id][frozen_path[cpu_id]].begin < 0) && + !(tp->flags & IPIPE_TFLG_FREEZING)) { + tp->post_trace = post_trace + 1; + tp->flags |= IPIPE_TFLG_FREEZING; + } + + /* enforce end of trace in case of overflow */ + if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) { + tp->end = pos; + goto enforce_end; + } + + /* stop tracing this path if we are in post-trace and + * a) that phase is over now or + * b) a new TRACE_BEGIN came in but we are not freezing this path */ + if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) || + ((type == IPIPE_TRACE_BEGIN) && + !(tp->flags & IPIPE_TFLG_FREEZING))))) { + /* store the path's end (i.e. excluding post-trace) */ + tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace); + + enforce_end: + if (tp->flags & IPIPE_TFLG_FREEZING) + tp = __ipipe_trace_freeze(cpu_id, tp, pos); + else + tp = __ipipe_trace_end(cpu_id, tp, pos); + + /* reset the active path, maybe already start a new one */ + tp->begin = (type == IPIPE_TRACE_BEGIN) ? + WRAP_POINT_NO(tp->trace_pos - 1) : -1; + tp->end = -1; + tp->post_trace = 0; + tp->flags = 0; + + /* update active_path not earlier to avoid races with NMIs */ + active_path[cpu_id] = tp - trace_paths[cpu_id]; + } + + /* we still have old_tp and point, + * let's reset NMI lock and check for catches */ + old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) { + /* well, this late tagging may not immediately be visible for + * other cpus already dumping this path - a minor issue */ + point->flags |= IPIPE_TFLG_NMI_HIT; + + /* handle deferred freezing from NMI context */ + if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ) + __ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip, + old_tp->nmi_saved_parent_eip, + old_tp->nmi_saved_v); + } + + local_irq_restore_hw_notrace(flags); +} + +static unsigned long __ipipe_global_path_lock(void) +{ + unsigned long flags; + int cpu_id; + struct ipipe_trace_path *tp; + + spin_lock_irqsave(&global_path_lock, flags); + + cpu_id = ipipe_processor_id(); + restart: + tp = &trace_paths[cpu_id][active_path[cpu_id]]; + + /* here is small race window with NMIs - catched below */ + + /* clear NMI events and set lock (atomically per cpu) */ + tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT | + IPIPE_TFLG_NMI_FREEZE_REQ)) + | IPIPE_TFLG_NMI_LOCK; + + /* check active_path again - some nasty NMI may have switched + * it meanwhile */ + if (tp != &trace_paths[cpu_id][active_path[cpu_id]]) { + /* release lock on wrong path and restart */ + tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + + /* there is no chance that the NMI got deferred + * => no need to check for pending freeze requests */ + goto restart; + } + + return flags; +} + +static void __ipipe_global_path_unlock(unsigned long flags) +{ + int cpu_id; + struct ipipe_trace_path *tp; + + /* release spinlock first - it's not involved in the NMI issue */ + spin_unlock(&global_path_lock); + + cpu_id = ipipe_processor_id(); + tp = &trace_paths[cpu_id][active_path[cpu_id]]; + + tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + + /* handle deferred freezing from NMI context */ + if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ) + __ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip, + tp->nmi_saved_parent_eip, tp->nmi_saved_v); + + local_irq_restore_hw(flags); +} + +void notrace ipipe_trace_begin(unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL(ipipe_trace_begin); + +void notrace ipipe_trace_end(unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL(ipipe_trace_end); + +void notrace ipipe_trace_freeze(unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL(ipipe_trace_freeze); + +void notrace ipipe_trace_special(unsigned char id, unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS), + __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL(ipipe_trace_special); + +void notrace ipipe_trace_pid(pid_t pid, short prio) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS), + __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, pid); +} +EXPORT_SYMBOL(ipipe_trace_pid); + +int ipipe_trace_max_reset(void) +{ + int cpu_id; + unsigned long flags; + struct ipipe_trace_path *path; + int ret = 0; + + flags = __ipipe_global_path_lock(); + + for_each_online_cpu(cpu_id) { + path = &trace_paths[cpu_id][max_path[cpu_id]]; + + if (path->dump_lock) { + ret = -EBUSY; + break; + } + + path->begin = -1; + path->end = -1; + path->trace_pos = 0; + path->length = 0; + } + + __ipipe_global_path_unlock(flags); + + return ret; +} +EXPORT_SYMBOL(ipipe_trace_max_reset); + +int ipipe_trace_frozen_reset(void) +{ + int cpu_id; + unsigned long flags; + struct ipipe_trace_path *path; + int ret = 0; + + flags = __ipipe_global_path_lock(); + + for_each_online_cpu(cpu_id) { + path = &trace_paths[cpu_id][frozen_path[cpu_id]]; + + if (path->dump_lock) { + ret = -EBUSY; + break; + } + + path->begin = -1; + path->end = -1; + path->trace_pos = 0; + path->length = 0; + } + + __ipipe_global_path_unlock(flags); + + return ret; +} +EXPORT_SYMBOL(ipipe_trace_frozen_reset); + +void ipipe_trace_panic_freeze(void) +{ + unsigned long flags; + int cpu_id; + + ipipe_trace_enable = 0; + local_irq_save_hw_notrace(flags); + + cpu_id = ipipe_processor_id(); + + panic_path = &trace_paths[cpu_id][active_path[cpu_id]]; + + local_irq_restore_hw(flags); +} +EXPORT_SYMBOL(ipipe_trace_panic_freeze); + +static void +__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point, + int trylock) +{ + struct task_struct *task = NULL; + char buf[8]; + int i; + int locked = 1; + + if (trylock && !read_trylock(&tasklist_lock)) + locked = 0; + else + read_lock(&tasklist_lock); + + if (locked) + task = find_task_by_pid((pid_t)point->v); + + if (task) + strncpy(task_info, task->comm, 11); + else + strcpy(task_info, "--"); + + if (locked) + read_unlock(&tasklist_lock); + + for (i = strlen(task_info); i < 11; i++) + task_info[i] = ' '; + + sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS); + strcpy(task_info + (11 - strlen(buf)), buf); +} + +void ipipe_trace_panic_dump(void) +{ + int cnt = back_trace; + int start, pos; + char task_info[12]; + + printk("I-pipe tracer log (%d points):\n", cnt); + + start = pos = WRAP_POINT_NO(panic_path->trace_pos-1); + + while (cnt-- > 0) { + struct ipipe_trace_point *point = &panic_path->point[pos]; + long time; + char buf[16]; + + if (!point->eip) + printk("--\n"); + else { + __ipipe_trace_point_type(buf, point); + printk(buf); + + switch (point->type & IPIPE_TYPE_MASK) { + case IPIPE_TRACE_FUNC: + printk(" "); + break; + + case IPIPE_TRACE_PID: + __ipipe_get_task_info(task_info, + point, 1); + printk(task_info); + break; + + default: + printk("0x%08lx ", point->v); + } + + time = __ipipe_signed_tsc2us(point->timestamp - + panic_path->point[start].timestamp); + printk(" %5ld ", time); + + __ipipe_print_symname(NULL, point->eip); + printk(" ("); + __ipipe_print_symname(NULL, point->parent_eip); + printk(")\n"); + } + pos = WRAP_POINT_NO(pos - 1); + } +} +EXPORT_SYMBOL(ipipe_trace_panic_dump); + + +/* --- /proc output --- */ + +static notrace int __ipipe_in_critical_trpath(long point_no) +{ + return ((WRAP_POINT_NO(point_no-print_path->begin) < + WRAP_POINT_NO(print_path->end-print_path->begin)) || + ((print_path->end == print_path->begin) && + (WRAP_POINT_NO(point_no-print_path->end) > + print_post_trace))); +} + +static long __ipipe_signed_tsc2us(long long tsc) +{ + unsigned long long abs_tsc; + long us; + + /* ipipe_tsc2us works on unsigned => handle sign separately */ + abs_tsc = (tsc >= 0) ? tsc : -tsc; + us = ipipe_tsc2us(abs_tsc); + if (tsc < 0) + return -us; + else + return us; +} + +static void +__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point) +{ + switch (point->type & IPIPE_TYPE_MASK) { + case IPIPE_TRACE_FUNC: + strcpy(buf, "func "); + break; + + case IPIPE_TRACE_BEGIN: + strcpy(buf, "begin "); + break; + + case IPIPE_TRACE_END: + strcpy(buf, "end "); + break; + + case IPIPE_TRACE_FREEZE: + strcpy(buf, "freeze "); + break; + + case IPIPE_TRACE_SPECIAL: + sprintf(buf, "(0x%02x) ", + point->type >> IPIPE_TYPE_BITS); + break; + + case IPIPE_TRACE_PID: + sprintf(buf, "[%5d] ", (pid_t)point->v); + break; + } +} + +static void +__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point) +{ + char mark = ' '; + int point_no = point - print_path->point; + int i; + + if (print_path->end == point_no) + mark = '<'; + else if (print_path->begin == point_no) + mark = '>'; + else if (__ipipe_in_critical_trpath(point_no)) + mark = ':'; + seq_printf(m, "%c%c", mark, + (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' '); + + if (!verbose_trace) + return; + + for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--) + seq_printf(m, "%c", + (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ? + (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? + '#' : '+') : + (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' ')); +} + +static void +__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point) +{ + unsigned long delay = 0; + int next; + char *mark = " "; + + next = WRAP_POINT_NO(point+1 - print_path->point); + + if (next != print_path->trace_pos) + delay = ipipe_tsc2ns(print_path->point[next].timestamp - + point->timestamp); + + if (__ipipe_in_critical_trpath(point - print_path->point)) { + if (delay > IPIPE_DELAY_WARN) + mark = "! "; + else if (delay > IPIPE_DELAY_NOTE) + mark = "+ "; + } + seq_puts(m, mark); + + if (verbose_trace) + seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000, + (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' '); + else + seq_puts(m, " "); +} + +static void __ipipe_print_symname(struct seq_file *m, unsigned long eip) +{ + char namebuf[KSYM_NAME_LEN+1]; + unsigned long size, offset; + const char *sym_name; + char *modname; + + sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf); + + /* printing to /proc? */ + if (m) { + if (sym_name) { + if (verbose_trace) { + seq_printf(m, "%s+0x%lx", sym_name, offset); + if (modname) + seq_printf(m, " [%s]", modname); + } else + seq_puts(m, sym_name); + } else + seq_printf(m, "<%08lx>", eip); + } else { + /* panic dump */ + if (sym_name) { + printk("%s+0x%lx", sym_name, offset); + if (modname) + printk(" [%s]", modname); + } + } +} + +#if defined(CONFIG_XENO_OPT_DEBUG) || defined(CONFIG_DEBUG_PREEMPT) +static void __ipipe_print_dbgwarning(struct seq_file *m) +{ + seq_puts(m, "\n******** WARNING ********\n" + "The following debugging options will increase the observed " + "latencies:\n" +#ifdef CONFIG_XENO_OPT_DEBUG + " o CONFIG_XENO_OPT_DEBUG\n" +#endif /* CONFIG_XENO_OPT_DEBUG */ +#ifdef CONFIG_XENO_OPT_DEBUG_QUEUES + " o CONFIG_XENO_OPT_DEBUG_QUEUES (very costly)\n" +#endif /* CONFIG_XENO_OPT_DEBUG */ +#ifdef CONFIG_DEBUG_PREEMPT + " o CONFIG_DEBUG_PREEMPT\n" +#endif /* CONFIG_DEBUG_PREEMPT */ + "\n"); +} +#else /* !WARN_ON_DEBUGGING_LATENCIES */ +# define __ipipe_print_dbgwarning(m) +#endif /* WARN_ON_DEBUGGING_LATENCIES */ + +static void __ipipe_print_headline(struct seq_file *m) +{ + if (verbose_trace) { + const char *name[4] = { [0 ... 3] = "" }; + struct list_head *pos; + int i = 0; + + list_for_each_prev(pos, &__ipipe_pipeline) { + struct ipipe_domain *ipd = + list_entry(pos, struct ipipe_domain, p_link); + + name[i] = ipd->name; + if (++i > 3) + break; + } + + seq_printf(m, + " +----- Hard IRQs ('|': locked)\n" + " |+---- %s\n" + " ||+--- %s\n" + " |||+-- %s\n" + " ||||+- %s%s\n" + " ||||| +---------- " + "Delay flag ('+': > %d us, '!': > %d us)\n" + " ||||| | +- " + "NMI noise ('N')\n" + " ||||| | |\n" + " Type User Val. Time Delay Function " + "(Parent)\n", + name[3], name[2], name[1], name[0], + name[0] ? " ('*': domain stalled, '+': current, " + "'#': current+stalled)" : "", + IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000); + } else + seq_printf(m, + " +--------------- Hard IRQs ('|': locked)\n" + " | +- Delay flag " + "('+': > %d us, '!': > %d us)\n" + " | |\n" + " Type Time Function (Parent)\n", + IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000); +} + +static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos) +{ + loff_t n = *pos; + + down(&out_mutex); + + if (!n) { + struct ipipe_trace_path *path; + unsigned long length_usecs; + int points, i; + unsigned long flags; + + /* protect against max_path/frozen_path updates while we + * haven't locked our target path, also avoid recursively + * taking global_path_lock from NMI context */ + flags = __ipipe_global_path_lock(); + + /* find the longest of all per-cpu paths */ + print_path = NULL; + for_each_online_cpu(i) { + path = &trace_paths[i][max_path[i]]; + if ((print_path == NULL) || + (path->length > print_path->length)) + print_path = path; + } + print_path->dump_lock = 1; + + __ipipe_global_path_unlock(flags); + + /* does this path actually contain data? */ + if (print_path->end == print_path->begin) + return NULL; + + /* number of points inside the critical path */ + points = WRAP_POINT_NO(print_path->end-print_path->begin+1); + + /* pre- and post-tracing length, post-trace length was frozen + in __ipipe_trace, pre-trace may have to be reduced due to + buffer overrun */ + print_pre_trace = pre_trace; + print_post_trace = WRAP_POINT_NO(print_path->trace_pos - + print_path->end - 1); + if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1) + print_pre_trace = IPIPE_TRACE_POINTS - 1 - points - + print_post_trace; + + length_usecs = ipipe_tsc2us(print_path->length); + seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe-%s\n" + "------------------------------------------------------------\n", + UTS_RELEASE, IPIPE_ARCH_STRING); + __ipipe_print_dbgwarning(m); + seq_printf(m, "Begin: %lld cycles, Trace Points: %d (-%d/+%d), " + "Length: %lu us\n\n", + print_path->point[print_path->begin].timestamp, + points, print_pre_trace, print_post_trace, length_usecs); + __ipipe_print_headline(m); + } + + /* check if we are inside the trace range */ + if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 + + print_pre_trace + print_post_trace)) + return NULL; + + /* return the next point to be shown */ + return &print_path->point[WRAP_POINT_NO(print_path->begin - + print_pre_trace + n)]; +} + +static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos) +{ + loff_t n = ++*pos; + + /* check if we are inside the trace range with the next entry */ + if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 + + print_pre_trace + print_post_trace)) + return NULL; + + /* return the next point to be shown */ + return &print_path->point[WRAP_POINT_NO(print_path->begin - + print_pre_trace + *pos)]; +} + +static void __ipipe_prtrace_stop(struct seq_file *m, void *p) +{ + if (print_path) + print_path->dump_lock = 0; + up(&out_mutex); +} + +static int __ipipe_prtrace_show(struct seq_file *m, void *p) +{ + long time; + struct ipipe_trace_point *point = p; + char buf[16]; + + if (!point->eip) { + seq_puts(m, "--\n"); + return 0; + } + + __ipipe_print_pathmark(m, point); + __ipipe_trace_point_type(buf, point); + seq_puts(m, buf); + if (verbose_trace) + switch (point->type & IPIPE_TYPE_MASK) { + case IPIPE_TRACE_FUNC: + seq_puts(m, " "); + break; + + case IPIPE_TRACE_PID: + __ipipe_get_task_info(buf, point, 0); + seq_puts(m, buf); + break; + + default: + seq_printf(m, "0x%08lx ", point->v); + } + + time = __ipipe_signed_tsc2us(point->timestamp - + print_path->point[print_path->begin].timestamp); + seq_printf(m, "%5ld", time); + + __ipipe_print_delay(m, point); + __ipipe_print_symname(m, point->eip); + seq_puts(m, " ("); + __ipipe_print_symname(m, point->parent_eip); + seq_puts(m, ")\n"); + + return 0; +} + +static struct seq_operations __ipipe_max_ptrace_ops = { + .start = __ipipe_max_prtrace_start, + .next = __ipipe_prtrace_next, + .stop = __ipipe_prtrace_stop, + .show = __ipipe_prtrace_show +}; + +static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &__ipipe_max_ptrace_ops); +} + +static ssize_t +__ipipe_max_reset(struct file *file, const char __user *pbuffer, + size_t count, loff_t *data) +{ + down(&out_mutex); + ipipe_trace_max_reset(); + up(&out_mutex); + + return count; +} + +struct file_operations __ipipe_max_prtrace_fops = { + .open = __ipipe_max_prtrace_open, + .read = seq_read, + .write = __ipipe_max_reset, + .llseek = seq_lseek, + .release = seq_release, +}; + +static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos) +{ + loff_t n = *pos; + + down(&out_mutex); + + if (!n) { + struct ipipe_trace_path *path; + int i; + unsigned long flags; + + /* protect against max_path/frozen_path updates while we + * haven't locked our target path, also avoid recursively + * taking global_path_lock from NMI context */ + flags = __ipipe_global_path_lock(); + + /* find the first of all per-cpu frozen paths */ + print_path = NULL; + for_each_online_cpu(i) { + path = &trace_paths[i][frozen_path[i]]; + if (path->end >= 0) + print_path = path; + } + if (print_path) + print_path->dump_lock = 1; + + __ipipe_global_path_unlock(flags); + + if (!print_path) + return NULL; + + /* back- and post-tracing length, post-trace length was frozen + in __ipipe_trace, back-trace may have to be reduced due to + buffer overrun */ + print_pre_trace = back_trace-1; /* substract freeze point */ + print_post_trace = WRAP_POINT_NO(print_path->trace_pos - + print_path->end - 1); + if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1) + print_pre_trace = IPIPE_TRACE_POINTS - 2 - + print_post_trace; + + seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe-%s\n" + "------------------------------------------------------" + "------\n", + UTS_RELEASE, IPIPE_ARCH_STRING); + __ipipe_print_dbgwarning(m); + seq_printf(m, "Freeze: %lld cycles, Trace Points: %d (+%d)\n\n", + print_path->point[print_path->begin].timestamp, + print_pre_trace+1, print_post_trace); + __ipipe_print_headline(m); + } + + /* check if we are inside the trace range */ + if (n >= print_pre_trace + 1 + print_post_trace) + return NULL; + + /* return the next point to be shown */ + return &print_path->point[WRAP_POINT_NO(print_path->begin- + print_pre_trace+n)]; +} + +static struct seq_operations __ipipe_frozen_ptrace_ops = { + .start = __ipipe_frozen_prtrace_start, + .next = __ipipe_prtrace_next, + .stop = __ipipe_prtrace_stop, + .show = __ipipe_prtrace_show +}; + +static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &__ipipe_frozen_ptrace_ops); +} + +static ssize_t +__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer, + size_t count, loff_t *data) +{ + char *end, buf[16]; + int val; + int n; + + n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count; + + if (copy_from_user(buf, pbuffer, n)) + return -EFAULT; + + buf[n] = '\0'; + val = simple_strtol(buf, &end, 0); + + if (((*end != '\0') && !isspace(*end)) || (val < 0)) + return -EINVAL; + + down(&out_mutex); + ipipe_trace_frozen_reset(); + if (val > 0) + ipipe_trace_freeze(-1); + up(&out_mutex); + + return count; +} + +struct file_operations __ipipe_frozen_prtrace_fops = { + .open = __ipipe_frozen_prtrace_open, + .read = seq_read, + .write = __ipipe_frozen_ctrl, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int __ipipe_rd_proc_val(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + int len; + + len = sprintf(page, "%u\n", *(int *)data); + len -= off; + if (len <= off + count) + *eof = 1; + *start = page + off; + if (len > count) + len = count; + if (len < 0) + len = 0; + + return len; +} + +static int __ipipe_wr_proc_val(struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + char *end, buf[16]; + int val; + int n; + + n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count; + + if (copy_from_user(buf, buffer, n)) + return -EFAULT; + + buf[n] = '\0'; + val = simple_strtol(buf, &end, 0); + + if (((*end != '\0') && !isspace(*end)) || (val < 0)) + return -EINVAL; + + down(&out_mutex); + *(int *)data = val; + up(&out_mutex); + + return count; +} + +extern struct proc_dir_entry *ipipe_proc_root; + +static void __init +__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir, + const char *name, int *value_ptr) +{ + struct proc_dir_entry *entry; + + entry = create_proc_entry(name, 0644, trace_dir); + if (entry) { + entry->data = value_ptr; + entry->read_proc = __ipipe_rd_proc_val; + entry->write_proc = __ipipe_wr_proc_val; + entry->owner = THIS_MODULE; + } +} + +void __init __ipipe_init_tracer(void) +{ + struct proc_dir_entry *trace_dir; + struct proc_dir_entry *entry; +#ifdef CONFIG_IPIPE_TRACE_VMALLOC + int cpu, path; + + for (cpu = 0; cpu < NR_CPUS; cpu++) { + trace_paths[cpu] = vmalloc( + sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS); + if (!trace_paths) { + printk(KERN_ERR "I-pipe: " + "insufficient memory for trace buffer.\n"); + return; + } + memset(trace_paths[cpu], 0, + sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS); + for (path = 0; path < IPIPE_TRACE_PATHS; path++) { + trace_paths[cpu][path].begin = -1; + trace_paths[cpu][path].end = -1; + } + } + ipipe_trace_enable = CONFIG_IPIPE_TRACE_ENABLE_VALUE; +#endif /* CONFIG_IPIPE_TRACE_VMALLOC */ + + trace_dir = create_proc_entry("trace", S_IFDIR, ipipe_proc_root); + + entry = create_proc_entry("max", 0644, trace_dir); + if (entry) + entry->proc_fops = &__ipipe_max_prtrace_fops; + + entry = create_proc_entry("frozen", 0644, trace_dir); + if (entry) + entry->proc_fops = &__ipipe_frozen_prtrace_fops; + + __ipipe_create_trace_proc_val(trace_dir, "pre_trace_points", + &pre_trace); + __ipipe_create_trace_proc_val(trace_dir, "post_trace_points", + &post_trace); + __ipipe_create_trace_proc_val(trace_dir, "back_trace_points", + &back_trace); + __ipipe_create_trace_proc_val(trace_dir, "verbose", + &verbose_trace); + __ipipe_create_trace_proc_val(trace_dir, "enable", + &ipipe_trace_enable); +} diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/irq/chip.c ipipe-2.6.19-arm/kernel/irq/chip.c --- linux-2.6.19/kernel/irq/chip.c 2007-01-04 22:05:14.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/irq/chip.c 2007-01-15 21:33:01.000000000 +0100 @@ -308,7 +308,9 @@ handle_level_irq(unsigned int irq, struc irqreturn_t action_ret; spin_lock(&desc->lock); +#ifndef CONFIG_IPIPE mask_ack_irq(desc, irq); +#endif /* CONFIG_IPIPE */ if (unlikely(desc->status & IRQ_INPROGRESS)) goto out_unlock; @@ -431,8 +433,10 @@ handle_edge_irq(unsigned int irq, struct kstat_cpu(cpu).irqs[irq]++; +#ifndef CONFIG_IPIPE /* Start handling the irq */ desc->chip->ack(irq); +#endif /* CONFIG_IPIPE */ /* Mark the IRQ currently in progress.*/ desc->status |= IRQ_INPROGRESS; @@ -472,6 +476,64 @@ out_unlock: spin_unlock(&desc->lock); } +#ifdef CONFIG_IPIPE + +void fastcall __ipipe_ack_simple_irq(unsigned irq, struct irq_desc *desc) +{ +} + +void fastcall __ipipe_end_simple_irq(unsigned irq, struct irq_desc *desc) +{ +} + +void fastcall __ipipe_ack_level_irq(unsigned irq, struct irq_desc *desc) +{ + mask_ack_irq(desc, irq); +} + +void fastcall __ipipe_end_level_irq(unsigned irq, struct irq_desc *desc) +{ + if (desc->chip->unmask) + desc->chip->unmask(irq); +} + +void fastcall __ipipe_ack_fasteoi_irq(unsigned irq, struct irq_desc *desc) +{ +} + +void fastcall __ipipe_end_fasteoi_irq(unsigned irq, struct irq_desc *desc) +{ + desc->chip->eoi(irq); +} + +void fastcall __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc) +{ + desc->chip->ack(irq); +} + +void fastcall __ipipe_end_edge_irq(unsigned irq, struct irq_desc *desc) +{ +} + +void fastcall __ipipe_ack_bad_irq(unsigned irq, struct irq_desc *desc) +{ + static int done; + + handle_bad_irq(irq, desc); + + if (!done) { + printk(KERN_WARNING "%s: unknown flow handler for IRQ %d\n", + __FUNCTION__, irq); + done = 1; + } +} + +void fastcall __ipipe_end_bad_irq(unsigned irq, struct irq_desc *desc) +{ +} + +#endif /* CONFIG_IPIPE */ + #ifdef CONFIG_SMP /** * handle_percpu_IRQ - Per CPU local irq handler @@ -487,8 +549,10 @@ handle_percpu_irq(unsigned int irq, stru kstat_this_cpu.irqs[irq]++; +#ifndef CONFIG_IPIPE if (desc->chip->ack) desc->chip->ack(irq); +#endif /* CONFIG_IPIPE */ action_ret = handle_IRQ_event(irq, desc->action); if (!noirqdebug) @@ -498,6 +562,22 @@ handle_percpu_irq(unsigned int irq, stru desc->chip->eoi(irq); } +#ifdef CONFIG_IPIPE + +void fastcall __ipipe_ack_percpu_irq(unsigned irq, struct irq_desc *desc) +{ + if (desc->chip->ack) + desc->chip->ack(irq); +} + +void fastcall __ipipe_end_percpu_irq(unsigned irq, struct irq_desc *desc) +{ + if (desc->chip->eoi) + desc->chip->eoi(irq); +} + +#endif /* CONFIG_IPIPE */ + #endif /* CONFIG_SMP */ void @@ -517,6 +597,34 @@ __set_irq_handler(unsigned int irq, irq_ if (!handle) handle = handle_bad_irq; +#ifdef CONFIG_IPIPE + else if (handle == &handle_simple_irq) { + desc->ipipe_ack = &__ipipe_ack_simple_irq; + desc->ipipe_end = &__ipipe_end_simple_irq; + } + else if (handle == &handle_level_irq) { + desc->ipipe_ack = &__ipipe_ack_level_irq; + desc->ipipe_end = &__ipipe_end_level_irq; + } + else if (handle == &handle_edge_irq) { + desc->ipipe_ack = &__ipipe_ack_edge_irq; + desc->ipipe_end = &__ipipe_end_edge_irq; + } + else if (handle == &handle_fasteoi_irq) { + desc->ipipe_ack = &__ipipe_ack_fasteoi_irq; + desc->ipipe_end = &__ipipe_end_fasteoi_irq; + } +#ifdef CONFIG_SMP + else if (handle == &handle_percpu_irq) { + desc->ipipe_ack = &__ipipe_ack_percpu_irq; + desc->ipipe_end = &__ipipe_end_percpu_irq; + } +#endif /* CONFIG_SMP */ + else { + desc->ipipe_ack = &__ipipe_ack_bad_irq; + desc->ipipe_end = &__ipipe_end_bad_irq; + } +#endif /* CONFIG_IPIPE */ if (desc->chip == &no_irq_chip) { printk(KERN_WARNING "Trying to install %sinterrupt handler " @@ -541,6 +649,10 @@ __set_irq_handler(unsigned int irq, irq_ } desc->status |= IRQ_DISABLED; desc->depth = 1; +#ifdef CONFIG_IPIPE + desc->ipipe_ack = &__ipipe_ack_bad_irq; + desc->ipipe_end = &__ipipe_end_bad_irq; +#endif /* CONFIG_IPIPE */ } desc->handle_irq = handle; desc->name = name; diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/printk.c ipipe-2.6.19-arm/kernel/printk.c --- linux-2.6.19/kernel/printk.c 2007-01-17 00:14:34.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/printk.c 2007-01-19 00:01:02.000000000 +0100 @@ -489,6 +489,78 @@ static int have_callable_console(void) * printf(3) */ +#ifdef CONFIG_IPIPE + +static ipipe_spinlock_t __ipipe_printk_lock = IPIPE_SPIN_LOCK_UNLOCKED; + +static int __ipipe_printk_fill; + +static char __ipipe_printk_buf[__LOG_BUF_LEN]; + +void __ipipe_flush_printk (unsigned virq, void *cookie) +{ + char *p = __ipipe_printk_buf; + int len, lmax, out = 0; + unsigned long flags; + + goto start; + + do { + spin_unlock_irqrestore(&__ipipe_printk_lock, flags); + start: + lmax = __ipipe_printk_fill; + while (out < lmax) { + len = strlen(p) + 1; + printk("%s",p); + p += len; + out += len; + } + spin_lock_irqsave(&__ipipe_printk_lock, flags); + } + while (__ipipe_printk_fill != lmax); + + __ipipe_printk_fill = 0; + + spin_unlock_irqrestore(&__ipipe_printk_lock, flags); +} + +asmlinkage int printk(const char *fmt, ...) +{ + int r, fbytes, oldcount; + unsigned long flags; + va_list args; + + va_start(args, fmt); + + if (ipipe_current_domain == ipipe_root_domain || + test_bit(IPIPE_SPRINTK_FLAG,&ipipe_current_domain->flags) || + oops_in_progress) { + r = vprintk(fmt, args); + goto out; + } + + spin_lock_irqsave(&__ipipe_printk_lock, flags); + + oldcount = __ipipe_printk_fill; + fbytes = __LOG_BUF_LEN - oldcount; + + if (fbytes > 1) { + r = vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill, + fbytes, fmt, args) + 1; /* account for the null byte */ + __ipipe_printk_fill += r; + } else + r = 0; + + spin_unlock_irqrestore(&__ipipe_printk_lock, flags); + + if (oldcount == 0) + ipipe_trigger_irq(__ipipe_printk_virq); +out: + va_end(args); + + return r; +} +#else /* !CONFIG_IPIPE */ asmlinkage int printk(const char *fmt, ...) { va_list args; @@ -500,6 +572,7 @@ asmlinkage int printk(const char *fmt, . return r; } +#endif /* CONFIG_IPIPE */ /* cpu currently holding logbuf_lock */ static volatile unsigned int printk_cpu = UINT_MAX; diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/sched.c ipipe-2.6.19-arm/kernel/sched.c --- linux-2.6.19/kernel/sched.c 2007-01-04 22:05:15.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/sched.c 2007-01-15 21:33:01.000000000 +0100 @@ -1391,7 +1391,7 @@ static int try_to_wake_up(struct task_st rq = task_rq_lock(p, &flags); old_state = p->state; - if (!(old_state & state)) + if (!(old_state & state) || (old_state & TASK_NOWAKEUP)) goto out; if (p->array) @@ -1809,6 +1809,8 @@ asmlinkage void schedule_tail(struct tas #endif if (current->set_child_tid) put_user(current->pid, current->set_child_tid); + + ipipe_init_notify(current); } /* @@ -3324,12 +3326,17 @@ asmlinkage void __sched schedule(void) long *switch_count; struct rq *rq; +#ifdef CONFIG_IPIPE + if (unlikely(!ipipe_root_domain_p)) + return; +#endif /* CONFIG_IPIPE */ /* * Test if we are atomic. Since do_exit() needs to call into * schedule() atomically, we ignore that path for now. * Otherwise, whine if we are scheduling when we should not be. */ - if (unlikely(in_atomic() && !current->exit_state)) { + if (unlikely(!(current->state & TASK_ATOMICSWITCH) && in_atomic() && + !current->exit_state)) { printk(KERN_ERR "BUG: scheduling while atomic: " "%s/0x%08x/%d\n", current->comm, preempt_count(), current->pid); @@ -3337,8 +3344,13 @@ asmlinkage void __sched schedule(void) } profile_hit(SCHED_PROFILING, __builtin_return_address(0)); + if (unlikely(current->state & TASK_ATOMICSWITCH)) { + current->state &= ~TASK_ATOMICSWITCH; + goto need_resched_nodisable; + } need_resched: preempt_disable(); +need_resched_nodisable: prev = current; release_kernel_lock(prev); need_resched_nonpreemptible: @@ -3456,6 +3468,8 @@ switch_tasks: prepare_task_switch(rq, next); prev = context_switch(rq, prev, next); barrier(); + if (task_hijacked(prev)) + return; /* * this_rq must be evaluated again because prev may have moved * CPUs since it called schedule(), thus the 'rq' on its stack @@ -3469,7 +3483,7 @@ switch_tasks: if (unlikely(reacquire_kernel_lock(prev) < 0)) goto need_resched_nonpreemptible; preempt_enable_no_resched(); - if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) + if (unlikely(test_thread_flag(TIF_NEED_RESCHED) && ipipe_root_domain_p)) goto need_resched; } EXPORT_SYMBOL(schedule); @@ -3487,6 +3501,11 @@ asmlinkage void __sched preempt_schedule struct task_struct *task = current; int saved_lock_depth; #endif +#ifdef CONFIG_IPIPE + /* Do not reschedule over non-Linux domains. */ + if (unlikely(!ipipe_root_domain_p)) + return; +#endif /* CONFIG_IPIPE */ /* * If there is a non-zero preempt_count or interrupts are disabled, * we do not want to preempt the current task. Just return.. @@ -4185,6 +4204,7 @@ recheck: deactivate_task(p, rq); oldprio = p->prio; __setscheduler(p, policy, param->sched_priority); + ipipe_setsched_notify(p); if (array) { __activate_task(p, rq); /* @@ -6950,3 +6970,50 @@ void set_curr_task(int cpu, struct task_ } #endif + +#ifdef CONFIG_IPIPE + +int ipipe_setscheduler_root (struct task_struct *p, int policy, int prio) +{ + struct prio_array *array; + unsigned long flags; + struct rq *rq; + int oldprio; + + rq = task_rq_lock(p, &flags); + array = p->array; + if (array) + deactivate_task(p, rq); + oldprio = p->prio; + __setscheduler(p, policy, prio); + if (array) { + __activate_task(p, rq); + if (task_running(rq, p)) { + if (p->prio > oldprio) + resched_task(rq->curr); + } else if (TASK_PREEMPTS_CURR(p, rq)) + resched_task(rq->curr); + } + task_rq_unlock(rq, &flags); + + return 0; +} + +EXPORT_SYMBOL(ipipe_setscheduler_root); + +int ipipe_reenter_root (struct task_struct *prev, int policy, int prio) +{ + finish_task_switch(this_rq(), prev); + if (reacquire_kernel_lock(current) < 0) + ; + preempt_enable_no_resched(); + + if (current->policy != policy || current->rt_priority != prio) + return ipipe_setscheduler_root(current,policy,prio); + + return 0; +} + +EXPORT_SYMBOL(ipipe_reenter_root); + +#endif /* CONFIG_IPIPE */ diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/signal.c ipipe-2.6.19-arm/kernel/signal.c --- linux-2.6.19/kernel/signal.c 2007-01-04 22:05:15.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/signal.c 2007-01-15 21:33:01.000000000 +0100 @@ -504,6 +504,7 @@ void signal_wake_up(struct task_struct * unsigned int mask; set_tsk_thread_flag(t, TIF_SIGPENDING); + ipipe_sigwake_notify(t); /* TIF_SIGPENDING must be set first. */ /* * For SIGKILL, we want to wake it up in the stopped/traced case. diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/kernel/spinlock.c ipipe-2.6.19-arm/kernel/spinlock.c --- linux-2.6.19/kernel/spinlock.c 2007-01-04 22:05:15.000000000 +0100 +++ ipipe-2.6.19-arm/kernel/spinlock.c 2007-01-15 21:33:01.000000000 +0100 @@ -88,7 +88,7 @@ unsigned long __lockfunc _spin_lock_irqs * _raw_spin_lock_flags() code, because lockdep assumes * that interrupts are not re-enabled during lock-acquire: */ -#ifdef CONFIG_PROVE_LOCKING +#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_IPIPE) _raw_spin_lock(lock); #else _raw_spin_lock_flags(lock, &flags); @@ -305,7 +305,7 @@ unsigned long __lockfunc _spin_lock_irqs * _raw_spin_lock_flags() code, because lockdep assumes * that interrupts are not re-enabled during lock-acquire: */ -#ifdef CONFIG_PROVE_SPIN_LOCKING +#if defined(CONFIG_PROVE_SPIN_LOCKING) || defined(CONFIG_IPIPE) _raw_spin_lock(lock); #else _raw_spin_lock_flags(lock, &flags); diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/lib/Kconfig.debug ipipe-2.6.19-arm/lib/Kconfig.debug --- linux-2.6.19/lib/Kconfig.debug 2007-01-04 22:05:15.000000000 +0100 +++ ipipe-2.6.19-arm/lib/Kconfig.debug 2007-01-15 21:33:01.000000000 +0100 @@ -46,6 +46,8 @@ config UNUSED_SYMBOLS you really need it, and what the merge plan to the mainline kernel for your module is. +source "kernel/ipipe/Kconfig.debug" + config DEBUG_KERNEL bool "Kernel debugging" help diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/lib/ioremap.c ipipe-2.6.19-arm/lib/ioremap.c --- linux-2.6.19/lib/ioremap.c 2007-01-04 22:05:15.000000000 +0100 +++ ipipe-2.6.19-arm/lib/ioremap.c 2007-01-15 21:33:01.000000000 +0100 @@ -11,6 +11,7 @@ #include #include +#include static int ioremap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, unsigned long phys_addr, pgprot_t prot) @@ -84,6 +85,7 @@ int ioremap_page_range(unsigned long add err = ioremap_pud_range(pgd, addr, next, phys_addr+addr, prot); if (err) break; + set_pgdir(addr, *pgd); } while (pgd++, addr = next, addr != end); flush_cache_vmap(start, end); diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/lib/smp_processor_id.c ipipe-2.6.19-arm/lib/smp_processor_id.c --- linux-2.6.19/lib/smp_processor_id.c 2006-05-07 15:37:02.000000000 +0200 +++ ipipe-2.6.19-arm/lib/smp_processor_id.c 2007-01-15 21:33:01.000000000 +0100 @@ -13,6 +13,11 @@ unsigned int debug_smp_processor_id(void int this_cpu = raw_smp_processor_id(); cpumask_t this_mask; +#ifdef CONFIG_IPIPE + if (ipipe_current_domain != ipipe_root_domain) + return this_cpu; +#endif /* CONFIG_IPIPE */ + if (likely(preempt_count)) goto out; diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/lib/spinlock_debug.c ipipe-2.6.19-arm/lib/spinlock_debug.c --- linux-2.6.19/lib/spinlock_debug.c 2007-01-04 22:05:15.000000000 +0100 +++ ipipe-2.6.19-arm/lib/spinlock_debug.c 2007-01-15 21:33:01.000000000 +0100 @@ -129,6 +129,8 @@ void _raw_spin_lock(spinlock_t *lock) debug_spin_lock_after(lock); } +EXPORT_SYMBOL(_raw_spin_lock); + int _raw_spin_trylock(spinlock_t *lock) { int ret = __raw_spin_trylock(&lock->raw_lock); @@ -144,12 +146,16 @@ int _raw_spin_trylock(spinlock_t *lock) return ret; } +EXPORT_SYMBOL(_raw_spin_trylock); + void _raw_spin_unlock(spinlock_t *lock) { debug_spin_unlock(lock); __raw_spin_unlock(&lock->raw_lock); } +EXPORT_SYMBOL(_raw_spin_unlock); + static void rwlock_bug(rwlock_t *lock, const char *msg) { if (!debug_locks_off()) @@ -195,6 +201,8 @@ void _raw_read_lock(rwlock_t *lock) __raw_read_lock(&lock->raw_lock); } +EXPORT_SYMBOL(_raw_read_lock); + int _raw_read_trylock(rwlock_t *lock) { int ret = __raw_read_trylock(&lock->raw_lock); @@ -208,12 +216,16 @@ int _raw_read_trylock(rwlock_t *lock) return ret; } +EXPORT_SYMBOL(_raw_read_trylock); + void _raw_read_unlock(rwlock_t *lock) { RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic"); __raw_read_unlock(&lock->raw_lock); } +EXPORT_SYMBOL(_raw_read_unlock); + static inline void debug_write_lock_before(rwlock_t *lock) { RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic"); @@ -271,6 +283,8 @@ void _raw_write_lock(rwlock_t *lock) debug_write_lock_after(lock); } +EXPORT_SYMBOL(_raw_write_lock); + int _raw_write_trylock(rwlock_t *lock) { int ret = __raw_write_trylock(&lock->raw_lock); @@ -286,8 +300,12 @@ int _raw_write_trylock(rwlock_t *lock) return ret; } +EXPORT_SYMBOL(_raw_write_trylock); + void _raw_write_unlock(rwlock_t *lock) { debug_write_unlock(lock); __raw_write_unlock(&lock->raw_lock); } + +EXPORT_SYMBOL(_raw_write_unlock); diff -x '*~' -x '*.orig' -x '*.rej' -Naurdp linux-2.6.19/mm/vmalloc.c ipipe-2.6.19-arm/mm/vmalloc.c --- linux-2.6.19/mm/vmalloc.c 2007-01-04 22:05:15.000000000 +0100 +++ ipipe-2.6.19-arm/mm/vmalloc.c 2007-01-15 21:33:01.000000000 +0100 @@ -19,6 +19,7 @@ #include #include +#include DEFINE_RWLOCK(vmlist_lock); @@ -151,10 +152,14 @@ int map_vm_area(struct vm_struct *area, BUG_ON(addr >= end); pgd = pgd_offset_k(addr); do { + pgd_t oldpgd; + memcpy(&oldpgd,pgd,sizeof(pgd_t)); next = pgd_addr_end(addr, end); err = vmap_pud_range(pgd, addr, next, prot, pages); if (err) break; + if (pgd_val(oldpgd) != pgd_val(*pgd)) + set_pgdir(addr, *pgd); } while (pgd++, addr = next, addr != end); flush_cache_vmap((unsigned long) area->addr, end); return err;