softirq parameters

softirq parameters

[Matthew Wilcox]

what do you guys think about this patch?  nobody's using the data argument
to the softirq routines, but most of the routines want to know which
CPU they're running on.

also, we have:

static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;

and i rather wonder why we cache-align them.  they only get initialised at
startup (kernel or module init), so this cacheline must be in shared state.

diff -urpNX dontdiff linux-2.5.30/drivers/scsi/scsi.c linux-2.5.30-willy/drivers/scsi/scsi.c
--- linux-2.5.30/drivers/scsi/scsi.c	2002-07-06 07:21:00.000000000 -0600
+++ linux-2.5.30-willy/drivers/scsi/scsi.c	2002-08-04 08:26:13.000000000 -0600
@@ -1192,9 +1192,8 @@ void scsi_done(Scsi_Cmnd * SCpnt)
  * interrupt latency, stack depth, and reentrancy of the low-level
  * drivers.
  */
-static void scsi_softirq(struct softirq_action *h)
+static void scsi_softirq(int cpu)
 {
-	int cpu = smp_processor_id();
 	struct softscsi_data *queue = &softscsi_data[cpu];
 
 	while (queue->head) {
@@ -2567,7 +2566,7 @@ static int __init init_scsi(void)
 	bus_register(&scsi_driverfs_bus_type);
 
 	/* Where we handle work queued by scsi_done */
-	open_softirq(SCSI_SOFTIRQ, scsi_softirq, NULL);
+	open_softirq(SCSI_SOFTIRQ, scsi_softirq);
 
 	return 0;
 }
diff -urpNX dontdiff linux-2.5.30/include/linux/interrupt.h linux-2.5.30-willy/include/linux/interrupt.h
--- linux-2.5.30/include/linux/interrupt.h	2002-07-27 12:09:21.000000000 -0600
+++ linux-2.5.30-willy/include/linux/interrupt.h	2002-07-27 12:12:52.000000000 -0600
@@ -77,12 +77,11 @@ enum
 
 struct softirq_action
 {
-	void	(*action)(struct softirq_action *);
-	void	*data;
+	void	(*action)(int cpu);
 };
 
 asmlinkage void do_softirq(void);
-extern void open_softirq(int nr, void (*action)(struct softirq_action*), void *data);
+extern void open_softirq(int nr, void (*action)(int cpu));
 extern void softirq_init(void);
 #define __cpu_raise_softirq(cpu, nr) do { softirq_pending(cpu) |= 1UL << (nr); } while (0)
 extern void FASTCALL(cpu_raise_softirq(unsigned int cpu, unsigned int nr));
diff -urpNX dontdiff linux-2.5.30/kernel/softirq.c linux-2.5.30-willy/kernel/softirq.c
--- linux-2.5.30/kernel/softirq.c	2002-08-02 05:44:53.000000000 -0600
+++ linux-2.5.30-willy/kernel/softirq.c	2002-08-02 05:45:35.000000000 -0600
@@ -86,7 +86,7 @@ restart:
 
 		do {
 			if (pending & 1)
-				h->action(h);
+				h->action(cpu);
 			h++;
 			pending >>= 1;
 		} while (pending);
@@ -136,9 +136,8 @@ void raise_softirq(unsigned int nr)
 	local_irq_restore(flags);
 }
 
-void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
+void open_softirq(int nr, void (*action)(int cpu))
 {
-	softirq_vec[nr].data = data;
 	softirq_vec[nr].action = action;
 }
 
@@ -176,7 +175,7 @@ void __tasklet_hi_schedule(struct taskle
 	local_irq_restore(flags);
 }
 
-static void tasklet_action(struct softirq_action *a)
+static void tasklet_action(int cpu)
 {
 	struct tasklet_struct *list;
 
@@ -209,7 +208,7 @@ static void tasklet_action(struct softir
 	}
 }
 
-static void tasklet_hi_action(struct softirq_action *a)
+static void tasklet_hi_action(int cpu)
 {
 	struct tasklet_struct *list;
 
@@ -321,8 +320,8 @@ void __init softirq_init()
 	for (i=0; i<32; i++)
 		tasklet_init(bh_task_vec+i, bh_action, i);
 
-	open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
-	open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
+	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
+	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 }
 
 void __run_task_queue(task_queue *list)
diff -urpNX dontdiff linux-2.5.30/net/core/dev.c linux-2.5.30-willy/net/core/dev.c
--- linux-2.5.30/net/core/dev.c	2002-06-20 16:53:53.000000000 -0600
+++ linux-2.5.30-willy/net/core/dev.c	2002-06-29 09:10:38.000000000 -0600
@@ -1333,10 +1333,8 @@ static __inline__ void skb_bond(struct s
 		skb->dev = dev->master;
 }
 
-static void net_tx_action(struct softirq_action *h)
+static void net_tx_action(int cpu)
 {
-	int cpu = smp_processor_id();
-
 	if (softnet_data[cpu].completion_queue) {
 		struct sk_buff *clist;
 
@@ -1573,9 +1571,8 @@ job_done:
 	return 0;
 }
 
-static void net_rx_action(struct softirq_action *h)
+static void net_rx_action(int this_cpu)
 {
-	int this_cpu = smp_processor_id();
 	struct softnet_data *queue = &softnet_data[this_cpu];
 	unsigned long start_time = jiffies;
 	int budget = netdev_max_backlog;
@@ -2816,8 +2813,8 @@ static int __init net_dev_init(void)
 
 	dev_boot_phase = 0;
 
-	open_softirq(NET_TX_SOFTIRQ, net_tx_action, NULL);
-	open_softirq(NET_RX_SOFTIRQ, net_rx_action, NULL);
+	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
+	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
 
 	dst_init();
 	dev_mcast_init();

-- 
Revolutions do not require corporate support.

Re: softirq parameters

[george anzinger]

Matthew Wilcox wrote:
> 
> what do you guys think about this patch?  nobody's using the data argument
> to the softirq routines, but most of the routines want to know which
> CPU they're running on.

I would vote no on this.  While no one is currently using
the data argument, it would be _hard_ to replace it if it
were needed.  The cpu, on the other hand, is available
regardless of it being passed or not and thus does not
_need_ to be passed.

-g
> 
> also, we have:
> 
> static struct softirq_action softirq_vec[32] __cacheline_aligned_in_smp;
> 
> and i rather wonder why we cache-align them.  they only get initialised at
> startup (kernel or module init), so this cacheline must be in shared state.
> 
> diff -urpNX dontdiff linux-2.5.30/drivers/scsi/scsi.c linux-2.5.30-willy/drivers/scsi/scsi.c
> --- linux-2.5.30/drivers/scsi/scsi.c    2002-07-06 07:21:00.000000000 -0600
> +++ linux-2.5.30-willy/drivers/scsi/scsi.c      2002-08-04 08:26:13.000000000 -0600
> @@ -1192,9 +1192,8 @@ void scsi_done(Scsi_Cmnd * SCpnt)
>   * interrupt latency, stack depth, and reentrancy of the low-level
>   * drivers.
>   */
> -static void scsi_softirq(struct softirq_action *h)
> +static void scsi_softirq(int cpu)
>  {
> -       int cpu = smp_processor_id();
>         struct softscsi_data *queue = &softscsi_data[cpu];
> 
>         while (queue->head) {
> @@ -2567,7 +2566,7 @@ static int __init init_scsi(void)
>         bus_register(&scsi_driverfs_bus_type);
> 
>         /* Where we handle work queued by scsi_done */
> -       open_softirq(SCSI_SOFTIRQ, scsi_softirq, NULL);
> +       open_softirq(SCSI_SOFTIRQ, scsi_softirq);
> 
>         return 0;
>  }
> diff -urpNX dontdiff linux-2.5.30/include/linux/interrupt.h linux-2.5.30-willy/include/linux/interrupt.h
> --- linux-2.5.30/include/linux/interrupt.h      2002-07-27 12:09:21.000000000 -0600
> +++ linux-2.5.30-willy/include/linux/interrupt.h        2002-07-27 12:12:52.000000000 -0600
> @@ -77,12 +77,11 @@ enum
> 
>  struct softirq_action
>  {
> -       void    (*action)(struct softirq_action *);
> -       void    *data;
> +       void    (*action)(int cpu);
>  };
> 
>  asmlinkage void do_softirq(void);
> -extern void open_softirq(int nr, void (*action)(struct softirq_action*), void *data);
> +extern void open_softirq(int nr, void (*action)(int cpu));
>  extern void softirq_init(void);
>  #define __cpu_raise_softirq(cpu, nr) do { softirq_pending(cpu) |= 1UL << (nr); } while (0)
>  extern void FASTCALL(cpu_raise_softirq(unsigned int cpu, unsigned int nr));
> diff -urpNX dontdiff linux-2.5.30/kernel/softirq.c linux-2.5.30-willy/kernel/softirq.c
> --- linux-2.5.30/kernel/softirq.c       2002-08-02 05:44:53.000000000 -0600
> +++ linux-2.5.30-willy/kernel/softirq.c 2002-08-02 05:45:35.000000000 -0600
> @@ -86,7 +86,7 @@ restart:
> 
>                 do {
>                         if (pending & 1)
> -                               h->action(h);
> +                               h->action(cpu);
>                         h++;
>                         pending >>= 1;
>                 } while (pending);
> @@ -136,9 +136,8 @@ void raise_softirq(unsigned int nr)
>         local_irq_restore(flags);
>  }
> 
> -void open_softirq(int nr, void (*action)(struct softirq_action*), void *data)
> +void open_softirq(int nr, void (*action)(int cpu))
>  {
> -       softirq_vec[nr].data = data;
>         softirq_vec[nr].action = action;
>  }
> 
> @@ -176,7 +175,7 @@ void __tasklet_hi_schedule(struct taskle
>         local_irq_restore(flags);
>  }
> 
> -static void tasklet_action(struct softirq_action *a)
> +static void tasklet_action(int cpu)
>  {
>         struct tasklet_struct *list;
> 
> @@ -209,7 +208,7 @@ static void tasklet_action(struct softir
>         }
>  }
> 
> -static void tasklet_hi_action(struct softirq_action *a)
> +static void tasklet_hi_action(int cpu)
>  {
>         struct tasklet_struct *list;
> 
> @@ -321,8 +320,8 @@ void __init softirq_init()
>         for (i=0; i<32; i++)
>                 tasklet_init(bh_task_vec+i, bh_action, i);
> 
> -       open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL);
> -       open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL);
> +       open_softirq(TASKLET_SOFTIRQ, tasklet_action);
> +       open_softirq(HI_SOFTIRQ, tasklet_hi_action);
>  }
> 
>  void __run_task_queue(task_queue *list)
> diff -urpNX dontdiff linux-2.5.30/net/core/dev.c linux-2.5.30-willy/net/core/dev.c
> --- linux-2.5.30/net/core/dev.c 2002-06-20 16:53:53.000000000 -0600
> +++ linux-2.5.30-willy/net/core/dev.c   2002-06-29 09:10:38.000000000 -0600
> @@ -1333,10 +1333,8 @@ static __inline__ void skb_bond(struct s
>                 skb->dev = dev->master;
>  }
> 
> -static void net_tx_action(struct softirq_action *h)
> +static void net_tx_action(int cpu)
>  {
> -       int cpu = smp_processor_id();
> -
>         if (softnet_data[cpu].completion_queue) {
>                 struct sk_buff *clist;
> 
> @@ -1573,9 +1571,8 @@ job_done:
>         return 0;
>  }
> 
> -static void net_rx_action(struct softirq_action *h)
> +static void net_rx_action(int this_cpu)
>  {
> -       int this_cpu = smp_processor_id();
>         struct softnet_data *queue = &softnet_data[this_cpu];
>         unsigned long start_time = jiffies;
>         int budget = netdev_max_backlog;
> @@ -2816,8 +2813,8 @@ static int __init net_dev_init(void)
> 
>         dev_boot_phase = 0;
> 
> -       open_softirq(NET_TX_SOFTIRQ, net_tx_action, NULL);
> -       open_softirq(NET_RX_SOFTIRQ, net_rx_action, NULL);
> +       open_softirq(NET_TX_SOFTIRQ, net_tx_action);
> +       open_softirq(NET_RX_SOFTIRQ, net_rx_action);
> 
>         dst_init();
>         dev_mcast_init();
> 
> --
> Revolutions do not require corporate support.
> -
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

-- 
George Anzinger   george@mvista.com
High-res-timers: 
http://sourceforge.net/projects/high-res-timers/
Real time sched:  http://sourceforge.net/projects/rtsched/
Preemption patch:
http://www.kernel.org/pub/linux/kernel/people/rml

Re: softirq parameters

[David S. Miller]

   From: george anzinger <george@mvista.com>
   Date: Sun, 04 Aug 2002 10:38:23 -0700

   Matthew Wilcox wrote:
   > what do you guys think about this patch?  nobody's using the data argument
   > to the softirq routines, but most of the routines want to know which
   > CPU they're running on.
   
   I would vote no on this.  While no one is currently using
   the data argument, it would be _hard_ to replace it if it
   were needed.  The cpu, on the other hand, is available
   regardless of it being passed or not and thus does not
   _need_ to be passed.

I totally disagree.  It is easy to put the specified argument back if
people really need it, because so FEW people use softirq's directly.

It's a 5 minute grep + edit job to put it back.  Prove me wrong.

Next, show me one case where it is actually useful to be able to
specify this argument even theoretically!  All such examples end
up being addresses or tables which could be made available to
the softint handler itself.  Remember, softint's are only for
core subsystems and are to be used rarely.  Tasklets foot the
bill for almost anything else.

Furthermore, this is one of the most important hot paths in
the entire kernel, any simplification and or improvement
in code generated to implement these paths is desirable.

I fully supporty Matthew's change.

Re: softirq parameters

[Rusty Russell]

On Sun, 04 Aug 2002 22:37:46 -0700 (PDT)
"David S. Miller" <davem@redhat.com> wrote:

>    From: george anzinger <george@mvista.com>
>    Date: Sun, 04 Aug 2002 10:38:23 -0700
> 
>    Matthew Wilcox wrote:
>    > what do you guys think about this patch?  nobody's using the data argument
>    > to the softirq routines, but most of the routines want to know which
>    > CPU they're running on.
>    
>    I would vote no on this.  While no one is currently using
>    the data argument, it would be _hard_ to replace it if it
>    were needed.  The cpu, on the other hand, is available
>    regardless of it being passed or not and thus does not
>    _need_ to be passed.
> 
> Furthermore, this is one of the most important hot paths in
> the entire kernel, any simplification and or improvement
> in code generated to implement these paths is desirable.
> 
> I fully supporty Matthew's change.

Partially agree.  Removing all args might be worthwhile.  But all these
softirqs use the "cpu" arg to access per-cpu data, which should be
changed to use the per_cpu_data mechanism anyway, which removes the
point of the arg.

Things haven't been changed over because I haven't pushed the per-cpu
interface changes (required for some archs 8() to Linus yet.  But you'll
want them so we can save space (you only need allocate per-cpu data for
cpus where cpu_possible(i) is true).

Clear?
Rusty.
-- 
   there are those who do and those who hang on and you don't see too
   many doers quoting their contemporaries.  -- Larry McVoy

Re: softirq parameters

[David S. Miller]

   From: Matthew Wilcox <willy@debian.org>
   Date: Wed, 7 Aug 2002 19:23:14 +0100

   On Wed, Aug 07, 2002 at 03:24:23PM +1000, Rusty Russell wrote:
   > Things haven't been changed over because I haven't pushed the per-cpu
   > interface changes (required for some archs 8() to Linus yet.  But you'll
   > want them so we can save space (you only need allocate per-cpu data for
   > cpus where cpu_possible(i) is true).
   
   So what we want is something more like:

Yes that would work.

I'm starting to become leery about this percpu stuff, which ends up
moving critical data structures (in this case softnet) out of the main
kernel image (and thus out of the single large PAGE_SIZE entry many
platforms use to map that part of the kernel).

Since all the per-cpu stuff ends up in the same cluster of bootmem
it probably doesn't matter so much.  Here's to hoping that's true :-)

Re: softirq parameters

[Matthew Wilcox]

On Wed, Aug 07, 2002 at 03:24:23PM +1000, Rusty Russell wrote:
> Partially agree.  Removing all args might be worthwhile.  But all these
> softirqs use the "cpu" arg to access per-cpu data, which should be
> changed to use the per_cpu_data mechanism anyway, which removes the
> point of the arg.

I see.  That makes a lot of sense.

> Things haven't been changed over because I haven't pushed the per-cpu
> interface changes (required for some archs 8() to Linus yet.  But you'll
> want them so we can save space (you only need allocate per-cpu data for
> cpus where cpu_possible(i) is true).

So what we want is something more like:

struct softnet_data softnet_data __per_cpu_data = { NULL };

static void void net_tx_action(void *arg)
{
	struct softnet_data *data = arg;
	if (arg->completion_queue) {
	...
}

	open_softirq(NET_TX_SOFTIRQ, net_tx_action, softnet_data);

and have kernel/softirq.c do:

                do {
                        if (pending & 1)
                                h->action(this_cpu(h->data));
                        h++;
                        pending >>= 1;
                } while (pending);

right?

-- 
Revolutions do not require corporate support.

Re: softirq parameters

[David S. Miller]

   From: Matthew Wilcox <willy@debian.org>
   Date: Wed, 7 Aug 2002 19:35:04 +0100
   
   Hrm, I didn't realise what the implementation of percpu was... does it
   work for modules?
   
Good question, I bet it doesn't based upon how I understand
it to work.

Rusty?

Re: softirq parameters

[Matthew Wilcox]

On Wed, Aug 07, 2002 at 11:18:43AM -0700, David S. Miller wrote:
> I'm starting to become leery about this percpu stuff, which ends up
> moving critical data structures (in this case softnet) out of the main
> kernel image (and thus out of the single large PAGE_SIZE entry many
> platforms use to map that part of the kernel).
> 
> Since all the per-cpu stuff ends up in the same cluster of bootmem
> it probably doesn't matter so much.  Here's to hoping that's true :-)

Hrm, I didn't realise what the implementation of percpu was... does it
work for modules?

-- 
Revolutions do not require corporate support.

Re: softirq parameters

[Rusty Russell]

In message <20020807.112424.60322440.davem@redhat.com> you write:
>    From: Matthew Wilcox <willy@debian.org>
>    Date: Wed, 7 Aug 2002 19:35:04 +0100
>    
>    Hrm, I didn't realise what the implementation of percpu was... does it
>    work for modules?
>    
> Good question, I bet it doesn't based upon how I understand
> it to work.

Yes, it doesn't work for modules.  The IBM guys have a kmalloc_percpu
thing which will, but I'm not trying to change the world.

This is one reason for changing to DECLARE_PER_CPU and DEFINE_PER_CPU
macros (the other is because RTH wants to use __thread for per-cpu
variables).

Rusty.
--
  Anyone who quotes me in their sig is an idiot. -- Rusty Russell.

Re: softirq parameters

[Rusty Russell]

In message <20020807192314.H24631@parcelfarce.linux.theplanet.co.uk> you write:
> So what we want is something more like:
> 
> struct softnet_data softnet_data __per_cpu_data = { NULL };
> 
>                 do {
>                         if (pending & 1)
>                                 h->action(this_cpu(h->data));
>                         h++;
>                         pending >>= 1;
>                 } while (pending);

Yep, a trivial change, but a nice one.

Of course, per-cpu changes needed first.  Linus?

Descriptions moved up here:

Name: get_cpu_var patch
Author: Rusty Russell
Status: Cleanup

D: This makes introduces get_cpu_var()/put_cpu_var() which gets a
D: per-cpu variable and disables preemption, and renames the (unsafe
D: under preemption) "this_cpu()" macro to __get_cpu_var().  It also
D: deletes the redundant definitions in linux/smp.h.

Name: Export __per_cpu_offset so modules can use per-cpu data.
Author: Rusty Russell
Status: Trivial

D: As per Andrew Morton's request.

Name: DECLARE_PER_CPU/DEFINE_PER_CPU patch
Author: Rusty Russell
Status: Cleanup
Depends: Misc/get-percpu.patch.gz Misc/percpu-export.patch.gz

D: This old __per_cpu_data define wasn't enough if an arch wants to
D: use the gcc __thread prefix (thread local storage), which needs to
D: go *before* the type in the definition.  So we have to go for a
D: DECLARE macro, and while we're there, separate DECLARE and DEFINE,
D: as definitions of per-cpu data cannot live in modules.  This also
D: means that accidental direct references to per-cpu variables will
D: be caught at compile time.

Rusty.
--
  Anyone who quotes me in their sig is an idiot. -- Rusty Russell.

diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.29.11839/arch/ia64/kernel/perfmon.c linux-2.5.29.11839.updated/arch/ia64/kernel/perfmon.c
--- linux-2.5.29.11839/arch/ia64/kernel/perfmon.c	Thu Jun 20 01:28:47 2002
+++ linux-2.5.29.11839.updated/arch/ia64/kernel/perfmon.c	Sat Jul 27 17:35:00 2002
@@ -1762,7 +1762,7 @@ pfm_stop(struct task_struct *task, pfm_c
 		ia64_srlz_i();
 
 #ifdef CONFIG_SMP
-		this_cpu(pfm_dcr_pp)  = 0;
+		__get_cpu_var(pfm_dcr_pp)  = 0;
 #else
 		pfm_tasklist_toggle_pp(0);
 #endif
@@ -2163,7 +2163,7 @@ pfm_start(struct task_struct *task, pfm_
 	if (ctx->ctx_fl_system) {
 		
 #ifdef CONFIG_SMP
-		this_cpu(pfm_dcr_pp)  = 1;
+		__get_cpu_var(pfm_dcr_pp)  = 1;
 #else
 		pfm_tasklist_toggle_pp(1);
 #endif
@@ -2218,8 +2218,8 @@ pfm_enable(struct task_struct *task, pfm
 		ia64_srlz_i();
 
 #ifdef CONFIG_SMP
-		this_cpu(pfm_syst_wide) = 1;
-		this_cpu(pfm_dcr_pp)    = 0;
+		__get_cpu_var(pfm_syst_wide) = 1;
+		__get_cpu_var(pfm_dcr_pp)    = 0;
 #endif
 	} else {
		/*
@@ -2973,9 +2973,9 @@ perfmon_proc_info(char *page)
 	p += sprintf(p, "CPU%d syst_wide   : %d\n"
 			"CPU%d dcr_pp      : %d\n", 
 			smp_processor_id(), 
-			this_cpu(pfm_syst_wide), 
+			__get_cpu_var(pfm_syst_wide), 
 			smp_processor_id(), 
-			this_cpu(pfm_dcr_pp));
+			__get_cpu_var(pfm_dcr_pp));
 #endif
 
 	LOCK_PFS();
@@ -3045,7 +3045,7 @@ pfm_syst_wide_update_task(struct task_st
 	/*
 	 * propagate the value of the dcr_pp bit to the psr
 	 */
-	ia64_psr(regs)->pp = mode ? this_cpu(pfm_dcr_pp) : 0;
+	ia64_psr(regs)->pp = mode ? __get_cpu_var(pfm_dcr_pp) : 0;
 }
 #endif
 
@@ -3539,8 +3539,8 @@ pfm_flush_regs (struct task_struct *task
 		ia64_srlz_i();
 
 #ifdef CONFIG_SMP
-		this_cpu(pfm_syst_wide) = 0;
-		this_cpu(pfm_dcr_pp)    = 0;
+		__get_cpu_var(pfm_syst_wide) = 0;
+		__get_cpu_var(pfm_dcr_pp)    = 0;
 #else
 		pfm_tasklist_toggle_pp(0);
 #endif
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.29.11839/arch/ia64/kernel/process.c linux-2.5.29.11839.updated/arch/ia64/kernel/process.c
--- linux-2.5.29.11839/arch/ia64/kernel/process.c	Thu May 30 10:00:47 2002
+++ linux-2.5.29.11839.updated/arch/ia64/kernel/process.c	Sat Jul 27 17:35:00 2002
@@ -194,7 +194,7 @@ ia64_save_extra (struct task_struct *tas
 		pfm_save_regs(task);
 
 # ifdef CONFIG_SMP
-	if (this_cpu(pfm_syst_wide))
+	if (__get_cpu_var(pfm_syst_wide))
 		pfm_syst_wide_update_task(task, 0);
 # endif
 #endif
@@ -216,7 +216,7 @@ ia64_load_extra (struct task_struct *tas
 		pfm_load_regs(task);
 
 # ifdef CONFIG_SMP
-	if (this_cpu(pfm_syst_wide)) pfm_syst_wide_update_task(task, 1);
+	if (__get_cpu_var(pfm_syst_wide)) pfm_syst_wide_update_task(task, 1);
 # endif
 #endif
 
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.29.11839/arch/ia64/kernel/smp.c linux-2.5.29.11839.updated/arch/ia64/kernel/smp.c
--- linux-2.5.29.11839/arch/ia64/kernel/smp.c	Thu Jul 25 10:13:01 2002
+++ linux-2.5.29.11839.updated/arch/ia64/kernel/smp.c	Sat Jul 27 17:35:00 2002
@@ -99,7 +99,7 @@ void
 handle_IPI (int irq, void *dev_id, struct pt_regs *regs)
 {
 	int this_cpu = smp_processor_id();
-	unsigned long *pending_ipis = &this_cpu(ipi_operation);
+	unsigned long *pending_ipis = &__get_cpu_var(ipi_operation);
 	unsigned long ops;
 
 	/* Count this now; we may make a call that never returns. */
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.29.11839/include/asm-generic/percpu.h linux-2.5.29.11839.updated/include/asm-generic/percpu.h
--- linux-2.5.29.11839/include/asm-generic/percpu.h	Mon Apr 15 11:47:44 2002
+++ linux-2.5.29.11839.updated/include/asm-generic/percpu.h	Sat Jul 27 17:35:00 2002
@@ -8,6 +8,6 @@ extern unsigned long __per_cpu_offset[NR
 
 /* var is in discarded region: offset to particular copy we want */
 #define per_cpu(var, cpu) (*RELOC_HIDE(&var, __per_cpu_offset[cpu]))
-#define this_cpu(var) per_cpu(var, smp_processor_id())
+#define __get_cpu_var(var) per_cpu(var, smp_processor_id())
 
 #endif /* _ASM_GENERIC_PERCPU_H_ */
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.29.11839/include/asm-ia64/percpu.h linux-2.5.29.11839.updated/include/asm-ia64/percpu.h
--- linux-2.5.29.11839/include/asm-ia64/percpu.h	Tue Apr 23 11:39:38 2002
+++ linux-2.5.29.11839.updated/include/asm-ia64/percpu.h	Sat Jul 27 17:35:00 2002
@@ -17,7 +17,7 @@
 extern unsigned long __per_cpu_offset[NR_CPUS];
 
 #define per_cpu(var, cpu)	(*(__typeof__(&(var))) ((void *) &(var) + __per_cpu_offset[cpu]))
-#define this_cpu(var)		(var)
+#define __get_cpu_var(var)	(var)
 
 #endif /* !__ASSEMBLY__ */
 
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.29.11839/include/asm-ia64/processor.h linux-2.5.29.11839.updated/include/asm-ia64/processor.h
--- linux-2.5.29.11839/include/asm-ia64/processor.h	Fri Jun 21 09:41:55 2002
+++ linux-2.5.29.11839.updated/include/asm-ia64/processor.h	Sat Jul 27 17:35:00 2002
@@ -181,7 +181,7 @@ extern struct cpuinfo_ia64 {
  * The "local" data pointer.  It points to the per-CPU data of the currently executing
  * CPU, much like "current" points to the per-task data of the currently executing task.
  */
-#define local_cpu_data		(&this_cpu(cpu_info))
+#define local_cpu_data		(&__get_cpu_var(cpu_info))
 #define cpu_data(cpu)		(&per_cpu(cpu_info, cpu))
 
 extern void identify_cpu (struct cpuinfo_ia64 *);
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.29.11839/include/asm-x86_64/percpu.h linux-2.5.29.11839.updated/include/asm-x86_64/percpu.h
--- linux-2.5.29.11839/include/asm-x86_64/percpu.h	Mon Jun 17 23:19:25 2002
+++ linux-2.5.29.11839.updated/include/asm-x86_64/percpu.h	Sat Jul 27 17:35:00 2002
@@ -4,7 +4,7 @@
 #include <asm/pda.h>
 
 /* var is in discarded region: offset to particular copy we want */
-#define this_cpu(var)     (*RELOC_HIDE(&var, read_pda(cpudata_offset)))
+#define __get_cpu_var(var)     (*RELOC_HIDE(&var, read_pda(cpudata_offset)))
 #define per_cpu(var, cpu) (*RELOC_HIDE(&var, per_cpu_pda[cpu]))
 
 void setup_per_cpu_areas(void);
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.29.11839/include/linux/percpu.h linux-2.5.29.11839.updated/include/linux/percpu.h
--- linux-2.5.29.11839/include/linux/percpu.h	Mon Jun 24 00:53:24 2002
+++ linux-2.5.29.11839.updated/include/linux/percpu.h	Sat Jul 27 17:35:00 2002
@@ -8,7 +8,9 @@
 #else
 #define __per_cpu_data
 #define per_cpu(var, cpu)			var
-#define this_cpu(var)				var
+#define __get_cpu_var(var)			var
 #endif
 
+#define get_cpu_var(var) ({ preempt_disable(); __get_cpu_var(var); })
+#define put_cpu_var(var) preempt_enable()
 #endif /* __LINUX_PERCPU_H */
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.29.11839/include/linux/smp.h linux-2.5.29.11839.updated/include/linux/smp.h
--- linux-2.5.29.11839/include/linux/smp.h	Sat Jul 27 15:24:39 2002
+++ linux-2.5.29.11839.updated/include/linux/smp.h	Sat Jul 27 17:35:30 2002
@@ -96,9 +96,6 @@ static inline void smp_send_reschedule_a
 #define cpu_online_map				1
 #define cpu_online(cpu)				({ cpu; 1; })
 #define num_online_cpus()			1
-#define __per_cpu_data
-#define per_cpu(var, cpu)			var
-#define this_cpu(var)				var
 
 /* Need to know about CPUs going up/down? */
 #define register_cpu_notifier(nb) 0
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.29.11839/kernel/softirq.c linux-2.5.29.11839.updated/kernel/softirq.c
--- linux-2.5.29.11839/kernel/softirq.c	Sat Jul 27 16:07:05 2002
+++ linux-2.5.29.11839.updated/kernel/softirq.c	Sat Jul 27 17:35:00 2002
@@ -159,8 +159,8 @@ void __tasklet_schedule(struct tasklet_s
 	unsigned long flags;
 
 	local_irq_save(flags);
-	t->next = this_cpu(tasklet_vec).list;
-	this_cpu(tasklet_vec).list = t;
+	t->next = __get_cpu_var(tasklet_vec).list;
+	__get_cpu_var(tasklet_vec).list = t;
 	cpu_raise_softirq(smp_processor_id(), TASKLET_SOFTIRQ);
 	local_irq_restore(flags);
 }
@@ -170,8 +170,8 @@ void __tasklet_hi_schedule(struct taskle
 	unsigned long flags;
 
 	local_irq_save(flags);
-	t->next = this_cpu(tasklet_hi_vec).list;
-	this_cpu(tasklet_hi_vec).list = t;
+	t->next = __get_cpu_var(tasklet_hi_vec).list;
+	__get_cpu_var(tasklet_hi_vec).list = t;
 	cpu_raise_softirq(smp_processor_id(), HI_SOFTIRQ);
 	local_irq_restore(flags);
 }
@@ -181,8 +181,8 @@ static void tasklet_action(struct softir
 	struct tasklet_struct *list;
 
 	local_irq_disable();
-	list = this_cpu(tasklet_vec).list;
-	this_cpu(tasklet_vec).list = NULL;
+	list = __get_cpu_var(tasklet_vec).list;
+	__get_cpu_var(tasklet_vec).list = NULL;
 	local_irq_enable();
 
 	while (list) {
@@ -202,8 +202,8 @@ static void tasklet_action(struct softir
 		}
 
 		local_irq_disable();
-		t->next = this_cpu(tasklet_vec).list;
-		this_cpu(tasklet_vec).list = t;
+		t->next = __get_cpu_var(tasklet_vec).list;
+		__get_cpu_var(tasklet_vec).list = t;
 		__cpu_raise_softirq(smp_processor_id(), TASKLET_SOFTIRQ);
 		local_irq_enable();
 	}
@@ -214,8 +214,8 @@ static void tasklet_hi_action(struct sof
 	struct tasklet_struct *list;
 
 	local_irq_disable();
-	list = this_cpu(tasklet_hi_vec).list;
-	this_cpu(tasklet_hi_vec).list = NULL;
+	list = __get_cpu_var(tasklet_hi_vec).list;
+	__get_cpu_var(tasklet_hi_vec).list = NULL;
 	local_irq_enable();
 
 	while (list) {
@@ -235,8 +235,8 @@ static void tasklet_hi_action(struct sof
 		}
 
 		local_irq_disable();
-		t->next = this_cpu(tasklet_hi_vec).list;
-		this_cpu(tasklet_hi_vec).list = t;
+		t->next = __get_cpu_var(tasklet_hi_vec).list;
+		__get_cpu_var(tasklet_hi_vec).list = t;
 		__cpu_raise_softirq(smp_processor_id(), HI_SOFTIRQ);
 		local_irq_enable();
 	}

diff -urN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.25/kernel/ksyms.c working-2.5.25-percpu/kernel/ksyms.c
--- linux-2.5.25/kernel/ksyms.c	Sun Jul  7 02:12:24 2002
+++ working-2.5.25-percpu/kernel/ksyms.c	Fri Jul 12 16:03:47 2002
@@ -50,6 +50,7 @@
 #include <linux/namei.h>
 #include <linux/buffer_head.h>
 #include <linux/root_dev.h>
+#include <linux/percpu.h>
 #include <asm/checksum.h>
 
 #if defined(CONFIG_PROC_FS)
@@ -597,3 +598,6 @@
 
 EXPORT_SYMBOL(tasklist_lock);
 EXPORT_SYMBOL(pidhash);
+#if defined(CONFIG_SMP) && defined(__GENERIC_PER_CPU)
+EXPORT_SYMBOL(__per_cpu_offset);
+#endif

diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.28.23931/arch/ia64/kernel/perfmon.c linux-2.5.28.23931.updated/arch/ia64/kernel/perfmon.c
--- linux-2.5.28.23931/arch/ia64/kernel/perfmon.c	Thu Jul 25 11:49:11 2002
+++ linux-2.5.28.23931.updated/arch/ia64/kernel/perfmon.c	Thu Jul 25 12:07:54 2002
@@ -369,8 +369,8 @@ static pmu_config_t	pmu_conf; 	/* PMU co
 static pfm_session_t	pfm_sessions;	/* global sessions information */
 static struct proc_dir_entry *perfmon_dir; /* for debug only */
 static pfm_stats_t	pfm_stats;
-int __per_cpu_data pfm_syst_wide;
-static int __per_cpu_data pfm_dcr_pp;
+DEFINE_PER_CPU(int, pfm_syst_wide);
+static DEFINE_PER_CPU(int, pfm_dcr_pp);
 
 /* sysctl() controls */
 static pfm_sysctl_t pfm_sysctl;
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.28.23931/arch/ia64/kernel/setup.c linux-2.5.28.23931.updated/arch/ia64/kernel/setup.c
--- linux-2.5.28.23931/arch/ia64/kernel/setup.c	Thu May 30 10:00:47 2002
+++ linux-2.5.28.23931.updated/arch/ia64/kernel/setup.c	Thu Jul 25 12:08:02 2002
@@ -58,7 +58,7 @@ extern char _end;
 unsigned long __per_cpu_offset[NR_CPUS];
 #endif
 
-struct cpuinfo_ia64 cpu_info __per_cpu_data;
+DECLARE_PER_CPU(struct cpuinfo_ia64, cpu_info);
 
 unsigned long ia64_phys_stacked_size_p8;
 unsigned long ia64_cycles_per_usec;
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.28.23931/arch/ia64/kernel/smp.c linux-2.5.28.23931.updated/arch/ia64/kernel/smp.c
--- linux-2.5.28.23931/arch/ia64/kernel/smp.c	Thu Jul 25 11:49:11 2002
+++ linux-2.5.28.23931.updated/arch/ia64/kernel/smp.c	Thu Jul 25 12:08:08 2002
@@ -80,7 +80,7 @@ static volatile struct call_data_struct 
 #define IPI_CPU_STOP		1
 
 /* This needs to be cacheline aligned because it is written to by *other* CPUs.  */
-static __u64 ipi_operation __per_cpu_data ____cacheline_aligned;
+static DECLARE_PER_CPU(__u64, ipi_operation) ____cacheline_aligned;
 
 static void
 stop_this_cpu (void)
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.28.23931/include/asm-generic/percpu.h linux-2.5.28.23931.updated/include/asm-generic/percpu.h
--- linux-2.5.28.23931/include/asm-generic/percpu.h	Thu Jul 25 11:49:11 2002
+++ linux-2.5.28.23931.updated/include/asm-generic/percpu.h	Thu Jul 25 12:01:29 2002
@@ -1,13 +1,34 @@
 #ifndef _ASM_GENERIC_PERCPU_H_
 #define _ASM_GENERIC_PERCPU_H_
+#include <linux/compiler.h>
 
 #define __GENERIC_PER_CPU
-#include <linux/compiler.h>
+#ifdef CONFIG_SMP
 
 extern unsigned long __per_cpu_offset[NR_CPUS];
 
+/* Separate out the type, so (int[3], foo) works. */
+#ifndef MODULE
+#define DEFINE_PER_CPU(type, name) \
+    __attribute__((__section__(".percpu"))) __typeof__(type) name##__per_cpu
+#endif
+
 /* var is in discarded region: offset to particular copy we want */
-#define per_cpu(var, cpu) (*RELOC_HIDE(&var, __per_cpu_offset[cpu]))
+#define per_cpu(var, cpu) (*RELOC_HIDE(&var##__per_cpu, __per_cpu_offset[cpu]))
 #define __get_cpu_var(var) per_cpu(var, smp_processor_id())
+
+#else /* ! SMP */
+
+/* Can't define per-cpu variables in modules.  Sorry --RR */
+#ifndef MODULE
+#define DEFINE_PER_CPU(type, name) \
+    __typeof__(type) name##__per_cpu
+#endif
+
+#define per_cpu(var, cpu)			var##__per_cpu
+#define __get_cpu_var(var)			var##__per_cpu
+#endif
+
+#define DECLARE_PER_CPU(type, name) extern __typeof__(type) name##__per_cpu
 
 #endif /* _ASM_GENERIC_PERCPU_H_ */
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.28.23931/include/asm-ia64/percpu.h linux-2.5.28.23931.updated/include/asm-ia64/percpu.h
--- linux-2.5.28.23931/include/asm-ia64/percpu.h	Thu Jul 25 11:49:10 2002
+++ linux-2.5.28.23931.updated/include/asm-ia64/percpu.h	Thu Jul 25 12:04:06 2002
@@ -8,7 +8,7 @@
 
 #ifdef __ASSEMBLY__
 
-#define THIS_CPU(var)	(var)	/* use this to mark accesses to per-CPU variables... */
+#define THIS_CPU(var)	(var##__per_cpu)	/* use this to mark accesses to per-CPU variables... */
 
 #else /* !__ASSEMBLY__ */
 
@@ -16,8 +16,14 @@
 
 extern unsigned long __per_cpu_offset[NR_CPUS];
 
-#define per_cpu(var, cpu)	(*(__typeof__(&(var))) ((void *) &(var) + __per_cpu_offset[cpu]))
-#define __get_cpu_var(var)	(var)
+#ifndef MODULE
+#define DEFINE_PER_CPU(type, name) \
+    __attribute__((__section__(".data.percpu"))) __typeof__(type) name##__per_cpu
+#endif
+#define DECLARE_PER_CPU(type, name) extern __typeof__(type) name##__per_cpu
+
+#define per_cpu(var, cpu) (*RELOC_HIDE(&var##__per_cpu, __per_cpu_offset[cpu]))
+#define __get_cpu_var(var)	(var##__per_cpu)
 
 #endif /* !__ASSEMBLY__ */
 
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.28.23931/include/asm-ia64/processor.h linux-2.5.28.23931.updated/include/asm-ia64/processor.h
--- linux-2.5.28.23931/include/asm-ia64/processor.h	Thu Jul 25 11:49:10 2002
+++ linux-2.5.28.23931.updated/include/asm-ia64/processor.h	Thu Jul 25 12:06:55 2002
@@ -134,7 +134,7 @@ struct ia64_psr {
  * CPU type, hardware bug flags, and per-CPU state.  Frequently used
  * state comes earlier:
  */
-extern struct cpuinfo_ia64 {
+struct cpuinfo_ia64 {
 	/* irq_stat must be 64-bit aligned */
 	union {
 		struct {
@@ -175,7 +175,9 @@ extern struct cpuinfo_ia64 {
 	__u64 prof_counter;
 	__u64 prof_multiplier;
 #endif
-} cpu_info __per_cpu_data;
+};
+
+DECLARE_PER_CPU(struct cpuinfo_ia64, cpu_info);
 
 /*
  * The "local" data pointer.  It points to the per-CPU data of the currently executing
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.28.23931/include/linux/percpu.h linux-2.5.28.23931.updated/include/linux/percpu.h
--- linux-2.5.28.23931/include/linux/percpu.h	Thu Jul 25 11:49:10 2002
+++ linux-2.5.28.23931.updated/include/linux/percpu.h	Thu Jul 25 11:49:28 2002
@@ -1,16 +1,9 @@
 #ifndef __LINUX_PERCPU_H
 #define __LINUX_PERCPU_H
-#include <linux/config.h>
-
-#ifdef CONFIG_SMP
-#define __per_cpu_data	__attribute__((section(".data.percpu")))
+#include <linux/spinlock.h> /* For preempt_disable() */
 #include <asm/percpu.h>
-#else
-#define __per_cpu_data
-#define per_cpu(var, cpu)			var
-#define __get_cpu_var(var)			var
-#endif
 
 #define get_cpu_var(var) ({ preempt_disable(); __get_cpu_var(var); })
 #define put_cpu_var(var) preempt_enable()
+
 #endif /* __LINUX_PERCPU_H */
diff -urpN -I \$.*\$ --exclude TAGS -X /home/rusty/devel/kernel/kernel-patches/current-dontdiff --minimal linux-2.5.28.23931/kernel/softirq.c linux-2.5.28.23931.updated/kernel/softirq.c
--- linux-2.5.28.23931/kernel/softirq.c	Thu Jul 25 11:49:11 2002
+++ linux-2.5.28.23931.updated/kernel/softirq.c	Thu Jul 25 12:05:55 2002
@@ -150,8 +150,8 @@ struct tasklet_head
 
 /* Some compilers disobey section attribute on statics when not
    initialized -- RR */
-static struct tasklet_head tasklet_vec __per_cpu_data = { NULL };
-static struct tasklet_head tasklet_hi_vec __per_cpu_data = { NULL };
+static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
+static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
 
 void __tasklet_schedule(struct tasklet_struct *t)
 {