diff -urNp 2.5.58/fs/exec.c 2.5.58-ms-ilb-nb/fs/exec.c --- 2.5.58/fs/exec.c 2003-01-14 06:58:33.000000000 +0100 +++ 2.5.58-ms-ilb-nb/fs/exec.c 2003-01-14 16:31:08.000000000 +0100 @@ -1031,6 +1031,8 @@ int do_execve(char * filename, char ** a int retval; int i; + sched_balance_exec(); + file = open_exec(filename); retval = PTR_ERR(file); diff -urNp 2.5.58/include/linux/sched.h 2.5.58-ms-ilb-nb/include/linux/sched.h --- 2.5.58/include/linux/sched.h 2003-01-14 06:58:06.000000000 +0100 +++ 2.5.58-ms-ilb-nb/include/linux/sched.h 2003-01-14 16:31:08.000000000 +0100 @@ -160,7 +160,6 @@ extern void update_one_process(struct ta extern void scheduler_tick(int user_tick, int system); extern unsigned long cache_decay_ticks; - #define MAX_SCHEDULE_TIMEOUT LONG_MAX extern signed long FASTCALL(schedule_timeout(signed long timeout)); asmlinkage void schedule(void); @@ -444,6 +443,28 @@ extern void set_cpus_allowed(task_t *p, # define set_cpus_allowed(p, new_mask) do { } while (0) #endif +#ifdef CONFIG_NUMA +extern void sched_balance_exec(void); +extern void node_nr_running_init(void); + +static inline void nr_running_inc(runqueue_t *rq) +{ + atomic_inc(rq->node_ptr); + rq->nr_running++; +} + +static inline void nr_running_dec(runqueue_t *rq) +{ + atomic_dec(rq->node_ptr); + rq->nr_running--; +} +#else +#define sched_balance_exec() {} +#define node_nr_running_init() {} +#define nr_running_inc(rq) rq->nr_running++ +#define nr_running_dec(rq) rq->nr_running-- +#endif + extern void set_user_nice(task_t *p, long nice); extern int task_prio(task_t *p); extern int task_nice(task_t *p); diff -urNp 2.5.58/init/main.c 2.5.58-ms-ilb-nb/init/main.c --- 2.5.58/init/main.c 2003-01-14 06:58:20.000000000 +0100 +++ 2.5.58-ms-ilb-nb/init/main.c 2003-01-14 16:31:08.000000000 +0100 @@ -495,6 +495,7 @@ static void do_pre_smp_initcalls(void) migration_init(); #endif + node_nr_running_init(); spawn_ksoftirqd(); } diff -urNp 2.5.58/kernel/sched.c 2.5.58-ms-ilb-nb/kernel/sched.c --- 2.5.58/kernel/sched.c 2003-01-14 06:59:11.000000000 +0100 +++ 2.5.58-ms-ilb-nb/kernel/sched.c 2003-01-14 16:31:57.000000000 +0100 @@ -67,6 +67,8 @@ #define INTERACTIVE_DELTA 2 #define MAX_SLEEP_AVG (2*HZ) #define STARVATION_LIMIT (2*HZ) +#define NODE_BALANCE_RATIO 10 +#define NODE_THRESHOLD 125 /* * If a task is 'interactive' then we reinsert it in the active @@ -153,7 +155,10 @@ struct runqueue { task_t *curr, *idle; prio_array_t *active, *expired, arrays[2]; int prev_nr_running[NR_CPUS]; - +#ifdef CONFIG_NUMA + atomic_t * node_ptr; + unsigned int nr_balanced; +#endif task_t *migration_thread; struct list_head migration_queue; @@ -294,7 +299,7 @@ static inline void activate_task(task_t p->prio = effective_prio(p); } enqueue_task(p, array); - rq->nr_running++; + nr_running_inc(rq); } /* @@ -302,7 +307,7 @@ static inline void activate_task(task_t */ static inline void deactivate_task(struct task_struct *p, runqueue_t *rq) { - rq->nr_running--; + nr_running_dec(rq); if (p->state == TASK_UNINTERRUPTIBLE) rq->nr_uninterruptible++; dequeue_task(p, p->array); @@ -624,6 +629,105 @@ static inline void double_rq_unlock(runq spin_unlock(&rq2->lock); } +#if CONFIG_NUMA +static atomic_t node_nr_running[MAX_NUMNODES] ____cacheline_maxaligned_in_smp = + {[0 ...MAX_NUMNODES-1] = ATOMIC_INIT(0)}; + +__init void node_nr_running_init(void) +{ + int i; + + for (i = 0; i < NR_CPUS; i++) + cpu_rq(i)->node_ptr = &node_nr_running[__cpu_to_node(i)]; +} + +/* + * If dest_cpu is allowed for this process, migrate the task to it. + * This is accomplished by forcing the cpu_allowed mask to only + * allow dest_cpu, which will force the cpu onto dest_cpu. Then + * the cpu_allowed mask is restored. + */ +static void sched_migrate_task(task_t *p, int dest_cpu) +{ + unsigned long old_mask; + + old_mask = p->cpus_allowed; + if (!(old_mask & (1UL << dest_cpu))) + return; + /* force the process onto the specified CPU */ + set_cpus_allowed(p, 1UL << dest_cpu); + + /* restore the cpus allowed mask */ + set_cpus_allowed(p, old_mask); +} + +/* + * Find the least loaded CPU. Slightly favor the current CPU by + * setting its runqueue length as the minimum to start. + */ +static int sched_best_cpu(struct task_struct *p) +{ + int i, minload, load, best_cpu, node = 0; + unsigned long cpumask; + + best_cpu = task_cpu(p); + if (cpu_rq(best_cpu)->nr_running <= 2) + return best_cpu; + + minload = 10000000; + for (i = 0; i < numnodes; i++) { + load = atomic_read(&node_nr_running[i]); + if (load < minload) { + minload = load; + node = i; + } + } + + minload = 10000000; + cpumask = __node_to_cpu_mask(node); + for (i = 0; i < NR_CPUS; ++i) { + if (!(cpumask & (1UL << i))) + continue; + if (cpu_rq(i)->nr_running < minload) { + best_cpu = i; + minload = cpu_rq(i)->nr_running; + } + } + return best_cpu; +} + +void sched_balance_exec(void) +{ + int new_cpu; + + if (numnodes > 1) { + new_cpu = sched_best_cpu(current); + if (new_cpu != smp_processor_id()) + sched_migrate_task(current, new_cpu); + } +} + +static int find_busiest_node(int this_node) +{ + int i, node = -1, load, this_load, maxload; + + this_load = maxload = atomic_read(&node_nr_running[this_node]); + for (i = 0; i < numnodes; i++) { + if (i == this_node) + continue; + load = atomic_read(&node_nr_running[i]); + if (load > maxload && + (100*load > ((NODE_THRESHOLD*100*this_load)/100))) { + maxload = load; + node = i; + } + } + if (maxload <= 4) + node = -1; + return node; +} +#endif /* CONFIG_NUMA */ + #if CONFIG_SMP /* @@ -652,9 +756,9 @@ static inline unsigned int double_lock_b } /* - * find_busiest_queue - find the busiest runqueue. + * find_busiest_queue - find the busiest runqueue among the cpus in cpumask */ -static inline runqueue_t *find_busiest_queue(runqueue_t *this_rq, int this_cpu, int idle, int *imbalance) +static inline runqueue_t *find_busiest_queue(runqueue_t *this_rq, int this_cpu, int idle, int *imbalance, unsigned long cpumask) { int nr_running, load, max_load, i; runqueue_t *busiest, *rq_src; @@ -689,7 +793,7 @@ static inline runqueue_t *find_busiest_q busiest = NULL; max_load = 1; for (i = 0; i < NR_CPUS; i++) { - if (!cpu_online(i)) + if (!cpu_online(i) || !((1UL << i) & cpumask) ) continue; rq_src = cpu_rq(i); @@ -736,9 +840,9 @@ out: static inline void pull_task(runqueue_t *src_rq, prio_array_t *src_array, task_t *p, runqueue_t *this_rq, int this_cpu) { dequeue_task(p, src_array); - src_rq->nr_running--; + nr_running_dec(src_rq); set_task_cpu(p, this_cpu); - this_rq->nr_running++; + nr_running_inc(this_rq); enqueue_task(p, this_rq->active); /* * Note that idle threads have a prio of MAX_PRIO, for this test @@ -763,8 +867,21 @@ static void load_balance(runqueue_t *thi prio_array_t *array; struct list_head *head, *curr; task_t *tmp; + int this_node = __cpu_to_node(this_cpu); + unsigned long cpumask = __node_to_cpu_mask(this_node); - busiest = find_busiest_queue(this_rq, this_cpu, idle, &imbalance); +#if CONFIG_NUMA + /* + * Avoid rebalancing between nodes too often. + */ + if (!(++(this_rq->nr_balanced) % NODE_BALANCE_RATIO)) { + int node = find_busiest_node(this_node); + if (node >= 0) + cpumask = __node_to_cpu_mask(node) | (1UL << this_cpu); + } +#endif + busiest = find_busiest_queue(this_rq, this_cpu, idle, &imbalance, + cpumask); if (!busiest) goto out; @@ -2231,6 +2348,10 @@ void __init sched_init(void) spin_lock_init(&rq->lock); INIT_LIST_HEAD(&rq->migration_queue); atomic_set(&rq->nr_iowait, 0); +#if CONFIG_NUMA + rq->node_ptr = &node_nr_running[0]; +#endif /* CONFIG_NUMA */ + for (j = 0; j < 2; j++) { array = rq->arrays + j;