[PATCH 0/7] RFC: __smp_call_function_single improvements

[PATCH 0/7] RFC: __smp_call_function_single improvements

[Christoph Hellwig]

This series is on top of the blk-mq series Jens just set out and changes
the generic IPI mechanism to be directly usable for block completions.

Besides always making __smp_call_function_single constantly available it
changes it to use a lockless list that avoids the overhead of disabling
irq and a lock every time it is called.

[PATCH 1/7] Revert: "softirq: Add support for triggering softirq work on softirqs"

[Christoph Hellwig]

[-- Attachment #1: 0001-Revert-softirq-Add-support-for-triggering-softirq-wo.patch --]
[-- Type: text/plain, Size: 6514 bytes --]

This commit was incomplete in that code to remove items from the per-cpu
lists was missing and never acquired a user in the 5 years it has been
in the tree.  We're going to implement what it seems to try to archive
in a simpler way, and this code is in the way of doing so.

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 include/linux/interrupt.h |   22 --------
 kernel/softirq.c          |  131 ---------------------------------------------
 2 files changed, 153 deletions(-)

diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 5e865b5..8c0d4ba 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -11,8 +11,6 @@
 #include <linux/irqnr.h>
 #include <linux/hardirq.h>
 #include <linux/irqflags.h>
-#include <linux/smp.h>
-#include <linux/percpu.h>
 #include <linux/hrtimer.h>
 #include <linux/kref.h>
 #include <linux/workqueue.h>
@@ -381,15 +379,6 @@ extern void __raise_softirq_irqoff(unsigned int nr);
 extern void raise_softirq_irqoff(unsigned int nr);
 extern void raise_softirq(unsigned int nr);
 
-/* This is the worklist that queues up per-cpu softirq work.
- *
- * send_remote_sendirq() adds work to these lists, and
- * the softirq handler itself dequeues from them.  The queues
- * are protected by disabling local cpu interrupts and they must
- * only be accessed by the local cpu that they are for.
- */
-DECLARE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
-
 DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
 
 static inline struct task_struct *this_cpu_ksoftirqd(void)
@@ -397,17 +386,6 @@ static inline struct task_struct *this_cpu_ksoftirqd(void)
 	return this_cpu_read(ksoftirqd);
 }
 
-/* Try to send a softirq to a remote cpu.  If this cannot be done, the
- * work will be queued to the local cpu.
- */
-extern void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq);
-
-/* Like send_remote_softirq(), but the caller must disable local cpu interrupts
- * and compute the current cpu, passed in as 'this_cpu'.
- */
-extern void __send_remote_softirq(struct call_single_data *cp, int cpu,
-				  int this_cpu, int softirq);
-
 /* Tasklets --- multithreaded analogue of BHs.
 
    Main feature differing them of generic softirqs: tasklet
diff --git a/kernel/softirq.c b/kernel/softirq.c
index d7d498d..c587b7f 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -6,8 +6,6 @@
  *	Distribute under GPLv2.
  *
  *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
- *
- *	Remote softirq infrastructure is by Jens Axboe.
  */
 
 #include <linux/export.h>
@@ -618,146 +616,17 @@ void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 }
 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 
-/*
- * Remote softirq bits
- */
-
-DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
-EXPORT_PER_CPU_SYMBOL(softirq_work_list);
-
-static void __local_trigger(struct call_single_data *cp, int softirq)
-{
-	struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
-
-	list_add_tail(&cp->list, head);
-
-	/* Trigger the softirq only if the list was previously empty.  */
-	if (head->next == &cp->list)
-		raise_softirq_irqoff(softirq);
-}
-
-#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
-static void remote_softirq_receive(void *data)
-{
-	struct call_single_data *cp = data;
-	unsigned long flags;
-	int softirq;
-
-	softirq = *(int *)cp->info;
-	local_irq_save(flags);
-	__local_trigger(cp, softirq);
-	local_irq_restore(flags);
-}
-
-static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
-{
-	if (cpu_online(cpu)) {
-		cp->func = remote_softirq_receive;
-		cp->info = &softirq;
-		cp->flags = 0;
-
-		__smp_call_function_single(cpu, cp, 0);
-		return 0;
-	}
-	return 1;
-}
-#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
-static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
-{
-	return 1;
-}
-#endif
-
-/**
- * __send_remote_softirq - try to schedule softirq work on a remote cpu
- * @cp: private SMP call function data area
- * @cpu: the remote cpu
- * @this_cpu: the currently executing cpu
- * @softirq: the softirq for the work
- *
- * Attempt to schedule softirq work on a remote cpu.  If this cannot be
- * done, the work is instead queued up on the local cpu.
- *
- * Interrupts must be disabled.
- */
-void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
-{
-	if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
-		__local_trigger(cp, softirq);
-}
-EXPORT_SYMBOL(__send_remote_softirq);
-
-/**
- * send_remote_softirq - try to schedule softirq work on a remote cpu
- * @cp: private SMP call function data area
- * @cpu: the remote cpu
- * @softirq: the softirq for the work
- *
- * Like __send_remote_softirq except that disabling interrupts and
- * computing the current cpu is done for the caller.
- */
-void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
-{
-	unsigned long flags;
-	int this_cpu;
-
-	local_irq_save(flags);
-	this_cpu = smp_processor_id();
-	__send_remote_softirq(cp, cpu, this_cpu, softirq);
-	local_irq_restore(flags);
-}
-EXPORT_SYMBOL(send_remote_softirq);
-
-static int remote_softirq_cpu_notify(struct notifier_block *self,
-					       unsigned long action, void *hcpu)
-{
-	/*
-	 * If a CPU goes away, splice its entries to the current CPU
-	 * and trigger a run of the softirq
-	 */
-	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
-		int cpu = (unsigned long) hcpu;
-		int i;
-
-		local_irq_disable();
-		for (i = 0; i < NR_SOFTIRQS; i++) {
-			struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
-			struct list_head *local_head;
-
-			if (list_empty(head))
-				continue;
-
-			local_head = &__get_cpu_var(softirq_work_list[i]);
-			list_splice_init(head, local_head);
-			raise_softirq_irqoff(i);
-		}
-		local_irq_enable();
-	}
-
-	return NOTIFY_OK;
-}
-
-static struct notifier_block remote_softirq_cpu_notifier = {
-	.notifier_call	= remote_softirq_cpu_notify,
-};
-
 void __init softirq_init(void)
 {
 	int cpu;
 
 	for_each_possible_cpu(cpu) {
-		int i;
-
 		per_cpu(tasklet_vec, cpu).tail =
 			&per_cpu(tasklet_vec, cpu).head;
 		per_cpu(tasklet_hi_vec, cpu).tail =
 			&per_cpu(tasklet_hi_vec, cpu).head;
-		for (i = 0; i < NR_SOFTIRQS; i++)
-			INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 	}
 
-	register_hotcpu_notifier(&remote_softirq_cpu_notifier);
-
 	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 }
-- 
1.7.10.4

[PATCH 2/7] kernel: remove CONFIG_USE_GENERIC_SMP_HELPERS

[Christoph Hellwig]

[-- Attachment #1: 0002-kernel-remove-CONFIG_USE_GENERIC_SMP_HELPERS.patch --]
[-- Type: text/plain, Size: 11983 bytes --]

We've switched over every architecture that supports SMP to it, so remove
the new useless config variable.

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 arch/Kconfig          |    3 ---
 arch/alpha/Kconfig    |    1 -
 arch/arc/Kconfig      |    1 -
 arch/arm/Kconfig      |    1 -
 arch/arm64/Kconfig    |    1 -
 arch/blackfin/Kconfig |    1 -
 arch/hexagon/Kconfig  |    1 -
 arch/ia64/Kconfig     |    1 -
 arch/m32r/Kconfig     |    1 -
 arch/metag/Kconfig    |    1 -
 arch/mips/Kconfig     |    1 -
 arch/mn10300/Kconfig  |    1 -
 arch/parisc/Kconfig   |    1 -
 arch/powerpc/Kconfig  |    1 -
 arch/s390/Kconfig     |    1 -
 arch/sh/Kconfig       |    1 -
 arch/sparc/Kconfig    |    1 -
 arch/tile/Kconfig     |    1 -
 arch/x86/Kconfig      |    1 -
 block/blk-mq.c        |    4 ++--
 block/blk-softirq.c   |    4 ++--
 block/blk-sysfs.c     |    2 +-
 include/linux/smp.h   |    4 ----
 kernel/Kconfig.hz     |    2 +-
 kernel/smp.c          |    2 --
 25 files changed, 6 insertions(+), 33 deletions(-)

diff --git a/arch/Kconfig b/arch/Kconfig
index af2cc6e..63c53c6 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -207,9 +207,6 @@ config HAVE_DMA_ATTRS
 config HAVE_DMA_CONTIGUOUS
 	bool
 
-config USE_GENERIC_SMP_HELPERS
-	bool
-
 config GENERIC_SMP_IDLE_THREAD
        bool
 
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index 35a300d..8d2a483 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -522,7 +522,6 @@ config ARCH_MAY_HAVE_PC_FDC
 config SMP
 	bool "Symmetric multi-processing support"
 	depends on ALPHA_SABLE || ALPHA_LYNX || ALPHA_RAWHIDE || ALPHA_DP264 || ALPHA_WILDFIRE || ALPHA_TITAN || ALPHA_GENERIC || ALPHA_SHARK || ALPHA_MARVEL
-	select USE_GENERIC_SMP_HELPERS
 	---help---
 	  This enables support for systems with more than one CPU. If you have
 	  a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index 91dbb27..9c2ddbf 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -119,7 +119,6 @@ config ARC_PLAT_NEEDS_CPU_TO_DMA
 config SMP
 	bool "Symmetric Multi-Processing (Incomplete)"
 	default n
-	select USE_GENERIC_SMP_HELPERS
 	help
 	  This enables support for systems with more than one CPU. If you have
 	  a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 1ad6fb6..a671b58 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1467,7 +1467,6 @@ config SMP
 	depends on GENERIC_CLOCKEVENTS
 	depends on HAVE_SMP
 	depends on MMU || ARM_MPU
-	select USE_GENERIC_SMP_HELPERS
 	help
 	  This enables support for systems with more than one CPU. If you have
 	  a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index c044548..7020613 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -140,7 +140,6 @@ config ARM64_64K_PAGES
 
 config SMP
 	bool "Symmetric Multi-Processing"
-	select USE_GENERIC_SMP_HELPERS
 	help
 	  This enables support for systems with more than one CPU.  If
 	  you say N here, the kernel will run on single and
diff --git a/arch/blackfin/Kconfig b/arch/blackfin/Kconfig
index f78c9a2..e06b5d8 100644
--- a/arch/blackfin/Kconfig
+++ b/arch/blackfin/Kconfig
@@ -34,7 +34,6 @@ config BLACKFIN
 	select ARCH_WANT_IPC_PARSE_VERSION
 	select GENERIC_ATOMIC64
 	select GENERIC_IRQ_PROBE
-	select USE_GENERIC_SMP_HELPERS if SMP
 	select HAVE_NMI_WATCHDOG if NMI_WATCHDOG
 	select GENERIC_SMP_IDLE_THREAD
 	select ARCH_USES_GETTIMEOFFSET if !GENERIC_CLOCKEVENTS
diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig
index 99041b0..09df260 100644
--- a/arch/hexagon/Kconfig
+++ b/arch/hexagon/Kconfig
@@ -4,7 +4,6 @@ comment "Linux Kernel Configuration for Hexagon"
 config HEXAGON
 	def_bool y
 	select HAVE_OPROFILE
-	select USE_GENERIC_SMP_HELPERS if SMP
 	# Other pending projects/to-do items.
 	# select HAVE_REGS_AND_STACK_ACCESS_API
 	# select HAVE_HW_BREAKPOINT if PERF_EVENTS
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 7740ab1..dfe85e9 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -343,7 +343,6 @@ config FORCE_MAX_ZONEORDER
 
 config SMP
 	bool "Symmetric multi-processing support"
-	select USE_GENERIC_SMP_HELPERS
 	help
 	  This enables support for systems with more than one CPU. If you have
 	  a system with only one CPU, say N.  If you have a system with more
diff --git a/arch/m32r/Kconfig b/arch/m32r/Kconfig
index 75661fb..09ef94a 100644
--- a/arch/m32r/Kconfig
+++ b/arch/m32r/Kconfig
@@ -275,7 +275,6 @@ source "kernel/Kconfig.preempt"
 
 config SMP
 	bool "Symmetric multi-processing support"
-	select USE_GENERIC_SMP_HELPERS
 	---help---
 	  This enables support for systems with more than one CPU. If you have
 	  a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/metag/Kconfig b/arch/metag/Kconfig
index 36368eb..e56abd2 100644
--- a/arch/metag/Kconfig
+++ b/arch/metag/Kconfig
@@ -111,7 +111,6 @@ config METAG_META21
 config SMP
 	bool "Symmetric multi-processing support"
 	depends on METAG_META21 && METAG_META21_MMU
-	select USE_GENERIC_SMP_HELPERS
 	help
 	  This enables support for systems with more than one thread running
 	  Linux. If you have a system with only one thread running Linux,
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index f75ab4a..f745dc3 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -2140,7 +2140,6 @@ source "mm/Kconfig"
 config SMP
 	bool "Multi-Processing support"
 	depends on SYS_SUPPORTS_SMP
-	select USE_GENERIC_SMP_HELPERS
 	help
 	  This enables support for systems with more than one CPU. If you have
 	  a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/mn10300/Kconfig b/arch/mn10300/Kconfig
index 6aaa160..8bde923 100644
--- a/arch/mn10300/Kconfig
+++ b/arch/mn10300/Kconfig
@@ -181,7 +181,6 @@ endmenu
 config SMP
 	bool "Symmetric multi-processing support"
 	default y
-	select USE_GENERIC_SMP_HELPERS
 	depends on MN10300_PROC_MN2WS0038 || MN10300_PROC_MN2WS0050
 	---help---
 	  This enables support for systems with more than one CPU. If you have
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index ad2ce8d..0c08dc8 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -226,7 +226,6 @@ endchoice
 
 config SMP
 	bool "Symmetric multi-processing support"
-	select USE_GENERIC_SMP_HELPERS
 	---help---
 	  This enables support for systems with more than one CPU. If you have
 	  a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 38f3b7e..2bc9c49 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -106,7 +106,6 @@ config PPC
 	select HAVE_MEMBLOCK_NODE_MAP
 	select HAVE_DMA_ATTRS
 	select HAVE_DMA_API_DEBUG
-	select USE_GENERIC_SMP_HELPERS if SMP
 	select HAVE_OPROFILE
 	select HAVE_DEBUG_KMEMLEAK
 	select GENERIC_ATOMIC64 if PPC32
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 7143793..8fed3f5 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -140,7 +140,6 @@ config S390
 	select OLD_SIGACTION
 	select OLD_SIGSUSPEND3
 	select SYSCTL_EXCEPTION_TRACE
-	select USE_GENERIC_SMP_HELPERS if SMP
 	select VIRT_CPU_ACCOUNTING
 	select VIRT_TO_BUS
 
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
index 224f4bc..e78561b 100644
--- a/arch/sh/Kconfig
+++ b/arch/sh/Kconfig
@@ -711,7 +711,6 @@ config CC_STACKPROTECTOR
 config SMP
 	bool "Symmetric multi-processing support"
 	depends on SYS_SUPPORTS_SMP
-	select USE_GENERIC_SMP_HELPERS
 	---help---
 	  This enables support for systems with more than one CPU. If you have
 	  a system with only one CPU, like most personal computers, say N. If
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index 78c4fdb..8591b20 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -28,7 +28,6 @@ config SPARC
 	select HAVE_ARCH_JUMP_LABEL
 	select GENERIC_IRQ_SHOW
 	select ARCH_WANT_IPC_PARSE_VERSION
-	select USE_GENERIC_SMP_HELPERS if SMP
 	select GENERIC_PCI_IOMAP
 	select HAVE_NMI_WATCHDOG if SPARC64
 	select HAVE_BPF_JIT
diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig
index d45a2c4..b3692ce 100644
--- a/arch/tile/Kconfig
+++ b/arch/tile/Kconfig
@@ -8,7 +8,6 @@ config TILE
 	select HAVE_KVM if !TILEGX
 	select GENERIC_FIND_FIRST_BIT
 	select SYSCTL_EXCEPTION_TRACE
-	select USE_GENERIC_SMP_HELPERS
 	select CC_OPTIMIZE_FOR_SIZE
 	select HAVE_DEBUG_KMEMLEAK
 	select GENERIC_IRQ_PROBE
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 145d703..f137df2 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -90,7 +90,6 @@ config X86
 	select GENERIC_IRQ_SHOW
 	select GENERIC_CLOCKEVENTS_MIN_ADJUST
 	select IRQ_FORCED_THREADING
-	select USE_GENERIC_SMP_HELPERS if SMP
 	select HAVE_BPF_JIT if X86_64
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
 	select CLKEVT_I8253
diff --git a/block/blk-mq.c b/block/blk-mq.c
index f21ec96..d43a7e8 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -318,7 +318,7 @@ void __blk_mq_end_io(struct request *rq, int error)
 		blk_mq_complete_request(rq, error);
 }
 
-#if defined(CONFIG_SMP) && defined(CONFIG_USE_GENERIC_SMP_HELPERS)
+#if defined(CONFIG_SMP)
 
 /*
  * Called with interrupts disabled.
@@ -360,7 +360,7 @@ static int ipi_remote_cpu(struct blk_mq_ctx *ctx, const int cpu,
 
 	return true;
 }
-#else /* CONFIG_SMP && CONFIG_USE_GENERIC_SMP_HELPERS */
+#else /* CONFIG_SMP */
 static int ipi_remote_cpu(struct blk_mq_ctx *ctx, const int cpu,
 			  struct request *rq, const int error)
 {
diff --git a/block/blk-softirq.c b/block/blk-softirq.c
index ec9e606..ed07d79 100644
--- a/block/blk-softirq.c
+++ b/block/blk-softirq.c
@@ -36,7 +36,7 @@ static void blk_done_softirq(struct softirq_action *h)
 	}
 }
 
-#if defined(CONFIG_SMP) && defined(CONFIG_USE_GENERIC_SMP_HELPERS)
+#ifdef CONFIG_SMP
 static void trigger_softirq(void *data)
 {
 	struct request *rq = data;
@@ -71,7 +71,7 @@ static int raise_blk_irq(int cpu, struct request *rq)
 
 	return 1;
 }
-#else /* CONFIG_SMP && CONFIG_USE_GENERIC_SMP_HELPERS */
+#else /* CONFIG_SMP */
 static int raise_blk_irq(int cpu, struct request *rq)
 {
 	return 1;
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 4f8c4d9..9777952 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -288,7 +288,7 @@ static ssize_t
 queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
 {
 	ssize_t ret = -EINVAL;
-#if defined(CONFIG_USE_GENERIC_SMP_HELPERS)
+#ifdef CONFIG_SMP
 	unsigned long val;
 
 	ret = queue_var_store(&val, page, count);
diff --git a/include/linux/smp.h b/include/linux/smp.h
index 731f523..7885151 100644
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -106,14 +106,10 @@ void kick_all_cpus_sync(void);
 /*
  * Generic and arch helpers
  */
-#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 void __init call_function_init(void);
 void generic_smp_call_function_single_interrupt(void);
 #define generic_smp_call_function_interrupt \
 	generic_smp_call_function_single_interrupt
-#else
-static inline void call_function_init(void) { }
-#endif
 
 /*
  * Mark the boot cpu "online" so that it can call console drivers in
diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz
index 94fabd5..2a202a8 100644
--- a/kernel/Kconfig.hz
+++ b/kernel/Kconfig.hz
@@ -55,4 +55,4 @@ config HZ
 	default 1000 if HZ_1000
 
 config SCHED_HRTICK
-	def_bool HIGH_RES_TIMERS && (!SMP || USE_GENERIC_SMP_HELPERS)
+	def_bool HIGH_RES_TIMERS
diff --git a/kernel/smp.c b/kernel/smp.c
index bc400e3..2b83645 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -15,7 +15,6 @@
 
 #include "smpboot.h"
 
-#ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 enum {
 	CSD_FLAG_LOCK		= 0x01,
 	CSD_FLAG_WAIT		= 0x02,
@@ -463,7 +462,6 @@ int smp_call_function(smp_call_func_t func, void *info, int wait)
 	return 0;
 }
 EXPORT_SYMBOL(smp_call_function);
-#endif /* USE_GENERIC_SMP_HELPERS */
 
 /* Setup configured maximum number of CPUs to activate */
 unsigned int setup_max_cpus = NR_CPUS;
-- 
1.7.10.4

[PATCH 3/7] kernel: provide a __smp_call_function_single stub for !CONFIG_SMP

[Christoph Hellwig]

[-- Attachment #1: 0003-kernel-provide-a-__smp_call_function_single-stub-for.patch --]
[-- Type: text/plain, Size: 708 bytes --]

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 kernel/up.c |   11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/kernel/up.c b/kernel/up.c
index 630d72b..509403e 100644
--- a/kernel/up.c
+++ b/kernel/up.c
@@ -22,6 +22,17 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
 }
 EXPORT_SYMBOL(smp_call_function_single);
 
+void __smp_call_function_single(int cpu, struct call_single_data *csd,
+				int wait)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	csd->func(csd->info);
+	local_irq_restore(flags);
+}
+EXPORT_SYMBOL(__smp_call_function_single);
+
 int on_each_cpu(smp_call_func_t func, void *info, int wait)
 {
 	unsigned long flags;
-- 
1.7.10.4

[PATCH 4/7] kernel: fix generic_exec_single indication

[Christoph Hellwig]

[-- Attachment #1: 0004-kernel-fix-generic_exec_single-indication.patch --]
[-- Type: text/plain, Size: 681 bytes --]

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 kernel/smp.c |    3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/kernel/smp.c b/kernel/smp.c
index 2b83645..53644e6 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -139,8 +139,7 @@ static void csd_unlock(struct call_single_data *csd)
  * for execution on the given CPU. data must already have
  * ->func, ->info, and ->flags set.
  */
-static
-void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
+static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
 {
 	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
 	unsigned long flags;
-- 
1.7.10.4

[PATCH 5/7] llists: move llist_reverse_order from raid5 to llist.c

[Christoph Hellwig]

[-- Attachment #1: 0005-llists-move-llist_reverse_order-from-raid5-to-llist..patch --]
[-- Type: text/plain, Size: 2070 bytes --]

Make this useful helper available for other users.

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 drivers/md/raid5.c    |   14 --------------
 include/linux/llist.h |    2 ++
 lib/llist.c           |   22 ++++++++++++++++++++++
 3 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 7ff4f25..046f1a9 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -293,20 +293,6 @@ static void __release_stripe(struct r5conf *conf, struct stripe_head *sh)
 		do_release_stripe(conf, sh);
 }
 
-static struct llist_node *llist_reverse_order(struct llist_node *head)
-{
-	struct llist_node *new_head = NULL;
-
-	while (head) {
-		struct llist_node *tmp = head;
-		head = head->next;
-		tmp->next = new_head;
-		new_head = tmp;
-	}
-
-	return new_head;
-}
-
 /* should hold conf->device_lock already */
 static int release_stripe_list(struct r5conf *conf)
 {
diff --git a/include/linux/llist.h b/include/linux/llist.h
index 8828a78..fbf10a0 100644
--- a/include/linux/llist.h
+++ b/include/linux/llist.h
@@ -195,4 +195,6 @@ static inline struct llist_node *llist_del_all(struct llist_head *head)
 
 extern struct llist_node *llist_del_first(struct llist_head *head);
 
+struct llist_node *llist_reverse_order(struct llist_node *head);
+
 #endif /* LLIST_H */
diff --git a/lib/llist.c b/lib/llist.c
index 4a70d12..ef48b87 100644
--- a/lib/llist.c
+++ b/lib/llist.c
@@ -81,3 +81,25 @@ struct llist_node *llist_del_first(struct llist_head *head)
 	return entry;
 }
 EXPORT_SYMBOL_GPL(llist_del_first);
+
+/**
+ * llist_reverse_order - reverse order of a llist chain
+ * @head:	first item of the list to be reversed
+ *
+ * Reverse the oder of a chain of llist entries and return the
+ * new first entry.
+ */
+struct llist_node *llist_reverse_order(struct llist_node *head)
+{
+	struct llist_node *new_head = NULL;
+
+	while (head) {
+		struct llist_node *tmp = head;
+		head = head->next;
+		tmp->next = new_head;
+		new_head = tmp;
+	}
+
+	return new_head;
+}
+EXPORT_SYMBOL_GPL(llist_reverse_order);
-- 
1.7.10.4

[PATCH 6/7] kernel: use lockless list for smp_call_function_single

[Christoph Hellwig]

[-- Attachment #1: 0006-kernel-use-lockless-list-for-smp_call_function_singl.patch --]
[-- Type: text/plain, Size: 4814 bytes --]

Make smp_call_function_single and friends more efficient by using
a lockless list.

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 include/linux/blkdev.h |    5 +----
 include/linux/smp.h    |    6 +++++-
 kernel/smp.c           |   51 ++++++++++++------------------------------------
 3 files changed, 19 insertions(+), 43 deletions(-)

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index f26ec20f..287bf7c 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -95,10 +95,7 @@ enum rq_cmd_type_bits {
  * as well!
  */
 struct request {
-	union {
-		struct list_head queuelist;
-		struct llist_node ll_list;
-	};
+	struct list_head queuelist;
 	union {
 		struct call_single_data csd;
 		struct work_struct mq_flush_data;
diff --git a/include/linux/smp.h b/include/linux/smp.h
index 7885151..10755dd 100644
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -11,12 +11,16 @@
 #include <linux/list.h>
 #include <linux/cpumask.h>
 #include <linux/init.h>
+#include <linux/llist.h>
 
 extern void cpu_idle(void);
 
 typedef void (*smp_call_func_t)(void *info);
 struct call_single_data {
-	struct list_head list;
+	union {
+		struct list_head list;
+		struct llist_node llist;
+	};
 	smp_call_func_t func;
 	void *info;
 	u16 flags;
diff --git a/kernel/smp.c b/kernel/smp.c
index 53644e6..a735c66 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -28,12 +28,7 @@ struct call_function_data {
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
 
-struct call_single_queue {
-	struct list_head	list;
-	raw_spinlock_t		lock;
-};
-
-static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_queue, call_single_queue);
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
 
 static int
 hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
@@ -85,12 +80,8 @@ void __init call_function_init(void)
 	void *cpu = (void *)(long)smp_processor_id();
 	int i;
 
-	for_each_possible_cpu(i) {
-		struct call_single_queue *q = &per_cpu(call_single_queue, i);
-
-		raw_spin_lock_init(&q->lock);
-		INIT_LIST_HEAD(&q->list);
-	}
+	for_each_possible_cpu(i)
+		init_llist_head(&per_cpu(call_single_queue, i));
 
 	hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);
 	register_cpu_notifier(&hotplug_cfd_notifier);
@@ -141,18 +132,9 @@ static void csd_unlock(struct call_single_data *csd)
  */
 static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
 {
-	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
-	unsigned long flags;
-	int ipi;
-
 	if (wait)
 		csd->flags |= CSD_FLAG_WAIT;
 
-	raw_spin_lock_irqsave(&dst->lock, flags);
-	ipi = list_empty(&dst->list);
-	list_add_tail(&csd->list, &dst->list);
-	raw_spin_unlock_irqrestore(&dst->lock, flags);
-
 	/*
 	 * The list addition should be visible before sending the IPI
 	 * handler locks the list to pull the entry off it because of
@@ -164,7 +146,7 @@ static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
 	 * locking and barrier primitives. Generic code isn't really
 	 * equipped to do the right thing...
 	 */
-	if (ipi)
+	if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
 		arch_send_call_function_single_ipi(cpu);
 
 	if (wait)
@@ -177,27 +159,26 @@ static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
  */
 void generic_smp_call_function_single_interrupt(void)
 {
-	struct call_single_queue *q = &__get_cpu_var(call_single_queue);
-	LIST_HEAD(list);
+	struct llist_node *entry, *next;
 
 	/*
 	 * Shouldn't receive this interrupt on a cpu that is not yet online.
 	 */
 	WARN_ON_ONCE(!cpu_online(smp_processor_id()));
 
-	raw_spin_lock(&q->lock);
-	list_replace_init(&q->list, &list);
-	raw_spin_unlock(&q->lock);
+	entry = llist_del_all(&__get_cpu_var(call_single_queue));
+	entry = llist_reverse_order(entry);
 
-	while (!list_empty(&list)) {
+	while (entry) {
 		struct call_single_data *csd;
 
-		csd = list_entry(list.next, struct call_single_data, list);
-		list_del(&csd->list);
+		next = entry->next;
 
+		csd = llist_entry(entry, struct call_single_data, llist);
 		csd->func(csd->info);
-
 		csd_unlock(csd);
+
+		entry = next;
 	}
 }
 
@@ -410,17 +391,11 @@ void smp_call_function_many(const struct cpumask *mask,
 
 	for_each_cpu(cpu, cfd->cpumask) {
 		struct call_single_data *csd = per_cpu_ptr(cfd->csd, cpu);
-		struct call_single_queue *dst =
-					&per_cpu(call_single_queue, cpu);
-		unsigned long flags;
 
 		csd_lock(csd);
 		csd->func = func;
 		csd->info = info;
-
-		raw_spin_lock_irqsave(&dst->lock, flags);
-		list_add_tail(&csd->list, &dst->list);
-		raw_spin_unlock_irqrestore(&dst->lock, flags);
+		llist_add(&csd->llist, &per_cpu(call_single_queue, cpu));
 	}
 
 	/* Send a message to all CPUs in the map */
-- 
1.7.10.4

[PATCH 7/7] blk-mq: use __smp_call_function_single directly

[Christoph Hellwig]

[-- Attachment #1: 0007-blk-mq-use-__smp_call_function_single-directly.patch --]
[-- Type: text/plain, Size: 4938 bytes --]

Now that __smp_call_function_single is available for all builds and
uses llists to queue up items without taking a lock or disabling
interrupts there is no need to wrap around it in the block code.

Signed-off-by: Christoph Hellwig <hch@lst.de>
---
 block/blk-mq-cpu.c |   31 ------------------------
 block/blk-mq.c     |   68 +++++++++-------------------------------------------
 block/blk-mq.h     |    1 -
 3 files changed, 11 insertions(+), 89 deletions(-)

diff --git a/block/blk-mq-cpu.c b/block/blk-mq-cpu.c
index f8ea39d..4f0c352 100644
--- a/block/blk-mq-cpu.c
+++ b/block/blk-mq-cpu.c
@@ -28,32 +28,6 @@ static int __cpuinit blk_mq_main_cpu_notify(struct notifier_block *self,
 	return NOTIFY_OK;
 }
 
-static void __cpuinit blk_mq_cpu_notify(void *data, unsigned long action,
-					unsigned int cpu)
-{
-	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
-		/*
-		 * If the CPU goes away, ensure that we run any pending
-		 * completions.
-		 */
-		struct llist_node *node;
-		struct request *rq;
-
-		local_irq_disable();
-
-		node = llist_del_all(&per_cpu(ipi_lists, cpu));
-		while (node) {
-			struct llist_node *next = node->next;
-
-			rq = llist_entry(node, struct request, ll_list);
-			__blk_mq_end_io(rq, rq->errors);
-			node = next;
-		}
-
-		local_irq_enable();
-	}
-}
-
 static struct notifier_block __cpuinitdata blk_mq_main_cpu_notifier = {
 	.notifier_call	= blk_mq_main_cpu_notify,
 };
@@ -82,12 +56,7 @@ void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier,
 	notifier->data = data;
 }
 
-static struct blk_mq_cpu_notifier __cpuinitdata cpu_notifier = {
-	.notify = blk_mq_cpu_notify,
-};
-
 void __init blk_mq_cpu_init(void)
 {
 	register_hotcpu_notifier(&blk_mq_main_cpu_notifier);
-	blk_mq_register_cpu_notifier(&cpu_notifier);
 }
diff --git a/block/blk-mq.c b/block/blk-mq.c
index d43a7e8..0b2b487 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -27,8 +27,6 @@ static LIST_HEAD(all_q_list);
 
 static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);
 
-DEFINE_PER_CPU(struct llist_head, ipi_lists);
-
 static struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
 					   unsigned int cpu)
 {
@@ -318,55 +316,12 @@ void __blk_mq_end_io(struct request *rq, int error)
 		blk_mq_complete_request(rq, error);
 }
 
-#if defined(CONFIG_SMP)
-
-/*
- * Called with interrupts disabled.
- */
-static void ipi_end_io(void *data)
+static void blk_mq_end_io_remote(void *data)
 {
-	struct llist_head *list = &per_cpu(ipi_lists, smp_processor_id());
-	struct llist_node *entry, *next;
-	struct request *rq;
-
-	entry = llist_del_all(list);
-
-	while (entry) {
-		next = entry->next;
-		rq = llist_entry(entry, struct request, ll_list);
-		__blk_mq_end_io(rq, rq->errors);
-		entry = next;
-	}
-}
-
-static int ipi_remote_cpu(struct blk_mq_ctx *ctx, const int cpu,
-			  struct request *rq, const int error)
-{
-	struct call_single_data *data = &rq->csd;
-
-	rq->errors = error;
-	rq->ll_list.next = NULL;
-
-	/*
-	 * If the list is non-empty, an existing IPI must already
-	 * be "in flight". If that is the case, we need not schedule
-	 * a new one.
-	 */
-	if (llist_add(&rq->ll_list, &per_cpu(ipi_lists, ctx->cpu))) {
-		data->func = ipi_end_io;
-		data->flags = 0;
-		__smp_call_function_single(ctx->cpu, data, 0);
-	}
+	struct request *rq = data;
 
-	return true;
+	__blk_mq_end_io(rq, rq->errors);
 }
-#else /* CONFIG_SMP */
-static int ipi_remote_cpu(struct blk_mq_ctx *ctx, const int cpu,
-			  struct request *rq, const int error)
-{
-	return false;
-}
-#endif
 
 /*
  * End IO on this request on a multiqueue enabled driver. We'll either do
@@ -382,11 +337,15 @@ void blk_mq_end_io(struct request *rq, int error)
 		return __blk_mq_end_io(rq, error);
 
 	cpu = get_cpu();
-
-	if (cpu == ctx->cpu || !cpu_online(ctx->cpu) ||
-	    !ipi_remote_cpu(ctx, cpu, rq, error))
+	if (cpu != ctx->cpu && cpu_online(ctx->cpu)) {
+		rq->errors = error;
+		rq->csd.func = blk_mq_end_io_remote;
+		rq->csd.info = rq;
+		rq->csd.flags = 0;
+		__smp_call_function_single(ctx->cpu, &rq->csd, 0);
+	} else {
 		__blk_mq_end_io(rq, error);
-
+	}
 	put_cpu();
 }
 EXPORT_SYMBOL(blk_mq_end_io);
@@ -1465,11 +1424,6 @@ static int __cpuinit blk_mq_queue_reinit_notify(struct notifier_block *nb,
 
 static int __init blk_mq_init(void)
 {
-	unsigned int i;
-
-	for_each_possible_cpu(i)
-		init_llist_head(&per_cpu(ipi_lists, i));
-
 	blk_mq_cpu_init();
 
 	/* Must be called after percpu_counter_hotcpu_callback() */
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 52bf1f9..5761eed 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -38,7 +38,6 @@ void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier,
 void blk_mq_register_cpu_notifier(struct blk_mq_cpu_notifier *notifier);
 void blk_mq_unregister_cpu_notifier(struct blk_mq_cpu_notifier *notifier);
 void blk_mq_cpu_init(void);
-DECLARE_PER_CPU(struct llist_head, ipi_lists);
 
 /*
  * CPU -> queue mappings
-- 
1.7.10.4

Re: [PATCH 5/7] llists: move llist_reverse_order from raid5 to llist.c

[Jan Kara]

On Thu 24-10-13 08:19:26, Christoph Hellwig wrote:
> Make this useful helper available for other users.
> 
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> ---
>  drivers/md/raid5.c    |   14 --------------
>  include/linux/llist.h |    2 ++
>  lib/llist.c           |   22 ++++++++++++++++++++++
>  3 files changed, 24 insertions(+), 14 deletions(-)
> 
> diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
> index 7ff4f25..046f1a9 100644
> --- a/drivers/md/raid5.c
> +++ b/drivers/md/raid5.c
> @@ -293,20 +293,6 @@ static void __release_stripe(struct r5conf *conf, struct stripe_head *sh)
>  		do_release_stripe(conf, sh);
>  }
>  
> -static struct llist_node *llist_reverse_order(struct llist_node *head)
> -{
> -	struct llist_node *new_head = NULL;
> -
> -	while (head) {
> -		struct llist_node *tmp = head;
> -		head = head->next;
> -		tmp->next = new_head;
> -		new_head = tmp;
> -	}
> -
> -	return new_head;
> -}
> -
>  /* should hold conf->device_lock already */
>  static int release_stripe_list(struct r5conf *conf)
>  {
> diff --git a/include/linux/llist.h b/include/linux/llist.h
> index 8828a78..fbf10a0 100644
> --- a/include/linux/llist.h
> +++ b/include/linux/llist.h
> @@ -195,4 +195,6 @@ static inline struct llist_node *llist_del_all(struct llist_head *head)
>  
>  extern struct llist_node *llist_del_first(struct llist_head *head);
>  
> +struct llist_node *llist_reverse_order(struct llist_node *head);
> +
>  #endif /* LLIST_H */
> diff --git a/lib/llist.c b/lib/llist.c
> index 4a70d12..ef48b87 100644
> --- a/lib/llist.c
> +++ b/lib/llist.c
> @@ -81,3 +81,25 @@ struct llist_node *llist_del_first(struct llist_head *head)
>  	return entry;
>  }
>  EXPORT_SYMBOL_GPL(llist_del_first);
> +
> +/**
> + * llist_reverse_order - reverse order of a llist chain
> + * @head:	first item of the list to be reversed
> + *
> + * Reverse the oder of a chain of llist entries and return the
                  ^^ order

  Otherwise the patch looks fine. You can add:
Reviewed-by: Jan Kara <jack@suse.cz>

								Honza

> + * new first entry.
> + */
> +struct llist_node *llist_reverse_order(struct llist_node *head)
> +{
> +	struct llist_node *new_head = NULL;
> +
> +	while (head) {
> +		struct llist_node *tmp = head;
> +		head = head->next;
> +		tmp->next = new_head;
> +		new_head = tmp;
> +	}
> +
> +	return new_head;
> +}
> +EXPORT_SYMBOL_GPL(llist_reverse_order);
> -- 
> 1.7.10.4
> 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
-- 
Jan Kara <jack@suse.cz>
SUSE Labs, CR

Re: [PATCH 6/7] kernel: use lockless list for smp_call_function_single

[Jan Kara]

On Thu 24-10-13 08:19:27, Christoph Hellwig wrote:
> Make smp_call_function_single and friends more efficient by using
> a lockless list.
> 
> Signed-off-by: Christoph Hellwig <hch@lst.de>
> ---
>  include/linux/blkdev.h |    5 +----
>  include/linux/smp.h    |    6 +++++-
>  kernel/smp.c           |   51 ++++++++++++------------------------------------
>  3 files changed, 19 insertions(+), 43 deletions(-)
> 
> diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
> index f26ec20f..287bf7c 100644
> --- a/include/linux/blkdev.h
> +++ b/include/linux/blkdev.h
> @@ -95,10 +95,7 @@ enum rq_cmd_type_bits {
>   * as well!
>   */
>  struct request {
> -	union {
> -		struct list_head queuelist;
> -		struct llist_node ll_list;
> -	};
> +	struct list_head queuelist;
>  	union {
>  		struct call_single_data csd;
>  		struct work_struct mq_flush_data;
> diff --git a/include/linux/smp.h b/include/linux/smp.h
> index 7885151..10755dd 100644
> --- a/include/linux/smp.h
> +++ b/include/linux/smp.h
> @@ -11,12 +11,16 @@
>  #include <linux/list.h>
>  #include <linux/cpumask.h>
>  #include <linux/init.h>
> +#include <linux/llist.h>
>  
>  extern void cpu_idle(void);
>  
>  typedef void (*smp_call_func_t)(void *info);
>  struct call_single_data {
> -	struct list_head list;
> +	union {
> +		struct list_head list;
> +		struct llist_node llist;
> +	};
  I'm wondering: Who's still using the normal list_head? I was grepping for
a while and I couldn't find any user. Otherwise the patch looks good to me.
You can add:
  Reviewed-by: Jan Kara <jack@suse.cz>

								Honza

>  	smp_call_func_t func;
>  	void *info;
>  	u16 flags;
> diff --git a/kernel/smp.c b/kernel/smp.c
> index 53644e6..a735c66 100644
> --- a/kernel/smp.c
> +++ b/kernel/smp.c
> @@ -28,12 +28,7 @@ struct call_function_data {
>  
>  static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_function_data, cfd_data);
>  
> -struct call_single_queue {
> -	struct list_head	list;
> -	raw_spinlock_t		lock;
> -};
> -
> -static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_queue, call_single_queue);
> +static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue);
>  
>  static int
>  hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
> @@ -85,12 +80,8 @@ void __init call_function_init(void)
>  	void *cpu = (void *)(long)smp_processor_id();
>  	int i;
>  
> -	for_each_possible_cpu(i) {
> -		struct call_single_queue *q = &per_cpu(call_single_queue, i);
> -
> -		raw_spin_lock_init(&q->lock);
> -		INIT_LIST_HEAD(&q->list);
> -	}
> +	for_each_possible_cpu(i)
> +		init_llist_head(&per_cpu(call_single_queue, i));
>  
>  	hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu);
>  	register_cpu_notifier(&hotplug_cfd_notifier);
> @@ -141,18 +132,9 @@ static void csd_unlock(struct call_single_data *csd)
>   */
>  static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
>  {
> -	struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
> -	unsigned long flags;
> -	int ipi;
> -
>  	if (wait)
>  		csd->flags |= CSD_FLAG_WAIT;
>  
> -	raw_spin_lock_irqsave(&dst->lock, flags);
> -	ipi = list_empty(&dst->list);
> -	list_add_tail(&csd->list, &dst->list);
> -	raw_spin_unlock_irqrestore(&dst->lock, flags);
> -
>  	/*
>  	 * The list addition should be visible before sending the IPI
>  	 * handler locks the list to pull the entry off it because of
> @@ -164,7 +146,7 @@ static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
>  	 * locking and barrier primitives. Generic code isn't really
>  	 * equipped to do the right thing...
>  	 */
> -	if (ipi)
> +	if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
>  		arch_send_call_function_single_ipi(cpu);
>  
>  	if (wait)
> @@ -177,27 +159,26 @@ static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
>   */
>  void generic_smp_call_function_single_interrupt(void)
>  {
> -	struct call_single_queue *q = &__get_cpu_var(call_single_queue);
> -	LIST_HEAD(list);
> +	struct llist_node *entry, *next;
>  
>  	/*
>  	 * Shouldn't receive this interrupt on a cpu that is not yet online.
>  	 */
>  	WARN_ON_ONCE(!cpu_online(smp_processor_id()));
>  
> -	raw_spin_lock(&q->lock);
> -	list_replace_init(&q->list, &list);
> -	raw_spin_unlock(&q->lock);
> +	entry = llist_del_all(&__get_cpu_var(call_single_queue));
> +	entry = llist_reverse_order(entry);
>  
> -	while (!list_empty(&list)) {
> +	while (entry) {
>  		struct call_single_data *csd;
>  
> -		csd = list_entry(list.next, struct call_single_data, list);
> -		list_del(&csd->list);
> +		next = entry->next;
>  
> +		csd = llist_entry(entry, struct call_single_data, llist);
>  		csd->func(csd->info);
> -
>  		csd_unlock(csd);
> +
> +		entry = next;
>  	}
>  }
>  
> @@ -410,17 +391,11 @@ void smp_call_function_many(const struct cpumask *mask,
>  
>  	for_each_cpu(cpu, cfd->cpumask) {
>  		struct call_single_data *csd = per_cpu_ptr(cfd->csd, cpu);
> -		struct call_single_queue *dst =
> -					&per_cpu(call_single_queue, cpu);
> -		unsigned long flags;
>  
>  		csd_lock(csd);
>  		csd->func = func;
>  		csd->info = info;
> -
> -		raw_spin_lock_irqsave(&dst->lock, flags);
> -		list_add_tail(&csd->list, &dst->list);
> -		raw_spin_unlock_irqrestore(&dst->lock, flags);
> +		llist_add(&csd->llist, &per_cpu(call_single_queue, cpu));
>  	}
>  
>  	/* Send a message to all CPUs in the map */
> -- 
> 1.7.10.4
> 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
-- 
Jan Kara <jack@suse.cz>
SUSE Labs, CR

Re: [PATCH 6/7] kernel: use lockless list for smp_call_function_single

[Christoph Hellwig]

On Tue, Oct 29, 2013 at 08:25:36PM +0100, Jan Kara wrote:
>   I'm wondering: Who's still using the normal list_head? I was grepping for
> a while and I couldn't find any user. Otherwise the patch looks good to me.

The block/blk-softirq.c code is abusing it in a fairly ugly way.

Re: [PATCH 6/7] kernel: use lockless list for smp_call_function_single

[Jan Kara]

On Tue 29-10-13 12:27:20, Christoph Hellwig wrote:
> On Tue, Oct 29, 2013 at 08:25:36PM +0100, Jan Kara wrote:
> >   I'm wondering: Who's still using the normal list_head? I was grepping for
> > a while and I couldn't find any user. Otherwise the patch looks good to me.
> 
> The block/blk-softirq.c code is abusing it in a fairly ugly way.
  Ah, thanks for the pointer. Looking into the code, that could easily use
llist as well, couldn't it?

								Honza

-- 
Jan Kara <jack@suse.cz>
SUSE Labs, CR

Re: [PATCH 6/7] kernel: use lockless list for smp_call_function_single

[Jens Axboe]

On 10/29/2013 03:17 PM, Jan Kara wrote:
> On Tue 29-10-13 12:27:20, Christoph Hellwig wrote:
>> On Tue, Oct 29, 2013 at 08:25:36PM +0100, Jan Kara wrote:
>>>   I'm wondering: Who's still using the normal list_head? I was grepping for
>>> a while and I couldn't find any user. Otherwise the patch looks good to me.
>>
>> The block/blk-softirq.c code is abusing it in a fairly ugly way.
>   Ah, thanks for the pointer. Looking into the code, that could easily use
> llist as well, couldn't it?

Sure, it'd be easy enough to do.

-- 
Jens Axboe