From: Atish Patra <atish.patra@wdc.com>
To: linux-kernel@vger.kernel.org
Cc: Atish Patra <atish.patra@wdc.com>,
Albert Ou <aou@eecs.berkeley.edu>,
Anup Patel <anup@brainfault.org>, Borislav Petkov <bp@suse.de>,
"Eric W. Biederman" <ebiederm@xmission.com>,
Geert Uytterhoeven <geert@linux-m68k.org>,
Greentime Hu <greentime.hu@sifive.com>,
Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
Heiko Carstens <heiko.carstens@de.ibm.com>,
Kees Cook <keescook@chromium.org>,
linux-riscv@lists.infradead.org, Mao Han <han_mao@c-sky.com>,
Mike Rapoport <rppt@linux.ibm.com>,
Palmer Dabbelt <palmer@dabbelt.com>,
Paul Walmsley <paul.walmsley@sifive.com>,
Thomas Gleixner <tglx@linutronix.de>,
Vincent Chen <vincent.chen@sifive.com>,
abner.chang@hpe.com, clin@suse.com, nickhu@andestech.com,
Palmer Dabbelt <palmerdabbelt@google.com>
Subject: [PATCH v7 10/10] RISC-V: Support cpu hotplug
Date: Mon, 27 Jan 2020 18:27:37 -0800 [thread overview]
Message-ID: <20200128022737.15371-11-atish.patra@wdc.com> (raw)
In-Reply-To: <20200128022737.15371-1-atish.patra@wdc.com>
This patch enable support for cpu hotplug in RISC-V. It uses SBI HSM
extension to online/offline any hart. As a result, the harts are
returned to firmware once they are offline. If the harts are brought
online afterwards, they re-enter Linux kernel as if a secondary hart
booted for the first time. All booting requirements are honored during
this process.
Tested both on QEMU and HighFive Unleashed board with. Test result follows.
---------------------------------------------------
Offline cpu 2
---------------------------------------------------
$ echo 0 > /sys/devices/system/cpu/cpu2/online
[ 32.828684] CPU2: off
$ cat /proc/cpuinfo
processor : 0
hart : 0
isa : rv64imafdcsu
mmu : sv48
processor : 1
hart : 1
isa : rv64imafdcsu
mmu : sv48
processor : 3
hart : 3
isa : rv64imafdcsu
mmu : sv48
processor : 4
hart : 4
isa : rv64imafdcsu
mmu : sv48
processor : 5
hart : 5
isa : rv64imafdcsu
mmu : sv48
processor : 6
hart : 6
isa : rv64imafdcsu
mmu : sv48
processor : 7
hart : 7
isa : rv64imafdcsu
mmu : sv48
---------------------------------------------------
online cpu 2
---------------------------------------------------
$ echo 1 > /sys/devices/system/cpu/cpu2/online
$ cat /proc/cpuinfo
processor : 0
hart : 0
isa : rv64imafdcsu
mmu : sv48
processor : 1
hart : 1
isa : rv64imafdcsu
mmu : sv48
processor : 2
hart : 2
isa : rv64imafdcsu
mmu : sv48
processor : 3
hart : 3
isa : rv64imafdcsu
mmu : sv48
processor : 4
hart : 4
isa : rv64imafdcsu
mmu : sv48
processor : 5
hart : 5
isa : rv64imafdcsu
mmu : sv48
processor : 6
hart : 6
isa : rv64imafdcsu
mmu : sv48
processor : 7
hart : 7
isa : rv64imafdcsu
mmu : sv48
Signed-off-by: Atish Patra <atish.patra@wdc.com>
---
arch/riscv/Kconfig | 12 ++++-
arch/riscv/include/asm/cpu_ops.h | 5 ++
arch/riscv/include/asm/smp.h | 7 +++
arch/riscv/kernel/Makefile | 1 +
arch/riscv/kernel/cpu-hotplug.c | 84 ++++++++++++++++++++++++++++++++
arch/riscv/kernel/cpu_ops.c | 34 +++++++++++++
arch/riscv/kernel/setup.c | 26 ++++++++++
7 files changed, 168 insertions(+), 1 deletion(-)
create mode 100644 arch/riscv/kernel/cpu-hotplug.c
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 518da42be545..99fb481dc805 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -22,7 +22,6 @@ config RISCV
select CLONE_BACKWARDS
select COMMON_CLK
select GENERIC_CLOCKEVENTS
- select GENERIC_CPU_DEVICES
select GENERIC_IRQ_SHOW
select GENERIC_PCI_IOMAP
select GENERIC_SCHED_CLOCK
@@ -247,6 +246,17 @@ config NR_CPUS
depends on SMP
default "8"
+config HOTPLUG_CPU
+ bool "Support for hot-pluggable CPUs"
+ depends on SMP
+ select GENERIC_IRQ_MIGRATION
+ help
+
+ Say Y here to experiment with turning CPUs off and on. CPUs
+ can be controlled through /sys/devices/system/cpu.
+
+ Say N if you want to disable CPU hotplug.
+
choice
prompt "CPU Tuning"
default TUNE_GENERIC
diff --git a/arch/riscv/include/asm/cpu_ops.h b/arch/riscv/include/asm/cpu_ops.h
index 27e9dfee5460..d53d7086f627 100644
--- a/arch/riscv/include/asm/cpu_ops.h
+++ b/arch/riscv/include/asm/cpu_ops.h
@@ -23,6 +23,11 @@ struct cpu_operations {
int (*cpu_prepare)(unsigned int cpu);
int (*cpu_start)(unsigned int cpu,
struct task_struct *tidle);
+#ifdef CONFIG_HOTPLUG_CPU
+ int (*cpu_disable)(unsigned int cpu);
+ void (*cpu_stop)(void);
+ int (*cpu_is_stopped)(unsigned int cpu);
+#endif
};
extern const struct cpu_operations *cpu_ops[NR_CPUS];
diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
index 023f74fb8b3b..8d9c50c0f91c 100644
--- a/arch/riscv/include/asm/smp.h
+++ b/arch/riscv/include/asm/smp.h
@@ -43,6 +43,13 @@ void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out);
*/
#define raw_smp_processor_id() (current_thread_info()->cpu)
+#if defined CONFIG_HOTPLUG_CPU
+int __cpu_disable(void);
+void __cpu_die(unsigned int cpu);
+void cpu_stop(void);
+#else
+#endif /* CONFIG_HOTPLUG_CPU */
+
#else
static inline void show_ipi_stats(struct seq_file *p, int prec)
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index d77def5b4e87..6fe35a719de1 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -43,5 +43,6 @@ obj-$(CONFIG_PERF_EVENTS) += perf_event.o
obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o
obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o
obj-$(CONFIG_RISCV_SBI) += sbi.o
+obj-$(CONFIG_HOTPLUG_CPU) += cpu-hotplug.o
clean:
diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c
new file mode 100644
index 000000000000..835b0747803e
--- /dev/null
+++ b/arch/riscv/kernel/cpu-hotplug.c
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Western Digital Corporation or its affiliates.
+ */
+
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/sched.h>
+#include <linux/err.h>
+#include <linux/irq.h>
+#include <linux/cpu.h>
+#include <linux/sched/hotplug.h>
+#include <asm/irq.h>
+#include <asm/cpu_ops.h>
+#include <asm/sbi.h>
+
+void cpu_stop(void);
+bool can_hotplug_cpu(void)
+{
+ return true;
+}
+
+void arch_cpu_idle_dead(void)
+{
+ cpu_stop();
+}
+
+/*
+ * __cpu_disable runs on the processor to be shutdown.
+ */
+int __cpu_disable(void)
+{
+ int ret = 0;
+ unsigned int cpu = smp_processor_id();
+
+ if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_stop)
+ return -EOPNOTSUPP;
+
+ if (cpu_ops[cpu]->cpu_disable)
+ ret = cpu_ops[cpu]->cpu_disable(cpu);
+
+ if (ret)
+ return ret;
+
+ remove_cpu_topology(cpu);
+ set_cpu_online(cpu, false);
+ irq_migrate_all_off_this_cpu();
+
+ return ret;
+}
+
+/*
+ * Called on the thread which is asking for a CPU to be shutdown.
+ */
+void __cpu_die(unsigned int cpu)
+{
+ int ret = 0;
+
+ if (!cpu_wait_death(cpu, 5)) {
+ pr_err("CPU %u: didn't die\n", cpu);
+ return;
+ }
+ pr_notice("CPU%u: off\n", cpu);
+
+ /* Verify from the firmware if the cpu is really stopped*/
+ if (cpu_ops[cpu]->cpu_is_stopped)
+ ret = cpu_ops[cpu]->cpu_is_stopped(cpu);
+ if (ret)
+ pr_warn("CPU%d may not have stopped: %d\n", cpu, ret);
+}
+
+/*
+ * Called from the idle thread for the CPU which has been shutdown.
+ */
+void cpu_stop(void)
+{
+ idle_task_exit();
+
+ (void)cpu_report_death();
+
+ cpu_ops[smp_processor_id()]->cpu_stop();
+ /* It should never reach here */
+ BUG();
+}
diff --git a/arch/riscv/kernel/cpu_ops.c b/arch/riscv/kernel/cpu_ops.c
index 454df032066f..9b315137b945 100644
--- a/arch/riscv/kernel/cpu_ops.c
+++ b/arch/riscv/kernel/cpu_ops.c
@@ -59,6 +59,34 @@ static int sbi_cpu_start(unsigned int cpuid, struct task_struct *tidle)
return rc;
}
+#ifdef CONFIG_HOTPLUG_CPU
+static int sbi_cpu_disable(unsigned int cpuid)
+{
+ if (!cpu_sbi_ops.cpu_stop)
+ return -EOPNOTSUPP;
+ return 0;
+}
+
+static void sbi_cpu_stop(void)
+{
+ int ret;
+
+ ret = sbi_hsm_hart_stop();
+ pr_crit("Unable to stop the cpu %u (%d)\n", smp_processor_id(), ret);
+}
+
+static int sbi_cpu_is_stopped(unsigned int cpuid)
+{
+ int rc;
+ int hartid = cpuid_to_hartid_map(cpuid);
+
+ rc = sbi_hsm_hart_get_status(hartid);
+
+ if (rc == RISCV_HART_FIRMWARE_STOPPED)
+ return 0;
+ return rc;
+}
+#endif
static int spinwait_cpu_start(unsigned int cpuid, struct task_struct *tidle)
{
int hartid = cpuid_to_hartid_map(cpuid);
@@ -82,6 +110,11 @@ const struct cpu_operations cpu_sbi_ops = {
.name = "sbi",
.cpu_prepare = sbi_cpu_prepare,
.cpu_start = sbi_cpu_start,
+#ifdef CONFIG_HOTPLUG_CPU
+ .cpu_disable = sbi_cpu_disable,
+ .cpu_stop = sbi_cpu_stop,
+ .cpu_is_stopped = sbi_cpu_is_stopped,
+#endif
};
const struct cpu_operations cpu_spinwait_ops = {
@@ -90,6 +123,7 @@ const struct cpu_operations cpu_spinwait_ops = {
.cpu_start = spinwait_cpu_start,
};
+
int __init cpu_set_ops(int cpuid)
{
if (sbi_hsm_is_available())
diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c
index 8208d1109ddb..dfab3bd40f2a 100644
--- a/arch/riscv/kernel/setup.c
+++ b/arch/riscv/kernel/setup.c
@@ -43,6 +43,7 @@ struct screen_info screen_info = {
/* The lucky hart to first increment this variable will boot the other cores */
atomic_t hart_lottery;
unsigned long boot_cpu_hartid;
+static DEFINE_PER_CPU(struct cpu, cpu_devices);
void __init parse_dtb(void)
{
@@ -90,3 +91,28 @@ void __init setup_arch(char **cmdline_p)
riscv_fill_hwcap();
}
+
+static inline bool can_hotplug_cpu(unsigned int cpu)
+{
+#ifdef CONFIG_HOTPLUG_CPU
+ if (cpu_ops[cpu]->cpu_stop)
+ return true;
+#endif
+ return false;
+}
+
+static int __init topology_init(void)
+{
+ int i;
+
+ pr_err("%s: In\n", __func__);
+ for_each_possible_cpu(i) {
+ struct cpu *cpu = &per_cpu(cpu_devices, i);
+
+ cpu->hotpluggable = can_hotplug_cpu(i);
+ register_cpu(cpu, i);
+ }
+
+ return 0;
+}
+subsys_initcall(topology_init);
--
2.24.0
next prev parent reply other threads:[~2020-01-28 2:28 UTC|newest]
Thread overview: 22+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-01-28 2:27 [PATCH v7 00/10] Add support for SBI v0.2 and CPU hotplug Atish Patra
2020-01-28 2:27 ` [PATCH v7 01/10] RISC-V: Mark existing SBI as 0.1 SBI Atish Patra
2020-03-06 3:32 ` Bin Meng
2020-01-28 2:27 ` [PATCH v7 02/10] RISC-V: Add basic support for SBI v0.2 Atish Patra
2020-01-28 4:25 ` Anup Patel
2020-01-28 19:12 ` Atish Patra
2020-02-06 6:24 ` Anup Patel
2020-01-28 2:27 ` [PATCH v7 03/10] RISC-V: Add SBI v0.2 extension definitions Atish Patra
2020-01-28 2:27 ` [PATCH v7 04/10] RISC-V: Introduce a new config for SBI v0.1 Atish Patra
2020-01-28 2:27 ` [PATCH v7 05/10] RISC-V: Implement new SBI v0.2 extensions Atish Patra
2020-01-28 2:27 ` [PATCH v7 06/10] RISC-V: Add cpu_ops and modify default booting method Atish Patra
2020-01-28 4:31 ` Anup Patel
2020-01-28 2:27 ` [PATCH v7 07/10] RISC-V: Move relocate and few other functions out of __init Atish Patra
2020-01-28 4:38 ` Anup Patel
2020-01-28 19:28 ` Atish Patra
2020-01-28 2:27 ` [PATCH v7 08/10] RISC-V: Add SBI HSM extension Atish Patra
2020-01-28 4:54 ` Anup Patel
2020-02-05 0:11 ` Atish Patra
2020-01-28 2:27 ` [PATCH v7 09/10] RISC-V: Add supported for ordered booting method using HSM Atish Patra
2020-01-28 2:27 ` Atish Patra [this message]
2020-01-28 5:00 ` [PATCH v7 10/10] RISC-V: Support cpu hotplug Anup Patel
2020-01-28 19:32 ` Atish Patra
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200128022737.15371-11-atish.patra@wdc.com \
--to=atish.patra@wdc.com \
--cc=abner.chang@hpe.com \
--cc=anup@brainfault.org \
--cc=aou@eecs.berkeley.edu \
--cc=bp@suse.de \
--cc=clin@suse.com \
--cc=ebiederm@xmission.com \
--cc=geert@linux-m68k.org \
--cc=greentime.hu@sifive.com \
--cc=gregkh@linuxfoundation.org \
--cc=han_mao@c-sky.com \
--cc=heiko.carstens@de.ibm.com \
--cc=keescook@chromium.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-riscv@lists.infradead.org \
--cc=nickhu@andestech.com \
--cc=palmer@dabbelt.com \
--cc=palmerdabbelt@google.com \
--cc=paul.walmsley@sifive.com \
--cc=rppt@linux.ibm.com \
--cc=tglx@linutronix.de \
--cc=vincent.chen@sifive.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).