Introduce SMP support for CI20 (based on JZ4780) v8.

Introduce SMP support for CI20 (based on JZ4780) v8.

[周琰杰 (Zhou Yanjie)]

Introduce SMP support for MIPS Creator CI20, which is
based on Ingenic JZ4780 SoC.

[PATCH v8 0/6] Introduce SMP support for CI20 (based on JZ4780).

[周琰杰 (Zhou Yanjie)]

Introduce SMP support for MIPS Creator CI20, which is
based on Ingenic JZ4780 SoC.

周琰杰 (Zhou Yanjie) (6):
  MIPS: JZ4780: Introduce SMP support.
  MIPS: CI20: Modify DTS to support high resolution timer for SMP.
  clocksource: Ingenic: Add high resolution timer support for SMP.
  dt-bindings: MIPS: Document Ingenic SoCs binding.
  MIPS: Ingenic: Add 'cpus' node for Ingenic SoCs.
  MIPS: CI20: Update defconfig to support SMP.

 .../bindings/mips/ingenic/ingenic,cpu.yaml         |  57 +++++
 arch/mips/boot/dts/ingenic/ci20.dts                |  11 +-
 arch/mips/boot/dts/ingenic/jz4740.dtsi             |  14 ++
 arch/mips/boot/dts/ingenic/jz4770.dtsi             |  15 +-
 arch/mips/boot/dts/ingenic/jz4780.dtsi             |  23 ++
 arch/mips/boot/dts/ingenic/x1000.dtsi              |  14 ++
 arch/mips/configs/ci20_defconfig                   |   2 +
 arch/mips/include/asm/mach-jz4740/smp.h            |  87 +++++++
 arch/mips/jz4740/Kconfig                           |   2 +
 arch/mips/jz4740/Makefile                          |   5 +
 arch/mips/jz4740/prom.c                            |   4 +
 arch/mips/jz4740/smp-entry.S                       |  57 +++++
 arch/mips/jz4740/smp.c                             | 258 +++++++++++++++++++++
 arch/mips/kernel/idle.c                            |  35 ++-
 drivers/clocksource/ingenic-timer.c                | 103 ++++++--
 15 files changed, 661 insertions(+), 26 deletions(-)
 create mode 100644 Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml
 create mode 100644 arch/mips/include/asm/mach-jz4740/smp.h
 create mode 100644 arch/mips/jz4740/smp-entry.S
 create mode 100644 arch/mips/jz4740/smp.c

-- 
2.7.4

[PATCH v8 1/6] MIPS: JZ4780: Introduce SMP support.

[周琰杰 (Zhou Yanjie)]

Forward port smp support from kernel 3.18.3 of CI20_linux
to upstream kernel 5.6.

Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
Tested-by: Paul Boddie <paul@boddie.org.uk>
Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
---

Notes:
    v1->v2:
    1.Remove unnecessary "plat_irq_dispatch(void)" in irq-ingenic.c.
    2.Add a timeout check for "jz4780_boot_secondary()" to avoid a dead loop.
    3.Replace hard code in smp.c with macro.
    
    v2->v3:
    1.Remove unnecessary "extern void (*r4k_blast_dcache)(void)" in smp.c.
    2.Use "for_each_of_cpu_node" instead "for_each_compatible_node" in smp.c.
    3.Use "of_cpu_node_to_id" instead "of_property_read_u32_index" in smp.c.
    4.Move LCR related operations to jz4780-cgu.c.
    
    v3->v4:
    Rebase on top of kernel 5.6-rc1.
    
    v4->v5:
    1.Splitting changes involving "jz4780-cgu.c" into separate commit.
    2.Use "request_irq()" replace "setup_irq()".
    
    v5->v6:
    In order to have a kernel that works on multiple SoCs at the same
    time, use "IS_ENABLED()" replace "#ifdef".
    
    v6->v7:
    1.SMP has be decoupled from the SoC version.
    2.Add mailboxes 3 and 4 for XBurst.
    3.Adjust code in "jz4780_smp_prepare_cpus()".
    4."jz4780_smp_init()" has be marked "__init".
    
    v7->v8:
    No change.

 arch/mips/include/asm/mach-jz4740/smp.h |  87 +++++++++++
 arch/mips/jz4740/Kconfig                |   2 +
 arch/mips/jz4740/Makefile               |   5 +
 arch/mips/jz4740/prom.c                 |   4 +
 arch/mips/jz4740/smp-entry.S            |  57 +++++++
 arch/mips/jz4740/smp.c                  | 258 ++++++++++++++++++++++++++++++++
 arch/mips/kernel/idle.c                 |  35 ++++-
 7 files changed, 447 insertions(+), 1 deletion(-)
 create mode 100644 arch/mips/include/asm/mach-jz4740/smp.h
 create mode 100644 arch/mips/jz4740/smp-entry.S
 create mode 100644 arch/mips/jz4740/smp.c

diff --git a/arch/mips/include/asm/mach-jz4740/smp.h b/arch/mips/include/asm/mach-jz4740/smp.h
new file mode 100644
index 00000000..86f660f
--- /dev/null
+++ b/arch/mips/include/asm/mach-jz4740/smp.h
@@ -0,0 +1,87 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
+ *  JZ4780 SMP definitions
+ */
+
+#ifndef __MIPS_ASM_MACH_JZ4740_SMP_H__
+#define __MIPS_ASM_MACH_JZ4740_SMP_H__
+
+#define read_c0_corectrl()		__read_32bit_c0_register($12, 2)
+#define write_c0_corectrl(val)		__write_32bit_c0_register($12, 2, val)
+
+#define read_c0_corestatus()		__read_32bit_c0_register($12, 3)
+#define write_c0_corestatus(val)	__write_32bit_c0_register($12, 3, val)
+
+#define read_c0_reim()			__read_32bit_c0_register($12, 4)
+#define write_c0_reim(val)		__write_32bit_c0_register($12, 4, val)
+
+#define read_c0_mailbox0()		__read_32bit_c0_register($20, 0)
+#define write_c0_mailbox0(val)		__write_32bit_c0_register($20, 0, val)
+
+#define read_c0_mailbox1()		__read_32bit_c0_register($20, 1)
+#define write_c0_mailbox1(val)		__write_32bit_c0_register($20, 1, val)
+
+#define read_c0_mailbox2()		__read_32bit_c0_register($20, 2)
+#define write_c0_mailbox2(val)		__write_32bit_c0_register($20, 2, val)
+
+#define read_c0_mailbox3()		__read_32bit_c0_register($20, 3)
+#define write_c0_mailbox3(val)		__write_32bit_c0_register($20, 3, val)
+
+#define smp_clr_pending(mask) do {		\
+		unsigned int stat;		\
+		stat = read_c0_corestatus();	\
+		stat &= ~((mask) & 0xff);	\
+		write_c0_corestatus(stat);	\
+	} while (0)
+
+/*
+ * Core Control register
+ */
+#define CORECTRL_SLEEP1M_SHIFT	17
+#define CORECTRL_SLEEP1M	(_ULCAST_(0x1) << CORECTRL_SLEEP1M_SHIFT)
+#define CORECTRL_SLEEP0M_SHIFT	16
+#define CORECTRL_SLEEP0M	(_ULCAST_(0x1) << CORECTRL_SLEEP0M_SHIFT)
+#define CORECTRL_RPC1_SHIFT	9
+#define CORECTRL_RPC1		(_ULCAST_(0x1) << CORECTRL_RPC1_SHIFT)
+#define CORECTRL_RPC0_SHIFT	8
+#define CORECTRL_RPC0		(_ULCAST_(0x1) << CORECTRL_RPC0_SHIFT)
+#define CORECTRL_SWRST1_SHIFT	1
+#define CORECTRL_SWRST1		(_ULCAST_(0x1) << CORECTRL_SWRST1_SHIFT)
+#define CORECTRL_SWRST0_SHIFT	0
+#define CORECTRL_SWRST0		(_ULCAST_(0x1) << CORECTRL_SWRST0_SHIFT)
+
+/*
+ * Core Status register
+ */
+#define CORESTATUS_SLEEP1_SHIFT	17
+#define CORESTATUS_SLEEP1	(_ULCAST_(0x1) << CORESTATUS_SLEEP1_SHIFT)
+#define CORESTATUS_SLEEP0_SHIFT	16
+#define CORESTATUS_SLEEP0	(_ULCAST_(0x1) << CORESTATUS_SLEEP0_SHIFT)
+#define CORESTATUS_IRQ1P_SHIFT	9
+#define CORESTATUS_IRQ1P	(_ULCAST_(0x1) << CORESTATUS_IRQ1P_SHIFT)
+#define CORESTATUS_IRQ0P_SHIFT	8
+#define CORESTATUS_IRQ0P	(_ULCAST_(0x1) << CORESTATUS_IRQ8P_SHIFT)
+#define CORESTATUS_MIRQ1P_SHIFT	1
+#define CORESTATUS_MIRQ1P	(_ULCAST_(0x1) << CORESTATUS_MIRQ1P_SHIFT)
+#define CORESTATUS_MIRQ0P_SHIFT	0
+#define CORESTATUS_MIRQ0P	(_ULCAST_(0x1) << CORESTATUS_MIRQ0P_SHIFT)
+
+/*
+ * Reset Entry & IRQ Mask register
+ */
+#define REIM_ENTRY_SHIFT	16
+#define REIM_ENTRY		(_ULCAST_(0xffff) << REIM_ENTRY_SHIFT)
+#define REIM_IRQ1M_SHIFT	9
+#define REIM_IRQ1M		(_ULCAST_(0x1) << REIM_IRQ1M_SHIFT)
+#define REIM_IRQ0M_SHIFT	8
+#define REIM_IRQ0M		(_ULCAST_(0x1) << REIM_IRQ0M_SHIFT)
+#define REIM_MBOXIRQ1M_SHIFT	1
+#define REIM_MBOXIRQ1M		(_ULCAST_(0x1) << REIM_MBOXIRQ1M_SHIFT)
+#define REIM_MBOXIRQ0M_SHIFT	0
+#define REIM_MBOXIRQ0M		(_ULCAST_(0x1) << REIM_MBOXIRQ0M_SHIFT)
+
+extern void jz4780_smp_init(void);
+extern void jz4780_secondary_cpu_entry(void);
+
+#endif /* __MIPS_ASM_MACH_JZ4740_SMP_H__ */
diff --git a/arch/mips/jz4740/Kconfig b/arch/mips/jz4740/Kconfig
index 412d2fa..2b88557 100644
--- a/arch/mips/jz4740/Kconfig
+++ b/arch/mips/jz4740/Kconfig
@@ -34,9 +34,11 @@ config MACH_JZ4770
 
 config MACH_JZ4780
 	bool
+	select GENERIC_CLOCKEVENTS_BROADCAST if SMP
 	select MIPS_CPU_SCACHE
 	select SYS_HAS_CPU_MIPS32_R2
 	select SYS_SUPPORTS_HIGHMEM
+	select SYS_SUPPORTS_SMP
 
 config MACH_X1000
 	bool
diff --git a/arch/mips/jz4740/Makefile b/arch/mips/jz4740/Makefile
index 6de14c0..0a0f024 100644
--- a/arch/mips/jz4740/Makefile
+++ b/arch/mips/jz4740/Makefile
@@ -12,3 +12,8 @@ CFLAGS_setup.o = -I$(src)/../../../scripts/dtc/libfdt
 # PM support
 
 obj-$(CONFIG_PM) += pm.o
+
+# SMP support
+
+obj-$(CONFIG_SMP) += smp.o
+obj-$(CONFIG_SMP) += smp-entry.o
diff --git a/arch/mips/jz4740/prom.c b/arch/mips/jz4740/prom.c
index ff4555c..4acf5c2c 100644
--- a/arch/mips/jz4740/prom.c
+++ b/arch/mips/jz4740/prom.c
@@ -8,10 +8,14 @@
 
 #include <asm/bootinfo.h>
 #include <asm/fw/fw.h>
+#include <asm/mach-jz4740/smp.h>
 
 void __init prom_init(void)
 {
 	fw_init_cmdline();
+
+	if (IS_ENABLED(CONFIG_SMP))
+		jz4780_smp_init();
 }
 
 void __init prom_free_prom_memory(void)
diff --git a/arch/mips/jz4740/smp-entry.S b/arch/mips/jz4740/smp-entry.S
new file mode 100644
index 00000000..20049a3
--- /dev/null
+++ b/arch/mips/jz4740/smp-entry.S
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
+ *  JZ4780 SMP entry point
+ */
+
+#include <asm/addrspace.h>
+#include <asm/asm.h>
+#include <asm/asmmacro.h>
+#include <asm/cacheops.h>
+#include <asm/mipsregs.h>
+
+#define CACHE_SIZE (32 * 1024)
+#define CACHE_LINESIZE 32
+
+.extern jz4780_cpu_entry_sp
+.extern jz4780_cpu_entry_gp
+
+.section .text.smp-entry
+.balign 0x10000
+.set noreorder
+LEAF(jz4780_secondary_cpu_entry)
+	mtc0	zero, CP0_CAUSE
+
+	li	t0, ST0_CU0
+	mtc0	t0, CP0_STATUS
+
+	/* cache setup */
+	li	t0, KSEG0
+	ori	t1, t0, CACHE_SIZE
+	mtc0	zero, CP0_TAGLO, 0
+1:	cache	Index_Store_Tag_I, 0(t0)
+	cache	Index_Store_Tag_D, 0(t0)
+	bne	t0, t1, 1b
+	 addiu	t0, t0, CACHE_LINESIZE
+
+	/* kseg0 cache attribute */
+	mfc0	t0, CP0_CONFIG, 0
+	ori	t0, t0, CONF_CM_CACHABLE_NONCOHERENT
+	mtc0	t0, CP0_CONFIG, 0
+
+	/* pagemask */
+	mtc0	zero, CP0_PAGEMASK, 0
+
+	/* retrieve sp */
+	la	t0, jz4780_cpu_entry_sp
+	lw	sp, 0(t0)
+
+	/* retrieve gp */
+	la	t0, jz4780_cpu_entry_gp
+	lw	gp, 0(t0)
+
+	/* jump to the kernel in kseg0 */
+	la	t0, smp_bootstrap
+	jr	t0
+	 nop
+	END(jz4780_secondary_cpu_entry)
diff --git a/arch/mips/jz4740/smp.c b/arch/mips/jz4740/smp.c
new file mode 100644
index 00000000..d95d22a
--- /dev/null
+++ b/arch/mips/jz4740/smp.c
@@ -0,0 +1,258 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
+ *  JZ4780 SMP
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/of.h>
+#include <linux/sched.h>
+#include <linux/sched/task_stack.h>
+#include <linux/smp.h>
+#include <linux/tick.h>
+#include <asm/mach-jz4740/smp.h>
+#include <asm/smp-ops.h>
+
+static struct clk *cpu_clock_gates[CONFIG_NR_CPUS] = { NULL };
+
+u32 jz4780_cpu_entry_sp;
+u32 jz4780_cpu_entry_gp;
+
+static struct cpumask cpu_running;
+
+static DEFINE_SPINLOCK(smp_lock);
+
+static irqreturn_t mbox_handler(int irq, void *dev_id)
+{
+	int cpu = smp_processor_id();
+	u32 action, status;
+
+	spin_lock(&smp_lock);
+
+	switch (cpu) {
+	case 0:
+		action = read_c0_mailbox0();
+		write_c0_mailbox0(0);
+		break;
+	case 1:
+		action = read_c0_mailbox1();
+		write_c0_mailbox1(0);
+		break;
+	case 2:
+		action = read_c0_mailbox2();
+		write_c0_mailbox2(0);
+		break;
+	case 3:
+		action = read_c0_mailbox3();
+		write_c0_mailbox3(0);
+		break;
+	default:
+		panic("unhandled cpu %d!", cpu);
+	}
+
+	/* clear pending mailbox interrupt */
+	status = read_c0_corestatus();
+	status &= ~(CORESTATUS_MIRQ0P << cpu);
+	write_c0_corestatus(status);
+
+	spin_unlock(&smp_lock);
+
+	if (action & SMP_RESCHEDULE_YOURSELF)
+		scheduler_ipi();
+	if (action & SMP_CALL_FUNCTION)
+		generic_smp_call_function_interrupt();
+
+	return IRQ_HANDLED;
+}
+
+static void jz4780_smp_setup(void)
+{
+	u32 addr, reim;
+	int cpu;
+
+	reim = read_c0_reim();
+
+	for (cpu = 0; cpu < NR_CPUS; cpu++) {
+		__cpu_number_map[cpu] = cpu;
+		__cpu_logical_map[cpu] = cpu;
+		set_cpu_possible(cpu, true);
+	}
+
+	/* mask mailbox interrupts for this core */
+	reim &= ~REIM_MBOXIRQ0M;
+	write_c0_reim(reim);
+
+	/* clear mailboxes & pending mailbox IRQs */
+	write_c0_mailbox0(0);
+	write_c0_mailbox1(0);
+	write_c0_corestatus(0);
+
+	/* set reset entry point */
+	addr = KSEG1ADDR((u32)&jz4780_secondary_cpu_entry);
+	WARN_ON(addr & ~REIM_ENTRY);
+	reim &= ~REIM_ENTRY;
+	reim |= addr & REIM_ENTRY;
+
+	/* unmask mailbox interrupts for this core */
+	reim |= REIM_MBOXIRQ0M;
+	write_c0_reim(reim);
+	set_c0_status(STATUSF_IP3);
+	irq_enable_hazard();
+
+	cpumask_set_cpu(cpu, &cpu_running);
+}
+
+static void jz4780_smp_prepare_cpus(unsigned int max_cpus)
+{
+	struct device_node *cpu_node;
+	unsigned cpu, ctrl;
+	int err;
+
+	/* setup the mailbox IRQ */
+	err = request_irq(MIPS_CPU_IRQ_BASE + 3, mbox_handler,
+			IRQF_PERCPU | IRQF_NO_THREAD, "core mailbox", NULL);
+	if (err)
+		pr_err("request_irq() on core mailbox failed\n");
+
+	ctrl = read_c0_corectrl();
+
+	for_each_of_cpu_node(cpu_node) {
+		cpu = of_cpu_node_to_id(cpu_node);
+		if (cpu < 0) {
+			pr_err("Failed to read index of %s\n",
+			       cpu_node->full_name);
+			continue;
+		}
+
+		/* use reset entry point from REIM register */
+		ctrl |= CORECTRL_RPC0 << cpu;
+
+		cpu_clock_gates[cpu] = of_clk_get(cpu_node, 0);
+		if (IS_ERR(cpu_clock_gates[cpu])) {
+			cpu_clock_gates[cpu] = NULL;
+			continue;
+		}
+
+		err = clk_prepare(cpu_clock_gates[cpu]);
+		if (err)
+			pr_err("Failed to prepare CPU clock gate\n");
+	}
+
+	write_c0_corectrl(ctrl);
+}
+
+static int jz4780_boot_secondary(int cpu, struct task_struct *idle)
+{
+	unsigned long flags;
+	u32 ctrl;
+
+	spin_lock_irqsave(&smp_lock, flags);
+
+	/* ensure the core is in reset */
+	ctrl = read_c0_corectrl();
+	ctrl |= CORECTRL_SWRST0 << cpu;
+	write_c0_corectrl(ctrl);
+
+	/* ungate core clock */
+	if (cpu_clock_gates[cpu])
+		clk_enable(cpu_clock_gates[cpu]);
+
+	/* set entry sp/gp register values */
+	jz4780_cpu_entry_sp = __KSTK_TOS(idle);
+	jz4780_cpu_entry_gp = (u32)task_thread_info(idle);
+	smp_wmb();
+
+	/* take the core out of reset */
+	ctrl &= ~(CORECTRL_SWRST0 << cpu);
+	write_c0_corectrl(ctrl);
+
+	cpumask_set_cpu(cpu, &cpu_running);
+
+	spin_unlock_irqrestore(&smp_lock, flags);
+
+	return 0;
+}
+
+static void jz4780_init_secondary(void)
+{
+}
+
+static void jz4780_smp_finish(void)
+{
+	u32 reim;
+
+	spin_lock(&smp_lock);
+
+	/* unmask mailbox interrupts for this core */
+	reim = read_c0_reim();
+	reim |= REIM_MBOXIRQ0M << smp_processor_id();
+	write_c0_reim(reim);
+
+	spin_unlock(&smp_lock);
+
+	/* unmask interrupts for this core */
+	change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP2 |
+			 STATUSF_IP1 | STATUSF_IP0);
+	irq_enable_hazard();
+
+	/* force broadcast timer */
+	tick_broadcast_force();
+}
+
+static void jz4780_send_ipi_single_locked(int cpu, unsigned int action)
+{
+	u32 mbox;
+
+	switch (cpu) {
+	case 0:
+		mbox = read_c0_mailbox0();
+		write_c0_mailbox0(mbox | action);
+		break;
+	case 1:
+		mbox = read_c0_mailbox1();
+		write_c0_mailbox1(mbox | action);
+		break;
+	default:
+		panic("unhandled cpu %d!", cpu);
+	}
+}
+
+static void jz4780_send_ipi_single(int cpu, unsigned int action)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&smp_lock, flags);
+	jz4780_send_ipi_single_locked(cpu, action);
+	spin_unlock_irqrestore(&smp_lock, flags);
+}
+
+static void jz4780_send_ipi_mask(const struct cpumask *mask,
+				 unsigned int action)
+{
+	unsigned long flags;
+	int cpu;
+
+	spin_lock_irqsave(&smp_lock, flags);
+
+	for_each_cpu(cpu, mask)
+		jz4780_send_ipi_single_locked(cpu, action);
+
+	spin_unlock_irqrestore(&smp_lock, flags);
+}
+
+static struct plat_smp_ops jz4780_smp_ops = {
+	.send_ipi_single = jz4780_send_ipi_single,
+	.send_ipi_mask = jz4780_send_ipi_mask,
+	.init_secondary = jz4780_init_secondary,
+	.smp_finish = jz4780_smp_finish,
+	.boot_secondary = jz4780_boot_secondary,
+	.smp_setup = jz4780_smp_setup,
+	.prepare_cpus = jz4780_smp_prepare_cpus,
+};
+
+void __init jz4780_smp_init(void)
+{
+	register_smp_ops(&jz4780_smp_ops);
+}
diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c
index 37f8e78..d33f2d4 100644
--- a/arch/mips/kernel/idle.c
+++ b/arch/mips/kernel/idle.c
@@ -18,6 +18,7 @@
 #include <asm/cpu-type.h>
 #include <asm/idle.h>
 #include <asm/mipsregs.h>
+#include <asm/r4kcache.h>
 
 /*
  * Not all of the MIPS CPUs have the "wait" instruction available. Moreover,
@@ -88,6 +89,34 @@ static void __cpuidle rm7k_wait_irqoff(void)
 }
 
 /*
+ * The Ingenic jz4780 SMP variant has to write back dirty cache lines before
+ * executing wait. The CPU & cache clock will be gated until we return from
+ * the wait, and if another core attempts to access data from our data cache
+ * during this time then it will lock up.
+ */
+void jz4780_smp_wait_irqoff(void)
+{
+	unsigned long pending = read_c0_cause() & read_c0_status() & CAUSEF_IP;
+
+	/*
+	 * Going to idle has a significant overhead due to the cache flush so
+	 * try to avoid it if we'll immediately be woken again due to an IRQ.
+	 */
+	if (!need_resched() && !pending) {
+		r4k_blast_dcache();
+
+		__asm__(
+		"	.set push	\n"
+		"	.set mips3	\n"
+		"	sync		\n"
+		"	wait		\n"
+		"	.set pop	\n");
+	}
+
+	local_irq_enable();
+}
+
+/*
  * Au1 'wait' is only useful when the 32kHz counter is used as timer,
  * since coreclock (and the cp0 counter) stops upon executing it. Only an
  * interrupt can wake it, so they must be enabled before entering idle modes.
@@ -172,7 +201,6 @@ void __init check_wait(void)
 	case CPU_CAVIUM_OCTEON_PLUS:
 	case CPU_CAVIUM_OCTEON2:
 	case CPU_CAVIUM_OCTEON3:
-	case CPU_XBURST:
 	case CPU_LOONGSON32:
 	case CPU_XLR:
 	case CPU_XLP:
@@ -246,6 +274,11 @@ void __init check_wait(void)
 		   cpu_wait = r4k_wait;
 		 */
 		break;
+	case CPU_XBURST:
+		if (IS_ENABLED(CONFIG_SMP))
+			cpu_wait = jz4780_smp_wait_irqoff;
+		else
+			cpu_wait = r4k_wait;
 	default:
 		break;
 	}
-- 
2.7.4

[PATCH v8 2/6] MIPS: CI20: Modify DTS to support high resolution timer for SMP.

[周琰杰 (Zhou Yanjie)]

Modify DTS, change tcu channel from 2 to 3, channel #0 and #1 for
per core local timer, #2 for clocksource.

Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
Tested-by: Paul Boddie <paul@boddie.org.uk>
Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
---

Notes:
    v1->v2:
    No change.
    
    v2->v3:
    No change.
    
    v3->v4:
    Rebase on top of kernel 5.6-rc1.
    
    v4->v5:
    Move [5/6] in v4 to this patch, to ensure that we can
    git-bisect without ending up with a broken kernel.
    
    v5->v6:
    No change.
    
    v6->v7:
    Remove unnecessary "ingenic,pwm-channels-mask".
    
    v7->v8:
    No change.

 arch/mips/boot/dts/ingenic/ci20.dts | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/arch/mips/boot/dts/ingenic/ci20.dts b/arch/mips/boot/dts/ingenic/ci20.dts
index db0ca25..06e3186 100644
--- a/arch/mips/boot/dts/ingenic/ci20.dts
+++ b/arch/mips/boot/dts/ingenic/ci20.dts
@@ -486,7 +486,12 @@
 };
 
 &tcu {
-	/* 3 MHz for the system timer and clocksource */
-	assigned-clocks = <&tcu TCU_CLK_TIMER0>, <&tcu TCU_CLK_TIMER1>;
-	assigned-clock-rates = <3000000>, <3000000>;
+	/*
+	 * 750 kHz for the system timers and clocksource,
+	 * use channel #0 and #1 for the per cpu system timers,
+	 * and use channel #2 for the clocksource.
+	 */
+	assigned-clocks = <&tcu TCU_CLK_TIMER0>, <&tcu TCU_CLK_TIMER1>,
+					  <&tcu TCU_CLK_TIMER2>;
+	assigned-clock-rates = <750000>, <750000>, <750000>;
 };
-- 
2.7.4

[PATCH v8 3/6] clocksource: Ingenic: Add high resolution timer support for SMP.

[周琰杰 (Zhou Yanjie)]

Enable clock event handling on per CPU core basis.
Make sure that interrupts raised on the first core execute
event handlers on the correct CPU core.

Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
Tested-by: Paul Boddie <paul@boddie.org.uk>
Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
---

Notes:
    v1->v2:
    1.Adjust function naming to make it more reasonable.
    2.Replace function smp_call_function_single() with
      smp_call_function_single_async() in order to resolve
      the warning below:
    
    [    0.350942] smp: Brought up 1 node, 2 CPUs
    [    0.365497] ------------[ cut here ]------------
    [    0.365522] WARNING: CPU: 0 PID: 1 at kernel/smp.c:300 smp_call_function_single+0x110/0x200
    [    0.365533] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.5.0-rc1+ #5
    [    0.365537] Stack : 00000000 59c73bcd 00000037 80074e80 80000000 80670000 805a0000 80620590
    [    0.365557]         8065ce38 8fc0dc8c 806d0000 00000000 80670000 00000001 8fc0dc20 59c73bcd
    [    0.365574]         00000000 00000000 806f0000 80670000 00000000 806dab00 00000000 2d302e35
    [    0.365591]         203a6d6d 806e0000 806e0000 70617773 80670000 00000000 00000000 00000009
    [    0.365610]         00000000 8fc94e20 8fc0de30 80690000 00000018 803592dc 00000000 806d0000
    [    0.365627]         ...
    [    0.365634] Call Trace:
    [    0.365647] [<8001b9a0>] show_stack+0x6c/0x12c
    [    0.365663] [<804aed20>] dump_stack+0x98/0xc8
    [    0.365673] [<8003044c>] __warn+0xc4/0xe8
    [    0.365682] [<800304f4>] warn_slowpath_fmt+0x84/0xb8
    [    0.365690] [<800a886c>] smp_call_function_single+0x110/0x200
    [    0.365703] ---[ end trace 5785856ca39c79d5 ]---
    [    0.365557]         8065ce38 8fc0dc8c 806d0000 00000000 80670000 00000001 8fc0dc20 59c73bcd
    [    0.365574]         00000000 00000000 806f0000 80670000 00000000 806dab00 00000000 2d302e35
    [    0.365591]         203a6d6d 806e0000 806e0000 70617773 80670000 00000000 00000000 00000009
    [    0.365610]         00000000 8fc94e20 8fc0de30 80690000 00000018 803592dc 00000000 806d0000
    [    0.365627]         ...
    [    0.365634] Call Trace:
    [    0.365647] [<8001b9a0>] show_stack+0x6c/0x12c
    [    0.365663] [<804aed20>] dump_stack+0x98/0xc8
    [    0.365673] [<8003044c>] __warn+0xc4/0xe8
    [    0.365682] [<800304f4>] warn_slowpath_fmt+0x84/0xb8
    [    0.365690] [<800a886c>] smp_call_function_single+0x110/0x200
    [    0.365703] ---[ end trace 5785856ca39c79d5 ]---
    
    v2->v3:
    No Change.
    
    v3->v4:
    Rebase on top of kernel 5.6-rc1.
    
    v4->v5:
    Move the check for (evt->event_handler) from "ingenic_per_cpu_event_handler"
    to "ingenic_tcu_cevt_cb".
    
    v5->v6:
    No change.
    
    v6->v7:
    Remove unnecessary check for "NR_CPUS > 1".
    
    v7->v8:
    Use "num_possible_cpus()" instead "NR_CPUS".
    Reported-by: kbuild test robot <lkp@intel.com>

 drivers/clocksource/ingenic-timer.c | 103 ++++++++++++++++++++++++++++--------
 1 file changed, 82 insertions(+), 21 deletions(-)

diff --git a/drivers/clocksource/ingenic-timer.c b/drivers/clocksource/ingenic-timer.c
index 4963336..230e996 100644
--- a/drivers/clocksource/ingenic-timer.c
+++ b/drivers/clocksource/ingenic-timer.c
@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * JZ47xx SoCs TCU IRQ driver
+ * XBurst SoCs TCU IRQ driver
  * Copyright (C) 2019 Paul Cercueil <paul@crapouillou.net>
+ * Copyright (C) 2020 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
  */
 
 #include <linux/bitops.h>
@@ -21,18 +22,23 @@
 
 #include <dt-bindings/clock/ingenic,tcu.h>
 
+static DEFINE_PER_CPU(call_single_data_t, ingenic_cevt_csd);
+
 struct ingenic_soc_info {
 	unsigned int num_channels;
 };
 
 struct ingenic_tcu {
 	struct regmap *map;
+	struct device_node *np;
 	struct clk *timer_clk, *cs_clk;
+	unsigned int timer_local[NR_CPUS];
 	unsigned int timer_channel, cs_channel;
 	struct clock_event_device cevt;
 	struct clocksource cs;
-	char name[4];
+	char name[8];
 	unsigned long pwm_channels_mask;
+	int cpu;
 };
 
 static struct ingenic_tcu *ingenic_tcu;
@@ -81,6 +87,24 @@ static int ingenic_tcu_cevt_set_next(unsigned long next,
 	return 0;
 }
 
+static void ingenic_per_cpu_event_handler(void *info)
+{
+	struct clock_event_device *cevt = (struct clock_event_device *) info;
+
+	cevt->event_handler(cevt);
+}
+
+static void ingenic_tcu_per_cpu_cb(struct clock_event_device *evt)
+{
+	struct ingenic_tcu *tcu = to_ingenic_tcu(evt);
+	call_single_data_t *csd;
+
+	csd = &per_cpu(ingenic_cevt_csd, tcu->cpu);
+	csd->info = (void *) evt;
+	csd->func = ingenic_per_cpu_event_handler;
+	smp_call_function_single_async(tcu->cpu, csd);
+}
+
 static irqreturn_t ingenic_tcu_cevt_cb(int irq, void *dev_id)
 {
 	struct clock_event_device *evt = dev_id;
@@ -89,7 +113,7 @@ static irqreturn_t ingenic_tcu_cevt_cb(int irq, void *dev_id)
 	regmap_write(tcu->map, TCU_REG_TECR, BIT(tcu->timer_channel));
 
 	if (evt->event_handler)
-		evt->event_handler(evt);
+		ingenic_tcu_per_cpu_cb(evt);
 
 	return IRQ_HANDLED;
 }
@@ -105,14 +129,21 @@ static struct clk * __init ingenic_tcu_get_clock(struct device_node *np, int id)
 	return of_clk_get_from_provider(&args);
 }
 
-static int __init ingenic_tcu_timer_init(struct device_node *np,
-					 struct ingenic_tcu *tcu)
+static int ingenic_tcu_setup_per_cpu_cevt(struct device_node *np,
+				     unsigned int channel)
 {
-	unsigned int timer_virq, channel = tcu->timer_channel;
+	unsigned int timer_virq;
 	struct irq_domain *domain;
+	struct ingenic_tcu *tcu;
 	unsigned long rate;
 	int err;
 
+	tcu = kzalloc(sizeof(*tcu), GFP_KERNEL);
+	if (!tcu)
+		return -ENOMEM;
+
+	tcu->map = ingenic_tcu->map;
+
 	tcu->timer_clk = ingenic_tcu_get_clock(np, channel);
 	if (IS_ERR(tcu->timer_clk))
 		return PTR_ERR(tcu->timer_clk);
@@ -139,13 +170,15 @@ static int __init ingenic_tcu_timer_init(struct device_node *np,
 		goto err_clk_disable;
 	}
 
-	snprintf(tcu->name, sizeof(tcu->name), "TCU");
+	snprintf(tcu->name, sizeof(tcu->name), "TCU%u", channel);
 
 	err = request_irq(timer_virq, ingenic_tcu_cevt_cb, IRQF_TIMER,
 			  tcu->name, &tcu->cevt);
 	if (err)
 		goto err_irq_dispose_mapping;
 
+	tcu->cpu = smp_processor_id();
+	tcu->timer_channel = channel;
 	tcu->cevt.cpumask = cpumask_of(smp_processor_id());
 	tcu->cevt.features = CLOCK_EVT_FEAT_ONESHOT;
 	tcu->cevt.name = tcu->name;
@@ -166,6 +199,25 @@ static int __init ingenic_tcu_timer_init(struct device_node *np,
 	return err;
 }
 
+static int ingenic_tcu_setup_cevt(unsigned int cpu)
+{
+	int ret;
+
+	ret = ingenic_tcu_setup_per_cpu_cevt(ingenic_tcu->np,
+						ingenic_tcu->timer_local[cpu]);
+	if (ret)
+		goto err_tcu_clocksource_cleanup;
+
+	return 0;
+
+err_tcu_clocksource_cleanup:
+	clocksource_unregister(&ingenic_tcu->cs);
+	clk_disable_unprepare(ingenic_tcu->cs_clk);
+	clk_put(ingenic_tcu->cs_clk);
+	kfree(ingenic_tcu);
+	return ret;
+}
+
 static int __init ingenic_tcu_clocksource_init(struct device_node *np,
 					       struct ingenic_tcu *tcu)
 {
@@ -240,6 +292,7 @@ static int __init ingenic_tcu_init(struct device_node *np)
 	const struct ingenic_soc_info *soc_info = id->data;
 	struct ingenic_tcu *tcu;
 	struct regmap *map;
+	unsigned cpu = 0;
 	long rate;
 	int ret;
 
@@ -253,13 +306,18 @@ static int __init ingenic_tcu_init(struct device_node *np)
 	if (!tcu)
 		return -ENOMEM;
 
-	/* Enable all TCU channels for PWM use by default except channels 0/1 */
-	tcu->pwm_channels_mask = GENMASK(soc_info->num_channels - 1, 2);
+	/*
+	 * Enable all TCU channels for PWM use by default except channels 0/1,
+	 * and channel 2 if target CPU is JZ4780 and SMP is selected.
+	 */
+	tcu->pwm_channels_mask = GENMASK(soc_info->num_channels - 1,
+								num_possible_cpus() + 1);
 	of_property_read_u32(np, "ingenic,pwm-channels-mask",
 			     (u32 *)&tcu->pwm_channels_mask);
 
-	/* Verify that we have at least two free channels */
-	if (hweight8(tcu->pwm_channels_mask) > soc_info->num_channels - 2) {
+	/* Verify that we have at least num_possible_cpus() + 1 free channels */
+	if (hweight8(tcu->pwm_channels_mask) >
+			soc_info->num_channels - num_possible_cpus() + 1) {
 		pr_crit("%s: Invalid PWM channel mask: 0x%02lx\n", __func__,
 			tcu->pwm_channels_mask);
 		ret = -EINVAL;
@@ -267,13 +325,19 @@ static int __init ingenic_tcu_init(struct device_node *np)
 	}
 
 	tcu->map = map;
+	tcu->np = np;
 	ingenic_tcu = tcu;
 
-	tcu->timer_channel = find_first_zero_bit(&tcu->pwm_channels_mask,
+	tcu->timer_local[cpu] = find_first_zero_bit(&tcu->pwm_channels_mask,
 						 soc_info->num_channels);
+
+	for (cpu = 1; cpu < num_possible_cpus(); cpu++)
+		tcu->timer_local[cpu] = find_next_zero_bit(
+				&tcu->pwm_channels_mask, soc_info->num_channels,
+				tcu->timer_local[cpu - 1] + 1);
+
 	tcu->cs_channel = find_next_zero_bit(&tcu->pwm_channels_mask,
-					     soc_info->num_channels,
-					     tcu->timer_channel + 1);
+			soc_info->num_channels, tcu->timer_local[cpu - 1] + 1);
 
 	ret = ingenic_tcu_clocksource_init(np, tcu);
 	if (ret) {
@@ -281,9 +345,10 @@ static int __init ingenic_tcu_init(struct device_node *np)
 		goto err_free_ingenic_tcu;
 	}
 
-	ret = ingenic_tcu_timer_init(np, tcu);
-	if (ret)
-		goto err_tcu_clocksource_cleanup;
+	/* Setup clock events on each CPU core */
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "Ingenic XBurst: online",
+				ingenic_tcu_setup_cevt, NULL);
+	WARN_ON(ret < 0);
 
 	/* Register the sched_clock at the end as there's no way to undo it */
 	rate = clk_get_rate(tcu->cs_clk);
@@ -291,10 +356,6 @@ static int __init ingenic_tcu_init(struct device_node *np)
 
 	return 0;
 
-err_tcu_clocksource_cleanup:
-	clocksource_unregister(&tcu->cs);
-	clk_disable_unprepare(tcu->cs_clk);
-	clk_put(tcu->cs_clk);
 err_free_ingenic_tcu:
 	kfree(tcu);
 	return ret;
-- 
2.7.4

[PATCH v8 4/6] dt-bindings: MIPS: Document Ingenic SoCs binding.

[周琰杰 (Zhou Yanjie)]

Document the available properties for the SoC root node and the
CPU nodes of the devicetree for the Ingenic XBurst SoCs.

Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
Tested-by: Paul Boddie <paul@boddie.org.uk>
Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
---

Notes:
    v1->v2:
    Change the two Document from txt to yaml.
    
    v2->v3:
    Fix formatting errors.
    
    v3->v4:
    Fix bugs in the two yaml files.
    
    v4->v5:
    No change.
    
    v5->v6:
    Rewrite the two yaml files.
    
    v6->v7:
    1.Update compatible strings in "ingenic,cpu.yaml".
    2.Fix formatting errors, and enum for compatible strings.
    3.Remove unnecessary "ingenic,soc.yaml".
    
    v7->v8:
    No change.

 .../bindings/mips/ingenic/ingenic,cpu.yaml         | 57 ++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml

diff --git a/Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml b/Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml
new file mode 100644
index 00000000..afb0207
--- /dev/null
+++ b/Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mips/ingenic/ingenic,cpu.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Bindings for Ingenic XBurst family CPUs
+
+maintainers:
+  - 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
+
+description:
+  Ingenic XBurst family CPUs shall have the following properties.
+
+properties:
+  compatible:
+    oneOf:
+
+      - description: Ingenic XBurst®1 CPU Cores
+        items:
+          enum:
+            - ingenic,xburst-mxu1.0
+            - ingenic,xburst-fpu1.0-mxu1.1
+            - ingenic,xburst-fpu2.0-mxu2.0
+
+      - description: Ingenic XBurst®2 CPU Cores
+        items:
+          enum:
+            - ingenic,xburst2-fpu2.1-mxu2.1-smt
+
+  reg:
+    maxItems: 1
+
+required:
+  - device_type
+  - compatible
+  - reg
+
+examples:
+  - |
+    cpus {
+    	#address-cells = <1>;
+    	#size-cells = <0>;
+
+    	cpu0: cpu@0 {
+    		device_type = "cpu";
+    		compatible = "ingenic,xburst-fpu1.0-mxu1.1";
+    		reg = <0>;
+    	};
+
+    	cpu1: cpu@1 {
+    		device_type = "cpu";
+    		compatible = "ingenic,xburst-fpu1.0-mxu1.1";
+    		reg = <1>;
+    	};
+    };
+...
-- 
2.7.4

[PATCH v8 5/6] MIPS: Ingenic: Add 'cpus' node for Ingenic SoCs.

[周琰杰 (Zhou Yanjie)]

Add 'cpus' node to the jz4740.dtsi, jz4770.dtsi, jz4780.dtsi
and x1000.dtsi files.

Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
Tested-by: Paul Boddie <paul@boddie.org.uk>
Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
---

Notes:
    v1->v2:
    No change.
    
    v2->v3:
    No change.
    
    v3->v4:
    Rebase on top of kernel 5.6-rc1.
    
    v4->v5:
    No change.
    
    v5->v6:
    No change.
    
    v6->v7:
    Update compatible strings.
    
    v7->v8:
    No change.

 arch/mips/boot/dts/ingenic/jz4740.dtsi | 14 ++++++++++++++
 arch/mips/boot/dts/ingenic/jz4770.dtsi | 15 ++++++++++++++-
 arch/mips/boot/dts/ingenic/jz4780.dtsi | 23 +++++++++++++++++++++++
 arch/mips/boot/dts/ingenic/x1000.dtsi  | 14 ++++++++++++++
 4 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/arch/mips/boot/dts/ingenic/jz4740.dtsi b/arch/mips/boot/dts/ingenic/jz4740.dtsi
index a3301ba..1f2f896 100644
--- a/arch/mips/boot/dts/ingenic/jz4740.dtsi
+++ b/arch/mips/boot/dts/ingenic/jz4740.dtsi
@@ -7,6 +7,20 @@
 	#size-cells = <1>;
 	compatible = "ingenic,jz4740";
 
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "ingenic,xburst-mxu1.0";
+			reg = <0>;
+
+			clocks = <&cgu JZ4740_CLK_CCLK>;
+			clock-names = "cpu";
+		};
+	};
+
 	cpuintc: interrupt-controller {
 		#address-cells = <0>;
 		#interrupt-cells = <1>;
diff --git a/arch/mips/boot/dts/ingenic/jz4770.dtsi b/arch/mips/boot/dts/ingenic/jz4770.dtsi
index 0bfb9ed..12c7101 100644
--- a/arch/mips/boot/dts/ingenic/jz4770.dtsi
+++ b/arch/mips/boot/dts/ingenic/jz4770.dtsi
@@ -1,5 +1,4 @@
 // SPDX-License-Identifier: GPL-2.0
-
 #include <dt-bindings/clock/jz4770-cgu.h>
 
 / {
@@ -7,6 +6,20 @@
 	#size-cells = <1>;
 	compatible = "ingenic,jz4770";
 
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
+			reg = <0>;
+
+			clocks = <&cgu JZ4770_CLK_CCLK>;
+			clock-names = "cpu";
+		};
+	};
+
 	cpuintc: interrupt-controller {
 		#address-cells = <0>;
 		#interrupt-cells = <1>;
diff --git a/arch/mips/boot/dts/ingenic/jz4780.dtsi b/arch/mips/boot/dts/ingenic/jz4780.dtsi
index bb89653..03aeeff 100644
--- a/arch/mips/boot/dts/ingenic/jz4780.dtsi
+++ b/arch/mips/boot/dts/ingenic/jz4780.dtsi
@@ -8,6 +8,29 @@
 	#size-cells = <1>;
 	compatible = "ingenic,jz4780";
 
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
+			reg = <0>;
+
+			clocks = <&cgu JZ4780_CLK_CPU>;
+			clock-names = "cpu";
+		};
+
+		cpu1: cpu@1 {
+			device_type = "cpu";
+			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
+			reg = <1>;
+
+			clocks = <&cgu JZ4780_CLK_CORE1>;
+			clock-names = "cpu";
+		};
+	};
+
 	cpuintc: interrupt-controller {
 		#address-cells = <0>;
 		#interrupt-cells = <1>;
diff --git a/arch/mips/boot/dts/ingenic/x1000.dtsi b/arch/mips/boot/dts/ingenic/x1000.dtsi
index 147f7d5..2205e1b 100644
--- a/arch/mips/boot/dts/ingenic/x1000.dtsi
+++ b/arch/mips/boot/dts/ingenic/x1000.dtsi
@@ -8,6 +8,20 @@
 	#size-cells = <1>;
 	compatible = "ingenic,x1000", "ingenic,x1000e";
 
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
+			reg = <0>;
+
+			clocks = <&cgu X1000_CLK_CPU>;
+			clock-names = "cpu";
+		};
+	};
+
 	cpuintc: interrupt-controller {
 		#address-cells = <0>;
 		#interrupt-cells = <1>;
-- 
2.7.4

[PATCH v8 6/6] MIPS: CI20: Update defconfig to support SMP.

[周琰杰 (Zhou Yanjie)]

Add "CONFIG_SMP=y" and "CONFIG_NR_CPUS=2" to support SMP.

Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
Tested-by: Paul Boddie <paul@boddie.org.uk>
Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
---

Notes:
    v1->v2:
    No change.
    
    v2->v3:
    No change.
    
    v3->v4:
    Rebase on top of kernel 5.6-rc1.
    
    v4->v5:
    No change.
    
    v5->v6:
    No change.
    
    v6->v7:
    No change.
    
    v7->v8:
    No change.

 arch/mips/configs/ci20_defconfig | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/arch/mips/configs/ci20_defconfig b/arch/mips/configs/ci20_defconfig
index 0db0088..c8dd136 100644
--- a/arch/mips/configs/ci20_defconfig
+++ b/arch/mips/configs/ci20_defconfig
@@ -1,3 +1,5 @@
+CONFIG_SMP=y
+CONFIG_NR_CPUS=2
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_MODULES=y
 CONFIG_KERNEL_XZ=y
-- 
2.7.4

Re: [PATCH v8 1/6] MIPS: JZ4780: Introduce SMP support.

[Paul Cercueil]

Hi Zhou,

Le mar. 19 mai 2020 à 22:35, 周琰杰 (Zhou Yanjie) 
<zhouyanjie@wanyeetech.com> a écrit :
> Forward port smp support from kernel 3.18.3 of CI20_linux
> to upstream kernel 5.6.
> 
> Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
> Tested-by: Paul Boddie <paul@boddie.org.uk>
> Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
> Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
> ---
> 
> Notes:
>     v1->v2:
>     1.Remove unnecessary "plat_irq_dispatch(void)" in irq-ingenic.c.
>     2.Add a timeout check for "jz4780_boot_secondary()" to avoid a 
> dead loop.
>     3.Replace hard code in smp.c with macro.
> 
>     v2->v3:
>     1.Remove unnecessary "extern void (*r4k_blast_dcache)(void)" in 
> smp.c.
>     2.Use "for_each_of_cpu_node" instead "for_each_compatible_node" 
> in smp.c.
>     3.Use "of_cpu_node_to_id" instead "of_property_read_u32_index" in 
> smp.c.
>     4.Move LCR related operations to jz4780-cgu.c.
> 
>     v3->v4:
>     Rebase on top of kernel 5.6-rc1.
> 
>     v4->v5:
>     1.Splitting changes involving "jz4780-cgu.c" into separate commit.
>     2.Use "request_irq()" replace "setup_irq()".
> 
>     v5->v6:
>     In order to have a kernel that works on multiple SoCs at the same
>     time, use "IS_ENABLED()" replace "#ifdef".
> 
>     v6->v7:
>     1.SMP has be decoupled from the SoC version.
>     2.Add mailboxes 3 and 4 for XBurst.
>     3.Adjust code in "jz4780_smp_prepare_cpus()".
>     4."jz4780_smp_init()" has be marked "__init".
> 
>     v7->v8:
>     No change.
> 
>  arch/mips/include/asm/mach-jz4740/smp.h |  87 +++++++++++
>  arch/mips/jz4740/Kconfig                |   2 +
>  arch/mips/jz4740/Makefile               |   5 +
>  arch/mips/jz4740/prom.c                 |   4 +
>  arch/mips/jz4740/smp-entry.S            |  57 +++++++
>  arch/mips/jz4740/smp.c                  | 258 
> ++++++++++++++++++++++++++++++++
>  arch/mips/kernel/idle.c                 |  35 ++++-
>  7 files changed, 447 insertions(+), 1 deletion(-)
>  create mode 100644 arch/mips/include/asm/mach-jz4740/smp.h
>  create mode 100644 arch/mips/jz4740/smp-entry.S
>  create mode 100644 arch/mips/jz4740/smp.c
> 
> diff --git a/arch/mips/include/asm/mach-jz4740/smp.h 
> b/arch/mips/include/asm/mach-jz4740/smp.h
> new file mode 100644
> index 00000000..86f660f
> --- /dev/null
> +++ b/arch/mips/include/asm/mach-jz4740/smp.h
> @@ -0,0 +1,87 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
> + *  JZ4780 SMP definitions
> + */
> +
> +#ifndef __MIPS_ASM_MACH_JZ4740_SMP_H__
> +#define __MIPS_ASM_MACH_JZ4740_SMP_H__
> +
> +#define read_c0_corectrl()		__read_32bit_c0_register($12, 2)
> +#define write_c0_corectrl(val)		__write_32bit_c0_register($12, 2, 
> val)
> +
> +#define read_c0_corestatus()		__read_32bit_c0_register($12, 3)
> +#define write_c0_corestatus(val)	__write_32bit_c0_register($12, 3, 
> val)
> +
> +#define read_c0_reim()			__read_32bit_c0_register($12, 4)
> +#define write_c0_reim(val)		__write_32bit_c0_register($12, 4, val)
> +
> +#define read_c0_mailbox0()		__read_32bit_c0_register($20, 0)
> +#define write_c0_mailbox0(val)		__write_32bit_c0_register($20, 0, 
> val)
> +
> +#define read_c0_mailbox1()		__read_32bit_c0_register($20, 1)
> +#define write_c0_mailbox1(val)		__write_32bit_c0_register($20, 1, 
> val)
> +
> +#define read_c0_mailbox2()		__read_32bit_c0_register($20, 2)
> +#define write_c0_mailbox2(val)		__write_32bit_c0_register($20, 2, 
> val)
> +
> +#define read_c0_mailbox3()		__read_32bit_c0_register($20, 3)
> +#define write_c0_mailbox3(val)		__write_32bit_c0_register($20, 3, 
> val)
> +
> +#define smp_clr_pending(mask) do {		\
> +		unsigned int stat;		\
> +		stat = read_c0_corestatus();	\
> +		stat &= ~((mask) & 0xff);	\
> +		write_c0_corestatus(stat);	\
> +	} while (0)
> +
> +/*
> + * Core Control register
> + */
> +#define CORECTRL_SLEEP1M_SHIFT	17
> +#define CORECTRL_SLEEP1M	(_ULCAST_(0x1) << CORECTRL_SLEEP1M_SHIFT)
> +#define CORECTRL_SLEEP0M_SHIFT	16
> +#define CORECTRL_SLEEP0M	(_ULCAST_(0x1) << CORECTRL_SLEEP0M_SHIFT)
> +#define CORECTRL_RPC1_SHIFT	9
> +#define CORECTRL_RPC1		(_ULCAST_(0x1) << CORECTRL_RPC1_SHIFT)
> +#define CORECTRL_RPC0_SHIFT	8
> +#define CORECTRL_RPC0		(_ULCAST_(0x1) << CORECTRL_RPC0_SHIFT)
> +#define CORECTRL_SWRST1_SHIFT	1
> +#define CORECTRL_SWRST1		(_ULCAST_(0x1) << CORECTRL_SWRST1_SHIFT)
> +#define CORECTRL_SWRST0_SHIFT	0
> +#define CORECTRL_SWRST0		(_ULCAST_(0x1) << CORECTRL_SWRST0_SHIFT)
> +
> +/*
> + * Core Status register
> + */
> +#define CORESTATUS_SLEEP1_SHIFT	17
> +#define CORESTATUS_SLEEP1	(_ULCAST_(0x1) << CORESTATUS_SLEEP1_SHIFT)
> +#define CORESTATUS_SLEEP0_SHIFT	16
> +#define CORESTATUS_SLEEP0	(_ULCAST_(0x1) << CORESTATUS_SLEEP0_SHIFT)
> +#define CORESTATUS_IRQ1P_SHIFT	9
> +#define CORESTATUS_IRQ1P	(_ULCAST_(0x1) << CORESTATUS_IRQ1P_SHIFT)
> +#define CORESTATUS_IRQ0P_SHIFT	8
> +#define CORESTATUS_IRQ0P	(_ULCAST_(0x1) << CORESTATUS_IRQ8P_SHIFT)
> +#define CORESTATUS_MIRQ1P_SHIFT	1
> +#define CORESTATUS_MIRQ1P	(_ULCAST_(0x1) << CORESTATUS_MIRQ1P_SHIFT)
> +#define CORESTATUS_MIRQ0P_SHIFT	0
> +#define CORESTATUS_MIRQ0P	(_ULCAST_(0x1) << CORESTATUS_MIRQ0P_SHIFT)
> +
> +/*
> + * Reset Entry & IRQ Mask register
> + */
> +#define REIM_ENTRY_SHIFT	16
> +#define REIM_ENTRY		(_ULCAST_(0xffff) << REIM_ENTRY_SHIFT)
> +#define REIM_IRQ1M_SHIFT	9
> +#define REIM_IRQ1M		(_ULCAST_(0x1) << REIM_IRQ1M_SHIFT)
> +#define REIM_IRQ0M_SHIFT	8
> +#define REIM_IRQ0M		(_ULCAST_(0x1) << REIM_IRQ0M_SHIFT)
> +#define REIM_MBOXIRQ1M_SHIFT	1
> +#define REIM_MBOXIRQ1M		(_ULCAST_(0x1) << REIM_MBOXIRQ1M_SHIFT)
> +#define REIM_MBOXIRQ0M_SHIFT	0
> +#define REIM_MBOXIRQ0M		(_ULCAST_(0x1) << REIM_MBOXIRQ0M_SHIFT)
> +
> +extern void jz4780_smp_init(void);
> +extern void jz4780_secondary_cpu_entry(void);
> +
> +#endif /* __MIPS_ASM_MACH_JZ4740_SMP_H__ */
> diff --git a/arch/mips/jz4740/Kconfig b/arch/mips/jz4740/Kconfig
> index 412d2fa..2b88557 100644
> --- a/arch/mips/jz4740/Kconfig
> +++ b/arch/mips/jz4740/Kconfig
> @@ -34,9 +34,11 @@ config MACH_JZ4770
> 
>  config MACH_JZ4780
>  	bool
> +	select GENERIC_CLOCKEVENTS_BROADCAST if SMP
>  	select MIPS_CPU_SCACHE
>  	select SYS_HAS_CPU_MIPS32_R2
>  	select SYS_SUPPORTS_HIGHMEM
> +	select SYS_SUPPORTS_SMP
> 
>  config MACH_X1000
>  	bool
> diff --git a/arch/mips/jz4740/Makefile b/arch/mips/jz4740/Makefile
> index 6de14c0..0a0f024 100644
> --- a/arch/mips/jz4740/Makefile
> +++ b/arch/mips/jz4740/Makefile
> @@ -12,3 +12,8 @@ CFLAGS_setup.o = 
> -I$(src)/../../../scripts/dtc/libfdt
>  # PM support
> 
>  obj-$(CONFIG_PM) += pm.o
> +
> +# SMP support
> +
> +obj-$(CONFIG_SMP) += smp.o
> +obj-$(CONFIG_SMP) += smp-entry.o
> diff --git a/arch/mips/jz4740/prom.c b/arch/mips/jz4740/prom.c
> index ff4555c..4acf5c2c 100644
> --- a/arch/mips/jz4740/prom.c
> +++ b/arch/mips/jz4740/prom.c

That file is gone in mips-next. You should rebase your patchset on top 
of mips-next.

Cheers,
-Paul

> @@ -8,10 +8,14 @@
> 
>  #include <asm/bootinfo.h>
>  #include <asm/fw/fw.h>
> +#include <asm/mach-jz4740/smp.h>
> 
>  void __init prom_init(void)
>  {
>  	fw_init_cmdline();
> +
> +	if (IS_ENABLED(CONFIG_SMP))
> +		jz4780_smp_init();
>  }
> 
>  void __init prom_free_prom_memory(void)
> diff --git a/arch/mips/jz4740/smp-entry.S 
> b/arch/mips/jz4740/smp-entry.S
> new file mode 100644
> index 00000000..20049a3
> --- /dev/null
> +++ b/arch/mips/jz4740/smp-entry.S
> @@ -0,0 +1,57 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
> + *  JZ4780 SMP entry point
> + */
> +
> +#include <asm/addrspace.h>
> +#include <asm/asm.h>
> +#include <asm/asmmacro.h>
> +#include <asm/cacheops.h>
> +#include <asm/mipsregs.h>
> +
> +#define CACHE_SIZE (32 * 1024)
> +#define CACHE_LINESIZE 32
> +
> +.extern jz4780_cpu_entry_sp
> +.extern jz4780_cpu_entry_gp
> +
> +.section .text.smp-entry
> +.balign 0x10000
> +.set noreorder
> +LEAF(jz4780_secondary_cpu_entry)
> +	mtc0	zero, CP0_CAUSE
> +
> +	li	t0, ST0_CU0
> +	mtc0	t0, CP0_STATUS
> +
> +	/* cache setup */
> +	li	t0, KSEG0
> +	ori	t1, t0, CACHE_SIZE
> +	mtc0	zero, CP0_TAGLO, 0
> +1:	cache	Index_Store_Tag_I, 0(t0)
> +	cache	Index_Store_Tag_D, 0(t0)
> +	bne	t0, t1, 1b
> +	 addiu	t0, t0, CACHE_LINESIZE
> +
> +	/* kseg0 cache attribute */
> +	mfc0	t0, CP0_CONFIG, 0
> +	ori	t0, t0, CONF_CM_CACHABLE_NONCOHERENT
> +	mtc0	t0, CP0_CONFIG, 0
> +
> +	/* pagemask */
> +	mtc0	zero, CP0_PAGEMASK, 0
> +
> +	/* retrieve sp */
> +	la	t0, jz4780_cpu_entry_sp
> +	lw	sp, 0(t0)
> +
> +	/* retrieve gp */
> +	la	t0, jz4780_cpu_entry_gp
> +	lw	gp, 0(t0)
> +
> +	/* jump to the kernel in kseg0 */
> +	la	t0, smp_bootstrap
> +	jr	t0
> +	 nop
> +	END(jz4780_secondary_cpu_entry)
> diff --git a/arch/mips/jz4740/smp.c b/arch/mips/jz4740/smp.c
> new file mode 100644
> index 00000000..d95d22a
> --- /dev/null
> +++ b/arch/mips/jz4740/smp.c
> @@ -0,0 +1,258 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
> + *  JZ4780 SMP
> + */
> +
> +#include <linux/clk.h>
> +#include <linux/delay.h>
> +#include <linux/interrupt.h>
> +#include <linux/of.h>
> +#include <linux/sched.h>
> +#include <linux/sched/task_stack.h>
> +#include <linux/smp.h>
> +#include <linux/tick.h>
> +#include <asm/mach-jz4740/smp.h>
> +#include <asm/smp-ops.h>
> +
> +static struct clk *cpu_clock_gates[CONFIG_NR_CPUS] = { NULL };
> +
> +u32 jz4780_cpu_entry_sp;
> +u32 jz4780_cpu_entry_gp;
> +
> +static struct cpumask cpu_running;
> +
> +static DEFINE_SPINLOCK(smp_lock);
> +
> +static irqreturn_t mbox_handler(int irq, void *dev_id)
> +{
> +	int cpu = smp_processor_id();
> +	u32 action, status;
> +
> +	spin_lock(&smp_lock);
> +
> +	switch (cpu) {
> +	case 0:
> +		action = read_c0_mailbox0();
> +		write_c0_mailbox0(0);
> +		break;
> +	case 1:
> +		action = read_c0_mailbox1();
> +		write_c0_mailbox1(0);
> +		break;
> +	case 2:
> +		action = read_c0_mailbox2();
> +		write_c0_mailbox2(0);
> +		break;
> +	case 3:
> +		action = read_c0_mailbox3();
> +		write_c0_mailbox3(0);
> +		break;
> +	default:
> +		panic("unhandled cpu %d!", cpu);
> +	}
> +
> +	/* clear pending mailbox interrupt */
> +	status = read_c0_corestatus();
> +	status &= ~(CORESTATUS_MIRQ0P << cpu);
> +	write_c0_corestatus(status);
> +
> +	spin_unlock(&smp_lock);
> +
> +	if (action & SMP_RESCHEDULE_YOURSELF)
> +		scheduler_ipi();
> +	if (action & SMP_CALL_FUNCTION)
> +		generic_smp_call_function_interrupt();
> +
> +	return IRQ_HANDLED;
> +}
> +
> +static void jz4780_smp_setup(void)
> +{
> +	u32 addr, reim;
> +	int cpu;
> +
> +	reim = read_c0_reim();
> +
> +	for (cpu = 0; cpu < NR_CPUS; cpu++) {
> +		__cpu_number_map[cpu] = cpu;
> +		__cpu_logical_map[cpu] = cpu;
> +		set_cpu_possible(cpu, true);
> +	}
> +
> +	/* mask mailbox interrupts for this core */
> +	reim &= ~REIM_MBOXIRQ0M;
> +	write_c0_reim(reim);
> +
> +	/* clear mailboxes & pending mailbox IRQs */
> +	write_c0_mailbox0(0);
> +	write_c0_mailbox1(0);
> +	write_c0_corestatus(0);
> +
> +	/* set reset entry point */
> +	addr = KSEG1ADDR((u32)&jz4780_secondary_cpu_entry);
> +	WARN_ON(addr & ~REIM_ENTRY);
> +	reim &= ~REIM_ENTRY;
> +	reim |= addr & REIM_ENTRY;
> +
> +	/* unmask mailbox interrupts for this core */
> +	reim |= REIM_MBOXIRQ0M;
> +	write_c0_reim(reim);
> +	set_c0_status(STATUSF_IP3);
> +	irq_enable_hazard();
> +
> +	cpumask_set_cpu(cpu, &cpu_running);
> +}
> +
> +static void jz4780_smp_prepare_cpus(unsigned int max_cpus)
> +{
> +	struct device_node *cpu_node;
> +	unsigned cpu, ctrl;
> +	int err;
> +
> +	/* setup the mailbox IRQ */
> +	err = request_irq(MIPS_CPU_IRQ_BASE + 3, mbox_handler,
> +			IRQF_PERCPU | IRQF_NO_THREAD, "core mailbox", NULL);
> +	if (err)
> +		pr_err("request_irq() on core mailbox failed\n");
> +
> +	ctrl = read_c0_corectrl();
> +
> +	for_each_of_cpu_node(cpu_node) {
> +		cpu = of_cpu_node_to_id(cpu_node);
> +		if (cpu < 0) {
> +			pr_err("Failed to read index of %s\n",
> +			       cpu_node->full_name);
> +			continue;
> +		}
> +
> +		/* use reset entry point from REIM register */
> +		ctrl |= CORECTRL_RPC0 << cpu;
> +
> +		cpu_clock_gates[cpu] = of_clk_get(cpu_node, 0);
> +		if (IS_ERR(cpu_clock_gates[cpu])) {
> +			cpu_clock_gates[cpu] = NULL;
> +			continue;
> +		}
> +
> +		err = clk_prepare(cpu_clock_gates[cpu]);
> +		if (err)
> +			pr_err("Failed to prepare CPU clock gate\n");
> +	}
> +
> +	write_c0_corectrl(ctrl);
> +}
> +
> +static int jz4780_boot_secondary(int cpu, struct task_struct *idle)
> +{
> +	unsigned long flags;
> +	u32 ctrl;
> +
> +	spin_lock_irqsave(&smp_lock, flags);
> +
> +	/* ensure the core is in reset */
> +	ctrl = read_c0_corectrl();
> +	ctrl |= CORECTRL_SWRST0 << cpu;
> +	write_c0_corectrl(ctrl);
> +
> +	/* ungate core clock */
> +	if (cpu_clock_gates[cpu])
> +		clk_enable(cpu_clock_gates[cpu]);
> +
> +	/* set entry sp/gp register values */
> +	jz4780_cpu_entry_sp = __KSTK_TOS(idle);
> +	jz4780_cpu_entry_gp = (u32)task_thread_info(idle);
> +	smp_wmb();
> +
> +	/* take the core out of reset */
> +	ctrl &= ~(CORECTRL_SWRST0 << cpu);
> +	write_c0_corectrl(ctrl);
> +
> +	cpumask_set_cpu(cpu, &cpu_running);
> +
> +	spin_unlock_irqrestore(&smp_lock, flags);
> +
> +	return 0;
> +}
> +
> +static void jz4780_init_secondary(void)
> +{
> +}
> +
> +static void jz4780_smp_finish(void)
> +{
> +	u32 reim;
> +
> +	spin_lock(&smp_lock);
> +
> +	/* unmask mailbox interrupts for this core */
> +	reim = read_c0_reim();
> +	reim |= REIM_MBOXIRQ0M << smp_processor_id();
> +	write_c0_reim(reim);
> +
> +	spin_unlock(&smp_lock);
> +
> +	/* unmask interrupts for this core */
> +	change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP2 |
> +			 STATUSF_IP1 | STATUSF_IP0);
> +	irq_enable_hazard();
> +
> +	/* force broadcast timer */
> +	tick_broadcast_force();
> +}
> +
> +static void jz4780_send_ipi_single_locked(int cpu, unsigned int 
> action)
> +{
> +	u32 mbox;
> +
> +	switch (cpu) {
> +	case 0:
> +		mbox = read_c0_mailbox0();
> +		write_c0_mailbox0(mbox | action);
> +		break;
> +	case 1:
> +		mbox = read_c0_mailbox1();
> +		write_c0_mailbox1(mbox | action);
> +		break;
> +	default:
> +		panic("unhandled cpu %d!", cpu);
> +	}
> +}
> +
> +static void jz4780_send_ipi_single(int cpu, unsigned int action)
> +{
> +	unsigned long flags;
> +
> +	spin_lock_irqsave(&smp_lock, flags);
> +	jz4780_send_ipi_single_locked(cpu, action);
> +	spin_unlock_irqrestore(&smp_lock, flags);
> +}
> +
> +static void jz4780_send_ipi_mask(const struct cpumask *mask,
> +				 unsigned int action)
> +{
> +	unsigned long flags;
> +	int cpu;
> +
> +	spin_lock_irqsave(&smp_lock, flags);
> +
> +	for_each_cpu(cpu, mask)
> +		jz4780_send_ipi_single_locked(cpu, action);
> +
> +	spin_unlock_irqrestore(&smp_lock, flags);
> +}
> +
> +static struct plat_smp_ops jz4780_smp_ops = {
> +	.send_ipi_single = jz4780_send_ipi_single,
> +	.send_ipi_mask = jz4780_send_ipi_mask,
> +	.init_secondary = jz4780_init_secondary,
> +	.smp_finish = jz4780_smp_finish,
> +	.boot_secondary = jz4780_boot_secondary,
> +	.smp_setup = jz4780_smp_setup,
> +	.prepare_cpus = jz4780_smp_prepare_cpus,
> +};
> +
> +void __init jz4780_smp_init(void)
> +{
> +	register_smp_ops(&jz4780_smp_ops);
> +}
> diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c
> index 37f8e78..d33f2d4 100644
> --- a/arch/mips/kernel/idle.c
> +++ b/arch/mips/kernel/idle.c
> @@ -18,6 +18,7 @@
>  #include <asm/cpu-type.h>
>  #include <asm/idle.h>
>  #include <asm/mipsregs.h>
> +#include <asm/r4kcache.h>
> 
>  /*
>   * Not all of the MIPS CPUs have the "wait" instruction available. 
> Moreover,
> @@ -88,6 +89,34 @@ static void __cpuidle rm7k_wait_irqoff(void)
>  }
> 
>  /*
> + * The Ingenic jz4780 SMP variant has to write back dirty cache 
> lines before
> + * executing wait. The CPU & cache clock will be gated until we 
> return from
> + * the wait, and if another core attempts to access data from our 
> data cache
> + * during this time then it will lock up.
> + */
> +void jz4780_smp_wait_irqoff(void)
> +{
> +	unsigned long pending = read_c0_cause() & read_c0_status() & 
> CAUSEF_IP;
> +
> +	/*
> +	 * Going to idle has a significant overhead due to the cache flush 
> so
> +	 * try to avoid it if we'll immediately be woken again due to an 
> IRQ.
> +	 */
> +	if (!need_resched() && !pending) {
> +		r4k_blast_dcache();
> +
> +		__asm__(
> +		"	.set push	\n"
> +		"	.set mips3	\n"
> +		"	sync		\n"
> +		"	wait		\n"
> +		"	.set pop	\n");
> +	}
> +
> +	local_irq_enable();
> +}
> +
> +/*
>   * Au1 'wait' is only useful when the 32kHz counter is used as timer,
>   * since coreclock (and the cp0 counter) stops upon executing it. 
> Only an
>   * interrupt can wake it, so they must be enabled before entering 
> idle modes.
> @@ -172,7 +201,6 @@ void __init check_wait(void)
>  	case CPU_CAVIUM_OCTEON_PLUS:
>  	case CPU_CAVIUM_OCTEON2:
>  	case CPU_CAVIUM_OCTEON3:
> -	case CPU_XBURST:
>  	case CPU_LOONGSON32:
>  	case CPU_XLR:
>  	case CPU_XLP:
> @@ -246,6 +274,11 @@ void __init check_wait(void)
>  		   cpu_wait = r4k_wait;
>  		 */
>  		break;
> +	case CPU_XBURST:
> +		if (IS_ENABLED(CONFIG_SMP))
> +			cpu_wait = jz4780_smp_wait_irqoff;
> +		else
> +			cpu_wait = r4k_wait;
>  	default:
>  		break;
>  	}
> --
> 2.7.4
> 

Re: [PATCH v8 3/6] clocksource: Ingenic: Add high resolution timer support for SMP.

[Paul Cercueil]

Hi Zhou,

Le mar. 19 mai 2020 à 22:35, 周琰杰 (Zhou Yanjie) 
<zhouyanjie@wanyeetech.com> a écrit :
> Enable clock event handling on per CPU core basis.
> Make sure that interrupts raised on the first core execute
> event handlers on the correct CPU core.
> 
> Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
> Tested-by: Paul Boddie <paul@boddie.org.uk>
> Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
> ---
> 
> Notes:
>     v1->v2:
>     1.Adjust function naming to make it more reasonable.
>     2.Replace function smp_call_function_single() with
>       smp_call_function_single_async() in order to resolve
>       the warning below:
> 
>     [    0.350942] smp: Brought up 1 node, 2 CPUs
>     [    0.365497] ------------[ cut here ]------------
>     [    0.365522] WARNING: CPU: 0 PID: 1 at kernel/smp.c:300 
> smp_call_function_single+0x110/0x200
>     [    0.365533] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 
> 5.5.0-rc1+ #5
>     [    0.365537] Stack : 00000000 59c73bcd 00000037 80074e80 
> 80000000 80670000 805a0000 80620590
>     [    0.365557]         8065ce38 8fc0dc8c 806d0000 00000000 
> 80670000 00000001 8fc0dc20 59c73bcd
>     [    0.365574]         00000000 00000000 806f0000 80670000 
> 00000000 806dab00 00000000 2d302e35
>     [    0.365591]         203a6d6d 806e0000 806e0000 70617773 
> 80670000 00000000 00000000 00000009
>     [    0.365610]         00000000 8fc94e20 8fc0de30 80690000 
> 00000018 803592dc 00000000 806d0000
>     [    0.365627]         ...
>     [    0.365634] Call Trace:
>     [    0.365647] [<8001b9a0>] show_stack+0x6c/0x12c
>     [    0.365663] [<804aed20>] dump_stack+0x98/0xc8
>     [    0.365673] [<8003044c>] __warn+0xc4/0xe8
>     [    0.365682] [<800304f4>] warn_slowpath_fmt+0x84/0xb8
>     [    0.365690] [<800a886c>] smp_call_function_single+0x110/0x200
>     [    0.365703] ---[ end trace 5785856ca39c79d5 ]---
>     [    0.365557]         8065ce38 8fc0dc8c 806d0000 00000000 
> 80670000 00000001 8fc0dc20 59c73bcd
>     [    0.365574]         00000000 00000000 806f0000 80670000 
> 00000000 806dab00 00000000 2d302e35
>     [    0.365591]         203a6d6d 806e0000 806e0000 70617773 
> 80670000 00000000 00000000 00000009
>     [    0.365610]         00000000 8fc94e20 8fc0de30 80690000 
> 00000018 803592dc 00000000 806d0000
>     [    0.365627]         ...
>     [    0.365634] Call Trace:
>     [    0.365647] [<8001b9a0>] show_stack+0x6c/0x12c
>     [    0.365663] [<804aed20>] dump_stack+0x98/0xc8
>     [    0.365673] [<8003044c>] __warn+0xc4/0xe8
>     [    0.365682] [<800304f4>] warn_slowpath_fmt+0x84/0xb8
>     [    0.365690] [<800a886c>] smp_call_function_single+0x110/0x200
>     [    0.365703] ---[ end trace 5785856ca39c79d5 ]---
> 
>     v2->v3:
>     No Change.
> 
>     v3->v4:
>     Rebase on top of kernel 5.6-rc1.
> 
>     v4->v5:
>     Move the check for (evt->event_handler) from 
> "ingenic_per_cpu_event_handler"
>     to "ingenic_tcu_cevt_cb".
> 
>     v5->v6:
>     No change.
> 
>     v6->v7:
>     Remove unnecessary check for "NR_CPUS > 1".
> 
>     v7->v8:
>     Use "num_possible_cpus()" instead "NR_CPUS".
>     Reported-by: kbuild test robot <lkp@intel.com>
> 
>  drivers/clocksource/ingenic-timer.c | 103 
> ++++++++++++++++++++++++++++--------
>  1 file changed, 82 insertions(+), 21 deletions(-)
> 
> diff --git a/drivers/clocksource/ingenic-timer.c 
> b/drivers/clocksource/ingenic-timer.c
> index 4963336..230e996 100644
> --- a/drivers/clocksource/ingenic-timer.c
> +++ b/drivers/clocksource/ingenic-timer.c
> @@ -1,7 +1,8 @@
>  // SPDX-License-Identifier: GPL-2.0
>  /*
> - * JZ47xx SoCs TCU IRQ driver
> + * XBurst SoCs TCU IRQ driver

If you want to get rid of the JZ47xx, then just write 'Ingenic SoCs TCU 
IRQ driver', since XBurst is only the name of the CPU. Also, this 
belongs to a separate patch.

>   * Copyright (C) 2019 Paul Cercueil <paul@crapouillou.net>
> + * Copyright (C) 2020 周琰杰 (Zhou Yanjie) 
> <zhouyanjie@wanyeetech.com>
>   */
> 
>  #include <linux/bitops.h>
> @@ -21,18 +22,23 @@
> 
>  #include <dt-bindings/clock/ingenic,tcu.h>
> 
> +static DEFINE_PER_CPU(call_single_data_t, ingenic_cevt_csd);
> +
>  struct ingenic_soc_info {
>  	unsigned int num_channels;
>  };
> 
>  struct ingenic_tcu {
>  	struct regmap *map;
> +	struct device_node *np;
>  	struct clk *timer_clk, *cs_clk;
> +	unsigned int timer_local[NR_CPUS];

NR_CPUS can be very big, so this would make the struct ingenic_tcu very 
large. What you could do, is have a variable array at the end of the 
struct:

unsigned int timer_local[];

Then you can create the ingenic_tcu struct using struct_size() from 
<linux/overflow.h>:

tcu = kzalloc(struct_size(tcu, timer_local, num_possible_cpus()), 
GFP_KERNEL);

-Paul

>  	unsigned int timer_channel, cs_channel;
>  	struct clock_event_device cevt;
>  	struct clocksource cs;
> -	char name[4];
> +	char name[8];
>  	unsigned long pwm_channels_mask;
> +	int cpu;
>  };
> 
>  static struct ingenic_tcu *ingenic_tcu;
> @@ -81,6 +87,24 @@ static int ingenic_tcu_cevt_set_next(unsigned long 
> next,
>  	return 0;
>  }
> 
> +static void ingenic_per_cpu_event_handler(void *info)
> +{
> +	struct clock_event_device *cevt = (struct clock_event_device *) 
> info;
> +
> +	cevt->event_handler(cevt);
> +}
> +
> +static void ingenic_tcu_per_cpu_cb(struct clock_event_device *evt)
> +{
> +	struct ingenic_tcu *tcu = to_ingenic_tcu(evt);
> +	call_single_data_t *csd;
> +
> +	csd = &per_cpu(ingenic_cevt_csd, tcu->cpu);
> +	csd->info = (void *) evt;
> +	csd->func = ingenic_per_cpu_event_handler;
> +	smp_call_function_single_async(tcu->cpu, csd);
> +}
> +
>  static irqreturn_t ingenic_tcu_cevt_cb(int irq, void *dev_id)
>  {
>  	struct clock_event_device *evt = dev_id;
> @@ -89,7 +113,7 @@ static irqreturn_t ingenic_tcu_cevt_cb(int irq, 
> void *dev_id)
>  	regmap_write(tcu->map, TCU_REG_TECR, BIT(tcu->timer_channel));
> 
>  	if (evt->event_handler)
> -		evt->event_handler(evt);
> +		ingenic_tcu_per_cpu_cb(evt);
> 
>  	return IRQ_HANDLED;
>  }
> @@ -105,14 +129,21 @@ static struct clk * __init 
> ingenic_tcu_get_clock(struct device_node *np, int id)
>  	return of_clk_get_from_provider(&args);
>  }
> 
> -static int __init ingenic_tcu_timer_init(struct device_node *np,
> -					 struct ingenic_tcu *tcu)
> +static int ingenic_tcu_setup_per_cpu_cevt(struct device_node *np,
> +				     unsigned int channel)
>  {
> -	unsigned int timer_virq, channel = tcu->timer_channel;
> +	unsigned int timer_virq;
>  	struct irq_domain *domain;
> +	struct ingenic_tcu *tcu;
>  	unsigned long rate;
>  	int err;
> 
> +	tcu = kzalloc(sizeof(*tcu), GFP_KERNEL);
> +	if (!tcu)
> +		return -ENOMEM;

I have no idea why you're doing that, but this is really wrong and 
highly confusing. The ingenic_tcu instance is already created in the 
probe. If you need per-timer data, then create another structure.

-Paul

> +
> +	tcu->map = ingenic_tcu->map;
> +
>  	tcu->timer_clk = ingenic_tcu_get_clock(np, channel);
>  	if (IS_ERR(tcu->timer_clk))
>  		return PTR_ERR(tcu->timer_clk);
> @@ -139,13 +170,15 @@ static int __init ingenic_tcu_timer_init(struct 
> device_node *np,
>  		goto err_clk_disable;
>  	}
> 
> -	snprintf(tcu->name, sizeof(tcu->name), "TCU");
> +	snprintf(tcu->name, sizeof(tcu->name), "TCU%u", channel);
> 
>  	err = request_irq(timer_virq, ingenic_tcu_cevt_cb, IRQF_TIMER,
>  			  tcu->name, &tcu->cevt);
>  	if (err)
>  		goto err_irq_dispose_mapping;
> 
> +	tcu->cpu = smp_processor_id();
> +	tcu->timer_channel = channel;
>  	tcu->cevt.cpumask = cpumask_of(smp_processor_id());
>  	tcu->cevt.features = CLOCK_EVT_FEAT_ONESHOT;
>  	tcu->cevt.name = tcu->name;
> @@ -166,6 +199,25 @@ static int __init ingenic_tcu_timer_init(struct 
> device_node *np,
>  	return err;
>  }
> 
> +static int ingenic_tcu_setup_cevt(unsigned int cpu)
> +{
> +	int ret;
> +
> +	ret = ingenic_tcu_setup_per_cpu_cevt(ingenic_tcu->np,
> +						ingenic_tcu->timer_local[cpu]);
> +	if (ret)
> +		goto err_tcu_clocksource_cleanup;
> +
> +	return 0;
> +
> +err_tcu_clocksource_cleanup:
> +	clocksource_unregister(&ingenic_tcu->cs);
> +	clk_disable_unprepare(ingenic_tcu->cs_clk);
> +	clk_put(ingenic_tcu->cs_clk);
> +	kfree(ingenic_tcu);
> +	return ret;
> +}
> +
>  static int __init ingenic_tcu_clocksource_init(struct device_node 
> *np,
>  					       struct ingenic_tcu *tcu)
>  {
> @@ -240,6 +292,7 @@ static int __init ingenic_tcu_init(struct 
> device_node *np)
>  	const struct ingenic_soc_info *soc_info = id->data;
>  	struct ingenic_tcu *tcu;
>  	struct regmap *map;
> +	unsigned cpu = 0;
>  	long rate;
>  	int ret;
> 
> @@ -253,13 +306,18 @@ static int __init ingenic_tcu_init(struct 
> device_node *np)
>  	if (!tcu)
>  		return -ENOMEM;
> 
> -	/* Enable all TCU channels for PWM use by default except channels 
> 0/1 */
> -	tcu->pwm_channels_mask = GENMASK(soc_info->num_channels - 1, 2);
> +	/*
> +	 * Enable all TCU channels for PWM use by default except channels 
> 0/1,
> +	 * and channel 2 if target CPU is JZ4780 and SMP is selected.
> +	 */
> +	tcu->pwm_channels_mask = GENMASK(soc_info->num_channels - 1,
> +								num_possible_cpus() + 1);
>  	of_property_read_u32(np, "ingenic,pwm-channels-mask",
>  			     (u32 *)&tcu->pwm_channels_mask);
> 
> -	/* Verify that we have at least two free channels */
> -	if (hweight8(tcu->pwm_channels_mask) > soc_info->num_channels - 2) {
> +	/* Verify that we have at least num_possible_cpus() + 1 free 
> channels */
> +	if (hweight8(tcu->pwm_channels_mask) >
> +			soc_info->num_channels - num_possible_cpus() + 1) {
>  		pr_crit("%s: Invalid PWM channel mask: 0x%02lx\n", __func__,
>  			tcu->pwm_channels_mask);
>  		ret = -EINVAL;
> @@ -267,13 +325,19 @@ static int __init ingenic_tcu_init(struct 
> device_node *np)
>  	}
> 
>  	tcu->map = map;
> +	tcu->np = np;
>  	ingenic_tcu = tcu;
> 
> -	tcu->timer_channel = find_first_zero_bit(&tcu->pwm_channels_mask,
> +	tcu->timer_local[cpu] = find_first_zero_bit(&tcu->pwm_channels_mask,
>  						 soc_info->num_channels);
> +
> +	for (cpu = 1; cpu < num_possible_cpus(); cpu++)
> +		tcu->timer_local[cpu] = find_next_zero_bit(
> +				&tcu->pwm_channels_mask, soc_info->num_channels,
> +				tcu->timer_local[cpu - 1] + 1);
> +
>  	tcu->cs_channel = find_next_zero_bit(&tcu->pwm_channels_mask,
> -					     soc_info->num_channels,
> -					     tcu->timer_channel + 1);
> +			soc_info->num_channels, tcu->timer_local[cpu - 1] + 1);
> 
>  	ret = ingenic_tcu_clocksource_init(np, tcu);
>  	if (ret) {
> @@ -281,9 +345,10 @@ static int __init ingenic_tcu_init(struct 
> device_node *np)
>  		goto err_free_ingenic_tcu;
>  	}
> 
> -	ret = ingenic_tcu_timer_init(np, tcu);
> -	if (ret)
> -		goto err_tcu_clocksource_cleanup;
> +	/* Setup clock events on each CPU core */
> +	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "Ingenic XBurst: 
> online",
> +				ingenic_tcu_setup_cevt, NULL);
> +	WARN_ON(ret < 0);
> 
>  	/* Register the sched_clock at the end as there's no way to undo it 
> */
>  	rate = clk_get_rate(tcu->cs_clk);
> @@ -291,10 +356,6 @@ static int __init ingenic_tcu_init(struct 
> device_node *np)
> 
>  	return 0;
> 
> -err_tcu_clocksource_cleanup:
> -	clocksource_unregister(&tcu->cs);
> -	clk_disable_unprepare(tcu->cs_clk);
> -	clk_put(tcu->cs_clk);
>  err_free_ingenic_tcu:
>  	kfree(tcu);
>  	return ret;
> --
> 2.7.4
> 

Re: [PATCH v8 1/6] MIPS: JZ4780: Introduce SMP support.

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 2668 bytes --]

Hi "周琰杰,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on robh/for-next]
[also build test WARNING on tip/timers/core linus/master v5.7-rc6]
[cannot apply to linux/master next-20200518]
[if your patch is applied to the wrong git tree, please drop us a note to help
improve the system. BTW, we also suggest to use '--base' option to specify the
base tree in git format-patch, please see https://stackoverflow.com/a/37406982]

url:    https://github.com/0day-ci/linux/commits/Zhou-Yanjie/Introduce-SMP-support-for-CI20-based-on-JZ4780/20200519-224008
base:   https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git for-next
config: mips-allyesconfig (attached as .config)
compiler: mips-linux-gcc (GCC) 9.3.0
reproduce:
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=mips 

If you fix the issue, kindly add following tag as appropriate
Reported-by: kbuild test robot <lkp@intel.com>

All warnings (new ones prefixed by >>, old ones prefixed by <<):

>> arch/mips/kernel/idle.c:97:6: warning: no previous prototype for 'jz4780_smp_wait_irqoff' [-Wmissing-prototypes]
97 | void jz4780_smp_wait_irqoff(void)
|      ^~~~~~~~~~~~~~~~~~~~~~
arch/mips/kernel/idle.c:155:13: warning: no previous prototype for 'check_wait' [-Wmissing-prototypes]
155 | void __init check_wait(void)
|             ^~~~~~~~~~

vim +/jz4780_smp_wait_irqoff +97 arch/mips/kernel/idle.c

    90	
    91	/*
    92	 * The Ingenic jz4780 SMP variant has to write back dirty cache lines before
    93	 * executing wait. The CPU & cache clock will be gated until we return from
    94	 * the wait, and if another core attempts to access data from our data cache
    95	 * during this time then it will lock up.
    96	 */
  > 97	void jz4780_smp_wait_irqoff(void)
    98	{
    99		unsigned long pending = read_c0_cause() & read_c0_status() & CAUSEF_IP;
   100	
   101		/*
   102		 * Going to idle has a significant overhead due to the cache flush so
   103		 * try to avoid it if we'll immediately be woken again due to an IRQ.
   104		 */
   105		if (!need_resched() && !pending) {
   106			r4k_blast_dcache();
   107	
   108			__asm__(
   109			"	.set push	\n"
   110			"	.set mips3	\n"
   111			"	sync		\n"
   112			"	wait		\n"
   113			"	.set pop	\n");
   114		}
   115	
   116		local_irq_enable();
   117	}
   118	

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 65515 bytes --]

Re: [PATCH v8 1/6] MIPS: JZ4780: Introduce SMP support.

[Paul Cercueil]

Hi Zhou,

Le mar. 19 mai 2020 à 22:35, 周琰杰 (Zhou Yanjie) 
<zhouyanjie@wanyeetech.com> a écrit :
> Forward port smp support from kernel 3.18.3 of CI20_linux
> to upstream kernel 5.6.
> 
> Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
> Tested-by: Paul Boddie <paul@boddie.org.uk>
> Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
> Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
> ---
> 
> Notes:
>     v1->v2:
>     1.Remove unnecessary "plat_irq_dispatch(void)" in irq-ingenic.c.
>     2.Add a timeout check for "jz4780_boot_secondary()" to avoid a 
> dead loop.
>     3.Replace hard code in smp.c with macro.
> 
>     v2->v3:
>     1.Remove unnecessary "extern void (*r4k_blast_dcache)(void)" in 
> smp.c.
>     2.Use "for_each_of_cpu_node" instead "for_each_compatible_node" 
> in smp.c.
>     3.Use "of_cpu_node_to_id" instead "of_property_read_u32_index" in 
> smp.c.
>     4.Move LCR related operations to jz4780-cgu.c.
> 
>     v3->v4:
>     Rebase on top of kernel 5.6-rc1.
> 
>     v4->v5:
>     1.Splitting changes involving "jz4780-cgu.c" into separate commit.
>     2.Use "request_irq()" replace "setup_irq()".
> 
>     v5->v6:
>     In order to have a kernel that works on multiple SoCs at the same
>     time, use "IS_ENABLED()" replace "#ifdef".
> 
>     v6->v7:
>     1.SMP has be decoupled from the SoC version.
>     2.Add mailboxes 3 and 4 for XBurst.
>     3.Adjust code in "jz4780_smp_prepare_cpus()".
>     4."jz4780_smp_init()" has be marked "__init".
> 
>     v7->v8:
>     No change.
> 
>  arch/mips/include/asm/mach-jz4740/smp.h |  87 +++++++++++
>  arch/mips/jz4740/Kconfig                |   2 +
>  arch/mips/jz4740/Makefile               |   5 +
>  arch/mips/jz4740/prom.c                 |   4 +
>  arch/mips/jz4740/smp-entry.S            |  57 +++++++
>  arch/mips/jz4740/smp.c                  | 258 
> ++++++++++++++++++++++++++++++++
>  arch/mips/kernel/idle.c                 |  35 ++++-
>  7 files changed, 447 insertions(+), 1 deletion(-)
>  create mode 100644 arch/mips/include/asm/mach-jz4740/smp.h
>  create mode 100644 arch/mips/jz4740/smp-entry.S
>  create mode 100644 arch/mips/jz4740/smp.c
> 
> diff --git a/arch/mips/include/asm/mach-jz4740/smp.h 
> b/arch/mips/include/asm/mach-jz4740/smp.h
> new file mode 100644
> index 00000000..86f660f
> --- /dev/null
> +++ b/arch/mips/include/asm/mach-jz4740/smp.h
> @@ -0,0 +1,87 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
> + *  JZ4780 SMP definitions
> + */
> +
> +#ifndef __MIPS_ASM_MACH_JZ4740_SMP_H__
> +#define __MIPS_ASM_MACH_JZ4740_SMP_H__
> +
> +#define read_c0_corectrl()		__read_32bit_c0_register($12, 2)
> +#define write_c0_corectrl(val)		__write_32bit_c0_register($12, 2, 
> val)
> +
> +#define read_c0_corestatus()		__read_32bit_c0_register($12, 3)
> +#define write_c0_corestatus(val)	__write_32bit_c0_register($12, 3, 
> val)
> +
> +#define read_c0_reim()			__read_32bit_c0_register($12, 4)
> +#define write_c0_reim(val)		__write_32bit_c0_register($12, 4, val)
> +
> +#define read_c0_mailbox0()		__read_32bit_c0_register($20, 0)
> +#define write_c0_mailbox0(val)		__write_32bit_c0_register($20, 0, 
> val)
> +
> +#define read_c0_mailbox1()		__read_32bit_c0_register($20, 1)
> +#define write_c0_mailbox1(val)		__write_32bit_c0_register($20, 1, 
> val)
> +
> +#define read_c0_mailbox2()		__read_32bit_c0_register($20, 2)
> +#define write_c0_mailbox2(val)		__write_32bit_c0_register($20, 2, 
> val)
> +
> +#define read_c0_mailbox3()		__read_32bit_c0_register($20, 3)
> +#define write_c0_mailbox3(val)		__write_32bit_c0_register($20, 3, 
> val)
> +
> +#define smp_clr_pending(mask) do {		\
> +		unsigned int stat;		\
> +		stat = read_c0_corestatus();	\
> +		stat &= ~((mask) & 0xff);	\
> +		write_c0_corestatus(stat);	\
> +	} while (0)
> +
> +/*
> + * Core Control register
> + */
> +#define CORECTRL_SLEEP1M_SHIFT	17
> +#define CORECTRL_SLEEP1M	(_ULCAST_(0x1) << CORECTRL_SLEEP1M_SHIFT)
> +#define CORECTRL_SLEEP0M_SHIFT	16
> +#define CORECTRL_SLEEP0M	(_ULCAST_(0x1) << CORECTRL_SLEEP0M_SHIFT)
> +#define CORECTRL_RPC1_SHIFT	9
> +#define CORECTRL_RPC1		(_ULCAST_(0x1) << CORECTRL_RPC1_SHIFT)
> +#define CORECTRL_RPC0_SHIFT	8
> +#define CORECTRL_RPC0		(_ULCAST_(0x1) << CORECTRL_RPC0_SHIFT)
> +#define CORECTRL_SWRST1_SHIFT	1
> +#define CORECTRL_SWRST1		(_ULCAST_(0x1) << CORECTRL_SWRST1_SHIFT)
> +#define CORECTRL_SWRST0_SHIFT	0
> +#define CORECTRL_SWRST0		(_ULCAST_(0x1) << CORECTRL_SWRST0_SHIFT)
> +
> +/*
> + * Core Status register
> + */
> +#define CORESTATUS_SLEEP1_SHIFT	17
> +#define CORESTATUS_SLEEP1	(_ULCAST_(0x1) << CORESTATUS_SLEEP1_SHIFT)
> +#define CORESTATUS_SLEEP0_SHIFT	16
> +#define CORESTATUS_SLEEP0	(_ULCAST_(0x1) << CORESTATUS_SLEEP0_SHIFT)
> +#define CORESTATUS_IRQ1P_SHIFT	9
> +#define CORESTATUS_IRQ1P	(_ULCAST_(0x1) << CORESTATUS_IRQ1P_SHIFT)
> +#define CORESTATUS_IRQ0P_SHIFT	8
> +#define CORESTATUS_IRQ0P	(_ULCAST_(0x1) << CORESTATUS_IRQ8P_SHIFT)
> +#define CORESTATUS_MIRQ1P_SHIFT	1
> +#define CORESTATUS_MIRQ1P	(_ULCAST_(0x1) << CORESTATUS_MIRQ1P_SHIFT)
> +#define CORESTATUS_MIRQ0P_SHIFT	0
> +#define CORESTATUS_MIRQ0P	(_ULCAST_(0x1) << CORESTATUS_MIRQ0P_SHIFT)
> +
> +/*
> + * Reset Entry & IRQ Mask register
> + */
> +#define REIM_ENTRY_SHIFT	16
> +#define REIM_ENTRY		(_ULCAST_(0xffff) << REIM_ENTRY_SHIFT)
> +#define REIM_IRQ1M_SHIFT	9
> +#define REIM_IRQ1M		(_ULCAST_(0x1) << REIM_IRQ1M_SHIFT)
> +#define REIM_IRQ0M_SHIFT	8
> +#define REIM_IRQ0M		(_ULCAST_(0x1) << REIM_IRQ0M_SHIFT)
> +#define REIM_MBOXIRQ1M_SHIFT	1
> +#define REIM_MBOXIRQ1M		(_ULCAST_(0x1) << REIM_MBOXIRQ1M_SHIFT)
> +#define REIM_MBOXIRQ0M_SHIFT	0
> +#define REIM_MBOXIRQ0M		(_ULCAST_(0x1) << REIM_MBOXIRQ0M_SHIFT)
> +
> +extern void jz4780_smp_init(void);
> +extern void jz4780_secondary_cpu_entry(void);
> +
> +#endif /* __MIPS_ASM_MACH_JZ4740_SMP_H__ */
> diff --git a/arch/mips/jz4740/Kconfig b/arch/mips/jz4740/Kconfig
> index 412d2fa..2b88557 100644
> --- a/arch/mips/jz4740/Kconfig
> +++ b/arch/mips/jz4740/Kconfig
> @@ -34,9 +34,11 @@ config MACH_JZ4770
> 
>  config MACH_JZ4780
>  	bool
> +	select GENERIC_CLOCKEVENTS_BROADCAST if SMP
>  	select MIPS_CPU_SCACHE
>  	select SYS_HAS_CPU_MIPS32_R2
>  	select SYS_SUPPORTS_HIGHMEM
> +	select SYS_SUPPORTS_SMP
> 
>  config MACH_X1000
>  	bool
> diff --git a/arch/mips/jz4740/Makefile b/arch/mips/jz4740/Makefile
> index 6de14c0..0a0f024 100644
> --- a/arch/mips/jz4740/Makefile
> +++ b/arch/mips/jz4740/Makefile
> @@ -12,3 +12,8 @@ CFLAGS_setup.o = 
> -I$(src)/../../../scripts/dtc/libfdt
>  # PM support
> 
>  obj-$(CONFIG_PM) += pm.o
> +
> +# SMP support
> +
> +obj-$(CONFIG_SMP) += smp.o
> +obj-$(CONFIG_SMP) += smp-entry.o
> diff --git a/arch/mips/jz4740/prom.c b/arch/mips/jz4740/prom.c
> index ff4555c..4acf5c2c 100644
> --- a/arch/mips/jz4740/prom.c
> +++ b/arch/mips/jz4740/prom.c
> @@ -8,10 +8,14 @@
> 
>  #include <asm/bootinfo.h>
>  #include <asm/fw/fw.h>
> +#include <asm/mach-jz4740/smp.h>
> 
>  void __init prom_init(void)
>  {
>  	fw_init_cmdline();
> +
> +	if (IS_ENABLED(CONFIG_SMP))
> +		jz4780_smp_init();
>  }
> 
>  void __init prom_free_prom_memory(void)
> diff --git a/arch/mips/jz4740/smp-entry.S 
> b/arch/mips/jz4740/smp-entry.S
> new file mode 100644
> index 00000000..20049a3
> --- /dev/null
> +++ b/arch/mips/jz4740/smp-entry.S
> @@ -0,0 +1,57 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
> + *  JZ4780 SMP entry point
> + */
> +
> +#include <asm/addrspace.h>
> +#include <asm/asm.h>
> +#include <asm/asmmacro.h>
> +#include <asm/cacheops.h>
> +#include <asm/mipsregs.h>
> +
> +#define CACHE_SIZE (32 * 1024)
> +#define CACHE_LINESIZE 32
> +
> +.extern jz4780_cpu_entry_sp
> +.extern jz4780_cpu_entry_gp
> +
> +.section .text.smp-entry
> +.balign 0x10000
> +.set noreorder
> +LEAF(jz4780_secondary_cpu_entry)
> +	mtc0	zero, CP0_CAUSE
> +
> +	li	t0, ST0_CU0
> +	mtc0	t0, CP0_STATUS
> +
> +	/* cache setup */
> +	li	t0, KSEG0
> +	ori	t1, t0, CACHE_SIZE
> +	mtc0	zero, CP0_TAGLO, 0
> +1:	cache	Index_Store_Tag_I, 0(t0)
> +	cache	Index_Store_Tag_D, 0(t0)
> +	bne	t0, t1, 1b
> +	 addiu	t0, t0, CACHE_LINESIZE
> +
> +	/* kseg0 cache attribute */
> +	mfc0	t0, CP0_CONFIG, 0
> +	ori	t0, t0, CONF_CM_CACHABLE_NONCOHERENT
> +	mtc0	t0, CP0_CONFIG, 0
> +
> +	/* pagemask */
> +	mtc0	zero, CP0_PAGEMASK, 0
> +
> +	/* retrieve sp */
> +	la	t0, jz4780_cpu_entry_sp
> +	lw	sp, 0(t0)
> +
> +	/* retrieve gp */
> +	la	t0, jz4780_cpu_entry_gp
> +	lw	gp, 0(t0)
> +
> +	/* jump to the kernel in kseg0 */
> +	la	t0, smp_bootstrap
> +	jr	t0
> +	 nop
> +	END(jz4780_secondary_cpu_entry)
> diff --git a/arch/mips/jz4740/smp.c b/arch/mips/jz4740/smp.c
> new file mode 100644
> index 00000000..d95d22a
> --- /dev/null
> +++ b/arch/mips/jz4740/smp.c
> @@ -0,0 +1,258 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
> + *  JZ4780 SMP
> + */
> +
> +#include <linux/clk.h>
> +#include <linux/delay.h>
> +#include <linux/interrupt.h>
> +#include <linux/of.h>
> +#include <linux/sched.h>
> +#include <linux/sched/task_stack.h>
> +#include <linux/smp.h>
> +#include <linux/tick.h>
> +#include <asm/mach-jz4740/smp.h>
> +#include <asm/smp-ops.h>
> +
> +static struct clk *cpu_clock_gates[CONFIG_NR_CPUS] = { NULL };
> +
> +u32 jz4780_cpu_entry_sp;
> +u32 jz4780_cpu_entry_gp;
> +
> +static struct cpumask cpu_running;

This cpumask is written, but never read anywhere. Since it's static, I 
believe it's dead code.

> +
> +static DEFINE_SPINLOCK(smp_lock);
> +
> +static irqreturn_t mbox_handler(int irq, void *dev_id)
> +{
> +	int cpu = smp_processor_id();
> +	u32 action, status;
> +
> +	spin_lock(&smp_lock);
> +
> +	switch (cpu) {
> +	case 0:
> +		action = read_c0_mailbox0();
> +		write_c0_mailbox0(0);
> +		break;
> +	case 1:
> +		action = read_c0_mailbox1();
> +		write_c0_mailbox1(0);
> +		break;
> +	case 2:
> +		action = read_c0_mailbox2();
> +		write_c0_mailbox2(0);
> +		break;
> +	case 3:
> +		action = read_c0_mailbox3();
> +		write_c0_mailbox3(0);
> +		break;
> +	default:
> +		panic("unhandled cpu %d!", cpu);
> +	}
> +
> +	/* clear pending mailbox interrupt */
> +	status = read_c0_corestatus();
> +	status &= ~(CORESTATUS_MIRQ0P << cpu);
> +	write_c0_corestatus(status);
> +
> +	spin_unlock(&smp_lock);
> +
> +	if (action & SMP_RESCHEDULE_YOURSELF)
> +		scheduler_ipi();
> +	if (action & SMP_CALL_FUNCTION)
> +		generic_smp_call_function_interrupt();
> +
> +	return IRQ_HANDLED;
> +}
> +
> +static void jz4780_smp_setup(void)
> +{
> +	u32 addr, reim;
> +	int cpu;
> +
> +	reim = read_c0_reim();
> +
> +	for (cpu = 0; cpu < NR_CPUS; cpu++) {
> +		__cpu_number_map[cpu] = cpu;
> +		__cpu_logical_map[cpu] = cpu;
> +		set_cpu_possible(cpu, true);

I assume if you do that, you will have num_possible_cpus() == NR_CPUS, 
which is not what you want.

Correct me if I'm wrong, but I think you would need to call 
set_cpu_possible() for each CPU node found.

> +	}
> +
> +	/* mask mailbox interrupts for this core */
> +	reim &= ~REIM_MBOXIRQ0M;
> +	write_c0_reim(reim);
> +
> +	/* clear mailboxes & pending mailbox IRQs */
> +	write_c0_mailbox0(0);
> +	write_c0_mailbox1(0);

Write mailbox2/3 too.

> +	write_c0_corestatus(0);
> +
> +	/* set reset entry point */
> +	addr = KSEG1ADDR((u32)&jz4780_secondary_cpu_entry);
> +	WARN_ON(addr & ~REIM_ENTRY);
> +	reim &= ~REIM_ENTRY;
> +	reim |= addr & REIM_ENTRY;
> +
> +	/* unmask mailbox interrupts for this core */
> +	reim |= REIM_MBOXIRQ0M;
> +	write_c0_reim(reim);
> +	set_c0_status(STATUSF_IP3);
> +	irq_enable_hazard();
> +
> +	cpumask_set_cpu(cpu, &cpu_running);
> +}
> +
> +static void jz4780_smp_prepare_cpus(unsigned int max_cpus)
> +{
> +	struct device_node *cpu_node;
> +	unsigned cpu, ctrl;
> +	int err;
> +
> +	/* setup the mailbox IRQ */
> +	err = request_irq(MIPS_CPU_IRQ_BASE + 3, mbox_handler,
> +			IRQF_PERCPU | IRQF_NO_THREAD, "core mailbox", NULL);

Please don't hardcode the IRQ number. Instead, it should be read from 
devicetree, maybe from the 'cpus' node (not sure).

> +	if (err)
> +		pr_err("request_irq() on core mailbox failed\n");
> +
> +	ctrl = read_c0_corectrl();
> +
> +	for_each_of_cpu_node(cpu_node) {
> +		cpu = of_cpu_node_to_id(cpu_node);
> +		if (cpu < 0) {
> +			pr_err("Failed to read index of %s\n",
> +			       cpu_node->full_name);
> +			continue;
> +		}
> +
> +		/* use reset entry point from REIM register */
> +		ctrl |= CORECTRL_RPC0 << cpu;
> +
> +		cpu_clock_gates[cpu] = of_clk_get(cpu_node, 0);
> +		if (IS_ERR(cpu_clock_gates[cpu])) {
> +			cpu_clock_gates[cpu] = NULL;
> +			continue;
> +		}
> +
> +		err = clk_prepare(cpu_clock_gates[cpu]);
> +		if (err)
> +			pr_err("Failed to prepare CPU clock gate\n");

I'd suggest to call clk_prepare() in jz4780_boot_secondary(), since you 
can't handle errors here. That would also avoid the static 
cpu_clock_gates[] array which can grow quite big since its size is 
given by NR_CPUS.

> +	}
> +
> +	write_c0_corectrl(ctrl);
> +}
> +
> +static int jz4780_boot_secondary(int cpu, struct task_struct *idle)
> +{
> +	unsigned long flags;
> +	u32 ctrl;
> +
> +	spin_lock_irqsave(&smp_lock, flags);
> +
> +	/* ensure the core is in reset */
> +	ctrl = read_c0_corectrl();
> +	ctrl |= CORECTRL_SWRST0 << cpu;
> +	write_c0_corectrl(ctrl);
> +
> +	/* ungate core clock */
> +	if (cpu_clock_gates[cpu])
> +		clk_enable(cpu_clock_gates[cpu]);

You should check the return value of clk_enable().

+		break;
> +
> +	/* set entry sp/gp register values */
> +	jz4780_cpu_entry_sp = __KSTK_TOS(idle);
> +	jz4780_cpu_entry_gp = (u32)task_thread_info(idle);
> +	smp_wmb();
> +
> +	/* take the core out of reset */
> +	ctrl &= ~(CORECTRL_SWRST0 << cpu);
> +	write_c0_corectrl(ctrl);
> +
> +	cpumask_set_cpu(cpu, &cpu_running);
> +
> +	spin_unlock_irqrestore(&smp_lock, flags);
> +
> +	return 0;
> +}
> +
> +static void jz4780_init_secondary(void)
> +{
> +}
> +
> +static void jz4780_smp_finish(void)
> +{
> +	u32 reim;
> +
> +	spin_lock(&smp_lock);
> +
> +	/* unmask mailbox interrupts for this core */
> +	reim = read_c0_reim();
> +	reim |= REIM_MBOXIRQ0M << smp_processor_id();
> +	write_c0_reim(reim);
> +
> +	spin_unlock(&smp_lock);
> +
> +	/* unmask interrupts for this core */
> +	change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP2 |
> +			 STATUSF_IP1 | STATUSF_IP0);
> +	irq_enable_hazard();
> +
> +	/* force broadcast timer */
> +	tick_broadcast_force();
> +}
> +
> +static void jz4780_send_ipi_single_locked(int cpu, unsigned int 
> action)
> +{
> +	u32 mbox;
> +
> +	switch (cpu) {
> +	case 0:
> +		mbox = read_c0_mailbox0();
> +		write_c0_mailbox0(mbox | action);
> +		break;
> +	case 1:
> +		mbox = read_c0_mailbox1();
> +		write_c0_mailbox1(mbox | action);

Handle mailboxes 2/3 too here.

> +	default:
> +		panic("unhandled cpu %d!", cpu);
> +	}
> +}
> +
> +static void jz4780_send_ipi_single(int cpu, unsigned int action)
> +{
> +	unsigned long flags;
> +
> +	spin_lock_irqsave(&smp_lock, flags);
> +	jz4780_send_ipi_single_locked(cpu, action);
> +	spin_unlock_irqrestore(&smp_lock, flags);
> +}
> +
> +static void jz4780_send_ipi_mask(const struct cpumask *mask,
> +				 unsigned int action)
> +{
> +	unsigned long flags;
> +	int cpu;
> +
> +	spin_lock_irqsave(&smp_lock, flags);
> +
> +	for_each_cpu(cpu, mask)
> +		jz4780_send_ipi_single_locked(cpu, action);
> +
> +	spin_unlock_irqrestore(&smp_lock, flags);
> +}
> +
> +static struct plat_smp_ops jz4780_smp_ops = {
> +	.send_ipi_single = jz4780_send_ipi_single,
> +	.send_ipi_mask = jz4780_send_ipi_mask,
> +	.init_secondary = jz4780_init_secondary,
> +	.smp_finish = jz4780_smp_finish,
> +	.boot_secondary = jz4780_boot_secondary,
> +	.smp_setup = jz4780_smp_setup,
> +	.prepare_cpus = jz4780_smp_prepare_cpus,
> +};
> +
> +void __init jz4780_smp_init(void)
> +{
> +	register_smp_ops(&jz4780_smp_ops);
> +}
> diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c
> index 37f8e78..d33f2d4 100644
> --- a/arch/mips/kernel/idle.c
> +++ b/arch/mips/kernel/idle.c
> @@ -18,6 +18,7 @@
>  #include <asm/cpu-type.h>
>  #include <asm/idle.h>
>  #include <asm/mipsregs.h>
> +#include <asm/r4kcache.h>
> 
>  /*
>   * Not all of the MIPS CPUs have the "wait" instruction available. 
> Moreover,
> @@ -88,6 +89,34 @@ static void __cpuidle rm7k_wait_irqoff(void)
>  }
> 
>  /*
> + * The Ingenic jz4780 SMP variant has to write back dirty cache 
> lines before
> + * executing wait. The CPU & cache clock will be gated until we 
> return from
> + * the wait, and if another core attempts to access data from our 
> data cache
> + * during this time then it will lock up.
> + */
> +void jz4780_smp_wait_irqoff(void)
> +{
> +	unsigned long pending = read_c0_cause() & read_c0_status() & 
> CAUSEF_IP;
> +
> +	/*
> +	 * Going to idle has a significant overhead due to the cache flush 
> so
> +	 * try to avoid it if we'll immediately be woken again due to an 
> IRQ.
> +	 */

You could add a fast path here where you just call r4k_wait() if 
num_online_cpus() < 2.

-Paul

> +	if (!need_resched() && !pending) {
> +		r4k_blast_dcache();
> +
> +		__asm__(
> +		"	.set push	\n"
> +		"	.set mips3	\n"
> +		"	sync		\n"
> +		"	wait		\n"
> +		"	.set pop	\n");
> +	}
> +
> +	local_irq_enable();
> +}
> +
> +/*
>   * Au1 'wait' is only useful when the 32kHz counter is used as timer,
>   * since coreclock (and the cp0 counter) stops upon executing it. 
> Only an
>   * interrupt can wake it, so they must be enabled before entering 
> idle modes.
> @@ -172,7 +201,6 @@ void __init check_wait(void)
>  	case CPU_CAVIUM_OCTEON_PLUS:
>  	case CPU_CAVIUM_OCTEON2:
>  	case CPU_CAVIUM_OCTEON3:
> -	case CPU_XBURST:
>  	case CPU_LOONGSON32:
>  	case CPU_XLR:
>  	case CPU_XLP:
> @@ -246,6 +274,11 @@ void __init check_wait(void)
>  		   cpu_wait = r4k_wait;
>  		 */
>  		break;
> +	case CPU_XBURST:
> +		if (IS_ENABLED(CONFIG_SMP))
> +			cpu_wait = jz4780_smp_wait_irqoff;
> +		else
> +			cpu_wait = r4k_wait;
>  	default:
>  		break;
>  	}
> --
> 2.7.4
> 

[PATCH] clocksource: Ingenic: Add high resolution timer support for SMP.

[Paul Cercueil]

From: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>

Enable clock event handling on per CPU core basis.
Make sure that interrupts raised on the first core execute
event handlers on the correct CPU core.

Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
Tested-by: Paul Boddie <paul@boddie.org.uk>
Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
Signed-off-by: Paul Cercueil <paul@crapouillou.net>
---

Zhou:

I took the liberty to clean your patch so that it doesn't create a
struct ingenic_tcu per CPU timer.

Tested, and fully working on the JZ4770 with CONFIG_SMP disabled, and
also with CONFIG_SMP enabled (even though JZ4770 has only one CPU) with
a fixed smp.c and USB disabled (otherwise it crashes at boot).

Cheers,
-Paul

 drivers/clocksource/ingenic-timer.c | 180 +++++++++++++++++++---------
 1 file changed, 123 insertions(+), 57 deletions(-)

diff --git a/drivers/clocksource/ingenic-timer.c b/drivers/clocksource/ingenic-timer.c
index 496333650de2..a068e4620c44 100644
--- a/drivers/clocksource/ingenic-timer.c
+++ b/drivers/clocksource/ingenic-timer.c
@@ -2,6 +2,7 @@
 /*
  * JZ47xx SoCs TCU IRQ driver
  * Copyright (C) 2019 Paul Cercueil <paul@crapouillou.net>
+ * Copyright (C) 2020 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
  */
 
 #include <linux/bitops.h>
@@ -15,24 +16,35 @@
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
+#include <linux/overflow.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
 #include <linux/sched_clock.h>
 
 #include <dt-bindings/clock/ingenic,tcu.h>
 
+static DEFINE_PER_CPU(call_single_data_t, ingenic_cevt_csd);
+
 struct ingenic_soc_info {
 	unsigned int num_channels;
 };
 
+struct ingenic_tcu_timer {
+	unsigned int cpu;
+	unsigned int channel;
+	struct clock_event_device cevt;
+	struct clk *clk;
+	char name[8];
+};
+
 struct ingenic_tcu {
 	struct regmap *map;
-	struct clk *timer_clk, *cs_clk;
-	unsigned int timer_channel, cs_channel;
-	struct clock_event_device cevt;
+	struct device_node *np;
+	struct clk *cs_clk;
+	unsigned int cs_channel;
 	struct clocksource cs;
-	char name[4];
 	unsigned long pwm_channels_mask;
+	struct ingenic_tcu_timer timers[];
 };
 
 static struct ingenic_tcu *ingenic_tcu;
@@ -52,16 +64,24 @@ static u64 notrace ingenic_tcu_timer_cs_read(struct clocksource *cs)
 	return ingenic_tcu_timer_read();
 }
 
-static inline struct ingenic_tcu *to_ingenic_tcu(struct clock_event_device *evt)
+static inline struct ingenic_tcu *
+to_ingenic_tcu(struct ingenic_tcu_timer *timer)
+{
+	return container_of(timer, struct ingenic_tcu, timers[timer->cpu]);
+}
+
+static inline struct ingenic_tcu_timer *
+to_ingenic_tcu_timer(struct clock_event_device *evt)
 {
-	return container_of(evt, struct ingenic_tcu, cevt);
+	return container_of(evt, struct ingenic_tcu_timer, cevt);
 }
 
 static int ingenic_tcu_cevt_set_state_shutdown(struct clock_event_device *evt)
 {
-	struct ingenic_tcu *tcu = to_ingenic_tcu(evt);
+	struct ingenic_tcu_timer *timer = to_ingenic_tcu_timer(evt);
+	struct ingenic_tcu *tcu = to_ingenic_tcu(timer);
 
-	regmap_write(tcu->map, TCU_REG_TECR, BIT(tcu->timer_channel));
+	regmap_write(tcu->map, TCU_REG_TECR, BIT(timer->channel));
 
 	return 0;
 }
@@ -69,27 +89,40 @@ static int ingenic_tcu_cevt_set_state_shutdown(struct clock_event_device *evt)
 static int ingenic_tcu_cevt_set_next(unsigned long next,
 				     struct clock_event_device *evt)
 {
-	struct ingenic_tcu *tcu = to_ingenic_tcu(evt);
+	struct ingenic_tcu_timer *timer = to_ingenic_tcu_timer(evt);
+	struct ingenic_tcu *tcu = to_ingenic_tcu(timer);
 
 	if (next > 0xffff)
 		return -EINVAL;
 
-	regmap_write(tcu->map, TCU_REG_TDFRc(tcu->timer_channel), next);
-	regmap_write(tcu->map, TCU_REG_TCNTc(tcu->timer_channel), 0);
-	regmap_write(tcu->map, TCU_REG_TESR, BIT(tcu->timer_channel));
+	regmap_write(tcu->map, TCU_REG_TDFRc(timer->channel), next);
+	regmap_write(tcu->map, TCU_REG_TCNTc(timer->channel), 0);
+	regmap_write(tcu->map, TCU_REG_TESR, BIT(timer->channel));
 
 	return 0;
 }
 
+static void ingenic_per_cpu_event_handler(void *info)
+{
+	struct clock_event_device *cevt = (struct clock_event_device *) info;
+
+	cevt->event_handler(cevt);
+}
+
 static irqreturn_t ingenic_tcu_cevt_cb(int irq, void *dev_id)
 {
-	struct clock_event_device *evt = dev_id;
-	struct ingenic_tcu *tcu = to_ingenic_tcu(evt);
+	struct ingenic_tcu_timer *timer = dev_id;
+	struct ingenic_tcu *tcu = to_ingenic_tcu(timer);
+	call_single_data_t *csd;
 
-	regmap_write(tcu->map, TCU_REG_TECR, BIT(tcu->timer_channel));
+	regmap_write(tcu->map, TCU_REG_TECR, BIT(timer->channel));
 
-	if (evt->event_handler)
-		evt->event_handler(evt);
+	if (timer->cevt.event_handler) {
+		csd = &per_cpu(ingenic_cevt_csd, timer->cpu);
+		csd->info = (void *) &timer->cevt;
+		csd->func = ingenic_per_cpu_event_handler;
+		smp_call_function_single_async(timer->cpu, csd);
+	}
 
 	return IRQ_HANDLED;
 }
@@ -105,64 +138,66 @@ static struct clk * __init ingenic_tcu_get_clock(struct device_node *np, int id)
 	return of_clk_get_from_provider(&args);
 }
 
-static int __init ingenic_tcu_timer_init(struct device_node *np,
-					 struct ingenic_tcu *tcu)
+static int ingenic_tcu_setup_cevt(unsigned int cpu)
 {
-	unsigned int timer_virq, channel = tcu->timer_channel;
+	struct ingenic_tcu *tcu = ingenic_tcu;
+	struct ingenic_tcu_timer *timer = &tcu->timers[cpu];
+	unsigned int timer_virq;
 	struct irq_domain *domain;
 	unsigned long rate;
 	int err;
 
-	tcu->timer_clk = ingenic_tcu_get_clock(np, channel);
-	if (IS_ERR(tcu->timer_clk))
-		return PTR_ERR(tcu->timer_clk);
+	timer->clk = ingenic_tcu_get_clock(tcu->np, timer->channel);
+	if (IS_ERR(timer->clk))
+		return PTR_ERR(timer->clk);
 
-	err = clk_prepare_enable(tcu->timer_clk);
+	err = clk_prepare_enable(timer->clk);
 	if (err)
 		goto err_clk_put;
 
-	rate = clk_get_rate(tcu->timer_clk);
+	rate = clk_get_rate(timer->clk);
 	if (!rate) {
 		err = -EINVAL;
 		goto err_clk_disable;
 	}
 
-	domain = irq_find_host(np);
+	domain = irq_find_host(tcu->np);
 	if (!domain) {
 		err = -ENODEV;
 		goto err_clk_disable;
 	}
 
-	timer_virq = irq_create_mapping(domain, channel);
+	timer_virq = irq_create_mapping(domain, timer->channel);
 	if (!timer_virq) {
 		err = -EINVAL;
 		goto err_clk_disable;
 	}
 
-	snprintf(tcu->name, sizeof(tcu->name), "TCU");
+	snprintf(timer->name, sizeof(timer->name), "TCU%u", timer->channel);
 
 	err = request_irq(timer_virq, ingenic_tcu_cevt_cb, IRQF_TIMER,
-			  tcu->name, &tcu->cevt);
+			  timer->name, timer);
 	if (err)
 		goto err_irq_dispose_mapping;
 
-	tcu->cevt.cpumask = cpumask_of(smp_processor_id());
-	tcu->cevt.features = CLOCK_EVT_FEAT_ONESHOT;
-	tcu->cevt.name = tcu->name;
-	tcu->cevt.rating = 200;
-	tcu->cevt.set_state_shutdown = ingenic_tcu_cevt_set_state_shutdown;
-	tcu->cevt.set_next_event = ingenic_tcu_cevt_set_next;
+	timer->cpu = smp_processor_id();
+	timer->cevt.cpumask = cpumask_of(smp_processor_id());
+	timer->cevt.features = CLOCK_EVT_FEAT_ONESHOT;
+	timer->cevt.name = timer->name;
+	timer->cevt.rating = 200;
+	timer->cevt.set_state_shutdown = ingenic_tcu_cevt_set_state_shutdown;
+	timer->cevt.set_next_event = ingenic_tcu_cevt_set_next;
 
-	clockevents_config_and_register(&tcu->cevt, rate, 10, 0xffff);
+	clockevents_config_and_register(&timer->cevt, rate, 10, 0xffff);
 
 	return 0;
 
 err_irq_dispose_mapping:
 	irq_dispose_mapping(timer_virq);
 err_clk_disable:
-	clk_disable_unprepare(tcu->timer_clk);
+	clk_disable_unprepare(timer->clk);
 err_clk_put:
-	clk_put(tcu->timer_clk);
+	clk_put(timer->clk);
 	return err;
 }
 
@@ -238,10 +273,12 @@ static int __init ingenic_tcu_init(struct device_node *np)
 {
 	const struct of_device_id *id = of_match_node(ingenic_tcu_of_match, np);
 	const struct ingenic_soc_info *soc_info = id->data;
+	struct ingenic_tcu_timer *timer;
 	struct ingenic_tcu *tcu;
 	struct regmap *map;
+	unsigned int cpu;
+	int ret, last_bit = -1;
 	long rate;
-	int ret;
 
 	of_node_clear_flag(np, OF_POPULATED);
 
@@ -249,17 +286,23 @@ static int __init ingenic_tcu_init(struct device_node *np)
 	if (IS_ERR(map))
 		return PTR_ERR(map);
 
-	tcu = kzalloc(sizeof(*tcu), GFP_KERNEL);
+	tcu = kzalloc(struct_size(tcu, timers, num_possible_cpus()),
+		      GFP_KERNEL);
 	if (!tcu)
 		return -ENOMEM;
 
-	/* Enable all TCU channels for PWM use by default except channels 0/1 */
-	tcu->pwm_channels_mask = GENMASK(soc_info->num_channels - 1, 2);
+	/*
+	 * Enable all TCU channels for PWM use by default except channels 0/1,
+	 * and channel 2 if target CPU is JZ4780 and SMP is selected.
+	 */
+	tcu->pwm_channels_mask = GENMASK(soc_info->num_channels - 1,
+					 num_possible_cpus() + 1);
 	of_property_read_u32(np, "ingenic,pwm-channels-mask",
 			     (u32 *)&tcu->pwm_channels_mask);
 
-	/* Verify that we have at least two free channels */
-	if (hweight8(tcu->pwm_channels_mask) > soc_info->num_channels - 2) {
+	/* Verify that we have at least num_possible_cpus() + 1 free channels */
+	if (hweight8(tcu->pwm_channels_mask) >
+			soc_info->num_channels - num_possible_cpus() + 1) {
 		pr_crit("%s: Invalid PWM channel mask: 0x%02lx\n", __func__,
 			tcu->pwm_channels_mask);
 		ret = -EINVAL;
@@ -267,13 +310,22 @@ static int __init ingenic_tcu_init(struct device_node *np)
 	}
 
 	tcu->map = map;
+	tcu->np = np;
 	ingenic_tcu = tcu;
 
-	tcu->timer_channel = find_first_zero_bit(&tcu->pwm_channels_mask,
-						 soc_info->num_channels);
+	for (cpu = 0; cpu < num_possible_cpus(); cpu++) {
+		timer = &tcu->timers[cpu];
+
+		timer->cpu = cpu;
+		timer->channel = find_next_zero_bit(&tcu->pwm_channels_mask,
+						  soc_info->num_channels,
+						  last_bit + 1);
+		last_bit = timer->channel;
+	}
+
 	tcu->cs_channel = find_next_zero_bit(&tcu->pwm_channels_mask,
 					     soc_info->num_channels,
-					     tcu->timer_channel + 1);
+					     last_bit + 1);
 
 	ret = ingenic_tcu_clocksource_init(np, tcu);
 	if (ret) {
@@ -281,9 +333,13 @@ static int __init ingenic_tcu_init(struct device_node *np)
 		goto err_free_ingenic_tcu;
 	}
 
-	ret = ingenic_tcu_timer_init(np, tcu);
-	if (ret)
+	/* Setup clock events on each CPU core */
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "Ingenic XBurst: online",
+				ingenic_tcu_setup_cevt, NULL);
+	if (ret < 0) {
+		pr_crit("%s: Unable to start CPU timers: %d\n", __func__, ret);
 		goto err_tcu_clocksource_cleanup;
+	}
 
 	/* Register the sched_clock at the end as there's no way to undo it */
 	rate = clk_get_rate(tcu->cs_clk);
@@ -315,28 +371,38 @@ static int __init ingenic_tcu_probe(struct platform_device *pdev)
 static int __maybe_unused ingenic_tcu_suspend(struct device *dev)
 {
 	struct ingenic_tcu *tcu = dev_get_drvdata(dev);
+	unsigned int cpu;
 
 	clk_disable(tcu->cs_clk);
-	clk_disable(tcu->timer_clk);
+
+	for (cpu = 0; cpu < num_online_cpus(); cpu++)
+		clk_disable(tcu->timers[cpu].clk);
+
 	return 0;
 }
 
 static int __maybe_unused ingenic_tcu_resume(struct device *dev)
 {
 	struct ingenic_tcu *tcu = dev_get_drvdata(dev);
+	unsigned int cpu;
 	int ret;
 
-	ret = clk_enable(tcu->timer_clk);
-	if (ret)
-		return ret;
+	for (cpu = 0; cpu < num_online_cpus(); cpu++) {
+		ret = clk_enable(tcu->timers[cpu].clk);
+		if (ret)
+			goto err_timer_clk_disable;
+	}
 
 	ret = clk_enable(tcu->cs_clk);
-	if (ret) {
-		clk_disable(tcu->timer_clk);
-		return ret;
-	}
+	if (ret)
+		goto err_timer_clk_disable;
 
 	return 0;
+
+err_timer_clk_disable:
+	for (; cpu > 0; cpu--)
+		clk_disable(tcu->timers[cpu - 1].clk);
+	return ret;
 }
 
 static const struct dev_pm_ops __maybe_unused ingenic_tcu_pm_ops = {
-- 
2.26.2

Re: [PATCH v8 1/6] MIPS: JZ4780: Introduce SMP support.

[Zhou Yanjie]

Hi Paul,

On 2020年05月20日 03:41, Paul Cercueil wrote:
> Hi Zhou,
>
> Le mar. 19 mai 2020 à 22:35, 周琰杰 (Zhou Yanjie) 
> <zhouyanjie@wanyeetech.com> a écrit :
>> Forward port smp support from kernel 3.18.3 of CI20_linux
>> to upstream kernel 5.6.
>>
>> Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
>> Tested-by: Paul Boddie <paul@boddie.org.uk>
>> Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
>> Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
>> ---
>>
>> Notes:
>>     v1->v2:
>>     1.Remove unnecessary "plat_irq_dispatch(void)" in irq-ingenic.c.
>>     2.Add a timeout check for "jz4780_boot_secondary()" to avoid a 
>> dead loop.
>>     3.Replace hard code in smp.c with macro.
>>
>>     v2->v3:
>>     1.Remove unnecessary "extern void (*r4k_blast_dcache)(void)" in 
>> smp.c.
>>     2.Use "for_each_of_cpu_node" instead "for_each_compatible_node" 
>> in smp.c.
>>     3.Use "of_cpu_node_to_id" instead "of_property_read_u32_index" in 
>> smp.c.
>>     4.Move LCR related operations to jz4780-cgu.c.
>>
>>     v3->v4:
>>     Rebase on top of kernel 5.6-rc1.
>>
>>     v4->v5:
>>     1.Splitting changes involving "jz4780-cgu.c" into separate commit.
>>     2.Use "request_irq()" replace "setup_irq()".
>>
>>     v5->v6:
>>     In order to have a kernel that works on multiple SoCs at the same
>>     time, use "IS_ENABLED()" replace "#ifdef".
>>
>>     v6->v7:
>>     1.SMP has be decoupled from the SoC version.
>>     2.Add mailboxes 3 and 4 for XBurst.
>>     3.Adjust code in "jz4780_smp_prepare_cpus()".
>>     4."jz4780_smp_init()" has be marked "__init".
>>
>>     v7->v8:
>>     No change.
>>
>>  arch/mips/include/asm/mach-jz4740/smp.h |  87 +++++++++++
>>  arch/mips/jz4740/Kconfig                |   2 +
>>  arch/mips/jz4740/Makefile               |   5 +
>>  arch/mips/jz4740/prom.c                 |   4 +
>>  arch/mips/jz4740/smp-entry.S            |  57 +++++++
>>  arch/mips/jz4740/smp.c                  | 258 
>> ++++++++++++++++++++++++++++++++
>>  arch/mips/kernel/idle.c                 |  35 ++++-
>>  7 files changed, 447 insertions(+), 1 deletion(-)
>>  create mode 100644 arch/mips/include/asm/mach-jz4740/smp.h
>>  create mode 100644 arch/mips/jz4740/smp-entry.S
>>  create mode 100644 arch/mips/jz4740/smp.c
>>
>> diff --git a/arch/mips/include/asm/mach-jz4740/smp.h 
>> b/arch/mips/include/asm/mach-jz4740/smp.h
>> new file mode 100644
>> index 00000000..86f660f
>> --- /dev/null
>> +++ b/arch/mips/include/asm/mach-jz4740/smp.h
>> @@ -0,0 +1,87 @@
>> +/* SPDX-License-Identifier: GPL-2.0-or-later */
>> +/*
>> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
>> + *  JZ4780 SMP definitions
>> + */
>> +
>> +#ifndef __MIPS_ASM_MACH_JZ4740_SMP_H__
>> +#define __MIPS_ASM_MACH_JZ4740_SMP_H__
>> +
>> +#define read_c0_corectrl()        __read_32bit_c0_register($12, 2)
>> +#define write_c0_corectrl(val) __write_32bit_c0_register($12, 2, val)
>> +
>> +#define read_c0_corestatus() __read_32bit_c0_register($12, 3)
>> +#define write_c0_corestatus(val) __write_32bit_c0_register($12, 3, val)
>> +
>> +#define read_c0_reim()            __read_32bit_c0_register($12, 4)
>> +#define write_c0_reim(val) __write_32bit_c0_register($12, 4, val)
>> +
>> +#define read_c0_mailbox0()        __read_32bit_c0_register($20, 0)
>> +#define write_c0_mailbox0(val) __write_32bit_c0_register($20, 0, val)
>> +
>> +#define read_c0_mailbox1()        __read_32bit_c0_register($20, 1)
>> +#define write_c0_mailbox1(val) __write_32bit_c0_register($20, 1, val)
>> +
>> +#define read_c0_mailbox2()        __read_32bit_c0_register($20, 2)
>> +#define write_c0_mailbox2(val) __write_32bit_c0_register($20, 2, val)
>> +
>> +#define read_c0_mailbox3()        __read_32bit_c0_register($20, 3)
>> +#define write_c0_mailbox3(val) __write_32bit_c0_register($20, 3, val)
>> +
>> +#define smp_clr_pending(mask) do {        \
>> +        unsigned int stat;        \
>> +        stat = read_c0_corestatus();    \
>> +        stat &= ~((mask) & 0xff);    \
>> +        write_c0_corestatus(stat);    \
>> +    } while (0)
>> +
>> +/*
>> + * Core Control register
>> + */
>> +#define CORECTRL_SLEEP1M_SHIFT    17
>> +#define CORECTRL_SLEEP1M    (_ULCAST_(0x1) << CORECTRL_SLEEP1M_SHIFT)
>> +#define CORECTRL_SLEEP0M_SHIFT    16
>> +#define CORECTRL_SLEEP0M    (_ULCAST_(0x1) << CORECTRL_SLEEP0M_SHIFT)
>> +#define CORECTRL_RPC1_SHIFT    9
>> +#define CORECTRL_RPC1        (_ULCAST_(0x1) << CORECTRL_RPC1_SHIFT)
>> +#define CORECTRL_RPC0_SHIFT    8
>> +#define CORECTRL_RPC0        (_ULCAST_(0x1) << CORECTRL_RPC0_SHIFT)
>> +#define CORECTRL_SWRST1_SHIFT    1
>> +#define CORECTRL_SWRST1        (_ULCAST_(0x1) << CORECTRL_SWRST1_SHIFT)
>> +#define CORECTRL_SWRST0_SHIFT    0
>> +#define CORECTRL_SWRST0        (_ULCAST_(0x1) << CORECTRL_SWRST0_SHIFT)
>> +
>> +/*
>> + * Core Status register
>> + */
>> +#define CORESTATUS_SLEEP1_SHIFT    17
>> +#define CORESTATUS_SLEEP1    (_ULCAST_(0x1) << CORESTATUS_SLEEP1_SHIFT)
>> +#define CORESTATUS_SLEEP0_SHIFT    16
>> +#define CORESTATUS_SLEEP0    (_ULCAST_(0x1) << CORESTATUS_SLEEP0_SHIFT)
>> +#define CORESTATUS_IRQ1P_SHIFT    9
>> +#define CORESTATUS_IRQ1P    (_ULCAST_(0x1) << CORESTATUS_IRQ1P_SHIFT)
>> +#define CORESTATUS_IRQ0P_SHIFT    8
>> +#define CORESTATUS_IRQ0P    (_ULCAST_(0x1) << CORESTATUS_IRQ8P_SHIFT)
>> +#define CORESTATUS_MIRQ1P_SHIFT    1
>> +#define CORESTATUS_MIRQ1P    (_ULCAST_(0x1) << CORESTATUS_MIRQ1P_SHIFT)
>> +#define CORESTATUS_MIRQ0P_SHIFT    0
>> +#define CORESTATUS_MIRQ0P    (_ULCAST_(0x1) << CORESTATUS_MIRQ0P_SHIFT)
>> +
>> +/*
>> + * Reset Entry & IRQ Mask register
>> + */
>> +#define REIM_ENTRY_SHIFT    16
>> +#define REIM_ENTRY        (_ULCAST_(0xffff) << REIM_ENTRY_SHIFT)
>> +#define REIM_IRQ1M_SHIFT    9
>> +#define REIM_IRQ1M        (_ULCAST_(0x1) << REIM_IRQ1M_SHIFT)
>> +#define REIM_IRQ0M_SHIFT    8
>> +#define REIM_IRQ0M        (_ULCAST_(0x1) << REIM_IRQ0M_SHIFT)
>> +#define REIM_MBOXIRQ1M_SHIFT    1
>> +#define REIM_MBOXIRQ1M        (_ULCAST_(0x1) << REIM_MBOXIRQ1M_SHIFT)
>> +#define REIM_MBOXIRQ0M_SHIFT    0
>> +#define REIM_MBOXIRQ0M        (_ULCAST_(0x1) << REIM_MBOXIRQ0M_SHIFT)
>> +
>> +extern void jz4780_smp_init(void);
>> +extern void jz4780_secondary_cpu_entry(void);
>> +
>> +#endif /* __MIPS_ASM_MACH_JZ4740_SMP_H__ */
>> diff --git a/arch/mips/jz4740/Kconfig b/arch/mips/jz4740/Kconfig
>> index 412d2fa..2b88557 100644
>> --- a/arch/mips/jz4740/Kconfig
>> +++ b/arch/mips/jz4740/Kconfig
>> @@ -34,9 +34,11 @@ config MACH_JZ4770
>>
>>  config MACH_JZ4780
>>      bool
>> +    select GENERIC_CLOCKEVENTS_BROADCAST if SMP
>>      select MIPS_CPU_SCACHE
>>      select SYS_HAS_CPU_MIPS32_R2
>>      select SYS_SUPPORTS_HIGHMEM
>> +    select SYS_SUPPORTS_SMP
>>
>>  config MACH_X1000
>>      bool
>> diff --git a/arch/mips/jz4740/Makefile b/arch/mips/jz4740/Makefile
>> index 6de14c0..0a0f024 100644
>> --- a/arch/mips/jz4740/Makefile
>> +++ b/arch/mips/jz4740/Makefile
>> @@ -12,3 +12,8 @@ CFLAGS_setup.o = -I$(src)/../../../scripts/dtc/libfdt
>>  # PM support
>>
>>  obj-$(CONFIG_PM) += pm.o
>> +
>> +# SMP support
>> +
>> +obj-$(CONFIG_SMP) += smp.o
>> +obj-$(CONFIG_SMP) += smp-entry.o
>> diff --git a/arch/mips/jz4740/prom.c b/arch/mips/jz4740/prom.c
>> index ff4555c..4acf5c2c 100644
>> --- a/arch/mips/jz4740/prom.c
>> +++ b/arch/mips/jz4740/prom.c
>> @@ -8,10 +8,14 @@
>>
>>  #include <asm/bootinfo.h>
>>  #include <asm/fw/fw.h>
>> +#include <asm/mach-jz4740/smp.h>
>>
>>  void __init prom_init(void)
>>  {
>>      fw_init_cmdline();
>> +
>> +    if (IS_ENABLED(CONFIG_SMP))
>> +        jz4780_smp_init();
>>  }
>>
>>  void __init prom_free_prom_memory(void)
>> diff --git a/arch/mips/jz4740/smp-entry.S b/arch/mips/jz4740/smp-entry.S
>> new file mode 100644
>> index 00000000..20049a3
>> --- /dev/null
>> +++ b/arch/mips/jz4740/smp-entry.S
>> @@ -0,0 +1,57 @@
>> +/* SPDX-License-Identifier: GPL-2.0-or-later */
>> +/*
>> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
>> + *  JZ4780 SMP entry point
>> + */
>> +
>> +#include <asm/addrspace.h>
>> +#include <asm/asm.h>
>> +#include <asm/asmmacro.h>
>> +#include <asm/cacheops.h>
>> +#include <asm/mipsregs.h>
>> +
>> +#define CACHE_SIZE (32 * 1024)
>> +#define CACHE_LINESIZE 32
>> +
>> +.extern jz4780_cpu_entry_sp
>> +.extern jz4780_cpu_entry_gp
>> +
>> +.section .text.smp-entry
>> +.balign 0x10000
>> +.set noreorder
>> +LEAF(jz4780_secondary_cpu_entry)
>> +    mtc0    zero, CP0_CAUSE
>> +
>> +    li    t0, ST0_CU0
>> +    mtc0    t0, CP0_STATUS
>> +
>> +    /* cache setup */
>> +    li    t0, KSEG0
>> +    ori    t1, t0, CACHE_SIZE
>> +    mtc0    zero, CP0_TAGLO, 0
>> +1:    cache    Index_Store_Tag_I, 0(t0)
>> +    cache    Index_Store_Tag_D, 0(t0)
>> +    bne    t0, t1, 1b
>> +     addiu    t0, t0, CACHE_LINESIZE
>> +
>> +    /* kseg0 cache attribute */
>> +    mfc0    t0, CP0_CONFIG, 0
>> +    ori    t0, t0, CONF_CM_CACHABLE_NONCOHERENT
>> +    mtc0    t0, CP0_CONFIG, 0
>> +
>> +    /* pagemask */
>> +    mtc0    zero, CP0_PAGEMASK, 0
>> +
>> +    /* retrieve sp */
>> +    la    t0, jz4780_cpu_entry_sp
>> +    lw    sp, 0(t0)
>> +
>> +    /* retrieve gp */
>> +    la    t0, jz4780_cpu_entry_gp
>> +    lw    gp, 0(t0)
>> +
>> +    /* jump to the kernel in kseg0 */
>> +    la    t0, smp_bootstrap
>> +    jr    t0
>> +     nop
>> +    END(jz4780_secondary_cpu_entry)
>> diff --git a/arch/mips/jz4740/smp.c b/arch/mips/jz4740/smp.c
>> new file mode 100644
>> index 00000000..d95d22a
>> --- /dev/null
>> +++ b/arch/mips/jz4740/smp.c
>> @@ -0,0 +1,258 @@
>> +// SPDX-License-Identifier: GPL-2.0
>> +/*
>> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
>> + *  JZ4780 SMP
>> + */
>> +
>> +#include <linux/clk.h>
>> +#include <linux/delay.h>
>> +#include <linux/interrupt.h>
>> +#include <linux/of.h>
>> +#include <linux/sched.h>
>> +#include <linux/sched/task_stack.h>
>> +#include <linux/smp.h>
>> +#include <linux/tick.h>
>> +#include <asm/mach-jz4740/smp.h>
>> +#include <asm/smp-ops.h>
>> +
>> +static struct clk *cpu_clock_gates[CONFIG_NR_CPUS] = { NULL };
>> +
>> +u32 jz4780_cpu_entry_sp;
>> +u32 jz4780_cpu_entry_gp;
>> +
>> +static struct cpumask cpu_running;
>
> This cpumask is written, but never read anywhere. Since it's static, I 
> believe it's dead code.
>

Sure, I will remove it.

>> +
>> +static DEFINE_SPINLOCK(smp_lock);
>> +
>> +static irqreturn_t mbox_handler(int irq, void *dev_id)
>> +{
>> +    int cpu = smp_processor_id();
>> +    u32 action, status;
>> +
>> +    spin_lock(&smp_lock);
>> +
>> +    switch (cpu) {
>> +    case 0:
>> +        action = read_c0_mailbox0();
>> +        write_c0_mailbox0(0);
>> +        break;
>> +    case 1:
>> +        action = read_c0_mailbox1();
>> +        write_c0_mailbox1(0);
>> +        break;
>> +    case 2:
>> +        action = read_c0_mailbox2();
>> +        write_c0_mailbox2(0);
>> +        break;
>> +    case 3:
>> +        action = read_c0_mailbox3();
>> +        write_c0_mailbox3(0);
>> +        break;
>> +    default:
>> +        panic("unhandled cpu %d!", cpu);
>> +    }
>> +
>> +    /* clear pending mailbox interrupt */
>> +    status = read_c0_corestatus();
>> +    status &= ~(CORESTATUS_MIRQ0P << cpu);
>> +    write_c0_corestatus(status);
>> +
>> +    spin_unlock(&smp_lock);
>> +
>> +    if (action & SMP_RESCHEDULE_YOURSELF)
>> +        scheduler_ipi();
>> +    if (action & SMP_CALL_FUNCTION)
>> +        generic_smp_call_function_interrupt();
>> +
>> +    return IRQ_HANDLED;
>> +}
>> +
>> +static void jz4780_smp_setup(void)
>> +{
>> +    u32 addr, reim;
>> +    int cpu;
>> +
>> +    reim = read_c0_reim();
>> +
>> +    for (cpu = 0; cpu < NR_CPUS; cpu++) {
>> +        __cpu_number_map[cpu] = cpu;
>> +        __cpu_logical_map[cpu] = cpu;
>> +        set_cpu_possible(cpu, true);
>
> I assume if you do that, you will have num_possible_cpus() == NR_CPUS, 
> which is not what you want.
>
> Correct me if I'm wrong, but I think you would need to call 
> set_cpu_possible() for each CPU node found.
>

Yes, the current way is indeed a little problem, it will cause 
num_possible_cpus() == NR_CPUS, I will try to find a better way.

>> +    }
>> +
>> +    /* mask mailbox interrupts for this core */
>> +    reim &= ~REIM_MBOXIRQ0M;
>> +    write_c0_reim(reim);
>> +
>> +    /* clear mailboxes & pending mailbox IRQs */
>> +    write_c0_mailbox0(0);
>> +    write_c0_mailbox1(0);
>
> Write mailbox2/3 too.
>

Although the XBurst1 architecture can have up to four cores, but JZ4780 
only has two. If we need to write all four mailboxes here, do we need 
change the function name to "xburst1_smp_setup" or other similar names? 
This seems more appropriate.

>> +    write_c0_corestatus(0);
>> +
>> +    /* set reset entry point */
>> +    addr = KSEG1ADDR((u32)&jz4780_secondary_cpu_entry);
>> +    WARN_ON(addr & ~REIM_ENTRY);
>> +    reim &= ~REIM_ENTRY;
>> +    reim |= addr & REIM_ENTRY;
>> +
>> +    /* unmask mailbox interrupts for this core */
>> +    reim |= REIM_MBOXIRQ0M;
>> +    write_c0_reim(reim);
>> +    set_c0_status(STATUSF_IP3);
>> +    irq_enable_hazard();
>> +
>> +    cpumask_set_cpu(cpu, &cpu_running);
>> +}
>> +
>> +static void jz4780_smp_prepare_cpus(unsigned int max_cpus)
>> +{
>> +    struct device_node *cpu_node;
>> +    unsigned cpu, ctrl;
>> +    int err;
>> +
>> +    /* setup the mailbox IRQ */
>> +    err = request_irq(MIPS_CPU_IRQ_BASE + 3, mbox_handler,
>> +            IRQF_PERCPU | IRQF_NO_THREAD, "core mailbox", NULL);
>
> Please don't hardcode the IRQ number. Instead, it should be read from 
> devicetree, maybe from the 'cpus' node (not sure).
>

OK, I'll try to figure it out.

>> +    if (err)
>> +        pr_err("request_irq() on core mailbox failed\n");
>> +
>> +    ctrl = read_c0_corectrl();
>> +
>> +    for_each_of_cpu_node(cpu_node) {
>> +        cpu = of_cpu_node_to_id(cpu_node);
>> +        if (cpu < 0) {
>> +            pr_err("Failed to read index of %s\n",
>> +                   cpu_node->full_name);
>> +            continue;
>> +        }
>> +
>> +        /* use reset entry point from REIM register */
>> +        ctrl |= CORECTRL_RPC0 << cpu;
>> +
>> +        cpu_clock_gates[cpu] = of_clk_get(cpu_node, 0);
>> +        if (IS_ERR(cpu_clock_gates[cpu])) {
>> +            cpu_clock_gates[cpu] = NULL;
>> +            continue;
>> +        }
>> +
>> +        err = clk_prepare(cpu_clock_gates[cpu]);
>> +        if (err)
>> +            pr_err("Failed to prepare CPU clock gate\n");
>
> I'd suggest to call clk_prepare() in jz4780_boot_secondary(), since 
> you can't handle errors here. That would also avoid the static 
> cpu_clock_gates[] array which can grow quite big since its size is 
> given by NR_CPUS.
>

Sure, I will move it to jz4780_boot_secondary().

>> +    }
>> +
>> +    write_c0_corectrl(ctrl);
>> +}
>> +
>> +static int jz4780_boot_secondary(int cpu, struct task_struct *idle)
>> +{
>> +    unsigned long flags;
>> +    u32 ctrl;
>> +
>> +    spin_lock_irqsave(&smp_lock, flags);
>> +
>> +    /* ensure the core is in reset */
>> +    ctrl = read_c0_corectrl();
>> +    ctrl |= CORECTRL_SWRST0 << cpu;
>> +    write_c0_corectrl(ctrl);
>> +
>> +    /* ungate core clock */
>> +    if (cpu_clock_gates[cpu])
>> +        clk_enable(cpu_clock_gates[cpu]);
>
> You should check the return value of clk_enable().
>
> +        break;

Sure.

>> +
>> +    /* set entry sp/gp register values */
>> +    jz4780_cpu_entry_sp = __KSTK_TOS(idle);
>> +    jz4780_cpu_entry_gp = (u32)task_thread_info(idle);
>> +    smp_wmb();
>> +
>> +    /* take the core out of reset */
>> +    ctrl &= ~(CORECTRL_SWRST0 << cpu);
>> +    write_c0_corectrl(ctrl);
>> +
>> +    cpumask_set_cpu(cpu, &cpu_running);
>> +
>> +    spin_unlock_irqrestore(&smp_lock, flags);
>> +
>> +    return 0;
>> +}
>> +
>> +static void jz4780_init_secondary(void)
>> +{
>> +}
>> +
>> +static void jz4780_smp_finish(void)
>> +{
>> +    u32 reim;
>> +
>> +    spin_lock(&smp_lock);
>> +
>> +    /* unmask mailbox interrupts for this core */
>> +    reim = read_c0_reim();
>> +    reim |= REIM_MBOXIRQ0M << smp_processor_id();
>> +    write_c0_reim(reim);
>> +
>> +    spin_unlock(&smp_lock);
>> +
>> +    /* unmask interrupts for this core */
>> +    change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP2 |
>> +             STATUSF_IP1 | STATUSF_IP0);
>> +    irq_enable_hazard();
>> +
>> +    /* force broadcast timer */
>> +    tick_broadcast_force();
>> +}
>> +
>> +static void jz4780_send_ipi_single_locked(int cpu, unsigned int action)
>> +{
>> +    u32 mbox;
>> +
>> +    switch (cpu) {
>> +    case 0:
>> +        mbox = read_c0_mailbox0();
>> +        write_c0_mailbox0(mbox | action);
>> +        break;
>> +    case 1:
>> +        mbox = read_c0_mailbox1();
>> +        write_c0_mailbox1(mbox | action);
>
> Handle mailboxes 2/3 too here.
>

Same to the above, do we need to change the function to a more 
appropriate name?

>> +    default:
>> +        panic("unhandled cpu %d!", cpu);
>> +    }
>> +}
>> +
>> +static void jz4780_send_ipi_single(int cpu, unsigned int action)
>> +{
>> +    unsigned long flags;
>> +
>> +    spin_lock_irqsave(&smp_lock, flags);
>> +    jz4780_send_ipi_single_locked(cpu, action);
>> +    spin_unlock_irqrestore(&smp_lock, flags);
>> +}
>> +
>> +static void jz4780_send_ipi_mask(const struct cpumask *mask,
>> +                 unsigned int action)
>> +{
>> +    unsigned long flags;
>> +    int cpu;
>> +
>> +    spin_lock_irqsave(&smp_lock, flags);
>> +
>> +    for_each_cpu(cpu, mask)
>> +        jz4780_send_ipi_single_locked(cpu, action);
>> +
>> +    spin_unlock_irqrestore(&smp_lock, flags);
>> +}
>> +
>> +static struct plat_smp_ops jz4780_smp_ops = {
>> +    .send_ipi_single = jz4780_send_ipi_single,
>> +    .send_ipi_mask = jz4780_send_ipi_mask,
>> +    .init_secondary = jz4780_init_secondary,
>> +    .smp_finish = jz4780_smp_finish,
>> +    .boot_secondary = jz4780_boot_secondary,
>> +    .smp_setup = jz4780_smp_setup,
>> +    .prepare_cpus = jz4780_smp_prepare_cpus,
>> +};
>> +
>> +void __init jz4780_smp_init(void)
>> +{
>> +    register_smp_ops(&jz4780_smp_ops);
>> +}
>> diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c
>> index 37f8e78..d33f2d4 100644
>> --- a/arch/mips/kernel/idle.c
>> +++ b/arch/mips/kernel/idle.c
>> @@ -18,6 +18,7 @@
>>  #include <asm/cpu-type.h>
>>  #include <asm/idle.h>
>>  #include <asm/mipsregs.h>
>> +#include <asm/r4kcache.h>
>>
>>  /*
>>   * Not all of the MIPS CPUs have the "wait" instruction available. 
>> Moreover,
>> @@ -88,6 +89,34 @@ static void __cpuidle rm7k_wait_irqoff(void)
>>  }
>>
>>  /*
>> + * The Ingenic jz4780 SMP variant has to write back dirty cache 
>> lines before
>> + * executing wait. The CPU & cache clock will be gated until we 
>> return from
>> + * the wait, and if another core attempts to access data from our 
>> data cache
>> + * during this time then it will lock up.
>> + */
>> +void jz4780_smp_wait_irqoff(void)
>> +{
>> +    unsigned long pending = read_c0_cause() & read_c0_status() & 
>> CAUSEF_IP;
>> +
>> +    /*
>> +     * Going to idle has a significant overhead due to the cache 
>> flush so
>> +     * try to avoid it if we'll immediately be woken again due to an 
>> IRQ.
>> +     */
>
> You could add a fast path here where you just call r4k_wait() if 
> num_online_cpus() < 2.
>

Please correct me if I'm wrong, if we add it here, when the number of 
CPU cores is greater than 1 (which should be the case on most 
occasions), each call to "jz4780_smp_wait_irqoff" will generate 
additional overhead (judging the number of CPUs), is it better to change 
"if (IS_ENABLED(CONFIG_SMP))" in "case CPU_XBURST" below to "if 
(IS_ENABLED(CONFIG_SMP) && (num_possible_cpus() > 1))"?

Thanks and best regards!

> -Paul
>
>> +    if (!need_resched() && !pending) {
>> +        r4k_blast_dcache();
>> +
>> +        __asm__(
>> +        "    .set push    \n"
>> +        "    .set mips3    \n"
>> +        "    sync        \n"
>> +        "    wait        \n"
>> +        "    .set pop    \n");
>> +    }
>> +
>> +    local_irq_enable();
>> +}
>> +
>> +/*
>>   * Au1 'wait' is only useful when the 32kHz counter is used as timer,
>>   * since coreclock (and the cp0 counter) stops upon executing it. 
>> Only an
>>   * interrupt can wake it, so they must be enabled before entering 
>> idle modes.
>> @@ -172,7 +201,6 @@ void __init check_wait(void)
>>      case CPU_CAVIUM_OCTEON_PLUS:
>>      case CPU_CAVIUM_OCTEON2:
>>      case CPU_CAVIUM_OCTEON3:
>> -    case CPU_XBURST:
>>      case CPU_LOONGSON32:
>>      case CPU_XLR:
>>      case CPU_XLP:
>> @@ -246,6 +274,11 @@ void __init check_wait(void)
>>             cpu_wait = r4k_wait;
>>           */
>>          break;
>> +    case CPU_XBURST:
>> +        if (IS_ENABLED(CONFIG_SMP))
>> +            cpu_wait = jz4780_smp_wait_irqoff;
>> +        else
>> +            cpu_wait = r4k_wait;
>>      default:
>>          break;
>>      }
>> -- 
>> 2.7.4
>>
>

Re: [PATCH v8 1/6] MIPS: JZ4780: Introduce SMP support.

[Zhou Yanjie]

On 2020年05月20日 00:09, Paul Cercueil wrote:
> Hi Zhou,
>
> Le mar. 19 mai 2020 à 22:35, 周琰杰 (Zhou Yanjie) 
> <zhouyanjie@wanyeetech.com> a écrit :
>> Forward port smp support from kernel 3.18.3 of CI20_linux
>> to upstream kernel 5.6.
>>
>> Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
>> Tested-by: Paul Boddie <paul@boddie.org.uk>
>> Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
>> Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
>> ---
>>
>> Notes:
>>     v1->v2:
>>     1.Remove unnecessary "plat_irq_dispatch(void)" in irq-ingenic.c.
>>     2.Add a timeout check for "jz4780_boot_secondary()" to avoid a 
>> dead loop.
>>     3.Replace hard code in smp.c with macro.
>>
>>     v2->v3:
>>     1.Remove unnecessary "extern void (*r4k_blast_dcache)(void)" in 
>> smp.c.
>>     2.Use "for_each_of_cpu_node" instead "for_each_compatible_node" 
>> in smp.c.
>>     3.Use "of_cpu_node_to_id" instead "of_property_read_u32_index" in 
>> smp.c.
>>     4.Move LCR related operations to jz4780-cgu.c.
>>
>>     v3->v4:
>>     Rebase on top of kernel 5.6-rc1.
>>
>>     v4->v5:
>>     1.Splitting changes involving "jz4780-cgu.c" into separate commit.
>>     2.Use "request_irq()" replace "setup_irq()".
>>
>>     v5->v6:
>>     In order to have a kernel that works on multiple SoCs at the same
>>     time, use "IS_ENABLED()" replace "#ifdef".
>>
>>     v6->v7:
>>     1.SMP has be decoupled from the SoC version.
>>     2.Add mailboxes 3 and 4 for XBurst.
>>     3.Adjust code in "jz4780_smp_prepare_cpus()".
>>     4."jz4780_smp_init()" has be marked "__init".
>>
>>     v7->v8:
>>     No change.
>>
>>  arch/mips/include/asm/mach-jz4740/smp.h |  87 +++++++++++
>>  arch/mips/jz4740/Kconfig                |   2 +
>>  arch/mips/jz4740/Makefile               |   5 +
>>  arch/mips/jz4740/prom.c                 |   4 +
>>  arch/mips/jz4740/smp-entry.S            |  57 +++++++
>>  arch/mips/jz4740/smp.c                  | 258 
>> ++++++++++++++++++++++++++++++++
>>  arch/mips/kernel/idle.c                 |  35 ++++-
>>  7 files changed, 447 insertions(+), 1 deletion(-)
>>  create mode 100644 arch/mips/include/asm/mach-jz4740/smp.h
>>  create mode 100644 arch/mips/jz4740/smp-entry.S
>>  create mode 100644 arch/mips/jz4740/smp.c
>>
>> diff --git a/arch/mips/include/asm/mach-jz4740/smp.h 
>> b/arch/mips/include/asm/mach-jz4740/smp.h
>> new file mode 100644
>> index 00000000..86f660f
>> --- /dev/null
>> +++ b/arch/mips/include/asm/mach-jz4740/smp.h
>> @@ -0,0 +1,87 @@
>> +/* SPDX-License-Identifier: GPL-2.0-or-later */
>> +/*
>> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
>> + *  JZ4780 SMP definitions
>> + */
>> +
>> +#ifndef __MIPS_ASM_MACH_JZ4740_SMP_H__
>> +#define __MIPS_ASM_MACH_JZ4740_SMP_H__
>> +
>> +#define read_c0_corectrl()        __read_32bit_c0_register($12, 2)
>> +#define write_c0_corectrl(val) __write_32bit_c0_register($12, 2, val)
>> +
>> +#define read_c0_corestatus() __read_32bit_c0_register($12, 3)
>> +#define write_c0_corestatus(val) __write_32bit_c0_register($12, 3, val)
>> +
>> +#define read_c0_reim()            __read_32bit_c0_register($12, 4)
>> +#define write_c0_reim(val) __write_32bit_c0_register($12, 4, val)
>> +
>> +#define read_c0_mailbox0()        __read_32bit_c0_register($20, 0)
>> +#define write_c0_mailbox0(val) __write_32bit_c0_register($20, 0, val)
>> +
>> +#define read_c0_mailbox1()        __read_32bit_c0_register($20, 1)
>> +#define write_c0_mailbox1(val) __write_32bit_c0_register($20, 1, val)
>> +
>> +#define read_c0_mailbox2()        __read_32bit_c0_register($20, 2)
>> +#define write_c0_mailbox2(val) __write_32bit_c0_register($20, 2, val)
>> +
>> +#define read_c0_mailbox3()        __read_32bit_c0_register($20, 3)
>> +#define write_c0_mailbox3(val) __write_32bit_c0_register($20, 3, val)
>> +
>> +#define smp_clr_pending(mask) do {        \
>> +        unsigned int stat;        \
>> +        stat = read_c0_corestatus();    \
>> +        stat &= ~((mask) & 0xff);    \
>> +        write_c0_corestatus(stat);    \
>> +    } while (0)
>> +
>> +/*
>> + * Core Control register
>> + */
>> +#define CORECTRL_SLEEP1M_SHIFT    17
>> +#define CORECTRL_SLEEP1M    (_ULCAST_(0x1) << CORECTRL_SLEEP1M_SHIFT)
>> +#define CORECTRL_SLEEP0M_SHIFT    16
>> +#define CORECTRL_SLEEP0M    (_ULCAST_(0x1) << CORECTRL_SLEEP0M_SHIFT)
>> +#define CORECTRL_RPC1_SHIFT    9
>> +#define CORECTRL_RPC1        (_ULCAST_(0x1) << CORECTRL_RPC1_SHIFT)
>> +#define CORECTRL_RPC0_SHIFT    8
>> +#define CORECTRL_RPC0        (_ULCAST_(0x1) << CORECTRL_RPC0_SHIFT)
>> +#define CORECTRL_SWRST1_SHIFT    1
>> +#define CORECTRL_SWRST1        (_ULCAST_(0x1) << CORECTRL_SWRST1_SHIFT)
>> +#define CORECTRL_SWRST0_SHIFT    0
>> +#define CORECTRL_SWRST0        (_ULCAST_(0x1) << CORECTRL_SWRST0_SHIFT)
>> +
>> +/*
>> + * Core Status register
>> + */
>> +#define CORESTATUS_SLEEP1_SHIFT    17
>> +#define CORESTATUS_SLEEP1    (_ULCAST_(0x1) << CORESTATUS_SLEEP1_SHIFT)
>> +#define CORESTATUS_SLEEP0_SHIFT    16
>> +#define CORESTATUS_SLEEP0    (_ULCAST_(0x1) << CORESTATUS_SLEEP0_SHIFT)
>> +#define CORESTATUS_IRQ1P_SHIFT    9
>> +#define CORESTATUS_IRQ1P    (_ULCAST_(0x1) << CORESTATUS_IRQ1P_SHIFT)
>> +#define CORESTATUS_IRQ0P_SHIFT    8
>> +#define CORESTATUS_IRQ0P    (_ULCAST_(0x1) << CORESTATUS_IRQ8P_SHIFT)
>> +#define CORESTATUS_MIRQ1P_SHIFT    1
>> +#define CORESTATUS_MIRQ1P    (_ULCAST_(0x1) << CORESTATUS_MIRQ1P_SHIFT)
>> +#define CORESTATUS_MIRQ0P_SHIFT    0
>> +#define CORESTATUS_MIRQ0P    (_ULCAST_(0x1) << CORESTATUS_MIRQ0P_SHIFT)
>> +
>> +/*
>> + * Reset Entry & IRQ Mask register
>> + */
>> +#define REIM_ENTRY_SHIFT    16
>> +#define REIM_ENTRY        (_ULCAST_(0xffff) << REIM_ENTRY_SHIFT)
>> +#define REIM_IRQ1M_SHIFT    9
>> +#define REIM_IRQ1M        (_ULCAST_(0x1) << REIM_IRQ1M_SHIFT)
>> +#define REIM_IRQ0M_SHIFT    8
>> +#define REIM_IRQ0M        (_ULCAST_(0x1) << REIM_IRQ0M_SHIFT)
>> +#define REIM_MBOXIRQ1M_SHIFT    1
>> +#define REIM_MBOXIRQ1M        (_ULCAST_(0x1) << REIM_MBOXIRQ1M_SHIFT)
>> +#define REIM_MBOXIRQ0M_SHIFT    0
>> +#define REIM_MBOXIRQ0M        (_ULCAST_(0x1) << REIM_MBOXIRQ0M_SHIFT)
>> +
>> +extern void jz4780_smp_init(void);
>> +extern void jz4780_secondary_cpu_entry(void);
>> +
>> +#endif /* __MIPS_ASM_MACH_JZ4740_SMP_H__ */
>> diff --git a/arch/mips/jz4740/Kconfig b/arch/mips/jz4740/Kconfig
>> index 412d2fa..2b88557 100644
>> --- a/arch/mips/jz4740/Kconfig
>> +++ b/arch/mips/jz4740/Kconfig
>> @@ -34,9 +34,11 @@ config MACH_JZ4770
>>
>>  config MACH_JZ4780
>>      bool
>> +    select GENERIC_CLOCKEVENTS_BROADCAST if SMP
>>      select MIPS_CPU_SCACHE
>>      select SYS_HAS_CPU_MIPS32_R2
>>      select SYS_SUPPORTS_HIGHMEM
>> +    select SYS_SUPPORTS_SMP
>>
>>  config MACH_X1000
>>      bool
>> diff --git a/arch/mips/jz4740/Makefile b/arch/mips/jz4740/Makefile
>> index 6de14c0..0a0f024 100644
>> --- a/arch/mips/jz4740/Makefile
>> +++ b/arch/mips/jz4740/Makefile
>> @@ -12,3 +12,8 @@ CFLAGS_setup.o = -I$(src)/../../../scripts/dtc/libfdt
>>  # PM support
>>
>>  obj-$(CONFIG_PM) += pm.o
>> +
>> +# SMP support
>> +
>> +obj-$(CONFIG_SMP) += smp.o
>> +obj-$(CONFIG_SMP) += smp-entry.o
>> diff --git a/arch/mips/jz4740/prom.c b/arch/mips/jz4740/prom.c
>> index ff4555c..4acf5c2c 100644
>> --- a/arch/mips/jz4740/prom.c
>> +++ b/arch/mips/jz4740/prom.c
>
> That file is gone in mips-next. You should rebase your patchset on top 
> of mips-next.
>

OK, I will fix it in the next version.

> Cheers,
> -Paul
>
>> @@ -8,10 +8,14 @@
>>
>>  #include <asm/bootinfo.h>
>>  #include <asm/fw/fw.h>
>> +#include <asm/mach-jz4740/smp.h>
>>
>>  void __init prom_init(void)
>>  {
>>      fw_init_cmdline();
>> +
>> +    if (IS_ENABLED(CONFIG_SMP))
>> +        jz4780_smp_init();
>>  }
>>
>>  void __init prom_free_prom_memory(void)
>> diff --git a/arch/mips/jz4740/smp-entry.S b/arch/mips/jz4740/smp-entry.S
>> new file mode 100644
>> index 00000000..20049a3
>> --- /dev/null
>> +++ b/arch/mips/jz4740/smp-entry.S
>> @@ -0,0 +1,57 @@
>> +/* SPDX-License-Identifier: GPL-2.0-or-later */
>> +/*
>> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
>> + *  JZ4780 SMP entry point
>> + */
>> +
>> +#include <asm/addrspace.h>
>> +#include <asm/asm.h>
>> +#include <asm/asmmacro.h>
>> +#include <asm/cacheops.h>
>> +#include <asm/mipsregs.h>
>> +
>> +#define CACHE_SIZE (32 * 1024)
>> +#define CACHE_LINESIZE 32
>> +
>> +.extern jz4780_cpu_entry_sp
>> +.extern jz4780_cpu_entry_gp
>> +
>> +.section .text.smp-entry
>> +.balign 0x10000
>> +.set noreorder
>> +LEAF(jz4780_secondary_cpu_entry)
>> +    mtc0    zero, CP0_CAUSE
>> +
>> +    li    t0, ST0_CU0
>> +    mtc0    t0, CP0_STATUS
>> +
>> +    /* cache setup */
>> +    li    t0, KSEG0
>> +    ori    t1, t0, CACHE_SIZE
>> +    mtc0    zero, CP0_TAGLO, 0
>> +1:    cache    Index_Store_Tag_I, 0(t0)
>> +    cache    Index_Store_Tag_D, 0(t0)
>> +    bne    t0, t1, 1b
>> +     addiu    t0, t0, CACHE_LINESIZE
>> +
>> +    /* kseg0 cache attribute */
>> +    mfc0    t0, CP0_CONFIG, 0
>> +    ori    t0, t0, CONF_CM_CACHABLE_NONCOHERENT
>> +    mtc0    t0, CP0_CONFIG, 0
>> +
>> +    /* pagemask */
>> +    mtc0    zero, CP0_PAGEMASK, 0
>> +
>> +    /* retrieve sp */
>> +    la    t0, jz4780_cpu_entry_sp
>> +    lw    sp, 0(t0)
>> +
>> +    /* retrieve gp */
>> +    la    t0, jz4780_cpu_entry_gp
>> +    lw    gp, 0(t0)
>> +
>> +    /* jump to the kernel in kseg0 */
>> +    la    t0, smp_bootstrap
>> +    jr    t0
>> +     nop
>> +    END(jz4780_secondary_cpu_entry)
>> diff --git a/arch/mips/jz4740/smp.c b/arch/mips/jz4740/smp.c
>> new file mode 100644
>> index 00000000..d95d22a
>> --- /dev/null
>> +++ b/arch/mips/jz4740/smp.c
>> @@ -0,0 +1,258 @@
>> +// SPDX-License-Identifier: GPL-2.0
>> +/*
>> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
>> + *  JZ4780 SMP
>> + */
>> +
>> +#include <linux/clk.h>
>> +#include <linux/delay.h>
>> +#include <linux/interrupt.h>
>> +#include <linux/of.h>
>> +#include <linux/sched.h>
>> +#include <linux/sched/task_stack.h>
>> +#include <linux/smp.h>
>> +#include <linux/tick.h>
>> +#include <asm/mach-jz4740/smp.h>
>> +#include <asm/smp-ops.h>
>> +
>> +static struct clk *cpu_clock_gates[CONFIG_NR_CPUS] = { NULL };
>> +
>> +u32 jz4780_cpu_entry_sp;
>> +u32 jz4780_cpu_entry_gp;
>> +
>> +static struct cpumask cpu_running;
>> +
>> +static DEFINE_SPINLOCK(smp_lock);
>> +
>> +static irqreturn_t mbox_handler(int irq, void *dev_id)
>> +{
>> +    int cpu = smp_processor_id();
>> +    u32 action, status;
>> +
>> +    spin_lock(&smp_lock);
>> +
>> +    switch (cpu) {
>> +    case 0:
>> +        action = read_c0_mailbox0();
>> +        write_c0_mailbox0(0);
>> +        break;
>> +    case 1:
>> +        action = read_c0_mailbox1();
>> +        write_c0_mailbox1(0);
>> +        break;
>> +    case 2:
>> +        action = read_c0_mailbox2();
>> +        write_c0_mailbox2(0);
>> +        break;
>> +    case 3:
>> +        action = read_c0_mailbox3();
>> +        write_c0_mailbox3(0);
>> +        break;
>> +    default:
>> +        panic("unhandled cpu %d!", cpu);
>> +    }
>> +
>> +    /* clear pending mailbox interrupt */
>> +    status = read_c0_corestatus();
>> +    status &= ~(CORESTATUS_MIRQ0P << cpu);
>> +    write_c0_corestatus(status);
>> +
>> +    spin_unlock(&smp_lock);
>> +
>> +    if (action & SMP_RESCHEDULE_YOURSELF)
>> +        scheduler_ipi();
>> +    if (action & SMP_CALL_FUNCTION)
>> +        generic_smp_call_function_interrupt();
>> +
>> +    return IRQ_HANDLED;
>> +}
>> +
>> +static void jz4780_smp_setup(void)
>> +{
>> +    u32 addr, reim;
>> +    int cpu;
>> +
>> +    reim = read_c0_reim();
>> +
>> +    for (cpu = 0; cpu < NR_CPUS; cpu++) {
>> +        __cpu_number_map[cpu] = cpu;
>> +        __cpu_logical_map[cpu] = cpu;
>> +        set_cpu_possible(cpu, true);
>> +    }
>> +
>> +    /* mask mailbox interrupts for this core */
>> +    reim &= ~REIM_MBOXIRQ0M;
>> +    write_c0_reim(reim);
>> +
>> +    /* clear mailboxes & pending mailbox IRQs */
>> +    write_c0_mailbox0(0);
>> +    write_c0_mailbox1(0);
>> +    write_c0_corestatus(0);
>> +
>> +    /* set reset entry point */
>> +    addr = KSEG1ADDR((u32)&jz4780_secondary_cpu_entry);
>> +    WARN_ON(addr & ~REIM_ENTRY);
>> +    reim &= ~REIM_ENTRY;
>> +    reim |= addr & REIM_ENTRY;
>> +
>> +    /* unmask mailbox interrupts for this core */
>> +    reim |= REIM_MBOXIRQ0M;
>> +    write_c0_reim(reim);
>> +    set_c0_status(STATUSF_IP3);
>> +    irq_enable_hazard();
>> +
>> +    cpumask_set_cpu(cpu, &cpu_running);
>> +}
>> +
>> +static void jz4780_smp_prepare_cpus(unsigned int max_cpus)
>> +{
>> +    struct device_node *cpu_node;
>> +    unsigned cpu, ctrl;
>> +    int err;
>> +
>> +    /* setup the mailbox IRQ */
>> +    err = request_irq(MIPS_CPU_IRQ_BASE + 3, mbox_handler,
>> +            IRQF_PERCPU | IRQF_NO_THREAD, "core mailbox", NULL);
>> +    if (err)
>> +        pr_err("request_irq() on core mailbox failed\n");
>> +
>> +    ctrl = read_c0_corectrl();
>> +
>> +    for_each_of_cpu_node(cpu_node) {
>> +        cpu = of_cpu_node_to_id(cpu_node);
>> +        if (cpu < 0) {
>> +            pr_err("Failed to read index of %s\n",
>> +                   cpu_node->full_name);
>> +            continue;
>> +        }
>> +
>> +        /* use reset entry point from REIM register */
>> +        ctrl |= CORECTRL_RPC0 << cpu;
>> +
>> +        cpu_clock_gates[cpu] = of_clk_get(cpu_node, 0);
>> +        if (IS_ERR(cpu_clock_gates[cpu])) {
>> +            cpu_clock_gates[cpu] = NULL;
>> +            continue;
>> +        }
>> +
>> +        err = clk_prepare(cpu_clock_gates[cpu]);
>> +        if (err)
>> +            pr_err("Failed to prepare CPU clock gate\n");
>> +    }
>> +
>> +    write_c0_corectrl(ctrl);
>> +}
>> +
>> +static int jz4780_boot_secondary(int cpu, struct task_struct *idle)
>> +{
>> +    unsigned long flags;
>> +    u32 ctrl;
>> +
>> +    spin_lock_irqsave(&smp_lock, flags);
>> +
>> +    /* ensure the core is in reset */
>> +    ctrl = read_c0_corectrl();
>> +    ctrl |= CORECTRL_SWRST0 << cpu;
>> +    write_c0_corectrl(ctrl);
>> +
>> +    /* ungate core clock */
>> +    if (cpu_clock_gates[cpu])
>> +        clk_enable(cpu_clock_gates[cpu]);
>> +
>> +    /* set entry sp/gp register values */
>> +    jz4780_cpu_entry_sp = __KSTK_TOS(idle);
>> +    jz4780_cpu_entry_gp = (u32)task_thread_info(idle);
>> +    smp_wmb();
>> +
>> +    /* take the core out of reset */
>> +    ctrl &= ~(CORECTRL_SWRST0 << cpu);
>> +    write_c0_corectrl(ctrl);
>> +
>> +    cpumask_set_cpu(cpu, &cpu_running);
>> +
>> +    spin_unlock_irqrestore(&smp_lock, flags);
>> +
>> +    return 0;
>> +}
>> +
>> +static void jz4780_init_secondary(void)
>> +{
>> +}
>> +
>> +static void jz4780_smp_finish(void)
>> +{
>> +    u32 reim;
>> +
>> +    spin_lock(&smp_lock);
>> +
>> +    /* unmask mailbox interrupts for this core */
>> +    reim = read_c0_reim();
>> +    reim |= REIM_MBOXIRQ0M << smp_processor_id();
>> +    write_c0_reim(reim);
>> +
>> +    spin_unlock(&smp_lock);
>> +
>> +    /* unmask interrupts for this core */
>> +    change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP2 |
>> +             STATUSF_IP1 | STATUSF_IP0);
>> +    irq_enable_hazard();
>> +
>> +    /* force broadcast timer */
>> +    tick_broadcast_force();
>> +}
>> +
>> +static void jz4780_send_ipi_single_locked(int cpu, unsigned int action)
>> +{
>> +    u32 mbox;
>> +
>> +    switch (cpu) {
>> +    case 0:
>> +        mbox = read_c0_mailbox0();
>> +        write_c0_mailbox0(mbox | action);
>> +        break;
>> +    case 1:
>> +        mbox = read_c0_mailbox1();
>> +        write_c0_mailbox1(mbox | action);
>> +        break;
>> +    default:
>> +        panic("unhandled cpu %d!", cpu);
>> +    }
>> +}
>> +
>> +static void jz4780_send_ipi_single(int cpu, unsigned int action)
>> +{
>> +    unsigned long flags;
>> +
>> +    spin_lock_irqsave(&smp_lock, flags);
>> +    jz4780_send_ipi_single_locked(cpu, action);
>> +    spin_unlock_irqrestore(&smp_lock, flags);
>> +}
>> +
>> +static void jz4780_send_ipi_mask(const struct cpumask *mask,
>> +                 unsigned int action)
>> +{
>> +    unsigned long flags;
>> +    int cpu;
>> +
>> +    spin_lock_irqsave(&smp_lock, flags);
>> +
>> +    for_each_cpu(cpu, mask)
>> +        jz4780_send_ipi_single_locked(cpu, action);
>> +
>> +    spin_unlock_irqrestore(&smp_lock, flags);
>> +}
>> +
>> +static struct plat_smp_ops jz4780_smp_ops = {
>> +    .send_ipi_single = jz4780_send_ipi_single,
>> +    .send_ipi_mask = jz4780_send_ipi_mask,
>> +    .init_secondary = jz4780_init_secondary,
>> +    .smp_finish = jz4780_smp_finish,
>> +    .boot_secondary = jz4780_boot_secondary,
>> +    .smp_setup = jz4780_smp_setup,
>> +    .prepare_cpus = jz4780_smp_prepare_cpus,
>> +};
>> +
>> +void __init jz4780_smp_init(void)
>> +{
>> +    register_smp_ops(&jz4780_smp_ops);
>> +}
>> diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c
>> index 37f8e78..d33f2d4 100644
>> --- a/arch/mips/kernel/idle.c
>> +++ b/arch/mips/kernel/idle.c
>> @@ -18,6 +18,7 @@
>>  #include <asm/cpu-type.h>
>>  #include <asm/idle.h>
>>  #include <asm/mipsregs.h>
>> +#include <asm/r4kcache.h>
>>
>>  /*
>>   * Not all of the MIPS CPUs have the "wait" instruction available. 
>> Moreover,
>> @@ -88,6 +89,34 @@ static void __cpuidle rm7k_wait_irqoff(void)
>>  }
>>
>>  /*
>> + * The Ingenic jz4780 SMP variant has to write back dirty cache 
>> lines before
>> + * executing wait. The CPU & cache clock will be gated until we 
>> return from
>> + * the wait, and if another core attempts to access data from our 
>> data cache
>> + * during this time then it will lock up.
>> + */
>> +void jz4780_smp_wait_irqoff(void)
>> +{
>> +    unsigned long pending = read_c0_cause() & read_c0_status() & 
>> CAUSEF_IP;
>> +
>> +    /*
>> +     * Going to idle has a significant overhead due to the cache 
>> flush so
>> +     * try to avoid it if we'll immediately be woken again due to an 
>> IRQ.
>> +     */
>> +    if (!need_resched() && !pending) {
>> +        r4k_blast_dcache();
>> +
>> +        __asm__(
>> +        "    .set push    \n"
>> +        "    .set mips3    \n"
>> +        "    sync        \n"
>> +        "    wait        \n"
>> +        "    .set pop    \n");
>> +    }
>> +
>> +    local_irq_enable();
>> +}
>> +
>> +/*
>>   * Au1 'wait' is only useful when the 32kHz counter is used as timer,
>>   * since coreclock (and the cp0 counter) stops upon executing it. 
>> Only an
>>   * interrupt can wake it, so they must be enabled before entering 
>> idle modes.
>> @@ -172,7 +201,6 @@ void __init check_wait(void)
>>      case CPU_CAVIUM_OCTEON_PLUS:
>>      case CPU_CAVIUM_OCTEON2:
>>      case CPU_CAVIUM_OCTEON3:
>> -    case CPU_XBURST:
>>      case CPU_LOONGSON32:
>>      case CPU_XLR:
>>      case CPU_XLP:
>> @@ -246,6 +274,11 @@ void __init check_wait(void)
>>             cpu_wait = r4k_wait;
>>           */
>>          break;
>> +    case CPU_XBURST:
>> +        if (IS_ENABLED(CONFIG_SMP))
>> +            cpu_wait = jz4780_smp_wait_irqoff;
>> +        else
>> +            cpu_wait = r4k_wait;
>>      default:
>>          break;
>>      }
>> -- 
>> 2.7.4
>>
>

Re: [PATCH v8 1/6] MIPS: JZ4780: Introduce SMP support.

[Paul Cercueil]

Hi Zhou,

Le mer. 20 mai 2020 à 15:23, Zhou Yanjie <zhouyanjie@wanyeetech.com> a 
écrit :
> Hi Paul,
> 
> On 2020年05月20日 03:41, Paul Cercueil wrote:
>> Hi Zhou,
>> 
>> Le mar. 19 mai 2020 à 22:35, 周琰杰 (Zhou Yanjie) 
>> \x7f<zhouyanjie@wanyeetech.com> a écrit :
>>> Forward port smp support from kernel 3.18.3 of CI20_linux
>>> to upstream kernel 5.6.
>>> 
>>> Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
>>> Tested-by: Paul Boddie <paul@boddie.org.uk>
>>> Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
>>> Reviewed-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
>>> ---
>>> 
>>> Notes:
>>>     v1->v2:
>>>     1.Remove unnecessary "plat_irq_dispatch(void)" in irq-ingenic.c.
>>>     2.Add a timeout check for "jz4780_boot_secondary()" to avoid a 
>>> \x7f\x7fdead loop.
>>>     3.Replace hard code in smp.c with macro.
>>> 
>>>     v2->v3:
>>>     1.Remove unnecessary "extern void (*r4k_blast_dcache)(void)" in 
>>> \x7f\x7fsmp.c.
>>>     2.Use "for_each_of_cpu_node" instead "for_each_compatible_node" 
>>> \x7f\x7fin smp.c.
>>>     3.Use "of_cpu_node_to_id" instead "of_property_read_u32_index" 
>>> in \x7f\x7fsmp.c.
>>>     4.Move LCR related operations to jz4780-cgu.c.
>>> 
>>>     v3->v4:
>>>     Rebase on top of kernel 5.6-rc1.
>>> 
>>>     v4->v5:
>>>     1.Splitting changes involving "jz4780-cgu.c" into separate 
>>> commit.
>>>     2.Use "request_irq()" replace "setup_irq()".
>>> 
>>>     v5->v6:
>>>     In order to have a kernel that works on multiple SoCs at the 
>>> same
>>>     time, use "IS_ENABLED()" replace "#ifdef".
>>> 
>>>     v6->v7:
>>>     1.SMP has be decoupled from the SoC version.
>>>     2.Add mailboxes 3 and 4 for XBurst.
>>>     3.Adjust code in "jz4780_smp_prepare_cpus()".
>>>     4."jz4780_smp_init()" has be marked "__init".
>>> 
>>>     v7->v8:
>>>     No change.
>>> 
>>>  arch/mips/include/asm/mach-jz4740/smp.h |  87 +++++++++++
>>>  arch/mips/jz4740/Kconfig                |   2 +
>>>  arch/mips/jz4740/Makefile               |   5 +
>>>  arch/mips/jz4740/prom.c                 |   4 +
>>>  arch/mips/jz4740/smp-entry.S            |  57 +++++++
>>>  arch/mips/jz4740/smp.c                  | 258 
>>> \x7f\x7f++++++++++++++++++++++++++++++++
>>>  arch/mips/kernel/idle.c                 |  35 ++++-
>>>  7 files changed, 447 insertions(+), 1 deletion(-)
>>>  create mode 100644 arch/mips/include/asm/mach-jz4740/smp.h
>>>  create mode 100644 arch/mips/jz4740/smp-entry.S
>>>  create mode 100644 arch/mips/jz4740/smp.c
>>> 
>>> diff --git a/arch/mips/include/asm/mach-jz4740/smp.h 
>>> \x7f\x7fb/arch/mips/include/asm/mach-jz4740/smp.h
>>> new file mode 100644
>>> index 00000000..86f660f
>>> --- /dev/null
>>> +++ b/arch/mips/include/asm/mach-jz4740/smp.h
>>> @@ -0,0 +1,87 @@
>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */
>>> +/*
>>> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
>>> + *  JZ4780 SMP definitions
>>> + */
>>> +
>>> +#ifndef __MIPS_ASM_MACH_JZ4740_SMP_H__
>>> +#define __MIPS_ASM_MACH_JZ4740_SMP_H__
>>> +
>>> +#define read_c0_corectrl()        __read_32bit_c0_register($12, 2)
>>> +#define write_c0_corectrl(val) __write_32bit_c0_register($12, 2, 
>>> val)
>>> +
>>> +#define read_c0_corestatus() __read_32bit_c0_register($12, 3)
>>> +#define write_c0_corestatus(val) __write_32bit_c0_register($12, 3, 
>>> val)
>>> +
>>> +#define read_c0_reim()            __read_32bit_c0_register($12, 4)
>>> +#define write_c0_reim(val) __write_32bit_c0_register($12, 4, val)
>>> +
>>> +#define read_c0_mailbox0()        __read_32bit_c0_register($20, 0)
>>> +#define write_c0_mailbox0(val) __write_32bit_c0_register($20, 0, 
>>> val)
>>> +
>>> +#define read_c0_mailbox1()        __read_32bit_c0_register($20, 1)
>>> +#define write_c0_mailbox1(val) __write_32bit_c0_register($20, 1, 
>>> val)
>>> +
>>> +#define read_c0_mailbox2()        __read_32bit_c0_register($20, 2)
>>> +#define write_c0_mailbox2(val) __write_32bit_c0_register($20, 2, 
>>> val)
>>> +
>>> +#define read_c0_mailbox3()        __read_32bit_c0_register($20, 3)
>>> +#define write_c0_mailbox3(val) __write_32bit_c0_register($20, 3, 
>>> val)
>>> +
>>> +#define smp_clr_pending(mask) do {        \
>>> +        unsigned int stat;        \
>>> +        stat = read_c0_corestatus();    \
>>> +        stat &= ~((mask) & 0xff);    \
>>> +        write_c0_corestatus(stat);    \
>>> +    } while (0)
>>> +
>>> +/*
>>> + * Core Control register
>>> + */
>>> +#define CORECTRL_SLEEP1M_SHIFT    17
>>> +#define CORECTRL_SLEEP1M    (_ULCAST_(0x1) << 
>>> CORECTRL_SLEEP1M_SHIFT)
>>> +#define CORECTRL_SLEEP0M_SHIFT    16
>>> +#define CORECTRL_SLEEP0M    (_ULCAST_(0x1) << 
>>> CORECTRL_SLEEP0M_SHIFT)
>>> +#define CORECTRL_RPC1_SHIFT    9
>>> +#define CORECTRL_RPC1        (_ULCAST_(0x1) << CORECTRL_RPC1_SHIFT)
>>> +#define CORECTRL_RPC0_SHIFT    8
>>> +#define CORECTRL_RPC0        (_ULCAST_(0x1) << CORECTRL_RPC0_SHIFT)
>>> +#define CORECTRL_SWRST1_SHIFT    1
>>> +#define CORECTRL_SWRST1        (_ULCAST_(0x1) << 
>>> CORECTRL_SWRST1_SHIFT)
>>> +#define CORECTRL_SWRST0_SHIFT    0
>>> +#define CORECTRL_SWRST0        (_ULCAST_(0x1) << 
>>> CORECTRL_SWRST0_SHIFT)
>>> +
>>> +/*
>>> + * Core Status register
>>> + */
>>> +#define CORESTATUS_SLEEP1_SHIFT    17
>>> +#define CORESTATUS_SLEEP1    (_ULCAST_(0x1) << 
>>> CORESTATUS_SLEEP1_SHIFT)
>>> +#define CORESTATUS_SLEEP0_SHIFT    16
>>> +#define CORESTATUS_SLEEP0    (_ULCAST_(0x1) << 
>>> CORESTATUS_SLEEP0_SHIFT)
>>> +#define CORESTATUS_IRQ1P_SHIFT    9
>>> +#define CORESTATUS_IRQ1P    (_ULCAST_(0x1) << 
>>> CORESTATUS_IRQ1P_SHIFT)
>>> +#define CORESTATUS_IRQ0P_SHIFT    8
>>> +#define CORESTATUS_IRQ0P    (_ULCAST_(0x1) << 
>>> CORESTATUS_IRQ8P_SHIFT)
>>> +#define CORESTATUS_MIRQ1P_SHIFT    1
>>> +#define CORESTATUS_MIRQ1P    (_ULCAST_(0x1) << 
>>> CORESTATUS_MIRQ1P_SHIFT)
>>> +#define CORESTATUS_MIRQ0P_SHIFT    0
>>> +#define CORESTATUS_MIRQ0P    (_ULCAST_(0x1) << 
>>> CORESTATUS_MIRQ0P_SHIFT)
>>> +
>>> +/*
>>> + * Reset Entry & IRQ Mask register
>>> + */
>>> +#define REIM_ENTRY_SHIFT    16
>>> +#define REIM_ENTRY        (_ULCAST_(0xffff) << REIM_ENTRY_SHIFT)
>>> +#define REIM_IRQ1M_SHIFT    9
>>> +#define REIM_IRQ1M        (_ULCAST_(0x1) << REIM_IRQ1M_SHIFT)
>>> +#define REIM_IRQ0M_SHIFT    8
>>> +#define REIM_IRQ0M        (_ULCAST_(0x1) << REIM_IRQ0M_SHIFT)
>>> +#define REIM_MBOXIRQ1M_SHIFT    1
>>> +#define REIM_MBOXIRQ1M        (_ULCAST_(0x1) << 
>>> REIM_MBOXIRQ1M_SHIFT)
>>> +#define REIM_MBOXIRQ0M_SHIFT    0
>>> +#define REIM_MBOXIRQ0M        (_ULCAST_(0x1) << 
>>> REIM_MBOXIRQ0M_SHIFT)
>>> +
>>> +extern void jz4780_smp_init(void);
>>> +extern void jz4780_secondary_cpu_entry(void);
>>> +
>>> +#endif /* __MIPS_ASM_MACH_JZ4740_SMP_H__ */
>>> diff --git a/arch/mips/jz4740/Kconfig b/arch/mips/jz4740/Kconfig
>>> index 412d2fa..2b88557 100644
>>> --- a/arch/mips/jz4740/Kconfig
>>> +++ b/arch/mips/jz4740/Kconfig
>>> @@ -34,9 +34,11 @@ config MACH_JZ4770
>>> 
>>>  config MACH_JZ4780
>>>      bool
>>> +    select GENERIC_CLOCKEVENTS_BROADCAST if SMP
>>>      select MIPS_CPU_SCACHE
>>>      select SYS_HAS_CPU_MIPS32_R2
>>>      select SYS_SUPPORTS_HIGHMEM
>>> +    select SYS_SUPPORTS_SMP
>>> 
>>>  config MACH_X1000
>>>      bool
>>> diff --git a/arch/mips/jz4740/Makefile b/arch/mips/jz4740/Makefile
>>> index 6de14c0..0a0f024 100644
>>> --- a/arch/mips/jz4740/Makefile
>>> +++ b/arch/mips/jz4740/Makefile
>>> @@ -12,3 +12,8 @@ CFLAGS_setup.o = 
>>> -I$(src)/../../../scripts/dtc/libfdt
>>>  # PM support
>>> 
>>>  obj-$(CONFIG_PM) += pm.o
>>> +
>>> +# SMP support
>>> +
>>> +obj-$(CONFIG_SMP) += smp.o
>>> +obj-$(CONFIG_SMP) += smp-entry.o
>>> diff --git a/arch/mips/jz4740/prom.c b/arch/mips/jz4740/prom.c
>>> index ff4555c..4acf5c2c 100644
>>> --- a/arch/mips/jz4740/prom.c
>>> +++ b/arch/mips/jz4740/prom.c
>>> @@ -8,10 +8,14 @@
>>> 
>>>  #include <asm/bootinfo.h>
>>>  #include <asm/fw/fw.h>
>>> +#include <asm/mach-jz4740/smp.h>
>>> 
>>>  void __init prom_init(void)
>>>  {
>>>      fw_init_cmdline();
>>> +
>>> +    if (IS_ENABLED(CONFIG_SMP))
>>> +        jz4780_smp_init();
>>>  }
>>> 
>>>  void __init prom_free_prom_memory(void)
>>> diff --git a/arch/mips/jz4740/smp-entry.S 
>>> b/arch/mips/jz4740/smp-entry.S
>>> new file mode 100644
>>> index 00000000..20049a3
>>> --- /dev/null
>>> +++ b/arch/mips/jz4740/smp-entry.S
>>> @@ -0,0 +1,57 @@
>>> +/* SPDX-License-Identifier: GPL-2.0-or-later */
>>> +/*
>>> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
>>> + *  JZ4780 SMP entry point
>>> + */
>>> +
>>> +#include <asm/addrspace.h>
>>> +#include <asm/asm.h>
>>> +#include <asm/asmmacro.h>
>>> +#include <asm/cacheops.h>
>>> +#include <asm/mipsregs.h>
>>> +
>>> +#define CACHE_SIZE (32 * 1024)
>>> +#define CACHE_LINESIZE 32
>>> +
>>> +.extern jz4780_cpu_entry_sp
>>> +.extern jz4780_cpu_entry_gp
>>> +
>>> +.section .text.smp-entry
>>> +.balign 0x10000
>>> +.set noreorder
>>> +LEAF(jz4780_secondary_cpu_entry)
>>> +    mtc0    zero, CP0_CAUSE
>>> +
>>> +    li    t0, ST0_CU0
>>> +    mtc0    t0, CP0_STATUS
>>> +
>>> +    /* cache setup */
>>> +    li    t0, KSEG0
>>> +    ori    t1, t0, CACHE_SIZE
>>> +    mtc0    zero, CP0_TAGLO, 0
>>> +1:    cache    Index_Store_Tag_I, 0(t0)
>>> +    cache    Index_Store_Tag_D, 0(t0)
>>> +    bne    t0, t1, 1b
>>> +     addiu    t0, t0, CACHE_LINESIZE
>>> +
>>> +    /* kseg0 cache attribute */
>>> +    mfc0    t0, CP0_CONFIG, 0
>>> +    ori    t0, t0, CONF_CM_CACHABLE_NONCOHERENT
>>> +    mtc0    t0, CP0_CONFIG, 0
>>> +
>>> +    /* pagemask */
>>> +    mtc0    zero, CP0_PAGEMASK, 0
>>> +
>>> +    /* retrieve sp */
>>> +    la    t0, jz4780_cpu_entry_sp
>>> +    lw    sp, 0(t0)
>>> +
>>> +    /* retrieve gp */
>>> +    la    t0, jz4780_cpu_entry_gp
>>> +    lw    gp, 0(t0)
>>> +
>>> +    /* jump to the kernel in kseg0 */
>>> +    la    t0, smp_bootstrap
>>> +    jr    t0
>>> +     nop
>>> +    END(jz4780_secondary_cpu_entry)
>>> diff --git a/arch/mips/jz4740/smp.c b/arch/mips/jz4740/smp.c
>>> new file mode 100644
>>> index 00000000..d95d22a
>>> --- /dev/null
>>> +++ b/arch/mips/jz4740/smp.c
>>> @@ -0,0 +1,258 @@
>>> +// SPDX-License-Identifier: GPL-2.0
>>> +/*
>>> + *  Copyright (C) 2013, Paul Burton <paul.burton@imgtec.com>
>>> + *  JZ4780 SMP
>>> + */
>>> +
>>> +#include <linux/clk.h>
>>> +#include <linux/delay.h>
>>> +#include <linux/interrupt.h>
>>> +#include <linux/of.h>
>>> +#include <linux/sched.h>
>>> +#include <linux/sched/task_stack.h>
>>> +#include <linux/smp.h>
>>> +#include <linux/tick.h>
>>> +#include <asm/mach-jz4740/smp.h>
>>> +#include <asm/smp-ops.h>
>>> +
>>> +static struct clk *cpu_clock_gates[CONFIG_NR_CPUS] = { NULL };
>>> +
>>> +u32 jz4780_cpu_entry_sp;
>>> +u32 jz4780_cpu_entry_gp;
>>> +
>>> +static struct cpumask cpu_running;
>> 
>> This cpumask is written, but never read anywhere. Since it's static, 
>> I \x7fbelieve it's dead code.
>> 
> 
> Sure, I will remove it.
> 
>>> +
>>> +static DEFINE_SPINLOCK(smp_lock);
>>> +
>>> +static irqreturn_t mbox_handler(int irq, void *dev_id)
>>> +{
>>> +    int cpu = smp_processor_id();
>>> +    u32 action, status;
>>> +
>>> +    spin_lock(&smp_lock);
>>> +
>>> +    switch (cpu) {
>>> +    case 0:
>>> +        action = read_c0_mailbox0();
>>> +        write_c0_mailbox0(0);
>>> +        break;
>>> +    case 1:
>>> +        action = read_c0_mailbox1();
>>> +        write_c0_mailbox1(0);
>>> +        break;
>>> +    case 2:
>>> +        action = read_c0_mailbox2();
>>> +        write_c0_mailbox2(0);
>>> +        break;
>>> +    case 3:
>>> +        action = read_c0_mailbox3();
>>> +        write_c0_mailbox3(0);
>>> +        break;
>>> +    default:
>>> +        panic("unhandled cpu %d!", cpu);
>>> +    }
>>> +
>>> +    /* clear pending mailbox interrupt */
>>> +    status = read_c0_corestatus();
>>> +    status &= ~(CORESTATUS_MIRQ0P << cpu);
>>> +    write_c0_corestatus(status);
>>> +
>>> +    spin_unlock(&smp_lock);
>>> +
>>> +    if (action & SMP_RESCHEDULE_YOURSELF)
>>> +        scheduler_ipi();
>>> +    if (action & SMP_CALL_FUNCTION)
>>> +        generic_smp_call_function_interrupt();
>>> +
>>> +    return IRQ_HANDLED;
>>> +}
>>> +
>>> +static void jz4780_smp_setup(void)
>>> +{
>>> +    u32 addr, reim;
>>> +    int cpu;
>>> +
>>> +    reim = read_c0_reim();
>>> +
>>> +    for (cpu = 0; cpu < NR_CPUS; cpu++) {
>>> +        __cpu_number_map[cpu] = cpu;
>>> +        __cpu_logical_map[cpu] = cpu;
>>> +        set_cpu_possible(cpu, true);
>> 
>> I assume if you do that, you will have num_possible_cpus() == 
>> NR_CPUS, \x7fwhich is not what you want.
>> 
>> Correct me if I'm wrong, but I think you would need to call 
>> \x7fset_cpu_possible() for each CPU node found.
>> 
> 
> Yes, the current way is indeed a little problem, it will cause 
> num_possible_cpus() == NR_CPUS, I will try to find a better way.

You can do:

for_each_of_cpu_node(cpu_node) {
       cpu = of_cpu_node_to_id(cpu_node);
       __cpu_number_map[cpu] = cpu;
       __cpu_logical_map[cpu] = cpu;
       set_cpu_possible(cpu, true);
}


>>> +    }
>>> +
>>> +    /* mask mailbox interrupts for this core */
>>> +    reim &= ~REIM_MBOXIRQ0M;
>>> +    write_c0_reim(reim);
>>> +
>>> +    /* clear mailboxes & pending mailbox IRQs */
>>> +    write_c0_mailbox0(0);
>>> +    write_c0_mailbox1(0);
>> 
>> Write mailbox2/3 too.
>> 
> 
> Although the XBurst1 architecture can have up to four cores, but 
> JZ4780 only has two. If we need to write all four mailboxes here, do 
> we need change the function name to "xburst1_smp_setup" or other 
> similar names? This seems more appropriate.

Yes, these functions are not jz4780-specific, you can rename them all.

>>> +    write_c0_corestatus(0);
>>> +
>>> +    /* set reset entry point */
>>> +    addr = KSEG1ADDR((u32)&jz4780_secondary_cpu_entry);
>>> +    WARN_ON(addr & ~REIM_ENTRY);
>>> +    reim &= ~REIM_ENTRY;
>>> +    reim |= addr & REIM_ENTRY;
>>> +
>>> +    /* unmask mailbox interrupts for this core */
>>> +    reim |= REIM_MBOXIRQ0M;
>>> +    write_c0_reim(reim);
>>> +    set_c0_status(STATUSF_IP3);
>>> +    irq_enable_hazard();
>>> +
>>> +    cpumask_set_cpu(cpu, &cpu_running);
>>> +}
>>> +
>>> +static void jz4780_smp_prepare_cpus(unsigned int max_cpus)
>>> +{
>>> +    struct device_node *cpu_node;
>>> +    unsigned cpu, ctrl;
>>> +    int err;
>>> +
>>> +    /* setup the mailbox IRQ */
>>> +    err = request_irq(MIPS_CPU_IRQ_BASE + 3, mbox_handler,
>>> +            IRQF_PERCPU | IRQF_NO_THREAD, "core mailbox", NULL);
>> 
>> Please don't hardcode the IRQ number. Instead, it should be read 
>> from \x7fdevicetree, maybe from the 'cpus' node (not sure).
>> 
> 
> OK, I'll try to figure it out.
> 
>>> +    if (err)
>>> +        pr_err("request_irq() on core mailbox failed\n");
>>> +
>>> +    ctrl = read_c0_corectrl();
>>> +
>>> +    for_each_of_cpu_node(cpu_node) {
>>> +        cpu = of_cpu_node_to_id(cpu_node);
>>> +        if (cpu < 0) {
>>> +            pr_err("Failed to read index of %s\n",
>>> +                   cpu_node->full_name);
>>> +            continue;
>>> +        }
>>> +
>>> +        /* use reset entry point from REIM register */
>>> +        ctrl |= CORECTRL_RPC0 << cpu;
>>> +
>>> +        cpu_clock_gates[cpu] = of_clk_get(cpu_node, 0);
>>> +        if (IS_ERR(cpu_clock_gates[cpu])) {
>>> +            cpu_clock_gates[cpu] = NULL;
>>> +            continue;
>>> +        }
>>> +
>>> +        err = clk_prepare(cpu_clock_gates[cpu]);
>>> +        if (err)
>>> +            pr_err("Failed to prepare CPU clock gate\n");
>> 
>> I'd suggest to call clk_prepare() in jz4780_boot_secondary(), since 
>> \x7fyou can't handle errors here. That would also avoid the static 
>> \x7fcpu_clock_gates[] array which can grow quite big since its size is 
>> \x7fgiven by NR_CPUS.
>> 
> 
> Sure, I will move it to jz4780_boot_secondary().
> 
>>> +    }
>>> +
>>> +    write_c0_corectrl(ctrl);
>>> +}
>>> +
>>> +static int jz4780_boot_secondary(int cpu, struct task_struct *idle)
>>> +{
>>> +    unsigned long flags;
>>> +    u32 ctrl;
>>> +
>>> +    spin_lock_irqsave(&smp_lock, flags);
>>> +
>>> +    /* ensure the core is in reset */
>>> +    ctrl = read_c0_corectrl();
>>> +    ctrl |= CORECTRL_SWRST0 << cpu;
>>> +    write_c0_corectrl(ctrl);
>>> +
>>> +    /* ungate core clock */
>>> +    if (cpu_clock_gates[cpu])
>>> +        clk_enable(cpu_clock_gates[cpu]);
>> 
>> You should check the return value of clk_enable().
>> 
>> +        break;
> 
> Sure.
> 
>>> +
>>> +    /* set entry sp/gp register values */
>>> +    jz4780_cpu_entry_sp = __KSTK_TOS(idle);
>>> +    jz4780_cpu_entry_gp = (u32)task_thread_info(idle);
>>> +    smp_wmb();
>>> +
>>> +    /* take the core out of reset */
>>> +    ctrl &= ~(CORECTRL_SWRST0 << cpu);
>>> +    write_c0_corectrl(ctrl);
>>> +
>>> +    cpumask_set_cpu(cpu, &cpu_running);
>>> +
>>> +    spin_unlock_irqrestore(&smp_lock, flags);
>>> +
>>> +    return 0;
>>> +}
>>> +
>>> +static void jz4780_init_secondary(void)
>>> +{
>>> +}
>>> +
>>> +static void jz4780_smp_finish(void)
>>> +{
>>> +    u32 reim;
>>> +
>>> +    spin_lock(&smp_lock);
>>> +
>>> +    /* unmask mailbox interrupts for this core */
>>> +    reim = read_c0_reim();
>>> +    reim |= REIM_MBOXIRQ0M << smp_processor_id();
>>> +    write_c0_reim(reim);
>>> +
>>> +    spin_unlock(&smp_lock);
>>> +
>>> +    /* unmask interrupts for this core */
>>> +    change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP2 |
>>> +             STATUSF_IP1 | STATUSF_IP0);
>>> +    irq_enable_hazard();
>>> +
>>> +    /* force broadcast timer */
>>> +    tick_broadcast_force();
>>> +}
>>> +
>>> +static void jz4780_send_ipi_single_locked(int cpu, unsigned int 
>>> action)
>>> +{
>>> +    u32 mbox;
>>> +
>>> +    switch (cpu) {
>>> +    case 0:
>>> +        mbox = read_c0_mailbox0();
>>> +        write_c0_mailbox0(mbox | action);
>>> +        break;
>>> +    case 1:
>>> +        mbox = read_c0_mailbox1();
>>> +        write_c0_mailbox1(mbox | action);
>> 
>> Handle mailboxes 2/3 too here.
>> 
> 
> Same to the above, do we need to change the function to a more 
> appropriate name?
> 
>>> +    default:
>>> +        panic("unhandled cpu %d!", cpu);
>>> +    }
>>> +}
>>> +
>>> +static void jz4780_send_ipi_single(int cpu, unsigned int action)
>>> +{
>>> +    unsigned long flags;
>>> +
>>> +    spin_lock_irqsave(&smp_lock, flags);
>>> +    jz4780_send_ipi_single_locked(cpu, action);
>>> +    spin_unlock_irqrestore(&smp_lock, flags);
>>> +}
>>> +
>>> +static void jz4780_send_ipi_mask(const struct cpumask *mask,
>>> +                 unsigned int action)
>>> +{
>>> +    unsigned long flags;
>>> +    int cpu;
>>> +
>>> +    spin_lock_irqsave(&smp_lock, flags);
>>> +
>>> +    for_each_cpu(cpu, mask)
>>> +        jz4780_send_ipi_single_locked(cpu, action);
>>> +
>>> +    spin_unlock_irqrestore(&smp_lock, flags);
>>> +}
>>> +
>>> +static struct plat_smp_ops jz4780_smp_ops = {
>>> +    .send_ipi_single = jz4780_send_ipi_single,
>>> +    .send_ipi_mask = jz4780_send_ipi_mask,
>>> +    .init_secondary = jz4780_init_secondary,
>>> +    .smp_finish = jz4780_smp_finish,
>>> +    .boot_secondary = jz4780_boot_secondary,
>>> +    .smp_setup = jz4780_smp_setup,
>>> +    .prepare_cpus = jz4780_smp_prepare_cpus,
>>> +};
>>> +
>>> +void __init jz4780_smp_init(void)
>>> +{
>>> +    register_smp_ops(&jz4780_smp_ops);
>>> +}
>>> diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c
>>> index 37f8e78..d33f2d4 100644
>>> --- a/arch/mips/kernel/idle.c
>>> +++ b/arch/mips/kernel/idle.c
>>> @@ -18,6 +18,7 @@
>>>  #include <asm/cpu-type.h>
>>>  #include <asm/idle.h>
>>>  #include <asm/mipsregs.h>
>>> +#include <asm/r4kcache.h>
>>> 
>>>  /*
>>>   * Not all of the MIPS CPUs have the "wait" instruction available. 
>>> \x7f\x7fMoreover,
>>> @@ -88,6 +89,34 @@ static void __cpuidle rm7k_wait_irqoff(void)
>>>  }
>>> 
>>>  /*
>>> + * The Ingenic jz4780 SMP variant has to write back dirty cache 
>>> \x7f\x7flines before
>>> + * executing wait. The CPU & cache clock will be gated until we 
>>> \x7f\x7freturn from
>>> + * the wait, and if another core attempts to access data from our 
>>> \x7f\x7fdata cache
>>> + * during this time then it will lock up.
>>> + */
>>> +void jz4780_smp_wait_irqoff(void)
>>> +{
>>> +    unsigned long pending = read_c0_cause() & read_c0_status() & 
>>> \x7f\x7fCAUSEF_IP;
>>> +
>>> +    /*
>>> +     * Going to idle has a significant overhead due to the cache 
>>> \x7f\x7fflush so
>>> +     * try to avoid it if we'll immediately be woken again due to 
>>> an \x7f\x7fIRQ.
>>> +     */
>> 
>> You could add a fast path here where you just call r4k_wait() if 
>> \x7fnum_online_cpus() < 2.
>> 
> 
> Please correct me if I'm wrong, if we add it here, when the number of 
> CPU cores is greater than 1 (which should be the case on most 
> occasions), each call to "jz4780_smp_wait_irqoff" will generate 
> additional overhead (judging the number of CPUs), is it better to 
> change "if (IS_ENABLED(CONFIG_SMP))" in "case CPU_XBURST" below to 
> "if (IS_ENABLED(CONFIG_SMP) && (num_possible_cpus() > 1))"?

Is the number of possible CPUs already known when cpu_wait is set?

Cheers,
-Paul

> Thanks and best regards!
> 
>> -Paul
>> 
>>> +    if (!need_resched() && !pending) {
>>> +        r4k_blast_dcache();
>>> +
>>> +        __asm__(
>>> +        "    .set push    \n"
>>> +        "    .set mips3    \n"
>>> +        "    sync        \n"
>>> +        "    wait        \n"
>>> +        "    .set pop    \n");
>>> +    }
>>> +
>>> +    local_irq_enable();
>>> +}
>>> +
>>> +/*
>>>   * Au1 'wait' is only useful when the 32kHz counter is used as 
>>> timer,
>>>   * since coreclock (and the cp0 counter) stops upon executing it. 
>>> \x7f\x7fOnly an
>>>   * interrupt can wake it, so they must be enabled before entering 
>>> \x7f\x7fidle modes.
>>> @@ -172,7 +201,6 @@ void __init check_wait(void)
>>>      case CPU_CAVIUM_OCTEON_PLUS:
>>>      case CPU_CAVIUM_OCTEON2:
>>>      case CPU_CAVIUM_OCTEON3:
>>> -    case CPU_XBURST:
>>>      case CPU_LOONGSON32:
>>>      case CPU_XLR:
>>>      case CPU_XLP:
>>> @@ -246,6 +274,11 @@ void __init check_wait(void)
>>>             cpu_wait = r4k_wait;
>>>           */
>>>          break;
>>> +    case CPU_XBURST:
>>> +        if (IS_ENABLED(CONFIG_SMP))
>>> +            cpu_wait = jz4780_smp_wait_irqoff;
>>> +        else
>>> +            cpu_wait = r4k_wait;
>>>      default:
>>>          break;
>>>      }
>>> --
>>> 2.7.4
>>> 
>> 
> 

Re: [PATCH v8 1/6] MIPS: JZ4780: Introduce SMP support.

[Jiaxun Yang]

于 2020年5月20日 GMT+08:00 下午7:33:22, Paul Cercueil <paul@crapouillou.net> 写到:
>> 
>> Yes, the current way is indeed a little problem, it will cause 
>> num_possible_cpus() == NR_CPUS, I will try to find a better way.
>
>You can do:
>
>for_each_of_cpu_node(cpu_node) {
>       cpu = of_cpu_node_to_id(cpu_node);
>       __cpu_number_map[cpu] = cpu;
>       __cpu_logical_map[cpu] = cpu;
>       set_cpu_possible(cpu, true);
>}
>

FYI: There is a abandoned DeviceTree[1], parser. You can take it.

I'm going to submit this topology clean-up for next release cycle
but you can pick it for now.

[...]

[1]: https://lkml.org/lkml/2020/4/11/1088
-- 
Jiaxun Yang

Re: [PATCH v8 4/6] dt-bindings: MIPS: Document Ingenic SoCs binding.

[Rob Herring]

On Tue, May 19, 2020 at 10:35:21PM +0800, 周琰杰 (Zhou Yanjie) wrote:
> Document the available properties for the SoC root node and the
> CPU nodes of the devicetree for the Ingenic XBurst SoCs.
> 
> Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
> Tested-by: Paul Boddie <paul@boddie.org.uk>
> Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
> ---
> 
> Notes:
>     v1->v2:
>     Change the two Document from txt to yaml.
>     
>     v2->v3:
>     Fix formatting errors.
>     
>     v3->v4:
>     Fix bugs in the two yaml files.
>     
>     v4->v5:
>     No change.
>     
>     v5->v6:
>     Rewrite the two yaml files.
>     
>     v6->v7:
>     1.Update compatible strings in "ingenic,cpu.yaml".
>     2.Fix formatting errors, and enum for compatible strings.
>     3.Remove unnecessary "ingenic,soc.yaml".
>     
>     v7->v8:
>     No change.
> 
>  .../bindings/mips/ingenic/ingenic,cpu.yaml         | 57 ++++++++++++++++++++++
>  1 file changed, 57 insertions(+)
>  create mode 100644 Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml
> 
> diff --git a/Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml b/Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml
> new file mode 100644
> index 00000000..afb0207
> --- /dev/null
> +++ b/Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml
> @@ -0,0 +1,57 @@
> +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
> +%YAML 1.2
> +---
> +$id: http://devicetree.org/schemas/mips/ingenic/ingenic,cpu.yaml#
> +$schema: http://devicetree.org/meta-schemas/core.yaml#
> +
> +title: Bindings for Ingenic XBurst family CPUs
> +
> +maintainers:
> +  - 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
> +
> +description:
> +  Ingenic XBurst family CPUs shall have the following properties.
> +
> +properties:
> +  compatible:
> +    oneOf:
> +
> +      - description: Ingenic XBurst®1 CPU Cores
> +        items:

This is a single compatible string, right? If so, drop items. 

> +          enum:
> +            - ingenic,xburst-mxu1.0
> +            - ingenic,xburst-fpu1.0-mxu1.1
> +            - ingenic,xburst-fpu2.0-mxu2.0
> +
> +      - description: Ingenic XBurst®2 CPU Cores
> +        items:
> +          enum:
> +            - ingenic,xburst2-fpu2.1-mxu2.1-smt

Just: const: ingenic,xburst2-fpu2.1-mxu2.1-smt

Continuing to append CPU features isn't going to scale well. Does 
'xburst2' imply certain features? If so, not really any need to have 
them be explicit.

> +
> +  reg:
> +    maxItems: 1
> +
> +required:
> +  - device_type
> +  - compatible
> +  - reg
> +
> +examples:
> +  - |
> +    cpus {
> +    	#address-cells = <1>;
> +    	#size-cells = <0>;
> +
> +    	cpu0: cpu@0 {
> +    		device_type = "cpu";
> +    		compatible = "ingenic,xburst-fpu1.0-mxu1.1";
> +    		reg = <0>;
> +    	};
> +
> +    	cpu1: cpu@1 {
> +    		device_type = "cpu";
> +    		compatible = "ingenic,xburst-fpu1.0-mxu1.1";
> +    		reg = <1>;
> +    	};
> +    };
> +...
> -- 
> 2.7.4
> 

Re: [PATCH v8 4/6] dt-bindings: MIPS: Document Ingenic SoCs binding.

[Zhou Yanjie]

在 2020/5/27 上午3:29, Rob Herring 写道:
> On Tue, May 19, 2020 at 10:35:21PM +0800, 周琰杰 (Zhou Yanjie) wrote:
>> Document the available properties for the SoC root node and the
>> CPU nodes of the devicetree for the Ingenic XBurst SoCs.
>>
>> Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
>> Tested-by: Paul Boddie <paul@boddie.org.uk>
>> Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
>> ---
>>
>> Notes:
>>      v1->v2:
>>      Change the two Document from txt to yaml.
>>      
>>      v2->v3:
>>      Fix formatting errors.
>>      
>>      v3->v4:
>>      Fix bugs in the two yaml files.
>>      
>>      v4->v5:
>>      No change.
>>      
>>      v5->v6:
>>      Rewrite the two yaml files.
>>      
>>      v6->v7:
>>      1.Update compatible strings in "ingenic,cpu.yaml".
>>      2.Fix formatting errors, and enum for compatible strings.
>>      3.Remove unnecessary "ingenic,soc.yaml".
>>      
>>      v7->v8:
>>      No change.
>>
>>   .../bindings/mips/ingenic/ingenic,cpu.yaml         | 57 ++++++++++++++++++++++
>>   1 file changed, 57 insertions(+)
>>   create mode 100644 Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml
>>
>> diff --git a/Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml b/Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml
>> new file mode 100644
>> index 00000000..afb0207
>> --- /dev/null
>> +++ b/Documentation/devicetree/bindings/mips/ingenic/ingenic,cpu.yaml
>> @@ -0,0 +1,57 @@
>> +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
>> +%YAML 1.2
>> +---
>> +$id: http://devicetree.org/schemas/mips/ingenic/ingenic,cpu.yaml#
>> +$schema: http://devicetree.org/meta-schemas/core.yaml#
>> +
>> +title: Bindings for Ingenic XBurst family CPUs
>> +
>> +maintainers:
>> +  - 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
>> +
>> +description:
>> +  Ingenic XBurst family CPUs shall have the following properties.
>> +
>> +properties:
>> +  compatible:
>> +    oneOf:
>> +
>> +      - description: Ingenic XBurst®1 CPU Cores
>> +        items:
> This is a single compatible string, right? If so, drop items.


Sure, I'll drop it. And because the SMP driver has some other work that 
can't be completed in a short time, so I will send this patch separately.


>> +          enum:
>> +            - ingenic,xburst-mxu1.0
>> +            - ingenic,xburst-fpu1.0-mxu1.1
>> +            - ingenic,xburst-fpu2.0-mxu2.0
>> +
>> +      - description: Ingenic XBurst®2 CPU Cores
>> +        items:
>> +          enum:
>> +            - ingenic,xburst2-fpu2.1-mxu2.1-smt
> Just: const: ingenic,xburst2-fpu2.1-mxu2.1-smt
>
> Continuing to append CPU features isn't going to scale well. Does
> 'xburst2' imply certain features? If so, not really any need to have
> them be explicit.


XBurst (XBurst1) is the first generation CPU core of Ingenic, its basic 
property is single-issue in-order execution, XBurst2 is the second 
generation CPU core of Ingenic, its basic property is daul-issue limited 
out-of-order execution, and both CPU cores can cooperate with some 
extended propeties,  such as different versions of FPU, different 
versions of MXU instruction set, with or without simultaneous 
multi-threading.

Currently there is only one processor entity based on XBurst2 is 
produced, so there is  only one string for now.


Thanks and best regards!


>> +
>> +  reg:
>> +    maxItems: 1
>> +
>> +required:
>> +  - device_type
>> +  - compatible
>> +  - reg
>> +
>> +examples:
>> +  - |
>> +    cpus {
>> +    	#address-cells = <1>;
>> +    	#size-cells = <0>;
>> +
>> +    	cpu0: cpu@0 {
>> +    		device_type = "cpu";
>> +    		compatible = "ingenic,xburst-fpu1.0-mxu1.1";
>> +    		reg = <0>;
>> +    	};
>> +
>> +    	cpu1: cpu@1 {
>> +    		device_type = "cpu";
>> +    		compatible = "ingenic,xburst-fpu1.0-mxu1.1";
>> +    		reg = <1>;
>> +    	};
>> +    };
>> +...
>> -- 
>> 2.7.4
>>

Re: [PATCH v8 5/6] MIPS: Ingenic: Add 'cpus' node for Ingenic SoCs.

[H. Nikolaus Schaller]

Hi Zhou Yanjie,
what is the status of this series? It does not seem to have arrived in linux-next for v5.10-rc1.

BR and thanks,
Nikolaus


> Am 19.05.2020 um 16:35 schrieb 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>:
> 
> Add 'cpus' node to the jz4740.dtsi, jz4770.dtsi, jz4780.dtsi
> and x1000.dtsi files.
> 
> Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
> Tested-by: Paul Boddie <paul@boddie.org.uk>
> Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
> ---
> 
> Notes:
>    v1->v2:
>    No change.
> 
>    v2->v3:
>    No change.
> 
>    v3->v4:
>    Rebase on top of kernel 5.6-rc1.
> 
>    v4->v5:
>    No change.
> 
>    v5->v6:
>    No change.
> 
>    v6->v7:
>    Update compatible strings.
> 
>    v7->v8:
>    No change.
> 
> arch/mips/boot/dts/ingenic/jz4740.dtsi | 14 ++++++++++++++
> arch/mips/boot/dts/ingenic/jz4770.dtsi | 15 ++++++++++++++-
> arch/mips/boot/dts/ingenic/jz4780.dtsi | 23 +++++++++++++++++++++++
> arch/mips/boot/dts/ingenic/x1000.dtsi  | 14 ++++++++++++++
> 4 files changed, 65 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/mips/boot/dts/ingenic/jz4740.dtsi b/arch/mips/boot/dts/ingenic/jz4740.dtsi
> index a3301ba..1f2f896 100644
> --- a/arch/mips/boot/dts/ingenic/jz4740.dtsi
> +++ b/arch/mips/boot/dts/ingenic/jz4740.dtsi
> @@ -7,6 +7,20 @@
> 	#size-cells = <1>;
> 	compatible = "ingenic,jz4740";
> 
> +	cpus {
> +		#address-cells = <1>;
> +		#size-cells = <0>;
> +
> +		cpu0: cpu@0 {
> +			device_type = "cpu";
> +			compatible = "ingenic,xburst-mxu1.0";
> +			reg = <0>;
> +
> +			clocks = <&cgu JZ4740_CLK_CCLK>;
> +			clock-names = "cpu";
> +		};
> +	};
> +
> 	cpuintc: interrupt-controller {
> 		#address-cells = <0>;
> 		#interrupt-cells = <1>;
> diff --git a/arch/mips/boot/dts/ingenic/jz4770.dtsi b/arch/mips/boot/dts/ingenic/jz4770.dtsi
> index 0bfb9ed..12c7101 100644
> --- a/arch/mips/boot/dts/ingenic/jz4770.dtsi
> +++ b/arch/mips/boot/dts/ingenic/jz4770.dtsi
> @@ -1,5 +1,4 @@
> // SPDX-License-Identifier: GPL-2.0
> -
> #include <dt-bindings/clock/jz4770-cgu.h>
> 
> / {
> @@ -7,6 +6,20 @@
> 	#size-cells = <1>;
> 	compatible = "ingenic,jz4770";
> 
> +	cpus {
> +		#address-cells = <1>;
> +		#size-cells = <0>;
> +
> +		cpu0: cpu@0 {
> +			device_type = "cpu";
> +			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
> +			reg = <0>;
> +
> +			clocks = <&cgu JZ4770_CLK_CCLK>;
> +			clock-names = "cpu";
> +		};
> +	};
> +
> 	cpuintc: interrupt-controller {
> 		#address-cells = <0>;
> 		#interrupt-cells = <1>;
> diff --git a/arch/mips/boot/dts/ingenic/jz4780.dtsi b/arch/mips/boot/dts/ingenic/jz4780.dtsi
> index bb89653..03aeeff 100644
> --- a/arch/mips/boot/dts/ingenic/jz4780.dtsi
> +++ b/arch/mips/boot/dts/ingenic/jz4780.dtsi
> @@ -8,6 +8,29 @@
> 	#size-cells = <1>;
> 	compatible = "ingenic,jz4780";
> 
> +	cpus {
> +		#address-cells = <1>;
> +		#size-cells = <0>;
> +
> +		cpu0: cpu@0 {
> +			device_type = "cpu";
> +			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
> +			reg = <0>;
> +
> +			clocks = <&cgu JZ4780_CLK_CPU>;
> +			clock-names = "cpu";
> +		};
> +
> +		cpu1: cpu@1 {
> +			device_type = "cpu";
> +			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
> +			reg = <1>;
> +
> +			clocks = <&cgu JZ4780_CLK_CORE1>;
> +			clock-names = "cpu";
> +		};
> +	};
> +
> 	cpuintc: interrupt-controller {
> 		#address-cells = <0>;
> 		#interrupt-cells = <1>;
> diff --git a/arch/mips/boot/dts/ingenic/x1000.dtsi b/arch/mips/boot/dts/ingenic/x1000.dtsi
> index 147f7d5..2205e1b 100644
> --- a/arch/mips/boot/dts/ingenic/x1000.dtsi
> +++ b/arch/mips/boot/dts/ingenic/x1000.dtsi
> @@ -8,6 +8,20 @@
> 	#size-cells = <1>;
> 	compatible = "ingenic,x1000", "ingenic,x1000e";
> 
> +	cpus {
> +		#address-cells = <1>;
> +		#size-cells = <0>;
> +
> +		cpu0: cpu@0 {
> +			device_type = "cpu";
> +			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
> +			reg = <0>;
> +
> +			clocks = <&cgu X1000_CLK_CPU>;
> +			clock-names = "cpu";
> +		};
> +	};
> +
> 	cpuintc: interrupt-controller {
> 		#address-cells = <0>;
> 		#interrupt-cells = <1>;
> -- 
> 2.7.4
> 

Re: [PATCH v8 5/6] MIPS: Ingenic: Add 'cpus' node for Ingenic SoCs.

[Zhou Yanjie]

Hi Nikolaus,

This series was temporarily shelved, because now it is necessary to add 
support for the new X2000 SoC, and it will continue to advance after all 
the functions related to X2000 are completed.

Thanks and best regards!

在 2020/9/10 下午3:52, H. Nikolaus Schaller 写道:
> Hi Zhou Yanjie,
> what is the status of this series? It does not seem to have arrived in linux-next for v5.10-rc1.
>
> BR and thanks,
> Nikolaus
>
>
>> Am 19.05.2020 um 16:35 schrieb 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>:
>>
>> Add 'cpus' node to the jz4740.dtsi, jz4770.dtsi, jz4780.dtsi
>> and x1000.dtsi files.
>>
>> Tested-by: H. Nikolaus Schaller <hns@goldelico.com>
>> Tested-by: Paul Boddie <paul@boddie.org.uk>
>> Signed-off-by: 周琰杰 (Zhou Yanjie) <zhouyanjie@wanyeetech.com>
>> ---
>>
>> Notes:
>>     v1->v2:
>>     No change.
>>
>>     v2->v3:
>>     No change.
>>
>>     v3->v4:
>>     Rebase on top of kernel 5.6-rc1.
>>
>>     v4->v5:
>>     No change.
>>
>>     v5->v6:
>>     No change.
>>
>>     v6->v7:
>>     Update compatible strings.
>>
>>     v7->v8:
>>     No change.
>>
>> arch/mips/boot/dts/ingenic/jz4740.dtsi | 14 ++++++++++++++
>> arch/mips/boot/dts/ingenic/jz4770.dtsi | 15 ++++++++++++++-
>> arch/mips/boot/dts/ingenic/jz4780.dtsi | 23 +++++++++++++++++++++++
>> arch/mips/boot/dts/ingenic/x1000.dtsi  | 14 ++++++++++++++
>> 4 files changed, 65 insertions(+), 1 deletion(-)
>>
>> diff --git a/arch/mips/boot/dts/ingenic/jz4740.dtsi b/arch/mips/boot/dts/ingenic/jz4740.dtsi
>> index a3301ba..1f2f896 100644
>> --- a/arch/mips/boot/dts/ingenic/jz4740.dtsi
>> +++ b/arch/mips/boot/dts/ingenic/jz4740.dtsi
>> @@ -7,6 +7,20 @@
>> 	#size-cells = <1>;
>> 	compatible = "ingenic,jz4740";
>>
>> +	cpus {
>> +		#address-cells = <1>;
>> +		#size-cells = <0>;
>> +
>> +		cpu0: cpu@0 {
>> +			device_type = "cpu";
>> +			compatible = "ingenic,xburst-mxu1.0";
>> +			reg = <0>;
>> +
>> +			clocks = <&cgu JZ4740_CLK_CCLK>;
>> +			clock-names = "cpu";
>> +		};
>> +	};
>> +
>> 	cpuintc: interrupt-controller {
>> 		#address-cells = <0>;
>> 		#interrupt-cells = <1>;
>> diff --git a/arch/mips/boot/dts/ingenic/jz4770.dtsi b/arch/mips/boot/dts/ingenic/jz4770.dtsi
>> index 0bfb9ed..12c7101 100644
>> --- a/arch/mips/boot/dts/ingenic/jz4770.dtsi
>> +++ b/arch/mips/boot/dts/ingenic/jz4770.dtsi
>> @@ -1,5 +1,4 @@
>> // SPDX-License-Identifier: GPL-2.0
>> -
>> #include <dt-bindings/clock/jz4770-cgu.h>
>>
>> / {
>> @@ -7,6 +6,20 @@
>> 	#size-cells = <1>;
>> 	compatible = "ingenic,jz4770";
>>
>> +	cpus {
>> +		#address-cells = <1>;
>> +		#size-cells = <0>;
>> +
>> +		cpu0: cpu@0 {
>> +			device_type = "cpu";
>> +			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
>> +			reg = <0>;
>> +
>> +			clocks = <&cgu JZ4770_CLK_CCLK>;
>> +			clock-names = "cpu";
>> +		};
>> +	};
>> +
>> 	cpuintc: interrupt-controller {
>> 		#address-cells = <0>;
>> 		#interrupt-cells = <1>;
>> diff --git a/arch/mips/boot/dts/ingenic/jz4780.dtsi b/arch/mips/boot/dts/ingenic/jz4780.dtsi
>> index bb89653..03aeeff 100644
>> --- a/arch/mips/boot/dts/ingenic/jz4780.dtsi
>> +++ b/arch/mips/boot/dts/ingenic/jz4780.dtsi
>> @@ -8,6 +8,29 @@
>> 	#size-cells = <1>;
>> 	compatible = "ingenic,jz4780";
>>
>> +	cpus {
>> +		#address-cells = <1>;
>> +		#size-cells = <0>;
>> +
>> +		cpu0: cpu@0 {
>> +			device_type = "cpu";
>> +			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
>> +			reg = <0>;
>> +
>> +			clocks = <&cgu JZ4780_CLK_CPU>;
>> +			clock-names = "cpu";
>> +		};
>> +
>> +		cpu1: cpu@1 {
>> +			device_type = "cpu";
>> +			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
>> +			reg = <1>;
>> +
>> +			clocks = <&cgu JZ4780_CLK_CORE1>;
>> +			clock-names = "cpu";
>> +		};
>> +	};
>> +
>> 	cpuintc: interrupt-controller {
>> 		#address-cells = <0>;
>> 		#interrupt-cells = <1>;
>> diff --git a/arch/mips/boot/dts/ingenic/x1000.dtsi b/arch/mips/boot/dts/ingenic/x1000.dtsi
>> index 147f7d5..2205e1b 100644
>> --- a/arch/mips/boot/dts/ingenic/x1000.dtsi
>> +++ b/arch/mips/boot/dts/ingenic/x1000.dtsi
>> @@ -8,6 +8,20 @@
>> 	#size-cells = <1>;
>> 	compatible = "ingenic,x1000", "ingenic,x1000e";
>>
>> +	cpus {
>> +		#address-cells = <1>;
>> +		#size-cells = <0>;
>> +
>> +		cpu0: cpu@0 {
>> +			device_type = "cpu";
>> +			compatible = "ingenic,xburst-fpu1.0-mxu1.1";
>> +			reg = <0>;
>> +
>> +			clocks = <&cgu X1000_CLK_CPU>;
>> +			clock-names = "cpu";
>> +		};
>> +	};
>> +
>> 	cpuintc: interrupt-controller {
>> 		#address-cells = <0>;
>> 		#interrupt-cells = <1>;
>> -- 
>> 2.7.4
>>