[PATCH v2 0/5] riscv: add PREEMPT_RT support

[PATCH v2 0/5] riscv: add PREEMPT_RT support

[Jisheng Zhang]

This series is to add PREEMPT_RT support to riscv:
patch1 adds the missing number of signal exits in vCPU stat
patch2 switches to the generic guest entry infrastructure
patch3 select HAVE_POSIX_CPU_TIMERS_TASK_WORK which is a requirement for
RT
patch4 adds lazy preempt support
patch5 allows to enable PREEMPT_RT

I assume patch1, patch2 and patch3 can be reviewed and merged for
riscv-next, patch4 and patch5 can be reviewed and maintained in rt tree,
and finally merged once the remaining patches in rt tree are all
mainlined.

Since v1:
  - send to related maillist, I press ENTER too quickly when sending v1
  - remove the signal_pending() handling because that's covered by
    generic guest entry infrastructure

Jisheng Zhang (5):
  RISC-V: KVM: Record number of signal exits as a vCPU stat
  RISC-V: KVM: Use generic guest entry infrastructure
  riscv: select HAVE_POSIX_CPU_TIMERS_TASK_WORK
  riscv: add lazy preempt support
  riscv: Allow to enable RT

 arch/riscv/Kconfig                   |  3 +++
 arch/riscv/include/asm/kvm_host.h    |  1 +
 arch/riscv/include/asm/thread_info.h |  7 +++++--
 arch/riscv/kernel/asm-offsets.c      |  1 +
 arch/riscv/kernel/entry.S            |  9 +++++++--
 arch/riscv/kvm/Kconfig               |  1 +
 arch/riscv/kvm/vcpu.c                | 18 +++++++-----------
 7 files changed, 25 insertions(+), 15 deletions(-)

-- 
2.34.1

[PATCH v2 1/5] RISC-V: KVM: Record number of signal exits as a vCPU stat

[Jisheng Zhang]

Record a statistic indicating the number of times a vCPU has exited
due to a pending signal.

Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
---
 arch/riscv/include/asm/kvm_host.h | 1 +
 arch/riscv/kvm/vcpu.c             | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h
index 60c517e4d576..dbbf43d52623 100644
--- a/arch/riscv/include/asm/kvm_host.h
+++ b/arch/riscv/include/asm/kvm_host.h
@@ -67,6 +67,7 @@ struct kvm_vcpu_stat {
 	u64 mmio_exit_kernel;
 	u64 csr_exit_user;
 	u64 csr_exit_kernel;
+	u64 signal_exits;
 	u64 exits;
 };
 
diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index d0f08d5b4282..3da459fedc28 100644
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -28,6 +28,7 @@ const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
 	STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
 	STATS_DESC_COUNTER(VCPU, csr_exit_user),
 	STATS_DESC_COUNTER(VCPU, csr_exit_kernel),
+	STATS_DESC_COUNTER(VCPU, signal_exits),
 	STATS_DESC_COUNTER(VCPU, exits)
 };
 
@@ -973,6 +974,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 		if (signal_pending(current)) {
 			ret = -EINTR;
 			run->exit_reason = KVM_EXIT_INTR;
+			++vcpu->stat.signal_exits;
 		}
 
 		/*
-- 
2.34.1

[PATCH v2 2/5] RISC-V: KVM: Use generic guest entry infrastructure

[Jisheng Zhang]

Use generic guest entry infrastructure to properly handle
TIF_NOTIFY_RESUME.

Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
---
 arch/riscv/kvm/Kconfig |  1 +
 arch/riscv/kvm/vcpu.c  | 18 ++++++------------
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig
index f5a342fa1b1d..f36a737d5f96 100644
--- a/arch/riscv/kvm/Kconfig
+++ b/arch/riscv/kvm/Kconfig
@@ -24,6 +24,7 @@ config KVM
 	select PREEMPT_NOTIFIERS
 	select KVM_MMIO
 	select KVM_GENERIC_DIRTYLOG_READ_PROTECT
+	select KVM_XFER_TO_GUEST_WORK
 	select HAVE_KVM_VCPU_ASYNC_IOCTL
 	select HAVE_KVM_EVENTFD
 	select SRCU
diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index 3da459fedc28..e3e6b8608288 100644
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/bitops.h>
+#include <linux/entry-kvm.h>
 #include <linux/errno.h>
 #include <linux/err.h>
 #include <linux/kdebug.h>
@@ -959,7 +960,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 	run->exit_reason = KVM_EXIT_UNKNOWN;
 	while (ret > 0) {
 		/* Check conditions before entering the guest */
-		cond_resched();
+		ret = xfer_to_guest_mode_handle_work(vcpu);
+		if (!ret)
+			ret = 1;
 
 		kvm_riscv_gstage_vmid_update(vcpu);
 
@@ -967,16 +970,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 
 		local_irq_disable();
 
-		/*
-		 * Exit if we have a signal pending so that we can deliver
-		 * the signal to user space.
-		 */
-		if (signal_pending(current)) {
-			ret = -EINTR;
-			run->exit_reason = KVM_EXIT_INTR;
-			++vcpu->stat.signal_exits;
-		}
-
 		/*
 		 * Ensure we set mode to IN_GUEST_MODE after we disable
 		 * interrupts and before the final VCPU requests check.
@@ -999,7 +992,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 
 		if (ret <= 0 ||
 		    kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||
-		    kvm_request_pending(vcpu)) {
+		    kvm_request_pending(vcpu) ||
+		    xfer_to_guest_mode_work_pending()) {
 			vcpu->mode = OUTSIDE_GUEST_MODE;
 			local_irq_enable();
 			kvm_vcpu_srcu_read_lock(vcpu);
-- 
2.34.1

[PATCH v2 3/5] riscv: select HAVE_POSIX_CPU_TIMERS_TASK_WORK

[Jisheng Zhang]

Move POSIX CPU timer expiry and signal delivery into task context to
allow PREEMPT_RT setups to coexist with KVM.

Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
---
 arch/riscv/Kconfig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 79e52441e18b..7a8134fd7ec9 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -104,6 +104,7 @@ config RISCV
 	select HAVE_PERF_EVENTS
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
+	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_FUNCTION_ARG_ACCESS_API
 	select HAVE_STACKPROTECTOR
-- 
2.34.1

[PATCH v2 4/5] riscv: add lazy preempt support

[Jisheng Zhang]

Implement the lazy preempt for riscv.

Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
---
 arch/riscv/Kconfig                   | 1 +
 arch/riscv/include/asm/thread_info.h | 7 +++++--
 arch/riscv/kernel/asm-offsets.c      | 1 +
 arch/riscv/kernel/entry.S            | 9 +++++++--
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 7a8134fd7ec9..9f2f1936b1b5 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -105,6 +105,7 @@ config RISCV
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
 	select HAVE_POSIX_CPU_TIMERS_TASK_WORK
+	select HAVE_PREEMPT_LAZY
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_FUNCTION_ARG_ACCESS_API
 	select HAVE_STACKPROTECTOR
diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
index 78933ac04995..471915b179a2 100644
--- a/arch/riscv/include/asm/thread_info.h
+++ b/arch/riscv/include/asm/thread_info.h
@@ -56,6 +56,7 @@
 struct thread_info {
 	unsigned long		flags;		/* low level flags */
 	int                     preempt_count;  /* 0=>preemptible, <0=>BUG */
+	int			preempt_lazy_count;  /* 0=>preemptible, <0=>BUG */
 	/*
 	 * These stack pointers are overwritten on every system call or
 	 * exception.  SP is also saved to the stack it can be recovered when
@@ -90,7 +91,7 @@ struct thread_info {
 #define TIF_NOTIFY_RESUME	1	/* callback before returning to user */
 #define TIF_SIGPENDING		2	/* signal pending */
 #define TIF_NEED_RESCHED	3	/* rescheduling necessary */
-#define TIF_RESTORE_SIGMASK	4	/* restore signal mask in do_signal() */
+#define TIF_NEED_RESCHED_LAZY	4	/* lazy rescheduling */
 #define TIF_MEMDIE		5	/* is terminating due to OOM killer */
 #define TIF_SYSCALL_TRACEPOINT  6       /* syscall tracepoint instrumentation */
 #define TIF_SYSCALL_AUDIT	7	/* syscall auditing */
@@ -98,6 +99,7 @@ struct thread_info {
 #define TIF_NOTIFY_SIGNAL	9	/* signal notifications exist */
 #define TIF_UPROBE		10	/* uprobe breakpoint or singlestep */
 #define TIF_32BIT		11	/* compat-mode 32bit process */
+#define TIF_RESTORE_SIGMASK	12	/* restore signal mask in do_signal() */
 
 #define _TIF_SYSCALL_TRACE	(1 << TIF_SYSCALL_TRACE)
 #define _TIF_NOTIFY_RESUME	(1 << TIF_NOTIFY_RESUME)
@@ -108,10 +110,11 @@ struct thread_info {
 #define _TIF_SECCOMP		(1 << TIF_SECCOMP)
 #define _TIF_NOTIFY_SIGNAL	(1 << TIF_NOTIFY_SIGNAL)
 #define _TIF_UPROBE		(1 << TIF_UPROBE)
+#define _TIF_NEED_RESCHED_LAZY	(1 << TIF_NEED_RESCHED_LAZY)
 
 #define _TIF_WORK_MASK \
 	(_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \
-	 _TIF_NOTIFY_SIGNAL | _TIF_UPROBE)
+	 _TIF_NEED_RESCHED_LAZY | _TIF_NOTIFY_SIGNAL | _TIF_UPROBE)
 
 #define _TIF_SYSCALL_WORK \
 	(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \
diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index df9444397908..e38e33822f72 100644
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -35,6 +35,7 @@ void asm_offsets(void)
 	OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]);
 	OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags);
 	OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
+	OFFSET(TASK_TI_PREEMPT_LAZY_COUNT, task_struct, thread_info.preempt_lazy_count);
 	OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
 	OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp);
 
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index b9eda3fcbd6d..595100a4c2c7 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -361,9 +361,14 @@ restore_all:
 resume_kernel:
 	REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
 	bnez s0, restore_all
-	REG_L s0, TASK_TI_FLAGS(tp)
-	andi s0, s0, _TIF_NEED_RESCHED
+	REG_L s1, TASK_TI_FLAGS(tp)
+	andi s0, s1, _TIF_NEED_RESCHED
+	bnez s0, 1f
+	REG_L s0, TASK_TI_PREEMPT_LAZY_COUNT(tp)
+	bnez s0, restore_all
+	andi s0, s1, _TIF_NEED_RESCHED_LAZY
 	beqz s0, restore_all
+1:
 	call preempt_schedule_irq
 	j restore_all
 #endif
-- 
2.34.1

[PATCH v2 5/5] riscv: Allow to enable RT

[Jisheng Zhang]

Allow to select RT.

Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
---
 arch/riscv/Kconfig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 9f2f1936b1b5..69cdcb3cf251 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -39,6 +39,7 @@ config RISCV
 	select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU
 	select ARCH_SUPPORTS_HUGETLBFS if MMU
 	select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU
+	select ARCH_SUPPORTS_RT
 	select ARCH_USE_MEMTEST
 	select ARCH_USE_QUEUED_RWLOCKS
 	select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU
-- 
2.34.1

Re: [PATCH v2 0/5] riscv: add PREEMPT_RT support

[Sebastian Andrzej Siewior]

On 2022-09-01 01:59:15 [+0800], Jisheng Zhang wrote:
> I assume patch1, patch2 and patch3 can be reviewed and merged for
> riscv-next, patch4 and patch5 can be reviewed and maintained in rt tree,
> and finally merged once the remaining patches in rt tree are all
> mainlined.

I would say so, yes.

What about JUMP_LABEL support? Do you halt all CPUs while patching the
code?

Sebastian

Re: [PATCH v2 0/5] riscv: add PREEMPT_RT support

[Jisheng Zhang]

On Thu, Sep 01, 2022 at 09:04:05AM +0200, Sebastian Andrzej Siewior wrote:
> On 2022-09-01 01:59:15 [+0800], Jisheng Zhang wrote:
> > I assume patch1, patch2 and patch3 can be reviewed and merged for
> > riscv-next, patch4 and patch5 can be reviewed and maintained in rt tree,
> > and finally merged once the remaining patches in rt tree are all
> > mainlined.
> 
> I would say so, yes.
> 
> What about JUMP_LABEL support? Do you halt all CPUs while patching the
> code?
> 

FWICT, riscv JUMP_LABEL implementation doesn't rely on stop all cpus while
patching text.

Thanks

Re: [PATCH v2 0/5] riscv: add PREEMPT_RT support

[Conor.Dooley]

On 31/08/2022 18:59, Jisheng Zhang wrote:
> EXTERNAL EMAIL: Do not click links or open attachments unless you know the content is safe
> 
> This series is to add PREEMPT_RT support to riscv:
> patch1 adds the missing number of signal exits in vCPU stat
> patch2 switches to the generic guest entry infrastructure
> patch3 select HAVE_POSIX_CPU_TIMERS_TASK_WORK which is a requirement for
> RT
> patch4 adds lazy preempt support
> patch5 allows to enable PREEMPT_RT
> 

What version of the preempt_rt patch did you test this with?

Maybe I am missing something, but I gave this a whirl with
v6.0-rc3 + v6.0-rc3-rt5 & was meant by a bunch of complaints.
I am not familiar with the preempt_rt patch, so I am not sure what
level of BUG()s or WARNING()s are to be expected, but I saw a fair
few...

Thanks,
Conor.



> I assume patch1, patch2 and patch3 can be reviewed and merged for
> riscv-next, patch4 and patch5 can be reviewed and maintained in rt tree,
> and finally merged once the remaining patches in rt tree are all
> mainlined.
> 
> Since v1:
>   - send to related maillist, I press ENTER too quickly when sending v1
>   - remove the signal_pending() handling because that's covered by
>     generic guest entry infrastructure
> 
> Jisheng Zhang (5):
>   RISC-V: KVM: Record number of signal exits as a vCPU stat
>   RISC-V: KVM: Use generic guest entry infrastructure
>   riscv: select HAVE_POSIX_CPU_TIMERS_TASK_WORK
>   riscv: add lazy preempt support
>   riscv: Allow to enable RT
> 
>  arch/riscv/Kconfig                   |  3 +++
>  arch/riscv/include/asm/kvm_host.h    |  1 +
>  arch/riscv/include/asm/thread_info.h |  7 +++++--
>  arch/riscv/kernel/asm-offsets.c      |  1 +
>  arch/riscv/kernel/entry.S            |  9 +++++++--
>  arch/riscv/kvm/Kconfig               |  1 +
>  arch/riscv/kvm/vcpu.c                | 18 +++++++-----------
>  7 files changed, 25 insertions(+), 15 deletions(-)
> 
> --
> 2.34.1
> 
> 
> _______________________________________________
> linux-riscv mailing list
> linux-riscv@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-riscv

Re: [PATCH v2 0/5] riscv: add PREEMPT_RT support

[Jisheng Zhang]

On Thu, Sep 01, 2022 at 04:41:52PM +0000, Conor.Dooley@microchip.com wrote:
> On 31/08/2022 18:59, Jisheng Zhang wrote:
> > EXTERNAL EMAIL: Do not click links or open attachments unless you know the content is safe
> > 
> > This series is to add PREEMPT_RT support to riscv:
> > patch1 adds the missing number of signal exits in vCPU stat
> > patch2 switches to the generic guest entry infrastructure
> > patch3 select HAVE_POSIX_CPU_TIMERS_TASK_WORK which is a requirement for
> > RT
> > patch4 adds lazy preempt support
> > patch5 allows to enable PREEMPT_RT
> > 
> 
> What version of the preempt_rt patch did you test this with?

v6.0-rc1 + v6.0-rc1-rt patch

> 
> Maybe I am missing something, but I gave this a whirl with
> v6.0-rc3 + v6.0-rc3-rt5 & was meant by a bunch of complaints.
> I am not familiar with the preempt_rt patch, so I am not sure what
> level of BUG()s or WARNING()s are to be expected, but I saw a fair
> few...

Could you please provide corresponding log? Usually, this means there's
a bug in related drivers, so it's better to fix them now rather than
wait for RT patches mainlined.

PS: which HW are you using?

Thanks

Re: [PATCH v2 0/5] riscv: add PREEMPT_RT support

[Conor.Dooley]

On 02/09/2022 14:09, Jisheng Zhang wrote:
> EXTERNAL EMAIL: Do not click links or open attachments unless you know the content is safe
> 
> On Thu, Sep 01, 2022 at 04:41:52PM +0000, Conor.Dooley@microchip.com wrote:
>> On 31/08/2022 18:59, Jisheng Zhang wrote:
>>> EXTERNAL EMAIL: Do not click links or open attachments unless you know the content is safe
>>>
>>> This series is to add PREEMPT_RT support to riscv:
>>> patch1 adds the missing number of signal exits in vCPU stat
>>> patch2 switches to the generic guest entry infrastructure
>>> patch3 select HAVE_POSIX_CPU_TIMERS_TASK_WORK which is a requirement for
>>> RT
>>> patch4 adds lazy preempt support
>>> patch5 allows to enable PREEMPT_RT
>>>
>>
>> What version of the preempt_rt patch did you test this with?
> 
> v6.0-rc1 + v6.0-rc1-rt patch
> 
>>
>> Maybe I am missing something, but I gave this a whirl with
>> v6.0-rc3 + v6.0-rc3-rt5 & was meant by a bunch of complaints.
>> I am not familiar with the preempt_rt patch, so I am not sure what
>> level of BUG()s or WARNING()s are to be expected, but I saw a fair
>> few...
> 
> Could you please provide corresponding log? Usually, this means there's
> a bug in related drivers, so it's better to fix them now rather than
> wait for RT patches mainlined.

I tried it on PolarFire SoC. I know that at least one of the problems
I found is down to drivers - specifically the system controller & hwrng.

The first issue that comes up is in early smp setup code - we call out
to update_siblings_masks() which does an alloc with preemption. It's
the same backtrace from here:

https://lore.kernel.org/all/0abd0acf-70a1-d546-a517-19efe60042d1@microchip.com/

I'll give it a run through tonight or tomorrow & give you a full log
of what I saw. There's some splats all over the place for me, but I
can't tell if that's just knock-on from the other issues.

Thanks,
Conor.

Re: [PATCH v2 4/5] riscv: add lazy preempt support

[Guo Ren]

On Thu, Sep 1, 2022 at 2:08 AM Jisheng Zhang <jszhang@kernel.org> wrote:
>
> Implement the lazy preempt for riscv.
>
> Signed-off-by: Jisheng Zhang <jszhang@kernel.org>
> ---
>  arch/riscv/Kconfig                   | 1 +
>  arch/riscv/include/asm/thread_info.h | 7 +++++--
>  arch/riscv/kernel/asm-offsets.c      | 1 +
>  arch/riscv/kernel/entry.S            | 9 +++++++--
>  4 files changed, 14 insertions(+), 4 deletions(-)
>
> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> index 7a8134fd7ec9..9f2f1936b1b5 100644
> --- a/arch/riscv/Kconfig
> +++ b/arch/riscv/Kconfig
> @@ -105,6 +105,7 @@ config RISCV
>         select HAVE_PERF_REGS
>         select HAVE_PERF_USER_STACK_DUMP
>         select HAVE_POSIX_CPU_TIMERS_TASK_WORK
> +       select HAVE_PREEMPT_LAZY
>         select HAVE_REGS_AND_STACK_ACCESS_API
>         select HAVE_FUNCTION_ARG_ACCESS_API
>         select HAVE_STACKPROTECTOR
> diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
> index 78933ac04995..471915b179a2 100644
> --- a/arch/riscv/include/asm/thread_info.h
> +++ b/arch/riscv/include/asm/thread_info.h
> @@ -56,6 +56,7 @@
>  struct thread_info {
>         unsigned long           flags;          /* low level flags */
>         int                     preempt_count;  /* 0=>preemptible, <0=>BUG */
> +       int                     preempt_lazy_count;  /* 0=>preemptible, <0=>BUG */
>         /*
>          * These stack pointers are overwritten on every system call or
>          * exception.  SP is also saved to the stack it can be recovered when
> @@ -90,7 +91,7 @@ struct thread_info {
>  #define TIF_NOTIFY_RESUME      1       /* callback before returning to user */
>  #define TIF_SIGPENDING         2       /* signal pending */
>  #define TIF_NEED_RESCHED       3       /* rescheduling necessary */
> -#define TIF_RESTORE_SIGMASK    4       /* restore signal mask in do_signal() */
> +#define TIF_NEED_RESCHED_LAZY  4       /* lazy rescheduling */
>  #define TIF_MEMDIE             5       /* is terminating due to OOM killer */
>  #define TIF_SYSCALL_TRACEPOINT  6       /* syscall tracepoint instrumentation */
>  #define TIF_SYSCALL_AUDIT      7       /* syscall auditing */
> @@ -98,6 +99,7 @@ struct thread_info {
>  #define TIF_NOTIFY_SIGNAL      9       /* signal notifications exist */
>  #define TIF_UPROBE             10      /* uprobe breakpoint or singlestep */
>  #define TIF_32BIT              11      /* compat-mode 32bit process */
> +#define TIF_RESTORE_SIGMASK    12      /* restore signal mask in do_signal() */
>
>  #define _TIF_SYSCALL_TRACE     (1 << TIF_SYSCALL_TRACE)
>  #define _TIF_NOTIFY_RESUME     (1 << TIF_NOTIFY_RESUME)
> @@ -108,10 +110,11 @@ struct thread_info {
>  #define _TIF_SECCOMP           (1 << TIF_SECCOMP)
>  #define _TIF_NOTIFY_SIGNAL     (1 << TIF_NOTIFY_SIGNAL)
>  #define _TIF_UPROBE            (1 << TIF_UPROBE)
> +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY)
>
>  #define _TIF_WORK_MASK \
>         (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \
> -        _TIF_NOTIFY_SIGNAL | _TIF_UPROBE)
> +        _TIF_NEED_RESCHED_LAZY | _TIF_NOTIFY_SIGNAL | _TIF_UPROBE)
>
>  #define _TIF_SYSCALL_WORK \
>         (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \
> diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
> index df9444397908..e38e33822f72 100644
> --- a/arch/riscv/kernel/asm-offsets.c
> +++ b/arch/riscv/kernel/asm-offsets.c
> @@ -35,6 +35,7 @@ void asm_offsets(void)
>         OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]);
>         OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags);
>         OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
> +       OFFSET(TASK_TI_PREEMPT_LAZY_COUNT, task_struct, thread_info.preempt_lazy_count);
>         OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
>         OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp);
>
> diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
> index b9eda3fcbd6d..595100a4c2c7 100644
> --- a/arch/riscv/kernel/entry.S
> +++ b/arch/riscv/kernel/entry.S
> @@ -361,9 +361,14 @@ restore_all:
>  resume_kernel:
>         REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
>         bnez s0, restore_all
> -       REG_L s0, TASK_TI_FLAGS(tp)
> -       andi s0, s0, _TIF_NEED_RESCHED
> +       REG_L s1, TASK_TI_FLAGS(tp)
> +       andi s0, s1, _TIF_NEED_RESCHED
> +       bnez s0, 1f
> +       REG_L s0, TASK_TI_PREEMPT_LAZY_COUNT(tp)
> +       bnez s0, restore_all
> +       andi s0, s1, _TIF_NEED_RESCHED_LAZY
Can you tell me, who increased/decreased the PREEMPT_LAZY_COUNT? And
who set NEED_RESCHED_LAZY?


>         beqz s0, restore_all
> +1:
>         call preempt_schedule_irq
>         j restore_all
>  #endif
> --
> 2.34.1
>
>
> --
> kvm-riscv mailing list
> kvm-riscv@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/kvm-riscv



-- 
Best Regards
 Guo Ren

Re: [PATCH v2 4/5] riscv: add lazy preempt support

[Sebastian Andrzej Siewior]

On 2022-09-04 23:16:12 [+0800], Guo Ren wrote:
> > diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
> > index b9eda3fcbd6d..595100a4c2c7 100644
> > --- a/arch/riscv/kernel/entry.S
> > +++ b/arch/riscv/kernel/entry.S
> > @@ -361,9 +361,14 @@ restore_all:
> >  resume_kernel:
> >         REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
> >         bnez s0, restore_all
> > -       REG_L s0, TASK_TI_FLAGS(tp)
> > -       andi s0, s0, _TIF_NEED_RESCHED
> > +       REG_L s1, TASK_TI_FLAGS(tp)
> > +       andi s0, s1, _TIF_NEED_RESCHED
> > +       bnez s0, 1f
> > +       REG_L s0, TASK_TI_PREEMPT_LAZY_COUNT(tp)
> > +       bnez s0, restore_all
> > +       andi s0, s1, _TIF_NEED_RESCHED_LAZY
> Can you tell me, who increased/decreased the PREEMPT_LAZY_COUNT? And
> who set NEED_RESCHED_LAZY?

There is "generic" code in the PREEMPT_RT patch doing that. The counter
is incremented/ decremented via preempt_lazy_enable()/disable() and one
of the user is migrate_disable()/enable().
Basically if a task is task_is_realtime() then NEED_RESCHED is set for
the wakeup. For the remaining states (SCHED_OTHER, …) NEED_RESCHED_LAZY
is set for the wakeup. This can be delayed if the task is in a "preempt
disable lazy" section (similar to a preempt_disable() section) but a
task_is_realtime() can still be scheduled if needed.
See details at
	https://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git/plain/patches/sched__Add_support_for_lazy_preemption.patch?h=linux-6.0.y-rt-patches

Sebastian

Re: [PATCH v2 4/5] riscv: add lazy preempt support

[Guo Ren]

On Mon, Sep 5, 2022 at 2:34 PM Sebastian Andrzej Siewior
<bigeasy@linutronix.de> wrote:
>
> On 2022-09-04 23:16:12 [+0800], Guo Ren wrote:
> > > diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
> > > index b9eda3fcbd6d..595100a4c2c7 100644
> > > --- a/arch/riscv/kernel/entry.S
> > > +++ b/arch/riscv/kernel/entry.S
> > > @@ -361,9 +361,14 @@ restore_all:
> > >  resume_kernel:
> > >         REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
> > >         bnez s0, restore_all
> > > -       REG_L s0, TASK_TI_FLAGS(tp)
> > > -       andi s0, s0, _TIF_NEED_RESCHED
> > > +       REG_L s1, TASK_TI_FLAGS(tp)
> > > +       andi s0, s1, _TIF_NEED_RESCHED
> > > +       bnez s0, 1f
> > > +       REG_L s0, TASK_TI_PREEMPT_LAZY_COUNT(tp)
> > > +       bnez s0, restore_all
> > > +       andi s0, s1, _TIF_NEED_RESCHED_LAZY
> > Can you tell me, who increased/decreased the PREEMPT_LAZY_COUNT? And
> > who set NEED_RESCHED_LAZY?
>
> There is "generic" code in the PREEMPT_RT patch doing that. The counter
> is incremented/ decremented via preempt_lazy_enable()/disable() and one
> of the user is migrate_disable()/enable().
> Basically if a task is task_is_realtime() then NEED_RESCHED is set for
> the wakeup. For the remaining states (SCHED_OTHER, …) NEED_RESCHED_LAZY
> is set for the wakeup. This can be delayed if the task is in a "preempt
> disable lazy" section (similar to a preempt_disable() section) but a
> task_is_realtime() can still be scheduled if needed.
Okay, It should be [PATCH RT]. RISC-V would also move to GENERIC_ENTRY
[1], so above assembly code would be replaced by generic one, right?

[1]: https://lore.kernel.org/linux-riscv/20220904072637.8619-3-guoren@kernel.org/T/#u


> See details at
>         https://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git/plain/patches/sched__Add_support_for_lazy_preemption.patch?h=linux-6.0.y-rt-patches


>
> Sebastian



-- 
Best Regards
 Guo Ren

Re: [PATCH v2 4/5] riscv: add lazy preempt support

[Sebastian Andrzej Siewior]

On 2022-09-05 16:33:54 [+0800], Guo Ren wrote:
> > There is "generic" code in the PREEMPT_RT patch doing that. The counter
> > is incremented/ decremented via preempt_lazy_enable()/disable() and one
> > of the user is migrate_disable()/enable().
> > Basically if a task is task_is_realtime() then NEED_RESCHED is set for
> > the wakeup. For the remaining states (SCHED_OTHER, …) NEED_RESCHED_LAZY
> > is set for the wakeup. This can be delayed if the task is in a "preempt
> > disable lazy" section (similar to a preempt_disable() section) but a
> > task_is_realtime() can still be scheduled if needed.
> Okay, It should be [PATCH RT]. RISC-V would also move to GENERIC_ENTRY
> [1], so above assembly code would be replaced by generic one, right?

correct.

Sebastian

Re: [PATCH v2 4/5] riscv: add lazy preempt support

[Jisheng Zhang]

On Mon, Sep 05, 2022 at 04:33:54PM +0800, Guo Ren wrote:
> On Mon, Sep 5, 2022 at 2:34 PM Sebastian Andrzej Siewior
> <bigeasy@linutronix.de> wrote:
> >
> > On 2022-09-04 23:16:12 [+0800], Guo Ren wrote:
> > > > diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
> > > > index b9eda3fcbd6d..595100a4c2c7 100644
> > > > --- a/arch/riscv/kernel/entry.S
> > > > +++ b/arch/riscv/kernel/entry.S
> > > > @@ -361,9 +361,14 @@ restore_all:
> > > >  resume_kernel:
> > > >         REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
> > > >         bnez s0, restore_all
> > > > -       REG_L s0, TASK_TI_FLAGS(tp)
> > > > -       andi s0, s0, _TIF_NEED_RESCHED
> > > > +       REG_L s1, TASK_TI_FLAGS(tp)
> > > > +       andi s0, s1, _TIF_NEED_RESCHED
> > > > +       bnez s0, 1f
> > > > +       REG_L s0, TASK_TI_PREEMPT_LAZY_COUNT(tp)
> > > > +       bnez s0, restore_all
> > > > +       andi s0, s1, _TIF_NEED_RESCHED_LAZY
> > > Can you tell me, who increased/decreased the PREEMPT_LAZY_COUNT? And
> > > who set NEED_RESCHED_LAZY?
> >
> > There is "generic" code in the PREEMPT_RT patch doing that. The counter
> > is incremented/ decremented via preempt_lazy_enable()/disable() and one
> > of the user is migrate_disable()/enable().
> > Basically if a task is task_is_realtime() then NEED_RESCHED is set for
> > the wakeup. For the remaining states (SCHED_OTHER, …) NEED_RESCHED_LAZY
> > is set for the wakeup. This can be delayed if the task is in a "preempt
> > disable lazy" section (similar to a preempt_disable() section) but a
> > task_is_realtime() can still be scheduled if needed.
> Okay, It should be [PATCH RT]. RISC-V would also move to GENERIC_ENTRY

As said in the cover letter, this patch is expected to reviewed and
maintained in RT tree. If your GENERIC_ENTRY patches are merged, I will
send an updated patch.

> [1], so above assembly code would be replaced by generic one, right?
> 
> [1]: https://lore.kernel.org/linux-riscv/20220904072637.8619-3-guoren@kernel.org/T/#u
> 
> 
> > See details at
> >         https://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git/plain/patches/sched__Add_support_for_lazy_preemption.patch?h=linux-6.0.y-rt-patches
> 
> 
> >
> > Sebastian
> 
> 
> 
> -- 
> Best Regards
>  Guo Ren

Re: [PATCH v2 4/5] riscv: add lazy preempt support

[Guo Ren]

On Mon, Sep 5, 2022 at 4:46 PM Sebastian Andrzej Siewior
<bigeasy@linutronix.de> wrote:
>
> On 2022-09-05 16:33:54 [+0800], Guo Ren wrote:
> > > There is "generic" code in the PREEMPT_RT patch doing that. The counter
> > > is incremented/ decremented via preempt_lazy_enable()/disable() and one
> > > of the user is migrate_disable()/enable().
> > > Basically if a task is task_is_realtime() then NEED_RESCHED is set for
> > > the wakeup. For the remaining states (SCHED_OTHER, …) NEED_RESCHED_LAZY
> > > is set for the wakeup. This can be delayed if the task is in a "preempt
> > > disable lazy" section (similar to a preempt_disable() section) but a
> > > task_is_realtime() can still be scheduled if needed.
> > Okay, It should be [PATCH RT]. RISC-V would also move to GENERIC_ENTRY
> > [1], so above assembly code would be replaced by generic one, right?
>
> correct.
Maybe TIF_XXX_RESCHED also could be merged into GENERIC_ENTRY, just
like what you've done in syscall.

struct thread_info {
          unsigned long           flags;
          unsigned long           syscall_work;   /* SYSCALL_WORK_ flags */
+        unsigned long           resched_work;   /* RESCHED flags */

Or merge them into one:
struct thread_info {
          unsigned long           flags;
-         unsigned long           syscall_work;   /* SYSCALL_WORK_ flags */
+         unsigned long           ge_flags; /* GENERIC_ENTRY flags */

>
> Sebastian



-- 
Best Regards
 Guo Ren

Re: [PATCH v2 0/5] riscv: add PREEMPT_RT support

[Sebastian Andrzej Siewior]

On 2022-09-02 13:29:23 [+0000], Conor.Dooley@microchip.com wrote:
> I'll give it a run through tonight or tomorrow & give you a full log
> of what I saw. There's some splats all over the place for me, but I
> can't tell if that's just knock-on from the other issues.

Is there an update to this or the series as a whole?

> Thanks,
> Conor.

Sebastian

Re: [PATCH v2 0/5] riscv: add PREEMPT_RT support

[Conor.Dooley]

On 11/11/2022 14:32, Sebastian Andrzej Siewior wrote:
> On 2022-09-02 13:29:23 [+0000], Conor.Dooley@microchip.com wrote:
>> I'll give it a run through tonight or tomorrow & give you a full log
>> of what I saw. There's some splats all over the place for me, but I
>> can't tell if that's just knock-on from the other issues.
> 
> Is there an update to this or the series as a whole?

Not from me.. completely forgot about it.
I'll put it back in my todo list, sorry.

Re: [PATCH v2 0/5] riscv: add PREEMPT_RT support

[Conor.Dooley]

On 11/11/2022 14:34, Conor Dooley - M52691 wrote:
> On 11/11/2022 14:32, Sebastian Andrzej Siewior wrote:
>> On 2022-09-02 13:29:23 [+0000], Conor.Dooley@microchip.com wrote:
>>> I'll give it a run through tonight or tomorrow & give you a full log
>>> of what I saw. There's some splats all over the place for me, but I
>>> can't tell if that's just knock-on from the other issues.
>>
>> Is there an update to this or the series as a whole?
> 
> Not from me.. completely forgot about it.
> I'll put it back in my todo list, sorry.
> 

I tried out v6.0.5-rc5 + this patchset (it doesnt apply to v6.1-rcN)
and rt14, got the following:
[    4.036667] smp: Bringing up secondary CPUs ...
[    4.069365] BUG: sleeping function called from invalid context at kernel/locking/spinlock_rt.c:46
[    4.069389] in_atomic(): 1, irqs_disabled(): 1, non_block: 0, pid: 0, name: swapper/1
[    4.069410] preempt_count: 1, expected: 0
[    4.069422] RCU nest depth: 1, expected: 1
[    4.069434] 3 locks held by swapper/1/0:
[    4.069449]  #0: ffffffd82cda3b58 (&pcp->lock){+.+.}-{2:2}, at: get_page_from_freelist+0x220/0x1094
[    4.069537]  #1: ffffffff8129b178 (rcu_read_lock){....}-{1:2}, at: rcu_lock_acquire+0x0/0x2e
[    4.069602]  #2: ffffffff813a3e38 (&zone->lock){+.+.}-{2:2}, at: __rmqueue_pcplist+0x156/0xc28
[    4.069662] irq event stamp: 0
[    4.069670] hardirqs last  enabled at (0): [<0000000000000000>] 0x0
[    4.069690] hardirqs last disabled at (0): [<ffffffff8000f0c8>] copy_process+0x50c/0xdaa
[    4.069727] softirqs last  enabled at (0): [<ffffffff8000f0d6>] copy_process+0x51a/0xdaa
[    4.069756] softirqs last disabled at (0): [<0000000000000000>] 0x0
[    4.069776] Preemption disabled at:
[    4.069782] [<ffffffff80041346>] migrate_enable+0x32/0x124
[    4.069819] CPU: 1 PID: 0 Comm: swapper/1 Not tainted 6.0.5-rt14-00006-g0fda08a972f4-dirty #1
[    4.069848] Hardware name: Microchip PolarFire-SoC Icicle Kit (DT)
[    4.069861] Call Trace:
[    4.069870] [<ffffffff80006628>] show_stack+0x2c/0x38
[    4.069907] [<ffffffff80900ad4>] dump_stack_lvl+0x64/0x86
[    4.069935] [<ffffffff80900b0a>] dump_stack+0x14/0x1c
[    4.069959] [<ffffffff80047534>] __might_resched+0x1bc/0x1c6
[    4.069995] [<ffffffff80908f7a>] rt_spin_lock+0x42/0xb8
[    4.070033] [<ffffffff801cab7a>] __rmqueue_pcplist+0x156/0xc28
[    4.070061] [<ffffffff801cbade>] get_page_from_freelist+0x24e/0x1094
[    4.070088] [<ffffffff801cb712>] __alloc_pages+0xc6/0x244
[    4.070113] [<ffffffff801ede42>] new_slab+0x8c/0x4a8
[    4.070153] [<ffffffff801e955a>] ___slab_alloc+0x5d4/0x9a4
[    4.070181] [<ffffffff801ea206>] __kmalloc+0xc0/0x1fc
[    4.070209] [<ffffffff80578296>] detect_cache_attributes+0xb4/0x470
[    4.070238] [<ffffffff80590520>] update_siblings_masks+0x2c/0x202
[    4.070270] [<ffffffff80590aa0>] store_cpu_topology+0x30/0x6a
[    4.070296] [<ffffffff80007756>] smp_callin+0x38/0x66
[    4.231582] smp: Brought up 1 node, 4 CPUs

There's other stuff that goes awry later on too:
https://gist.githubusercontent.com/ConchuOD/47fd47dfa1f49eb4b0f2fb2a68852a7c/raw/b109b83eec6caa1d67cbb156c6f3e671c10aefe9/gistfile1.txt

The SDHCI stuff I am seeing without rt & in v6.1-rc4 so is
unrelated, but the rest resembles what I saw previously.
idk anything about -rt so if there's something blatant that
I've missed here, please lmk.

Re: [PATCH v2 0/5] riscv: add PREEMPT_RT support

[Schaffner, Tobias]

On 31/08/2022 18:59, Jisheng Zhang wrote:
 > This series is to add PREEMPT_RT support to riscv:
 > patch1 adds the missing number of signal exits in vCPU stat
 > patch2 switches to the generic guest entry infrastructure
 > patch3 select HAVE_POSIX_CPU_TIMERS_TASK_WORK which is a requirement for
 > RT
 > patch4 adds lazy preempt support
 > patch5 allows to enable PREEMPT_RT
 >
 > I assume patch1, patch2 and patch3 can be reviewed and merged for
 > riscv-next, patch4 and patch5 can be reviewed and maintained in rt tree,
 > and finally merged once the remaining patches in rt tree are all
 > mainlined.

I tested the last two patches on a StarFive VisionFive V2 (DT) board 
with 6.1.12-rt7-gdfa52cc14f3b today and the results looked pretty good 
for a first run.

root@StarFive:~# lscpu
Architecture:          riscv64
   Byte Order:          Little Endian
CPU(s):                4
   On-line CPU(s) list: 0-3

root@StarFive:~# uname -a
Linux StarFive 6.1.12-rt7-gdfa52cc14f3b #1 SMP PREEMPT_RT Thu, 01 Jan 
1970 01:00:00 +0000 riscv64 GNU/Linuxb

root@StarFive:~# cat /proc/cmdline
initrd=\initrd.img-6.1.12-rt7-gdfa52cc14f3b LABEL=Boot 
root=PARTUUID=7176479f-eeea-46ac-afb6-7ec47ff7c390 console=tty0 
console=ttyS0,115200 earlycon rootwait isolcpus=2-3 rcu_nocbs=2-3 
nohz_full=2-3 irqaffinity=0-1

root@StarFive:~# cyclictest -m -S -p 90 -i 50 -d 0 -q -D 10m
WARN: stat /dev/cpu_dma_latency failed: No such file or directory
T: 0 (  358) P:90 I:50 C:11999999 Min:     11 Act:   11 Avg:   11 Max: 
    55
T: 1 (  359) P:90 I:50 C:11999241 Min:     11 Act:   11 Avg:   11 Max: 
    60

Feel free to reach out for further tests or logs.

Best,
Tobias

 > Since v1:
 >   - send to related maillist, I press ENTER too quickly when sending v1
 >   - remove the signal_pending() handling because that's covered by
 >     generic guest entry infrastructure
 >
 > Jisheng Zhang (5):
 >   RISC-V: KVM: Record number of signal exits as a vCPU stat
 >   RISC-V: KVM: Use generic guest entry infrastructure
 >   riscv: select HAVE_POSIX_CPU_TIMERS_TASK_WORK
 >   riscv: add lazy preempt support
 >   riscv: Allow to enable RT
 >
 >  arch/riscv/Kconfig                   |  3 +++
 >  arch/riscv/include/asm/kvm_host.h    |  1 +
 >  arch/riscv/include/asm/thread_info.h |  7 +++++--
 >  arch/riscv/kernel/asm-offsets.c      |  1 +
 >  arch/riscv/kernel/entry.S            |  9 +++++++--
 >  arch/riscv/kvm/Kconfig               |  1 +
 >  arch/riscv/kvm/vcpu.c                | 18 +++++++-----------
 >  7 files changed, 25 insertions(+), 15 deletions(-)
 >
 > --
 > 2.34.1
 >
 >
 > _______________________________________________
 > linux-riscv mailing list
 > linux-riscv@lists.infradead.org
 > http://lists.infradead.org/mailman/listinfo/linux-riscv