[RFC] arm: Handle starting up in secure mode

[RFC] arm: Handle starting up in secure mode

[Christopher Covington]

ARM Linux currently has the most features available to it in hypervisor
(HYP) mode, so switch to it when possible. This can also ensure proper
reset of newer registers such as CNTVOFF.

The permissions on the Non-Secure Access Control Register (NSACR) are
used to probe what the security setting currently is when in supervisor
(SVC) mode.

Signed-off-by: Christopher Covington <cov@codeaurora.org>
---
 arch/arm/kernel/head.S     |  1 +
 arch/arm/kernel/hyp-stub.S | 71 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)

diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S
index 664eee8..6fe2387 100644
--- a/arch/arm/kernel/head.S
+++ b/arch/arm/kernel/head.S
@@ -85,6 +85,7 @@ ENTRY(stext)
  THUMB(	.thumb			)	@ switch to Thumb now.
  THUMB(1:			)
 
+	bl	__mon_stub_install
 #ifdef CONFIG_ARM_VIRT_EXT
 	bl	__hyp_stub_install
 #endif
diff --git a/arch/arm/kernel/hyp-stub.S b/arch/arm/kernel/hyp-stub.S
index 2a55373..36d1a9c 100644
--- a/arch/arm/kernel/hyp-stub.S
+++ b/arch/arm/kernel/hyp-stub.S
@@ -20,6 +20,7 @@
 #include <linux/linkage.h>
 #include <asm/assembler.h>
 #include <asm/virt.h>
+#include <asm/opcodes-sec.h>
 
 #ifndef ZIMAGE
 /*
@@ -76,6 +77,64 @@ ENTRY(__boot_cpu_mode)
 #endif /* ZIMAGE */
 
 /*
+ * Detect whether the system is in secure supervisor mode, and if it is,
+ * switch to hypervisor mode by way of secure monitor mode.
+ */
+ENTRY(__mon_stub_install)
+	mrs	r4, cpsr
+	and	r4, r4, #MODE_MASK
+	cmp	r4, #SVC_MODE
+	movne	pc, lr
+
+	/*
+	 * Set things up so that if an NSACR access causes an undefined
+	 * instruction exception, we return. safe_svcmode_maskall called
+	 * just after this will get us back into supervisor mode.
+	 */
+	adr	r4, __mon_stub_vectors
+	mcr	p15, 0, r4, c12, c0, 0	@ set vector base address (VBAR)
+	mov	r4, lr
+
+	/*
+	 * Writing the NSACR will only succeed if we're in a secure mode.
+	 */
+	mrc	p15, 0, r5, c1, c1, 2	@ get non-secure access control (NSACR)
+	mcr	p15, 0, r5, c1, c1, 2	@ set non-secure access control (NSACR)
+
+	/*
+	 * If we get here, we know we're in secure supervisor mode, so make the
+	 * switch to secure monitor mode.
+	 *
+	 * TODO: make sure this doesn't trap to A64 EL3.
+	 */
+	adr	r4, __mon_stub_vectors
+	mcr	p15, 0, r4, c12, c0, 1	@ set monitor vector base (MVBAR)
+	adr	r4, mon_settings
+	__SMC(0)
+
+	/*
+	 * Now, from non-secure supervisor mode, transition to hypervisor mode
+	 * and return via the exception vector.
+	 */
+	mov	r4, lr
+	__HVC(0)
+ENDPROC(__mon_stub_install)
+
+ENTRY(mon_settings)
+	/*
+	 * Prepare for hypervisor mode by setting the HCE and NS bits.
+	 */
+	mrc	p15, 0, r4, c1, c1, 0	@ get secure configuration (SCR)
+	orr	r4, r4, #0x100
+	orr	r4, r4, #1
+	mcr	p15, 0, r4, c1, c1, 0	@ set secure configuration (SCR)
+
+	adr	r4, __mon_stub_vectors
+	mcr	p15, 4, r4, c12, c0, 0	@ set hypervisor vectors (HVBAR)
+	__ERET
+ENDPROC(mon_settings)
+
+/*
  * Hypervisor stub installation functions.
  *
  * These must be called with the MMU and D-cache off.
@@ -209,6 +268,18 @@ ENDPROC(__hyp_set_vectors)
 #endif
 
 .align 5
+__mon_stub_vectors:
+__mon_stub_reset:	W(b)	.
+__mon_stub_und:		mov	pc, r4
+__mon_stub_call:	mov	pc, r4
+__mon_stub_pabort:	W(b)	.
+__mon_stub_dabort:	W(b)	.
+__mon_stub_trap:	mov	pc, r4
+__mon_stub_irq:		W(b)	.
+__mon_stub_fiq:		W(b)	.
+ENDPROC(__hyp_stub_vectors)
+
+.align 5
 __hyp_stub_vectors:
 __hyp_stub_reset:	W(b)	.
 __hyp_stub_und:		W(b)	.
-- 
Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum,
hosted by the Linux Foundation.

[RFC] arm: Handle starting up in secure mode

[Christopher Covington]

ARM Linux currently has the most features available to it in hypervisor
(HYP) mode, so switch to it when possible. This can also ensure proper
reset of newer registers such as CNTVOFF.

The permissions on the Non-Secure Access Control Register (NSACR) are
used to probe what the security setting currently is when in supervisor
(SVC) mode.

Signed-off-by: Christopher Covington <cov@codeaurora.org>
---
 arch/arm/kernel/head.S     |  1 +
 arch/arm/kernel/hyp-stub.S | 71 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)

diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S
index 664eee8..6fe2387 100644
--- a/arch/arm/kernel/head.S
+++ b/arch/arm/kernel/head.S
@@ -85,6 +85,7 @@ ENTRY(stext)
  THUMB(	.thumb			)	@ switch to Thumb now.
  THUMB(1:			)
 
+	bl	__mon_stub_install
 #ifdef CONFIG_ARM_VIRT_EXT
 	bl	__hyp_stub_install
 #endif
diff --git a/arch/arm/kernel/hyp-stub.S b/arch/arm/kernel/hyp-stub.S
index 2a55373..36d1a9c 100644
--- a/arch/arm/kernel/hyp-stub.S
+++ b/arch/arm/kernel/hyp-stub.S
@@ -20,6 +20,7 @@
 #include <linux/linkage.h>
 #include <asm/assembler.h>
 #include <asm/virt.h>
+#include <asm/opcodes-sec.h>
 
 #ifndef ZIMAGE
 /*
@@ -76,6 +77,64 @@ ENTRY(__boot_cpu_mode)
 #endif /* ZIMAGE */
 
 /*
+ * Detect whether the system is in secure supervisor mode, and if it is,
+ * switch to hypervisor mode by way of secure monitor mode.
+ */
+ENTRY(__mon_stub_install)
+	mrs	r4, cpsr
+	and	r4, r4, #MODE_MASK
+	cmp	r4, #SVC_MODE
+	movne	pc, lr
+
+	/*
+	 * Set things up so that if an NSACR access causes an undefined
+	 * instruction exception, we return. safe_svcmode_maskall called
+	 * just after this will get us back into supervisor mode.
+	 */
+	adr	r4, __mon_stub_vectors
+	mcr	p15, 0, r4, c12, c0, 0	@ set vector base address (VBAR)
+	mov	r4, lr
+
+	/*
+	 * Writing the NSACR will only succeed if we're in a secure mode.
+	 */
+	mrc	p15, 0, r5, c1, c1, 2	@ get non-secure access control (NSACR)
+	mcr	p15, 0, r5, c1, c1, 2	@ set non-secure access control (NSACR)
+
+	/*
+	 * If we get here, we know we're in secure supervisor mode, so make the
+	 * switch to secure monitor mode.
+	 *
+	 * TODO: make sure this doesn't trap to A64 EL3.
+	 */
+	adr	r4, __mon_stub_vectors
+	mcr	p15, 0, r4, c12, c0, 1	@ set monitor vector base (MVBAR)
+	adr	r4, mon_settings
+	__SMC(0)
+
+	/*
+	 * Now, from non-secure supervisor mode, transition to hypervisor mode
+	 * and return via the exception vector.
+	 */
+	mov	r4, lr
+	__HVC(0)
+ENDPROC(__mon_stub_install)
+
+ENTRY(mon_settings)
+	/*
+	 * Prepare for hypervisor mode by setting the HCE and NS bits.
+	 */
+	mrc	p15, 0, r4, c1, c1, 0	@ get secure configuration (SCR)
+	orr	r4, r4, #0x100
+	orr	r4, r4, #1
+	mcr	p15, 0, r4, c1, c1, 0	@ set secure configuration (SCR)
+
+	adr	r4, __mon_stub_vectors
+	mcr	p15, 4, r4, c12, c0, 0	@ set hypervisor vectors (HVBAR)
+	__ERET
+ENDPROC(mon_settings)
+
+/*
  * Hypervisor stub installation functions.
  *
  * These must be called with the MMU and D-cache off.
@@ -209,6 +268,18 @@ ENDPROC(__hyp_set_vectors)
 #endif
 
 .align 5
+__mon_stub_vectors:
+__mon_stub_reset:	W(b)	.
+__mon_stub_und:		mov	pc, r4
+__mon_stub_call:	mov	pc, r4
+__mon_stub_pabort:	W(b)	.
+__mon_stub_dabort:	W(b)	.
+__mon_stub_trap:	mov	pc, r4
+__mon_stub_irq:		W(b)	.
+__mon_stub_fiq:		W(b)	.
+ENDPROC(__hyp_stub_vectors)
+
+.align 5
 __hyp_stub_vectors:
 __hyp_stub_reset:	W(b)	.
 __hyp_stub_und:		W(b)	.
-- 
Qualcomm Innovation Center, Inc. is a member of the Code Aurora Forum,
hosted by the Linux Foundation.

Re: [RFC] arm: Handle starting up in secure mode

[Christopher Covington]

On 09/16/2014 05:09 PM, Christopher Covington wrote:
> ARM Linux currently has the most features available to it in hypervisor
> (HYP) mode, so switch to it when possible. This can also ensure proper
> reset of newer registers such as CNTVOFF.
> 
> The permissions on the Non-Secure Access Control Register (NSACR) are
> used to probe what the security setting currently is when in supervisor
> (SVC) mode.

Sorry, this doesn't work yet. I was misinterpreting my test results. For what
it's worth, my testing and development methodology is to run it after hacked
up versions of the semihosting bootwrapper on the simulator that corresponds
to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.

Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

[RFC] arm: Handle starting up in secure mode

[Christopher Covington]

On 09/16/2014 05:09 PM, Christopher Covington wrote:
> ARM Linux currently has the most features available to it in hypervisor
> (HYP) mode, so switch to it when possible. This can also ensure proper
> reset of newer registers such as CNTVOFF.
> 
> The permissions on the Non-Secure Access Control Register (NSACR) are
> used to probe what the security setting currently is when in supervisor
> (SVC) mode.

Sorry, this doesn't work yet. I was misinterpreting my test results. For what
it's worth, my testing and development methodology is to run it after hacked
up versions of the semihosting bootwrapper on the simulator that corresponds
to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.

Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

Re: [RFC] arm: Handle starting up in secure mode

[Christopher Covington]

On 09/16/2014 05:24 PM, Christopher Covington wrote:
> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>> ARM Linux currently has the most features available to it in hypervisor
>> (HYP) mode, so switch to it when possible. This can also ensure proper
>> reset of newer registers such as CNTVOFF.
>>
>> The permissions on the Non-Secure Access Control Register (NSACR) are
>> used to probe what the security setting currently is when in supervisor
>> (SVC) mode.
> 
> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
> it's worth, my testing and development methodology is to run it after hacked
> up versions of the semihosting bootwrapper on the simulator that corresponds
> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.

Looks like the real problem was that I was hacking up the bootwrapper
incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
GIC for non-secure access. With that changed, I've tested the following
variations using the Image file in a single core configuration.

Start in non-secure SVC with non-secure access to GIC configured.

Start in secure SVC with non-secure access to GIC configured.

Start in secure SVC with non-secure access to GIC configured and hypervisor
support disabled in the model (-C cluster.has_el2=0). This required setting
the VBAR again in non-secure SVC but with that fix it seems to work. I'll
include this change in v2.

I have not been able to start up the bootwrapper with secure monitor support
disabled in the model (-C cluster.has_el3=0) because of faults during GIC
configuration.

So, any thoughts on the code? I have some questions.

What initialization am I missing?

What's the right ifdefery? I'm thinking a CONFIG_ARM_SEC_EXT that is
independent of CONFIG_ARM_VIRT_EXT.

Are there folks who want to run Linux in secure mode? Would documenting in
Kconfig that they can do this by leaving CONFIG_ARM_SEC_EXT undefined be
sufficient support for this?

Is it fine or bad to leave the CPU in undefined mode when one of the extension
or privileged operations fails? It's only for maybe a dozen instructions until
safe_svcmode_maskall.

How should CPU mode recording be handled? Should the CONFIG_ARM_SEC_EXT code
set __boot_cpu_mode with an extra bit specifying the security setting and then
the CONFIG_ARM_VIRT_EXT code set-if-unset?

What should the secure monitor stub leave behind? Right now it's different
than the hypervisor stub: put the address of the code you want run in monitor
mode into r4, make the secure monitor call, and make sure your code makes an
exception return when finished.

Whatever is left behind, I think it should allow the GIC driver to enable
non-secure access to the hardware if Linux was booted in secure mode.

If this code can be evolved and tested to the point where it's mergeable, then
Linux can finally interact intelligently with the features introduced by the
security extensions and only has to depend on firmware and bootloaders getting
the implementation defined aspects of a system right, taking care of the
architectually specified stuff itself when sufficiently privileged. Among
other things, this should lead towards the bootwrapper no longer being
required to boot Linux on ARM's virtual platforms, which would make my life
easier.

Thanks,
Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

[RFC] arm: Handle starting up in secure mode

[Christopher Covington]

On 09/16/2014 05:24 PM, Christopher Covington wrote:
> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>> ARM Linux currently has the most features available to it in hypervisor
>> (HYP) mode, so switch to it when possible. This can also ensure proper
>> reset of newer registers such as CNTVOFF.
>>
>> The permissions on the Non-Secure Access Control Register (NSACR) are
>> used to probe what the security setting currently is when in supervisor
>> (SVC) mode.
> 
> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
> it's worth, my testing and development methodology is to run it after hacked
> up versions of the semihosting bootwrapper on the simulator that corresponds
> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.

Looks like the real problem was that I was hacking up the bootwrapper
incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
GIC for non-secure access. With that changed, I've tested the following
variations using the Image file in a single core configuration.

Start in non-secure SVC with non-secure access to GIC configured.

Start in secure SVC with non-secure access to GIC configured.

Start in secure SVC with non-secure access to GIC configured and hypervisor
support disabled in the model (-C cluster.has_el2=0). This required setting
the VBAR again in non-secure SVC but with that fix it seems to work. I'll
include this change in v2.

I have not been able to start up the bootwrapper with secure monitor support
disabled in the model (-C cluster.has_el3=0) because of faults during GIC
configuration.

So, any thoughts on the code? I have some questions.

What initialization am I missing?

What's the right ifdefery? I'm thinking a CONFIG_ARM_SEC_EXT that is
independent of CONFIG_ARM_VIRT_EXT.

Are there folks who want to run Linux in secure mode? Would documenting in
Kconfig that they can do this by leaving CONFIG_ARM_SEC_EXT undefined be
sufficient support for this?

Is it fine or bad to leave the CPU in undefined mode when one of the extension
or privileged operations fails? It's only for maybe a dozen instructions until
safe_svcmode_maskall.

How should CPU mode recording be handled? Should the CONFIG_ARM_SEC_EXT code
set __boot_cpu_mode with an extra bit specifying the security setting and then
the CONFIG_ARM_VIRT_EXT code set-if-unset?

What should the secure monitor stub leave behind? Right now it's different
than the hypervisor stub: put the address of the code you want run in monitor
mode into r4, make the secure monitor call, and make sure your code makes an
exception return when finished.

Whatever is left behind, I think it should allow the GIC driver to enable
non-secure access to the hardware if Linux was booted in secure mode.

If this code can be evolved and tested to the point where it's mergeable, then
Linux can finally interact intelligently with the features introduced by the
security extensions and only has to depend on firmware and bootloaders getting
the implementation defined aspects of a system right, taking care of the
architectually specified stuff itself when sufficiently privileged. Among
other things, this should lead towards the bootwrapper no longer being
required to boot Linux on ARM's virtual platforms, which would make my life
easier.

Thanks,
Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

Re: [RFC] arm: Handle starting up in secure mode

[Sonny Rao]

On Wed, Sep 17, 2014 at 6:25 AM, Christopher Covington
<cov@codeaurora.org> wrote:
> On 09/16/2014 05:24 PM, Christopher Covington wrote:
>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>>> ARM Linux currently has the most features available to it in hypervisor
>>> (HYP) mode, so switch to it when possible. This can also ensure proper
>>> reset of newer registers such as CNTVOFF.
>>>
>>> The permissions on the Non-Secure Access Control Register (NSACR) are
>>> used to probe what the security setting currently is when in supervisor
>>> (SVC) mode.
>>
>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
>> it's worth, my testing and development methodology is to run it after hacked
>> up versions of the semihosting bootwrapper on the simulator that corresponds
>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
>
> Looks like the real problem was that I was hacking up the bootwrapper
> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
> GIC for non-secure access. With that changed, I've tested the following
> variations using the Image file in a single core configuration.
>
> Start in non-secure SVC with non-secure access to GIC configured.
>
> Start in secure SVC with non-secure access to GIC configured.

I tried this on my rk3288 which boots in secure SVC mode, but I must
be missing the GIC configuration for non-secure access because I get a
message like this:
[    0.000000] GIC CPU mask not found - kernel will fail to boot.
[    0.000000] GIC CPU mask not found - kernel will fail to boot.
....
and then it hangs here:
[    0.116871] CPU0: thread -1, cpu 0, socket 5, mpidr 80000500
[    0.123274] Setting up static identity map for 0x5e3bf8 - 0x5e3c50

So I need to figure out how to enable non-secure access to the GIC work.

> Start in secure SVC with non-secure access to GIC configured and hypervisor
> support disabled in the model (-C cluster.has_el2=0). This required setting
> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
> include this change in v2.
>
> I have not been able to start up the bootwrapper with secure monitor support
> disabled in the model (-C cluster.has_el3=0) because of faults during GIC
> configuration.
>
> So, any thoughts on the code? I have some questions.
>
> What initialization am I missing?
>
> What's the right ifdefery? I'm thinking a CONFIG_ARM_SEC_EXT that is
> independent of CONFIG_ARM_VIRT_EXT.
>
> Are there folks who want to run Linux in secure mode? Would documenting in
> Kconfig that they can do this by leaving CONFIG_ARM_SEC_EXT undefined be
> sufficient support for this?

We currently tend to run Linux in Secure SVC mode, and this has worked
fine for a long time, and I think this is because these chips have
reset defaults that allow us to access everything in secure mode, but
I don't know if we'd need to suddenly deal with having to configure
everything to also handle non-secure mode as well.

I our hope here is that if you know you're starting in secure svc mode
on armv7a, and have virtualization extensions, the kernel could
"elevate" itself to hyp mode, without too much work, but it looks like
on (at least) rk3288 we need to do something to the GIC at a
minimum.

>
> Is it fine or bad to leave the CPU in undefined mode when one of the extension
> or privileged operations fails? It's only for maybe a dozen instructions until
> safe_svcmode_maskall.
>
> How should CPU mode recording be handled? Should the CONFIG_ARM_SEC_EXT code
> set __boot_cpu_mode with an extra bit specifying the security setting and then
> the CONFIG_ARM_VIRT_EXT code set-if-unset?
>
> What should the secure monitor stub leave behind? Right now it's different
> than the hypervisor stub: put the address of the code you want run in monitor
> mode into r4, make the secure monitor call, and make sure your code makes an
> exception return when finished.
>
> Whatever is left behind, I think it should allow the GIC driver to enable
> non-secure access to the hardware if Linux was booted in secure mode.
>
> If this code can be evolved and tested to the point where it's mergeable, then
> Linux can finally interact intelligently with the features introduced by the
> security extensions and only has to depend on firmware and bootloaders getting
> the implementation defined aspects of a system right, taking care of the
> architectually specified stuff itself when sufficiently privileged. Among
> other things, this should lead towards the bootwrapper no longer being
> required to boot Linux on ARM's virtual platforms, which would make my life
> easier.

This would be great for us, because we tend to have our firmware do
the bare minimum possible, and this would allow people running
upstream kernels on ChromeOS systems to easily use virtualization if
they wish.

>
> Thanks,
> Christopher
>
> --
> Employee of Qualcomm Innovation Center, Inc.
> Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
> hosted by the Linux Foundation.

[RFC] arm: Handle starting up in secure mode

[Sonny Rao]

On Wed, Sep 17, 2014 at 6:25 AM, Christopher Covington
<cov@codeaurora.org> wrote:
> On 09/16/2014 05:24 PM, Christopher Covington wrote:
>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>>> ARM Linux currently has the most features available to it in hypervisor
>>> (HYP) mode, so switch to it when possible. This can also ensure proper
>>> reset of newer registers such as CNTVOFF.
>>>
>>> The permissions on the Non-Secure Access Control Register (NSACR) are
>>> used to probe what the security setting currently is when in supervisor
>>> (SVC) mode.
>>
>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
>> it's worth, my testing and development methodology is to run it after hacked
>> up versions of the semihosting bootwrapper on the simulator that corresponds
>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
>
> Looks like the real problem was that I was hacking up the bootwrapper
> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
> GIC for non-secure access. With that changed, I've tested the following
> variations using the Image file in a single core configuration.
>
> Start in non-secure SVC with non-secure access to GIC configured.
>
> Start in secure SVC with non-secure access to GIC configured.

I tried this on my rk3288 which boots in secure SVC mode, but I must
be missing the GIC configuration for non-secure access because I get a
message like this:
[    0.000000] GIC CPU mask not found - kernel will fail to boot.
[    0.000000] GIC CPU mask not found - kernel will fail to boot.
....
and then it hangs here:
[    0.116871] CPU0: thread -1, cpu 0, socket 5, mpidr 80000500
[    0.123274] Setting up static identity map for 0x5e3bf8 - 0x5e3c50

So I need to figure out how to enable non-secure access to the GIC work.

> Start in secure SVC with non-secure access to GIC configured and hypervisor
> support disabled in the model (-C cluster.has_el2=0). This required setting
> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
> include this change in v2.
>
> I have not been able to start up the bootwrapper with secure monitor support
> disabled in the model (-C cluster.has_el3=0) because of faults during GIC
> configuration.
>
> So, any thoughts on the code? I have some questions.
>
> What initialization am I missing?
>
> What's the right ifdefery? I'm thinking a CONFIG_ARM_SEC_EXT that is
> independent of CONFIG_ARM_VIRT_EXT.
>
> Are there folks who want to run Linux in secure mode? Would documenting in
> Kconfig that they can do this by leaving CONFIG_ARM_SEC_EXT undefined be
> sufficient support for this?

We currently tend to run Linux in Secure SVC mode, and this has worked
fine for a long time, and I think this is because these chips have
reset defaults that allow us to access everything in secure mode, but
I don't know if we'd need to suddenly deal with having to configure
everything to also handle non-secure mode as well.

I our hope here is that if you know you're starting in secure svc mode
on armv7a, and have virtualization extensions, the kernel could
"elevate" itself to hyp mode, without too much work, but it looks like
on (at least) rk3288 we need to do something to the GIC at a
minimum.

>
> Is it fine or bad to leave the CPU in undefined mode when one of the extension
> or privileged operations fails? It's only for maybe a dozen instructions until
> safe_svcmode_maskall.
>
> How should CPU mode recording be handled? Should the CONFIG_ARM_SEC_EXT code
> set __boot_cpu_mode with an extra bit specifying the security setting and then
> the CONFIG_ARM_VIRT_EXT code set-if-unset?
>
> What should the secure monitor stub leave behind? Right now it's different
> than the hypervisor stub: put the address of the code you want run in monitor
> mode into r4, make the secure monitor call, and make sure your code makes an
> exception return when finished.
>
> Whatever is left behind, I think it should allow the GIC driver to enable
> non-secure access to the hardware if Linux was booted in secure mode.
>
> If this code can be evolved and tested to the point where it's mergeable, then
> Linux can finally interact intelligently with the features introduced by the
> security extensions and only has to depend on firmware and bootloaders getting
> the implementation defined aspects of a system right, taking care of the
> architectually specified stuff itself when sufficiently privileged. Among
> other things, this should lead towards the bootwrapper no longer being
> required to boot Linux on ARM's virtual platforms, which would make my life
> easier.

This would be great for us, because we tend to have our firmware do
the bare minimum possible, and this would allow people running
upstream kernels on ChromeOS systems to easily use virtualization if
they wish.

>
> Thanks,
> Christopher
>
> --
> Employee of Qualcomm Innovation Center, Inc.
> Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
> hosted by the Linux Foundation.

Re: [RFC] arm: Handle starting up in secure mode

[Catalin Marinas]

On Tue, Sep 16, 2014 at 10:09:16PM +0100, Christopher Covington wrote:
> diff --git a/arch/arm/kernel/hyp-stub.S b/arch/arm/kernel/hyp-stub.S
> index 2a55373..36d1a9c 100644
> --- a/arch/arm/kernel/hyp-stub.S
> +++ b/arch/arm/kernel/hyp-stub.S
> @@ -20,6 +20,7 @@
>  #include <linux/linkage.h>
>  #include <asm/assembler.h>
>  #include <asm/virt.h>
> +#include <asm/opcodes-sec.h>
>  
>  #ifndef ZIMAGE
>  /*
> @@ -76,6 +77,64 @@ ENTRY(__boot_cpu_mode)
>  #endif /* ZIMAGE */
>  
>  /*
> + * Detect whether the system is in secure supervisor mode, and if it is,
> + * switch to hypervisor mode by way of secure monitor mode.
> + */
> +ENTRY(__mon_stub_install)
> +	mrs	r4, cpsr
> +	and	r4, r4, #MODE_MASK
> +	cmp	r4, #SVC_MODE
> +	movne	pc, lr
> +
> +	/*
> +	 * Set things up so that if an NSACR access causes an undefined
> +	 * instruction exception, we return. safe_svcmode_maskall called
> +	 * just after this will get us back into supervisor mode.
> +	 */
> +	adr	r4, __mon_stub_vectors
> +	mcr	p15, 0, r4, c12, c0, 0	@ set vector base address (VBAR)
> +	mov	r4, lr
> +
> +	/*
> +	 * Writing the NSACR will only succeed if we're in a secure mode.
> +	 */
> +	mrc	p15, 0, r5, c1, c1, 2	@ get non-secure access control (NSACR)
> +	mcr	p15, 0, r5, c1, c1, 2	@ set non-secure access control (NSACR)

Since you talk about ARMv8 further down, if you run in AArch32 mode in
secure EL1 it will trap to EL3 (if it was non-secure EL1/EL2, write
would undef and read returns 0xc00). Therefore for ARMv8 my
recommendation is to run Linux (whether AArch64 or AArch32) only on the
non-secure side.

> +	/*
> +	 * If we get here, we know we're in secure supervisor mode, so make the
> +	 * switch to secure monitor mode.
> +	 *
> +	 * TODO: make sure this doesn't trap to A64 EL3.
> +	 */
> +	adr	r4, __mon_stub_vectors
> +	mcr	p15, 0, r4, c12, c0, 1	@ set monitor vector base (MVBAR)
> +	adr	r4, mon_settings
> +	__SMC(0)

I think from secure EL1 you can switch to monitor simply by setting the
CPSR mode bits (though IIRC on ARMv8 would also trap to EL3).

> +
> +	/*
> +	 * Now, from non-secure supervisor mode, transition to hypervisor mode
> +	 * and return via the exception vector.
> +	 */
> +	mov	r4, lr
> +	__HVC(0)

>From monitor mode can you return directly to HVC mode via ERET and avoid
another HVC?

-- 
Catalin

[RFC] arm: Handle starting up in secure mode

[Catalin Marinas]

On Tue, Sep 16, 2014 at 10:09:16PM +0100, Christopher Covington wrote:
> diff --git a/arch/arm/kernel/hyp-stub.S b/arch/arm/kernel/hyp-stub.S
> index 2a55373..36d1a9c 100644
> --- a/arch/arm/kernel/hyp-stub.S
> +++ b/arch/arm/kernel/hyp-stub.S
> @@ -20,6 +20,7 @@
>  #include <linux/linkage.h>
>  #include <asm/assembler.h>
>  #include <asm/virt.h>
> +#include <asm/opcodes-sec.h>
>  
>  #ifndef ZIMAGE
>  /*
> @@ -76,6 +77,64 @@ ENTRY(__boot_cpu_mode)
>  #endif /* ZIMAGE */
>  
>  /*
> + * Detect whether the system is in secure supervisor mode, and if it is,
> + * switch to hypervisor mode by way of secure monitor mode.
> + */
> +ENTRY(__mon_stub_install)
> +	mrs	r4, cpsr
> +	and	r4, r4, #MODE_MASK
> +	cmp	r4, #SVC_MODE
> +	movne	pc, lr
> +
> +	/*
> +	 * Set things up so that if an NSACR access causes an undefined
> +	 * instruction exception, we return. safe_svcmode_maskall called
> +	 * just after this will get us back into supervisor mode.
> +	 */
> +	adr	r4, __mon_stub_vectors
> +	mcr	p15, 0, r4, c12, c0, 0	@ set vector base address (VBAR)
> +	mov	r4, lr
> +
> +	/*
> +	 * Writing the NSACR will only succeed if we're in a secure mode.
> +	 */
> +	mrc	p15, 0, r5, c1, c1, 2	@ get non-secure access control (NSACR)
> +	mcr	p15, 0, r5, c1, c1, 2	@ set non-secure access control (NSACR)

Since you talk about ARMv8 further down, if you run in AArch32 mode in
secure EL1 it will trap to EL3 (if it was non-secure EL1/EL2, write
would undef and read returns 0xc00). Therefore for ARMv8 my
recommendation is to run Linux (whether AArch64 or AArch32) only on the
non-secure side.

> +	/*
> +	 * If we get here, we know we're in secure supervisor mode, so make the
> +	 * switch to secure monitor mode.
> +	 *
> +	 * TODO: make sure this doesn't trap to A64 EL3.
> +	 */
> +	adr	r4, __mon_stub_vectors
> +	mcr	p15, 0, r4, c12, c0, 1	@ set monitor vector base (MVBAR)
> +	adr	r4, mon_settings
> +	__SMC(0)

I think from secure EL1 you can switch to monitor simply by setting the
CPSR mode bits (though IIRC on ARMv8 would also trap to EL3).

> +
> +	/*
> +	 * Now, from non-secure supervisor mode, transition to hypervisor mode
> +	 * and return via the exception vector.
> +	 */
> +	mov	r4, lr
> +	__HVC(0)

>From monitor mode can you return directly to HVC mode via ERET and avoid
another HVC?

-- 
Catalin

Re: [RFC] arm: Handle starting up in secure mode

[Peter Maydell]

On 17 September 2014 06:25, Christopher Covington <cov@codeaurora.org> wrote:
> On 09/16/2014 05:24 PM, Christopher Covington wrote:
>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>>> ARM Linux currently has the most features available to it in hypervisor
>>> (HYP) mode, so switch to it when possible. This can also ensure proper
>>> reset of newer registers such as CNTVOFF.
>>>
>>> The permissions on the Non-Secure Access Control Register (NSACR) are
>>> used to probe what the security setting currently is when in supervisor
>>> (SVC) mode.
>>
>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
>> it's worth, my testing and development methodology is to run it after hacked
>> up versions of the semihosting bootwrapper on the simulator that corresponds
>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
>
> Looks like the real problem was that I was hacking up the bootwrapper
> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
> GIC for non-secure access. With that changed, I've tested the following
> variations using the Image file in a single core configuration.
>
> Start in non-secure SVC with non-secure access to GIC configured.
>
> Start in secure SVC with non-secure access to GIC configured.
>
> Start in secure SVC with non-secure access to GIC configured and hypervisor
> support disabled in the model (-C cluster.has_el2=0). This required setting
> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
> include this change in v2.

If you're relying on the boot loader to set up the GIC to support
non-secure access anyway, why not just have it boot the kernel in Hyp
like the boot protocol document recommends? (The same thing as the GIC
is going to apply for any other hardware that needs configuration to
allow NS access; if we need the firmware to deal with this we might as
well just have it boot us in the right mode too.)

Incidentally, on a v7 CPU without the Security Extensions the VBAR
doesn't exist at all, so your code is going to UNDEF at an earlier
point than you think it is...

-- PMM

[RFC] arm: Handle starting up in secure mode

[Peter Maydell]

On 17 September 2014 06:25, Christopher Covington <cov@codeaurora.org> wrote:
> On 09/16/2014 05:24 PM, Christopher Covington wrote:
>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>>> ARM Linux currently has the most features available to it in hypervisor
>>> (HYP) mode, so switch to it when possible. This can also ensure proper
>>> reset of newer registers such as CNTVOFF.
>>>
>>> The permissions on the Non-Secure Access Control Register (NSACR) are
>>> used to probe what the security setting currently is when in supervisor
>>> (SVC) mode.
>>
>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
>> it's worth, my testing and development methodology is to run it after hacked
>> up versions of the semihosting bootwrapper on the simulator that corresponds
>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
>
> Looks like the real problem was that I was hacking up the bootwrapper
> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
> GIC for non-secure access. With that changed, I've tested the following
> variations using the Image file in a single core configuration.
>
> Start in non-secure SVC with non-secure access to GIC configured.
>
> Start in secure SVC with non-secure access to GIC configured.
>
> Start in secure SVC with non-secure access to GIC configured and hypervisor
> support disabled in the model (-C cluster.has_el2=0). This required setting
> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
> include this change in v2.

If you're relying on the boot loader to set up the GIC to support
non-secure access anyway, why not just have it boot the kernel in Hyp
like the boot protocol document recommends? (The same thing as the GIC
is going to apply for any other hardware that needs configuration to
allow NS access; if we need the firmware to deal with this we might as
well just have it boot us in the right mode too.)

Incidentally, on a v7 CPU without the Security Extensions the VBAR
doesn't exist at all, so your code is going to UNDEF at an earlier
point than you think it is...

-- PMM

Re: [RFC] arm: Handle starting up in secure mode

[Christopher Covington]

Hi Peter,

On 09/19/2014 01:56 AM, Peter Maydell wrote:
> On 17 September 2014 06:25, Christopher Covington <cov@codeaurora.org> wrote:
>> On 09/16/2014 05:24 PM, Christopher Covington wrote:
>>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>>>> ARM Linux currently has the most features available to it in hypervisor
>>>> (HYP) mode, so switch to it when possible. This can also ensure proper
>>>> reset of newer registers such as CNTVOFF.
>>>>
>>>> The permissions on the Non-Secure Access Control Register (NSACR) are
>>>> used to probe what the security setting currently is when in supervisor
>>>> (SVC) mode.
>>>
>>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
>>> it's worth, my testing and development methodology is to run it after hacked
>>> up versions of the semihosting bootwrapper on the simulator that corresponds
>>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
>>
>> Looks like the real problem was that I was hacking up the bootwrapper
>> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
>> GIC for non-secure access. With that changed, I've tested the following
>> variations using the Image file in a single core configuration.
>>
>> Start in non-secure SVC with non-secure access to GIC configured.
>>
>> Start in secure SVC with non-secure access to GIC configured.
>>
>> Start in secure SVC with non-secure access to GIC configured and hypervisor
>> support disabled in the model (-C cluster.has_el2=0). This required setting
>> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
>> include this change in v2.
> 
> If you're relying on the boot loader to set up the GIC to support
> non-secure access anyway, why not just have it boot the kernel in Hyp
> like the boot protocol document recommends? (The same thing as the GIC
> is going to apply for any other hardware that needs configuration to
> allow NS access; if we need the firmware to deal with this we might as
> well just have it boot us in the right mode too.)

I'd like to get rid of as much of the bootwrapper as possible (having gotten
spoiled by using QEMU's built-in bootloader). I'm just taking it one step at a
time. Handling GIC initialization in the kernel is probably the next step.

> Incidentally, on a v7 CPU without the Security Extensions the VBAR
> doesn't exist at all, so your code is going to UNDEF at an earlier
> point than you think it is...

Thanks for pointing this out. I recently was digging through the ID registers
and figured it would be good idea to guard attempting the SVC_S -> HYP switch
with some PFR checks that Security and Virtualization extensions are
implemented just right: PFR1.Virt_fac == 1 && PFR.Sec_frac == 1. (How to
handle separate secure physical memory, PFR1.Sec_frac > 1, is a challenge I
think I'd like to defer to a later patch).

Thanks,
Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

[RFC] arm: Handle starting up in secure mode

[Christopher Covington]

Hi Peter,

On 09/19/2014 01:56 AM, Peter Maydell wrote:
> On 17 September 2014 06:25, Christopher Covington <cov@codeaurora.org> wrote:
>> On 09/16/2014 05:24 PM, Christopher Covington wrote:
>>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>>>> ARM Linux currently has the most features available to it in hypervisor
>>>> (HYP) mode, so switch to it when possible. This can also ensure proper
>>>> reset of newer registers such as CNTVOFF.
>>>>
>>>> The permissions on the Non-Secure Access Control Register (NSACR) are
>>>> used to probe what the security setting currently is when in supervisor
>>>> (SVC) mode.
>>>
>>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
>>> it's worth, my testing and development methodology is to run it after hacked
>>> up versions of the semihosting bootwrapper on the simulator that corresponds
>>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
>>
>> Looks like the real problem was that I was hacking up the bootwrapper
>> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
>> GIC for non-secure access. With that changed, I've tested the following
>> variations using the Image file in a single core configuration.
>>
>> Start in non-secure SVC with non-secure access to GIC configured.
>>
>> Start in secure SVC with non-secure access to GIC configured.
>>
>> Start in secure SVC with non-secure access to GIC configured and hypervisor
>> support disabled in the model (-C cluster.has_el2=0). This required setting
>> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
>> include this change in v2.
> 
> If you're relying on the boot loader to set up the GIC to support
> non-secure access anyway, why not just have it boot the kernel in Hyp
> like the boot protocol document recommends? (The same thing as the GIC
> is going to apply for any other hardware that needs configuration to
> allow NS access; if we need the firmware to deal with this we might as
> well just have it boot us in the right mode too.)

I'd like to get rid of as much of the bootwrapper as possible (having gotten
spoiled by using QEMU's built-in bootloader). I'm just taking it one step at a
time. Handling GIC initialization in the kernel is probably the next step.

> Incidentally, on a v7 CPU without the Security Extensions the VBAR
> doesn't exist at all, so your code is going to UNDEF at an earlier
> point than you think it is...

Thanks for pointing this out. I recently was digging through the ID registers
and figured it would be good idea to guard attempting the SVC_S -> HYP switch
with some PFR checks that Security and Virtualization extensions are
implemented just right: PFR1.Virt_fac == 1 && PFR.Sec_frac == 1. (How to
handle separate secure physical memory, PFR1.Sec_frac > 1, is a challenge I
think I'd like to defer to a later patch).

Thanks,
Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

Re: [RFC] arm: Handle starting up in secure mode

[Catalin Marinas]

On Fri, Sep 19, 2014 at 02:22:10PM +0100, Christopher Covington wrote:
> On 09/19/2014 01:56 AM, Peter Maydell wrote:
> > On 17 September 2014 06:25, Christopher Covington <cov@codeaurora.org> wrote:
> >> On 09/16/2014 05:24 PM, Christopher Covington wrote:
> >>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
> >>>> ARM Linux currently has the most features available to it in hypervisor
> >>>> (HYP) mode, so switch to it when possible. This can also ensure proper
> >>>> reset of newer registers such as CNTVOFF.
> >>>>
> >>>> The permissions on the Non-Secure Access Control Register (NSACR) are
> >>>> used to probe what the security setting currently is when in supervisor
> >>>> (SVC) mode.
> >>>
> >>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
> >>> it's worth, my testing and development methodology is to run it after hacked
> >>> up versions of the semihosting bootwrapper on the simulator that corresponds
> >>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
> >>
> >> Looks like the real problem was that I was hacking up the bootwrapper
> >> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
> >> GIC for non-secure access. With that changed, I've tested the following
> >> variations using the Image file in a single core configuration.
> >>
> >> Start in non-secure SVC with non-secure access to GIC configured.
> >>
> >> Start in secure SVC with non-secure access to GIC configured.
> >>
> >> Start in secure SVC with non-secure access to GIC configured and hypervisor
> >> support disabled in the model (-C cluster.has_el2=0). This required setting
> >> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
> >> include this change in v2.
> > 
> > If you're relying on the boot loader to set up the GIC to support
> > non-secure access anyway, why not just have it boot the kernel in Hyp
> > like the boot protocol document recommends? (The same thing as the GIC
> > is going to apply for any other hardware that needs configuration to
> > allow NS access; if we need the firmware to deal with this we might as
> > well just have it boot us in the right mode too.)
> 
> I'd like to get rid of as much of the bootwrapper as possible (having gotten
> spoiled by using QEMU's built-in bootloader). I'm just taking it one step at a
> time. Handling GIC initialization in the kernel is probably the next step.

The problem is that the kernel doesn't know about GIC until much later.
So I don't see an easy workaround, other than relying on the boot-loader
to do the right thing (and then we go to the point Peter made about
changing it to start Linux in Hyp mode directly).

-- 
Catalin

[RFC] arm: Handle starting up in secure mode

[Catalin Marinas]

On Fri, Sep 19, 2014 at 02:22:10PM +0100, Christopher Covington wrote:
> On 09/19/2014 01:56 AM, Peter Maydell wrote:
> > On 17 September 2014 06:25, Christopher Covington <cov@codeaurora.org> wrote:
> >> On 09/16/2014 05:24 PM, Christopher Covington wrote:
> >>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
> >>>> ARM Linux currently has the most features available to it in hypervisor
> >>>> (HYP) mode, so switch to it when possible. This can also ensure proper
> >>>> reset of newer registers such as CNTVOFF.
> >>>>
> >>>> The permissions on the Non-Secure Access Control Register (NSACR) are
> >>>> used to probe what the security setting currently is when in supervisor
> >>>> (SVC) mode.
> >>>
> >>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
> >>> it's worth, my testing and development methodology is to run it after hacked
> >>> up versions of the semihosting bootwrapper on the simulator that corresponds
> >>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
> >>
> >> Looks like the real problem was that I was hacking up the bootwrapper
> >> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
> >> GIC for non-secure access. With that changed, I've tested the following
> >> variations using the Image file in a single core configuration.
> >>
> >> Start in non-secure SVC with non-secure access to GIC configured.
> >>
> >> Start in secure SVC with non-secure access to GIC configured.
> >>
> >> Start in secure SVC with non-secure access to GIC configured and hypervisor
> >> support disabled in the model (-C cluster.has_el2=0). This required setting
> >> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
> >> include this change in v2.
> > 
> > If you're relying on the boot loader to set up the GIC to support
> > non-secure access anyway, why not just have it boot the kernel in Hyp
> > like the boot protocol document recommends? (The same thing as the GIC
> > is going to apply for any other hardware that needs configuration to
> > allow NS access; if we need the firmware to deal with this we might as
> > well just have it boot us in the right mode too.)
> 
> I'd like to get rid of as much of the bootwrapper as possible (having gotten
> spoiled by using QEMU's built-in bootloader). I'm just taking it one step at a
> time. Handling GIC initialization in the kernel is probably the next step.

The problem is that the kernel doesn't know about GIC until much later.
So I don't see an easy workaround, other than relying on the boot-loader
to do the right thing (and then we go to the point Peter made about
changing it to start Linux in Hyp mode directly).

-- 
Catalin

Re: [RFC] arm: Handle starting up in secure mode

[Christopher Covington]

On 09/19/2014 09:30 AM, Catalin Marinas wrote:
> On Fri, Sep 19, 2014 at 02:22:10PM +0100, Christopher Covington wrote:
>> On 09/19/2014 01:56 AM, Peter Maydell wrote:
>>> On 17 September 2014 06:25, Christopher Covington <cov@codeaurora.org> wrote:
>>>> On 09/16/2014 05:24 PM, Christopher Covington wrote:
>>>>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>>>>>> ARM Linux currently has the most features available to it in hypervisor
>>>>>> (HYP) mode, so switch to it when possible. This can also ensure proper
>>>>>> reset of newer registers such as CNTVOFF.
>>>>>>
>>>>>> The permissions on the Non-Secure Access Control Register (NSACR) are
>>>>>> used to probe what the security setting currently is when in supervisor
>>>>>> (SVC) mode.
>>>>>
>>>>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
>>>>> it's worth, my testing and development methodology is to run it after hacked
>>>>> up versions of the semihosting bootwrapper on the simulator that corresponds
>>>>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
>>>>
>>>> Looks like the real problem was that I was hacking up the bootwrapper
>>>> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
>>>> GIC for non-secure access. With that changed, I've tested the following
>>>> variations using the Image file in a single core configuration.
>>>>
>>>> Start in non-secure SVC with non-secure access to GIC configured.
>>>>
>>>> Start in secure SVC with non-secure access to GIC configured.
>>>>
>>>> Start in secure SVC with non-secure access to GIC configured and hypervisor
>>>> support disabled in the model (-C cluster.has_el2=0). This required setting
>>>> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
>>>> include this change in v2.
>>>
>>> If you're relying on the boot loader to set up the GIC to support
>>> non-secure access anyway, why not just have it boot the kernel in Hyp
>>> like the boot protocol document recommends? (The same thing as the GIC
>>> is going to apply for any other hardware that needs configuration to
>>> allow NS access; if we need the firmware to deal with this we might as
>>> well just have it boot us in the right mode too.)
>>
>> I'd like to get rid of as much of the bootwrapper as possible (having gotten
>> spoiled by using QEMU's built-in bootloader). I'm just taking it one step at a
>> time. Handling GIC initialization in the kernel is probably the next step.
> 
> The problem is that the kernel doesn't know about GIC until much later.
> So I don't see an easy workaround, other than relying on the boot-loader
> to do the right thing (and then we go to the point Peter made about
> changing it to start Linux in Hyp mode directly).

My initial thought was to have the later-running GIC driver capable of running
some initialization code in monitor mode. However in reading through the ID
registers, keying off of PFR1.GIC might allow for early initialization on
newer systems.

Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

[RFC] arm: Handle starting up in secure mode

[Christopher Covington]

On 09/19/2014 09:30 AM, Catalin Marinas wrote:
> On Fri, Sep 19, 2014 at 02:22:10PM +0100, Christopher Covington wrote:
>> On 09/19/2014 01:56 AM, Peter Maydell wrote:
>>> On 17 September 2014 06:25, Christopher Covington <cov@codeaurora.org> wrote:
>>>> On 09/16/2014 05:24 PM, Christopher Covington wrote:
>>>>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>>>>>> ARM Linux currently has the most features available to it in hypervisor
>>>>>> (HYP) mode, so switch to it when possible. This can also ensure proper
>>>>>> reset of newer registers such as CNTVOFF.
>>>>>>
>>>>>> The permissions on the Non-Secure Access Control Register (NSACR) are
>>>>>> used to probe what the security setting currently is when in supervisor
>>>>>> (SVC) mode.
>>>>>
>>>>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
>>>>> it's worth, my testing and development methodology is to run it after hacked
>>>>> up versions of the semihosting bootwrapper on the simulator that corresponds
>>>>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
>>>>
>>>> Looks like the real problem was that I was hacking up the bootwrapper
>>>> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
>>>> GIC for non-secure access. With that changed, I've tested the following
>>>> variations using the Image file in a single core configuration.
>>>>
>>>> Start in non-secure SVC with non-secure access to GIC configured.
>>>>
>>>> Start in secure SVC with non-secure access to GIC configured.
>>>>
>>>> Start in secure SVC with non-secure access to GIC configured and hypervisor
>>>> support disabled in the model (-C cluster.has_el2=0). This required setting
>>>> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
>>>> include this change in v2.
>>>
>>> If you're relying on the boot loader to set up the GIC to support
>>> non-secure access anyway, why not just have it boot the kernel in Hyp
>>> like the boot protocol document recommends? (The same thing as the GIC
>>> is going to apply for any other hardware that needs configuration to
>>> allow NS access; if we need the firmware to deal with this we might as
>>> well just have it boot us in the right mode too.)
>>
>> I'd like to get rid of as much of the bootwrapper as possible (having gotten
>> spoiled by using QEMU's built-in bootloader). I'm just taking it one step at a
>> time. Handling GIC initialization in the kernel is probably the next step.
> 
> The problem is that the kernel doesn't know about GIC until much later.
> So I don't see an easy workaround, other than relying on the boot-loader
> to do the right thing (and then we go to the point Peter made about
> changing it to start Linux in Hyp mode directly).

My initial thought was to have the later-running GIC driver capable of running
some initialization code in monitor mode. However in reading through the ID
registers, keying off of PFR1.GIC might allow for early initialization on
newer systems.

Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

Re: [RFC] arm: Handle starting up in secure mode

[Christopher Covington]

Hi Catalin,

On 09/18/2014 01:23 PM, Catalin Marinas wrote:
> On Tue, Sep 16, 2014 at 10:09:16PM +0100, Christopher Covington wrote:
>> diff --git a/arch/arm/kernel/hyp-stub.S b/arch/arm/kernel/hyp-stub.S
>> index 2a55373..36d1a9c 100644
>> --- a/arch/arm/kernel/hyp-stub.S
>> +++ b/arch/arm/kernel/hyp-stub.S
>> @@ -20,6 +20,7 @@
>>  #include <linux/linkage.h>
>>  #include <asm/assembler.h>
>>  #include <asm/virt.h>
>> +#include <asm/opcodes-sec.h>
>>  
>>  #ifndef ZIMAGE
>>  /*
>> @@ -76,6 +77,64 @@ ENTRY(__boot_cpu_mode)
>>  #endif /* ZIMAGE */
>>  
>>  /*
>> + * Detect whether the system is in secure supervisor mode, and if it is,
>> + * switch to hypervisor mode by way of secure monitor mode.
>> + */
>> +ENTRY(__mon_stub_install)
>> +	mrs	r4, cpsr
>> +	and	r4, r4, #MODE_MASK
>> +	cmp	r4, #SVC_MODE
>> +	movne	pc, lr
>> +
>> +	/*
>> +	 * Set things up so that if an NSACR access causes an undefined
>> +	 * instruction exception, we return. safe_svcmode_maskall called
>> +	 * just after this will get us back into supervisor mode.
>> +	 */
>> +	adr	r4, __mon_stub_vectors
>> +	mcr	p15, 0, r4, c12, c0, 0	@ set vector base address (VBAR)
>> +	mov	r4, lr
>> +
>> +	/*
>> +	 * Writing the NSACR will only succeed if we're in a secure mode.
>> +	 */
>> +	mrc	p15, 0, r5, c1, c1, 2	@ get non-secure access control (NSACR)
>> +	mcr	p15, 0, r5, c1, c1, 2	@ set non-secure access control (NSACR)
> 
> Since you talk about ARMv8 further down, if you run in AArch32 mode in
> secure EL1 it will trap to EL3.

If EL3/MON is A64, yes. If EL3/MON is A32 (the case I care most about for A32
kernels), my simulator experiments and reading of the documentation suggest
the write succeeds. I'll have to make an A64 EL3 test case and see what I can
do about that.

>> +	/*
>> +	 * If we get here, we know we're in secure supervisor mode, so make the
>> +	 * switch to secure monitor mode.
>> +	 *
>> +	 * TODO: make sure this doesn't trap to A64 EL3.
>> +	 */
>> +	adr	r4, __mon_stub_vectors
>> +	mcr	p15, 0, r4, c12, c0, 1	@ set monitor vector base (MVBAR)
>> +	adr	r4, mon_settings
>> +	__SMC(0)
> 
> I think from secure EL1 you can switch to monitor simply by setting the
> CPSR mode bits (though IIRC on ARMv8 would also trap to EL3).
> 
>> +
>> +	/*
>> +	 * Now, from non-secure supervisor mode, transition to hypervisor mode
>> +	 * and return via the exception vector.
>> +	 */
>> +	mov	r4, lr
>> +	__HVC(0)
> 
> From monitor mode can you return directly to HVC mode via ERET and avoid
> another HVC?

I'll experiment with switching modes through [CS]PSR writes.

Thanks,
Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

[RFC] arm: Handle starting up in secure mode

[Christopher Covington]

Hi Catalin,

On 09/18/2014 01:23 PM, Catalin Marinas wrote:
> On Tue, Sep 16, 2014 at 10:09:16PM +0100, Christopher Covington wrote:
>> diff --git a/arch/arm/kernel/hyp-stub.S b/arch/arm/kernel/hyp-stub.S
>> index 2a55373..36d1a9c 100644
>> --- a/arch/arm/kernel/hyp-stub.S
>> +++ b/arch/arm/kernel/hyp-stub.S
>> @@ -20,6 +20,7 @@
>>  #include <linux/linkage.h>
>>  #include <asm/assembler.h>
>>  #include <asm/virt.h>
>> +#include <asm/opcodes-sec.h>
>>  
>>  #ifndef ZIMAGE
>>  /*
>> @@ -76,6 +77,64 @@ ENTRY(__boot_cpu_mode)
>>  #endif /* ZIMAGE */
>>  
>>  /*
>> + * Detect whether the system is in secure supervisor mode, and if it is,
>> + * switch to hypervisor mode by way of secure monitor mode.
>> + */
>> +ENTRY(__mon_stub_install)
>> +	mrs	r4, cpsr
>> +	and	r4, r4, #MODE_MASK
>> +	cmp	r4, #SVC_MODE
>> +	movne	pc, lr
>> +
>> +	/*
>> +	 * Set things up so that if an NSACR access causes an undefined
>> +	 * instruction exception, we return. safe_svcmode_maskall called
>> +	 * just after this will get us back into supervisor mode.
>> +	 */
>> +	adr	r4, __mon_stub_vectors
>> +	mcr	p15, 0, r4, c12, c0, 0	@ set vector base address (VBAR)
>> +	mov	r4, lr
>> +
>> +	/*
>> +	 * Writing the NSACR will only succeed if we're in a secure mode.
>> +	 */
>> +	mrc	p15, 0, r5, c1, c1, 2	@ get non-secure access control (NSACR)
>> +	mcr	p15, 0, r5, c1, c1, 2	@ set non-secure access control (NSACR)
> 
> Since you talk about ARMv8 further down, if you run in AArch32 mode in
> secure EL1 it will trap to EL3.

If EL3/MON is A64, yes. If EL3/MON is A32 (the case I care most about for A32
kernels), my simulator experiments and reading of the documentation suggest
the write succeeds. I'll have to make an A64 EL3 test case and see what I can
do about that.

>> +	/*
>> +	 * If we get here, we know we're in secure supervisor mode, so make the
>> +	 * switch to secure monitor mode.
>> +	 *
>> +	 * TODO: make sure this doesn't trap to A64 EL3.
>> +	 */
>> +	adr	r4, __mon_stub_vectors
>> +	mcr	p15, 0, r4, c12, c0, 1	@ set monitor vector base (MVBAR)
>> +	adr	r4, mon_settings
>> +	__SMC(0)
> 
> I think from secure EL1 you can switch to monitor simply by setting the
> CPSR mode bits (though IIRC on ARMv8 would also trap to EL3).
> 
>> +
>> +	/*
>> +	 * Now, from non-secure supervisor mode, transition to hypervisor mode
>> +	 * and return via the exception vector.
>> +	 */
>> +	mov	r4, lr
>> +	__HVC(0)
> 
> From monitor mode can you return directly to HVC mode via ERET and avoid
> another HVC?

I'll experiment with switching modes through [CS]PSR writes.

Thanks,
Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

Re: [RFC] arm: Handle starting up in secure mode

[Sonny Rao]

On Fri, Sep 19, 2014 at 6:30 AM, Catalin Marinas
<catalin.marinas@arm.com> wrote:
> On Fri, Sep 19, 2014 at 02:22:10PM +0100, Christopher Covington wrote:
>> On 09/19/2014 01:56 AM, Peter Maydell wrote:
>> > On 17 September 2014 06:25, Christopher Covington <cov@codeaurora.org> wrote:
>> >> On 09/16/2014 05:24 PM, Christopher Covington wrote:
>> >>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>> >>>> ARM Linux currently has the most features available to it in hypervisor
>> >>>> (HYP) mode, so switch to it when possible. This can also ensure proper
>> >>>> reset of newer registers such as CNTVOFF.
>> >>>>
>> >>>> The permissions on the Non-Secure Access Control Register (NSACR) are
>> >>>> used to probe what the security setting currently is when in supervisor
>> >>>> (SVC) mode.
>> >>>
>> >>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
>> >>> it's worth, my testing and development methodology is to run it after hacked
>> >>> up versions of the semihosting bootwrapper on the simulator that corresponds
>> >>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
>> >>
>> >> Looks like the real problem was that I was hacking up the bootwrapper
>> >> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
>> >> GIC for non-secure access. With that changed, I've tested the following
>> >> variations using the Image file in a single core configuration.
>> >>
>> >> Start in non-secure SVC with non-secure access to GIC configured.
>> >>
>> >> Start in secure SVC with non-secure access to GIC configured.
>> >>
>> >> Start in secure SVC with non-secure access to GIC configured and hypervisor
>> >> support disabled in the model (-C cluster.has_el2=0). This required setting
>> >> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
>> >> include this change in v2.
>> >
>> > If you're relying on the boot loader to set up the GIC to support
>> > non-secure access anyway, why not just have it boot the kernel in Hyp
>> > like the boot protocol document recommends? (The same thing as the GIC
>> > is going to apply for any other hardware that needs configuration to
>> > allow NS access; if we need the firmware to deal with this we might as
>> > well just have it boot us in the right mode too.)
>>
>> I'd like to get rid of as much of the bootwrapper as possible (having gotten
>> spoiled by using QEMU's built-in bootloader). I'm just taking it one step at a
>> time. Handling GIC initialization in the kernel is probably the next step.
>
> The problem is that the kernel doesn't know about GIC until much later.
> So I don't see an easy workaround, other than relying on the boot-loader
> to do the right thing (and then we go to the point Peter made about
> changing it to start Linux in Hyp mode directly).

Well, for us, the issue is that our boot-loader isn't involved in
secondary cpu startup, either at boot time nor suspend/resume or cpu
hotplug/power gating.
So we certainly could have the boot loader set up the GIC for
non-secure access and then this type of solution would work, though
I'm not sure what else might need to be set up for non-secure access.

>
> --
> Catalin

[RFC] arm: Handle starting up in secure mode

[Sonny Rao]

On Fri, Sep 19, 2014 at 6:30 AM, Catalin Marinas
<catalin.marinas@arm.com> wrote:
> On Fri, Sep 19, 2014 at 02:22:10PM +0100, Christopher Covington wrote:
>> On 09/19/2014 01:56 AM, Peter Maydell wrote:
>> > On 17 September 2014 06:25, Christopher Covington <cov@codeaurora.org> wrote:
>> >> On 09/16/2014 05:24 PM, Christopher Covington wrote:
>> >>> On 09/16/2014 05:09 PM, Christopher Covington wrote:
>> >>>> ARM Linux currently has the most features available to it in hypervisor
>> >>>> (HYP) mode, so switch to it when possible. This can also ensure proper
>> >>>> reset of newer registers such as CNTVOFF.
>> >>>>
>> >>>> The permissions on the Non-Secure Access Control Register (NSACR) are
>> >>>> used to probe what the security setting currently is when in supervisor
>> >>>> (SVC) mode.
>> >>>
>> >>> Sorry, this doesn't work yet. I was misinterpreting my test results. For what
>> >>> it's worth, my testing and development methodology is to run it after hacked
>> >>> up versions of the semihosting bootwrapper on the simulator that corresponds
>> >>> to rtsm_ve-aemv8a.dtb (AEM VE FVP these days?) and examine the instruction traces.
>> >>
>> >> Looks like the real problem was that I was hacking up the bootwrapper
>> >> incorrectly--my start-in-secure-mode bootwrapper variant wasn't setting up the
>> >> GIC for non-secure access. With that changed, I've tested the following
>> >> variations using the Image file in a single core configuration.
>> >>
>> >> Start in non-secure SVC with non-secure access to GIC configured.
>> >>
>> >> Start in secure SVC with non-secure access to GIC configured.
>> >>
>> >> Start in secure SVC with non-secure access to GIC configured and hypervisor
>> >> support disabled in the model (-C cluster.has_el2=0). This required setting
>> >> the VBAR again in non-secure SVC but with that fix it seems to work. I'll
>> >> include this change in v2.
>> >
>> > If you're relying on the boot loader to set up the GIC to support
>> > non-secure access anyway, why not just have it boot the kernel in Hyp
>> > like the boot protocol document recommends? (The same thing as the GIC
>> > is going to apply for any other hardware that needs configuration to
>> > allow NS access; if we need the firmware to deal with this we might as
>> > well just have it boot us in the right mode too.)
>>
>> I'd like to get rid of as much of the bootwrapper as possible (having gotten
>> spoiled by using QEMU's built-in bootloader). I'm just taking it one step at a
>> time. Handling GIC initialization in the kernel is probably the next step.
>
> The problem is that the kernel doesn't know about GIC until much later.
> So I don't see an easy workaround, other than relying on the boot-loader
> to do the right thing (and then we go to the point Peter made about
> changing it to start Linux in Hyp mode directly).

Well, for us, the issue is that our boot-loader isn't involved in
secondary cpu startup, either at boot time nor suspend/resume or cpu
hotplug/power gating.
So we certainly could have the boot loader set up the GIC for
non-secure access and then this type of solution would work, though
I'm not sure what else might need to be set up for non-secure access.

>
> --
> Catalin