[PATCH v1 0/2] relocate Xen in over 4GB space for arm32

[PATCH v1 0/2] relocate Xen in over 4GB space for arm32

[Iurii Konovalenko]

From: Iurii Konovalenko <iurii.konovalenko@globallogic.com>

The following patch series adds ability to relocate Xen in over 4GB space
for 32-bit arm  cores with LPAE.

Iurii Konovalenko (2):
  arm: Add ability to relocate Xen in over 4GB space
  arm: skip verifying memory continuity

 xen/Rules.mk                   |  1 +
 xen/arch/arm/arm32/head.S      | 21 ++++++++++++++++++++-
 xen/arch/arm/platforms/rcar2.c |  9 ++++++++-
 xen/arch/arm/setup.c           | 21 ++++++++++++++++-----
 xen/arch/arm/smpboot.c         | 33 +++++++++++++++++++++++++++++----
 5 files changed, 74 insertions(+), 11 deletions(-)

-- 
1.9.1

[PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Iurii Konovalenko]

From: Iurii Konovalenko <iurii.konovalenko@globallogic.com>

Primary CPU relocate Xen in over 4GB space and wake up seondary CPUs.
Secondary CPUs run on unrelocated copy of Xen until turning on MMU.
After turning on MMU secondary CPUs run on relocated copy of Xen.

To add ability to relocate Xen in over 4GB space add following to
compilation command: ARM32_RELOCATE_OVER_4GB=y

Signed-off-by: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
Signed-off-by: Andrii Anisov <andrii.anisov@globallogic.com>
---
 xen/Rules.mk                   |  1 +
 xen/arch/arm/arm32/head.S      | 21 ++++++++++++++++++++-
 xen/arch/arm/platforms/rcar2.c |  9 ++++++++-
 xen/arch/arm/setup.c           | 17 ++++++++++++++++-
 xen/arch/arm/smpboot.c         | 33 +++++++++++++++++++++++++++++----
 5 files changed, 74 insertions(+), 7 deletions(-)

diff --git a/xen/Rules.mk b/xen/Rules.mk
index feb08d6..fbd34a5 100644
--- a/xen/Rules.mk
+++ b/xen/Rules.mk
@@ -64,6 +64,7 @@ CFLAGS-$(HAS_PCI)       += -DHAS_PCI
 CFLAGS-$(HAS_IOPORTS)   += -DHAS_IOPORTS
 CFLAGS-$(HAS_PDX)       += -DHAS_PDX
 CFLAGS-$(frame_pointer) += -fno-omit-frame-pointer -DCONFIG_FRAME_POINTER
+CFLAGS-$(ARM32_RELOCATE_OVER_4GB) += -DARM32_RELOCATE_OVER_4GB
 
 ifneq ($(max_phys_cpus),)
 CFLAGS-y                += -DMAX_PHYS_CPUS=$(max_phys_cpus)
diff --git a/xen/arch/arm/arm32/head.S b/xen/arch/arm/arm32/head.S
index 5c0263e..26be1d0 100644
--- a/xen/arch/arm/arm32/head.S
+++ b/xen/arch/arm/arm32/head.S
@@ -262,8 +262,21 @@ cpu_init_done:
         add   r4, r4, r10            /* r4 := paddr (boot_pagetable) */
         mov   r5, #0                 /* r4:r5 is paddr (boot_pagetable) */
         mcrr  CP64(r4, r5, HTTBR)
+#ifdef ARM32_RELOCATE_OVER_4GB
+        teq   r7, #0
+        beq   1f                     /* construct pagetable if CPU0 */
 
-        /* Setup boot_pgtable: */
+        /*Skip constructing TLBs for secondary CPUs, use constructed by CPU0*/
+        PRINT("- Skip construction pagetable, using CPU0 table @")
+        mov   r0, r5
+        bl    putn
+        mov   r0, r4
+        bl    putn
+        PRINT("  -\r\n")
+        b   skip_constructing
+#endif
+
+1:      /* Setup boot_pgtable: */
         ldr   r1, =boot_second
         add   r1, r1, r10            /* r1 := paddr (boot_second) */
 
@@ -346,6 +359,7 @@ virtphys_clash:
         PRINT("- Unable to build boot page tables - virt and phys addresses clash. -\r\n")
         b     fail
 
+skip_constructing:
 1:
         PRINT("- Turning on paging -\r\n")
 
@@ -427,6 +441,11 @@ paging:
          * setup in init_secondary_pagetables. */
 
         ldr   r4, =init_ttbr         /* VA of HTTBR value stashed by CPU 0 */
+#ifdef ARM32_RELOCATE_OVER_4GB
+        ldr   r1, =_start
+        sub r4, r1
+        add r4, #BOOT_RELOC_VIRT_START
+#endif
         ldrd  r4, r5, [r4]           /* Actual value */
         dsb
         mcrr  CP64(r4, r5, HTTBR)
diff --git a/xen/arch/arm/platforms/rcar2.c b/xen/arch/arm/platforms/rcar2.c
index aef544c..4365166 100644
--- a/xen/arch/arm/platforms/rcar2.c
+++ b/xen/arch/arm/platforms/rcar2.c
@@ -25,6 +25,9 @@
 #define RCAR2_RAM_SIZE                         0x1000
 #define RCAR2_SMP_START_OFFSET                 0xFFC
 
+#ifdef ARM32_RELOCATE_OVER_4GB
+extern paddr_t xen_relocation_offset;
+#endif
 static int __init rcar2_smp_init(void)
 {
     void __iomem *pram;
@@ -38,7 +41,11 @@ static int __init rcar2_smp_init(void)
     }
 
     /* setup reset vectors */
-    writel(__pa(init_secondary), pram + RCAR2_SMP_START_OFFSET);
+    writel(__pa(init_secondary)
+#ifdef ARM32_RELOCATE_OVER_4GB
+            - xen_relocation_offset
+#endif
+            , pram + RCAR2_SMP_START_OFFSET);
     iounmap(pram);
 
     sev();
diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index 4ec7c13..66e2834 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -407,7 +407,7 @@ static paddr_t __init get_xen_paddr(void)
             if ( !e )
                 continue;
 
-#ifdef CONFIG_ARM_32
+#if defined (CONFIG_ARM_32) && !defined(ARM32_RELOCATE_OVER_4GB)
             /* Xen must be under 4GB */
             if ( e > 0x100000000ULL )
                 e = 0x100000000ULL;
@@ -699,6 +699,9 @@ void __init setup_cache(void)
     cacheline_bytes = 1U << (4 + (ccsid & 0x7));
 }
 
+#ifdef ARM32_RELOCATE_OVER_4GB
+paddr_t xen_relocation_offset;
+#endif
 /* C entry point for boot CPU */
 void __init start_xen(unsigned long boot_phys_offset,
                       unsigned long fdt_paddr,
@@ -740,8 +743,20 @@ void __init start_xen(unsigned long boot_phys_offset,
                              (paddr_t)(uintptr_t)(_end - _start + 1), NULL);
     BUG_ON(!xen_bootmodule);
 
+#ifdef ARM32_RELOCATE_OVER_4GB
+    //save physical address of init_secondary
+    xen_relocation_offset = __pa(init_secondary);
+#endif
     xen_paddr = get_xen_paddr();
     setup_pagetables(boot_phys_offset, xen_paddr);
+#ifdef ARM32_RELOCATE_OVER_4GB
+    //Now Xen is relocated
+    //Calculate offset of Xen relocation
+    //It is difference between new physical address of init_secondary an old one
+    //This offset is needed in several places when we have to write to old Xen location
+    //(secondary CPUs run on old-located Xen and rely on some variables from CPU0)
+    xen_relocation_offset = __pa(init_secondary) - xen_relocation_offset;
+#endif
 
     /* Update Xen's address now that we have relocated. */
     printk("Update BOOTMOD_XEN from %"PRIpaddr"-%"PRIpaddr" => %"PRIpaddr"-%"PRIpaddr"\n",
diff --git a/xen/arch/arm/smpboot.c b/xen/arch/arm/smpboot.c
index a96cda2..731144c 100644
--- a/xen/arch/arm/smpboot.c
+++ b/xen/arch/arm/smpboot.c
@@ -31,6 +31,9 @@
 #include <xen/console.h>
 #include <asm/gic.h>
 #include <asm/psci.h>
+#ifdef ARM32_RELOCATE_OVER_4GB
+#include <xen/vmap.h>
+#endif
 
 cpumask_t cpu_online_map;
 cpumask_t cpu_present_map;
@@ -353,17 +356,33 @@ int __init cpu_up_send_sgi(int cpu)
     return 0;
 }
 
+#ifdef ARM32_RELOCATE_OVER_4GB
+extern paddr_t xen_relocation_offset;
+#endif
 /* Bring up a remote CPU */
 int __cpu_up(unsigned int cpu)
 {
     int rc;
     s_time_t deadline;
+#ifdef ARM32_RELOCATE_OVER_4GB
+    paddr_t p_info = __pa(&smp_up_cpu) - xen_relocation_offset;
+    unsigned long* info = ioremap_nocache(p_info, sizeof(unsigned long));
+#else
+    unsigned long* info = &smp_up_cpu;
+#endif
 
     printk("Bringing up CPU%d\n", cpu);
 
     rc = init_secondary_pagetables(cpu);
     if ( rc < 0 )
+#ifdef ARM32_RELOCATE_OVER_4GB
+    {
+        iounmap(info);
+        return rc;
+    }
+#else
         return rc;
+#endif
 
     console_start_sync(); /* Secondary may use early_printk */
 
@@ -374,8 +393,8 @@ int __cpu_up(unsigned int cpu)
     init_data.cpuid = cpu;
 
     /* Open the gate for this CPU */
-    smp_up_cpu = cpu_logical_map(cpu);
-    clean_dcache(smp_up_cpu);
+    *info = cpu_logical_map(cpu);
+    clean_dcache(*info);
 
     rc = arch_cpu_up(cpu);
 
@@ -383,6 +402,9 @@ int __cpu_up(unsigned int cpu)
 
     if ( rc < 0 )
     {
+#ifdef ARM32_RELOCATE_OVER_4GB
+        iounmap(info);
+#endif
         printk("Failed to bring up CPU%d\n", cpu);
         return rc;
     }
@@ -406,8 +428,11 @@ int __cpu_up(unsigned int cpu)
      */
     init_data.stack = NULL;
     init_data.cpuid = ~0;
-    smp_up_cpu = MPIDR_INVALID;
-    clean_dcache(smp_up_cpu);
+    *info = MPIDR_INVALID;
+    clean_dcache(*info);
+#ifdef ARM32_RELOCATE_OVER_4GB
+    iounmap(info);
+#endif
 
     if ( !cpu_online(cpu) )
     {
-- 
1.9.1

[PATCH v1 2/2] arm: skip verifying memory continuity

[Iurii Konovalenko]

From: Iurii Konovalenko <iurii.konovalenko@globallogic.com>

Odd check.

Signed-off-by: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
Signed-off-by: Andrii Anisov <andrii.anisov@globallogic.com>
---
 xen/arch/arm/setup.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index 66e2834..a7fcbb6 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -394,10 +394,6 @@ static paddr_t __init get_xen_paddr(void)
         const struct membank *bank = &mi->bank[i];
         paddr_t s, e;
 
-        /* We can only deal with contiguous memory at the moment */
-        if ( last_end != bank->start )
-            break;
-
         last_end = bank->start + bank->size;
 
         if ( bank->size >= min_size )
-- 
1.9.1

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Julien Grall]

Hi Iurii,

OOI, can you give more context why you need to relocate Xen over 4GB?

On 08/04/15 13:36, Iurii Konovalenko wrote:
> From: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
> 
> Primary CPU relocate Xen in over 4GB space and wake up seondary CPUs.

Typoes:

s/relocate/relocates
s/seondary/secondary

> Secondary CPUs run on unrelocated copy of Xen until turning on MMU.
> After turning on MMU secondary CPUs run on relocated copy of Xen.

The non-relocated copy of Xen is not marked as reserved. So Xen may
allocate the area for his own purpose before the secondary CPU has boot.

> 
> To add ability to relocate Xen in over 4GB space add following to
> compilation command: ARM32_RELOCATE_OVER_4GB=y

The virtualization extension requires LPAE. Any reason to make this
optional?

> Signed-off-by: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
> Signed-off-by: Andrii Anisov <andrii.anisov@globallogic.com>
> ---
>  xen/Rules.mk                   |  1 +
>  xen/arch/arm/arm32/head.S      | 21 ++++++++++++++++++++-
>  xen/arch/arm/platforms/rcar2.c |  9 ++++++++-
>  xen/arch/arm/setup.c           | 17 ++++++++++++++++-
>  xen/arch/arm/smpboot.c         | 33 +++++++++++++++++++++++++++++----
>  5 files changed, 74 insertions(+), 7 deletions(-)
> 
> diff --git a/xen/Rules.mk b/xen/Rules.mk
> index feb08d6..fbd34a5 100644
> --- a/xen/Rules.mk
> +++ b/xen/Rules.mk
> @@ -64,6 +64,7 @@ CFLAGS-$(HAS_PCI)       += -DHAS_PCI
>  CFLAGS-$(HAS_IOPORTS)   += -DHAS_IOPORTS
>  CFLAGS-$(HAS_PDX)       += -DHAS_PDX
>  CFLAGS-$(frame_pointer) += -fno-omit-frame-pointer -DCONFIG_FRAME_POINTER
> +CFLAGS-$(ARM32_RELOCATE_OVER_4GB) += -DARM32_RELOCATE_OVER_4GB
>  
>  ifneq ($(max_phys_cpus),)
>  CFLAGS-y                += -DMAX_PHYS_CPUS=$(max_phys_cpus)
> diff --git a/xen/arch/arm/arm32/head.S b/xen/arch/arm/arm32/head.S
> index 5c0263e..26be1d0 100644
> --- a/xen/arch/arm/arm32/head.S
> +++ b/xen/arch/arm/arm32/head.S
> @@ -262,8 +262,21 @@ cpu_init_done:
>          add   r4, r4, r10            /* r4 := paddr (boot_pagetable) */
>          mov   r5, #0                 /* r4:r5 is paddr (boot_pagetable) */
>          mcrr  CP64(r4, r5, HTTBR)

Newline here.

> +#ifdef ARM32_RELOCATE_OVER_4GB
> +        teq   r7, #0
> +        beq   1f                     /* construct pagetable if CPU0 */
>  
> -        /* Setup boot_pgtable: */
> +        /*Skip constructing TLBs for secondary CPUs, use constructed by CPU0*/

/* ... */

TLBs is confusing. I first though you were talking to Translation
Lookaside Buffer.

> +        PRINT("- Skip construction pagetable, using CPU0 table @")

This very fragile. The CPU0 may drop a part of the 1:1 mapping if it's
clashes with the fixmap and DTB mappings (see after the label "paging").

Therefore secondary CPUs may hang during boot.

> +        mov   r0, r5
> +        bl    putn
> +        mov   r0, r4
> +        bl    putn
> +        PRINT("  -\r\n")
> +        b   skip_constructing
> +#endif
> +
> +1:      /* Setup boot_pgtable: */
>          ldr   r1, =boot_second
>          add   r1, r1, r10            /* r1 := paddr (boot_second) */
>  
> @@ -346,6 +359,7 @@ virtphys_clash:
>          PRINT("- Unable to build boot page tables - virt and phys addresses clash. -\r\n")
>          b     fail
>  
> +skip_constructing:
>  1:
>          PRINT("- Turning on paging -\r\n")
>  
> @@ -427,6 +441,11 @@ paging:
>           * setup in init_secondary_pagetables. */
>  
>          ldr   r4, =init_ttbr         /* VA of HTTBR value stashed by CPU 0 */
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +        ldr   r1, =_start
> +        sub r4, r1
> +        add r4, #BOOT_RELOC_VIRT_START
> +#endif

This chunk need some comment in order to explain what it's doing.

AFAIU the resulting value in r4, will be exactly the same as  "ldr r4,
=init_ttbr".

>          ldrd  r4, r5, [r4]           /* Actual value */
>          dsb
>          mcrr  CP64(r4, r5, HTTBR)
> diff --git a/xen/arch/arm/platforms/rcar2.c b/xen/arch/arm/platforms/rcar2.c
> index aef544c..4365166 100644
> --- a/xen/arch/arm/platforms/rcar2.c
> +++ b/xen/arch/arm/platforms/rcar2.c
> @@ -25,6 +25,9 @@
>  #define RCAR2_RAM_SIZE                         0x1000
>  #define RCAR2_SMP_START_OFFSET                 0xFFC
>  
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +extern paddr_t xen_relocation_offset;
> +#endif

Please define in an header.

>  static int __init rcar2_smp_init(void)
>  {
>      void __iomem *pram;
> @@ -38,7 +41,11 @@ static int __init rcar2_smp_init(void)
>      }
>  
>      /* setup reset vectors */
> -    writel(__pa(init_secondary), pram + RCAR2_SMP_START_OFFSET);
> +    writel(__pa(init_secondary)
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +            - xen_relocation_offset
> +#endif
> +            , pram + RCAR2_SMP_START_OFFSET);
>      iounmap(pram);
>  
>      sev();
> diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
> index 4ec7c13..66e2834 100644
> --- a/xen/arch/arm/setup.c
> +++ b/xen/arch/arm/setup.c
> @@ -407,7 +407,7 @@ static paddr_t __init get_xen_paddr(void)
>              if ( !e )
>                  continue;
>  
> -#ifdef CONFIG_ARM_32
> +#if defined (CONFIG_ARM_32) && !defined(ARM32_RELOCATE_OVER_4GB)
>              /* Xen must be under 4GB */
>              if ( e > 0x100000000ULL )
>                  e = 0x100000000ULL;
> @@ -699,6 +699,9 @@ void __init setup_cache(void)
>      cacheline_bytes = 1U << (4 + (ccsid & 0x7));
>  }
>  
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +paddr_t xen_relocation_offset;
> +#endif
>  /* C entry point for boot CPU */
>  void __init start_xen(unsigned long boot_phys_offset,
>                        unsigned long fdt_paddr,
> @@ -740,8 +743,20 @@ void __init start_xen(unsigned long boot_phys_offset,
>                               (paddr_t)(uintptr_t)(_end - _start + 1), NULL);
>      BUG_ON(!xen_bootmodule);
>  
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +    //save physical address of init_secondary

Xen is using /* ... */ for the comment.

> +    xen_relocation_offset = __pa(init_secondary);
> +#endif
>      xen_paddr = get_xen_paddr();
>      setup_pagetables(boot_phys_offset, xen_paddr);
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +    //Now Xen is relocated
> +    //Calculate offset of Xen relocation
> +    //It is difference between new physical address of init_secondary an old one
> +    //This offset is needed in several places when we have to write to old Xen location
> +    //(secondary CPUs run on old-located Xen and rely on some variables from CPU0)

Ditto

> +    xen_relocation_offset = __pa(init_secondary) - xen_relocation_offset;

The Xen may be relocated below the boot copy. So the final result may be
negative.

> +#endif
>  
>      /* Update Xen's address now that we have relocated. */
>      printk("Update BOOTMOD_XEN from %"PRIpaddr"-%"PRIpaddr" => %"PRIpaddr"-%"PRIpaddr"\n",
> diff --git a/xen/arch/arm/smpboot.c b/xen/arch/arm/smpboot.c
> index a96cda2..731144c 100644
> --- a/xen/arch/arm/smpboot.c
> +++ b/xen/arch/arm/smpboot.c
> @@ -31,6 +31,9 @@
>  #include <xen/console.h>
>  #include <asm/gic.h>
>  #include <asm/psci.h>
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +#include <xen/vmap.h>
> +#endif
>  
>  cpumask_t cpu_online_map;
>  cpumask_t cpu_present_map;
> @@ -353,17 +356,33 @@ int __init cpu_up_send_sgi(int cpu)
>      return 0;
>  }
>  
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +extern paddr_t xen_relocation_offset;
> +#endif

This can be avoid by defining xen_relocation_offset in an header.

>  /* Bring up a remote CPU */
>  int __cpu_up(unsigned int cpu)
>  {
>      int rc;
>      s_time_t deadline;
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +    paddr_t p_info = __pa(&smp_up_cpu) - xen_relocation_offset;
> +    unsigned long* info = ioremap_nocache(p_info, sizeof(unsigned long));

unsigned long *info and sizeof(smp_up_cpu).

ioremap_nocache may fail and "info" doesn't have a real meaning.

Regards,

-- 
Julien Grall

Re: [PATCH v1 2/2] arm: skip verifying memory continuity

[Julien Grall]

Hi Iurii,

On 08/04/15 13:36, Iurii Konovalenko wrote:
> From: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
> 
> Odd check.

Can you expand the commit explaining that Xen supports non-contiguous
bank since commit e01fa4e2 "xen: arm: Enable physical address space
compression (PDX) on arm"?

Regards,

-- 
Julien Grall

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Andrii Anisov]

[-- Attachment #1.1: Type: text/plain, Size: 9342 bytes --]

> OOI, can you give more context why you need to relocate Xen over 4GB?

Julien,

The context here is pretty simple: just a workaround of the real system
limitations:

   1. Renesas R-Car H2 evaluation board lager has 4GB of RAM, 2GB mapped
   under 4GB boundary and 2GB over.
   2. The R-Car2 chip has IOMMU's for almost all active devices except GPU.
   3. Within our system we need one user domain with 2GB of RAM and
   dedicated GPU.
   4. While doing PoC we have no access to the GPU driver(and firmware)
   sources to introduce swiotlb here.

Having no option to handle GPU driver addressing with iommu (2) or
swiotlb(4), it was decided to place the user domain described in 3 within
RAM below 4GB with 1:1 mapping. Therefore we needed to move all the rest
(XEN, rest of domains) under 4GB.

Andrii Anisov | Team Lead
GlobalLogic
Kyiv, 03038, Protasov Business Park, M.Grinchenka, 2/1
P +38.044.492.9695x3664  M +380505738852  S andriyanisov
www.globallogic.com

http://www.globallogic.com/email_disclaimer.txt


On Wed, Apr 8, 2015 at 7:05 PM, Julien Grall <julien.grall@citrix.com>
wrote:
> Hi Iurii,
>
> OOI, can you give more context why you need to relocate Xen over 4GB?
>
> On 08/04/15 13:36, Iurii Konovalenko wrote:
>> From: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
>>
>> Primary CPU relocate Xen in over 4GB space and wake up seondary CPUs.
>
> Typoes:
>
> s/relocate/relocates
> s/seondary/secondary
>
>> Secondary CPUs run on unrelocated copy of Xen until turning on MMU.
>> After turning on MMU secondary CPUs run on relocated copy of Xen.
>
> The non-relocated copy of Xen is not marked as reserved. So Xen may
> allocate the area for his own purpose before the secondary CPU has boot.
>
>>
>> To add ability to relocate Xen in over 4GB space add following to
>> compilation command: ARM32_RELOCATE_OVER_4GB=y
>
> The virtualization extension requires LPAE. Any reason to make this
> optional?
>
>> Signed-off-by: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
>> Signed-off-by: Andrii Anisov <andrii.anisov@globallogic.com>
>> ---
>>  xen/Rules.mk                   |  1 +
>>  xen/arch/arm/arm32/head.S      | 21 ++++++++++++++++++++-
>>  xen/arch/arm/platforms/rcar2.c |  9 ++++++++-
>>  xen/arch/arm/setup.c           | 17 ++++++++++++++++-
>>  xen/arch/arm/smpboot.c         | 33 +++++++++++++++++++++++++++++----
>>  5 files changed, 74 insertions(+), 7 deletions(-)
>>
>> diff --git a/xen/Rules.mk b/xen/Rules.mk
>> index feb08d6..fbd34a5 100644
>> --- a/xen/Rules.mk
>> +++ b/xen/Rules.mk
>> @@ -64,6 +64,7 @@ CFLAGS-$(HAS_PCI)       += -DHAS_PCI
>>  CFLAGS-$(HAS_IOPORTS)   += -DHAS_IOPORTS
>>  CFLAGS-$(HAS_PDX)       += -DHAS_PDX
>>  CFLAGS-$(frame_pointer) += -fno-omit-frame-pointer
-DCONFIG_FRAME_POINTER
>> +CFLAGS-$(ARM32_RELOCATE_OVER_4GB) += -DARM32_RELOCATE_OVER_4GB
>>
>>  ifneq ($(max_phys_cpus),)
>>  CFLAGS-y                += -DMAX_PHYS_CPUS=$(max_phys_cpus)
>> diff --git a/xen/arch/arm/arm32/head.S b/xen/arch/arm/arm32/head.S
>> index 5c0263e..26be1d0 100644
>> --- a/xen/arch/arm/arm32/head.S
>> +++ b/xen/arch/arm/arm32/head.S
>> @@ -262,8 +262,21 @@ cpu_init_done:
>>          add   r4, r4, r10            /* r4 := paddr (boot_pagetable) */
>>          mov   r5, #0                 /* r4:r5 is paddr (boot_pagetable)
*/
>>          mcrr  CP64(r4, r5, HTTBR)
>
> Newline here.
>
>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +        teq   r7, #0
>> +        beq   1f                     /* construct pagetable if CPU0 */
>>
>> -        /* Setup boot_pgtable: */
>> +        /*Skip constructing TLBs for secondary CPUs, use constructed by
CPU0*/
>
> /* ... */
>
> TLBs is confusing. I first though you were talking to Translation
> Lookaside Buffer.
>
>> +        PRINT("- Skip construction pagetable, using CPU0 table @")
>
> This very fragile. The CPU0 may drop a part of the 1:1 mapping if it's
> clashes with the fixmap and DTB mappings (see after the label "paging").
>
> Therefore secondary CPUs may hang during boot.
>
>> +        mov   r0, r5
>> +        bl    putn
>> +        mov   r0, r4
>> +        bl    putn
>> +        PRINT("  -\r\n")
>> +        b   skip_constructing
>> +#endif
>> +
>> +1:      /* Setup boot_pgtable: */
>>          ldr   r1, =boot_second
>>          add   r1, r1, r10            /* r1 := paddr (boot_second) */
>>
>> @@ -346,6 +359,7 @@ virtphys_clash:
>>          PRINT("- Unable to build boot page tables - virt and phys
addresses clash. -\r\n")
>>          b     fail
>>
>> +skip_constructing:
>>  1:
>>          PRINT("- Turning on paging -\r\n")
>>
>> @@ -427,6 +441,11 @@ paging:
>>           * setup in init_secondary_pagetables. */
>>
>>          ldr   r4, =init_ttbr         /* VA of HTTBR value stashed by
CPU 0 */
>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +        ldr   r1, =_start
>> +        sub r4, r1
>> +        add r4, #BOOT_RELOC_VIRT_START
>> +#endif
>
> This chunk need some comment in order to explain what it's doing.
>
> AFAIU the resulting value in r4, will be exactly the same as  "ldr r4,
> =init_ttbr".
>
>>          ldrd  r4, r5, [r4]           /* Actual value */
>>          dsb
>>          mcrr  CP64(r4, r5, HTTBR)
>> diff --git a/xen/arch/arm/platforms/rcar2.c
b/xen/arch/arm/platforms/rcar2.c
>> index aef544c..4365166 100644
>> --- a/xen/arch/arm/platforms/rcar2.c
>> +++ b/xen/arch/arm/platforms/rcar2.c
>> @@ -25,6 +25,9 @@
>>  #define RCAR2_RAM_SIZE                         0x1000
>>  #define RCAR2_SMP_START_OFFSET                 0xFFC
>>
>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +extern paddr_t xen_relocation_offset;
>> +#endif
>
> Please define in an header.
>
>>  static int __init rcar2_smp_init(void)
>>  {
>>      void __iomem *pram;
>> @@ -38,7 +41,11 @@ static int __init rcar2_smp_init(void)
>>      }
>>
>>      /* setup reset vectors */
>> -    writel(__pa(init_secondary), pram + RCAR2_SMP_START_OFFSET);
>> +    writel(__pa(init_secondary)
>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +            - xen_relocation_offset
>> +#endif
>> +            , pram + RCAR2_SMP_START_OFFSET);
>>      iounmap(pram);
>>
>>      sev();
>> diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
>> index 4ec7c13..66e2834 100644
>> --- a/xen/arch/arm/setup.c
>> +++ b/xen/arch/arm/setup.c
>> @@ -407,7 +407,7 @@ static paddr_t __init get_xen_paddr(void)
>>              if ( !e )
>>                  continue;
>>
>> -#ifdef CONFIG_ARM_32
>> +#if defined (CONFIG_ARM_32) && !defined(ARM32_RELOCATE_OVER_4GB)
>>              /* Xen must be under 4GB */
>>              if ( e > 0x100000000ULL )
>>                  e = 0x100000000ULL;
>> @@ -699,6 +699,9 @@ void __init setup_cache(void)
>>      cacheline_bytes = 1U << (4 + (ccsid & 0x7));
>>  }
>>
>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +paddr_t xen_relocation_offset;
>> +#endif
>>  /* C entry point for boot CPU */
>>  void __init start_xen(unsigned long boot_phys_offset,
>>                        unsigned long fdt_paddr,
>> @@ -740,8 +743,20 @@ void __init start_xen(unsigned long
boot_phys_offset,
>>                               (paddr_t)(uintptr_t)(_end - _start + 1),
NULL);
>>      BUG_ON(!xen_bootmodule);
>>
>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +    //save physical address of init_secondary
>
> Xen is using /* ... */ for the comment.
>
>> +    xen_relocation_offset = __pa(init_secondary);
>> +#endif
>>      xen_paddr = get_xen_paddr();
>>      setup_pagetables(boot_phys_offset, xen_paddr);
>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +    //Now Xen is relocated
>> +    //Calculate offset of Xen relocation
>> +    //It is difference between new physical address of init_secondary
an old one
>> +    //This offset is needed in several places when we have to write to
old Xen location
>> +    //(secondary CPUs run on old-located Xen and rely on some variables
from CPU0)
>
> Ditto
>
>> +    xen_relocation_offset = __pa(init_secondary) -
xen_relocation_offset;
>
> The Xen may be relocated below the boot copy. So the final result may be
> negative.
>
>> +#endif
>>
>>      /* Update Xen's address now that we have relocated. */
>>      printk("Update BOOTMOD_XEN from %"PRIpaddr"-%"PRIpaddr" =>
%"PRIpaddr"-%"PRIpaddr"\n",
>> diff --git a/xen/arch/arm/smpboot.c b/xen/arch/arm/smpboot.c
>> index a96cda2..731144c 100644
>> --- a/xen/arch/arm/smpboot.c
>> +++ b/xen/arch/arm/smpboot.c
>> @@ -31,6 +31,9 @@
>>  #include <xen/console.h>
>>  #include <asm/gic.h>
>>  #include <asm/psci.h>
>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +#include <xen/vmap.h>
>> +#endif
>>
>>  cpumask_t cpu_online_map;
>>  cpumask_t cpu_present_map;
>> @@ -353,17 +356,33 @@ int __init cpu_up_send_sgi(int cpu)
>>      return 0;
>>  }
>>
>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +extern paddr_t xen_relocation_offset;
>> +#endif
>
> This can be avoid by defining xen_relocation_offset in an header.
>
>>  /* Bring up a remote CPU */
>>  int __cpu_up(unsigned int cpu)
>>  {
>>      int rc;
>>      s_time_t deadline;
>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +    paddr_t p_info = __pa(&smp_up_cpu) - xen_relocation_offset;
>> +    unsigned long* info = ioremap_nocache(p_info, sizeof(unsigned
long));
>
> unsigned long *info and sizeof(smp_up_cpu).
>
> ioremap_nocache may fail and "info" doesn't have a real meaning.
>
> Regards,
>
> --
> Julien Grall
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xen.org
> http://lists.xen.org/xen-devel

[-- Attachment #1.2: Type: text/html, Size: 12503 bytes --]

[-- Attachment #2: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Andrii Anisov]

[-- Attachment #1.1: Type: text/plain, Size: 326 bytes --]

>>Therefore we needed to move all the rest (XEN, rest of domains) OVER 4GB.


Andrii Anisov | Team Lead
GlobalLogic
Kyiv, 03038, Protasov Business Park, M.Grinchenka, 2/1
P +38.044.492.9695x3664  M +380505738852  S andriyanisov
www.globallogic.com
<http://www.globallogic.com/>
http://www.globallogic.com/email_disclaimer.txt

[-- Attachment #1.2: Type: text/html, Size: 2635 bytes --]

[-- Attachment #2: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Iurii Konovalenko]

Hi, Julien!

On Wed, Apr 8, 2015 at 7:05 PM, Julien Grall <julien.grall@citrix.com> wrote:

> The virtualization extension requires LPAE. Any reason to make this
> optional?
I agree with you - there is no real reason to make it optional. I will
rework patch
to include functionality without any flags by default.

>> +#ifdef ARM32_RELOCATE_OVER_4GB
>> +        ldr   r1, =_start
>> +        sub r4, r1
>> +        add r4, #BOOT_RELOC_VIRT_START
>> +#endif
>
> This chunk need some comment in order to explain what it's doing.
>
> AFAIU the resulting value in r4, will be exactly the same as  "ldr r4,
> =init_ttbr".
Not exactly. Virtual address of _start is 0x00200000, but
#BOOT_RELOC_VIRT_START is 0x00800000.
This part of code is needed to get value, that was "stashed by CPU 0" in
relocated copy of Xen when secondary CPUs run in non-relocated.

>> +    xen_relocation_offset = __pa(init_secondary) - xen_relocation_offset;
>
> The Xen may be relocated below the boot copy. So the final result may be
> negative.
This offset is used to get physical address of non-relocated variables
and write there values.
If it is negative, then calculating non-relocated address gives bigger
number (substraction of  negative value).
In addition, it's hard to imagine when it can be negative if we take
highest region of highest memory bank.

> ioremap_nocache may fail and "info" doesn't have a real meaning.
Ok, I will add verifying block and rename variable.

> Regards,
>
> --
> Julien Grall


Best regards.

Iurii Konovalenko | Senior Software Engineer
GlobalLogic

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Julien Grall]

On 10/04/15 14:58, Iurii Konovalenko wrote:
> Hi, Julien!

Hi Iurii,

> On Wed, Apr 8, 2015 at 7:05 PM, Julien Grall <julien.grall@citrix.com> wrote:
> 
>> The virtualization extension requires LPAE. Any reason to make this
>> optional?
> I agree with you - there is no real reason to make it optional. I will
> rework patch
> to include functionality without any flags by default.

Before doing a such change, I'd like the point of view of Stefano or Ian
on this patch.

>>> +#ifdef ARM32_RELOCATE_OVER_4GB
>>> +        ldr   r1, =_start
>>> +        sub r4, r1
>>> +        add r4, #BOOT_RELOC_VIRT_START
>>> +#endif
>>
>> This chunk need some comment in order to explain what it's doing.
>>
>> AFAIU the resulting value in r4, will be exactly the same as  "ldr r4,
>> =init_ttbr".
> Not exactly. Virtual address of _start is 0x00200000, but
> #BOOT_RELOC_VIRT_START is 0x00800000.
> This part of code is needed to get value, that was "stashed by CPU 0" in
> relocated copy of Xen when secondary CPUs run in non-relocated.

And the reloc Xen is mapped by setup_page_tables in mm.c, right?

If so, this definitely need a comment explaining why and how it works.
Because you are relying on multiple things in the code.
	- 1:1 mapping is not overlapped (see my comment on the previous mail)
	- reloc Xen will never overlap the boot Xen (for instance we could
decide that Xen doesn't need to be relocated).
	- After relocation, Xen is still modifying the boot page table (clean
them) but hopefully it's only the relocated version

This is making the code in setup_page_tables quite difficult to
understand now.

>>> +    xen_relocation_offset = __pa(init_secondary) - xen_relocation_offset;
>>
>> The Xen may be relocated below the boot copy. So the final result may be
>> negative.
> This offset is used to get physical address of non-relocated variables
> and write there values.
> If it is negative, then calculating non-relocated address gives bigger
> number (substraction of  negative value).
> In addition, it's hard to imagine when it can be negative if we take
> highest region of highest memory bank.

Simple, the board has all the memory banks below 4G and Xen has been
loaded by the bootloader at a very high address.

Regards,

-- 
Julien Grall

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Andrii Anisov]

Julien,

> > Secondary CPUs run on unrelocated copy of Xen until turning on MMU.
> > After turning on MMU secondary CPUs run on relocated copy of Xen.
>
> The non-relocated copy of Xen is not marked as reserved. So Xen may
> allocate the area for his own purpose before the secondary CPU has boot.

Could you please clarify how the same issue resolved with the piece of
bootloader code which implements secondary processors wfe in HYP mode
and jump to hypervisor?

Andrii Anisov | Team Lead
GlobalLogic
Kyiv, 03038, Protasov Business Park, M.Grinchenka, 2/1
P +38.044.492.9695x3664  M +380505738852  S andriyanisov
www.globallogic.com

http://www.globallogic.com/email_disclaimer.txt

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Julien Grall]

On 10/04/15 15:38, Andrii Anisov wrote:
> Julien,
> 
>>> Secondary CPUs run on unrelocated copy of Xen until turning on MMU.
>>> After turning on MMU secondary CPUs run on relocated copy of Xen.
>>
>> The non-relocated copy of Xen is not marked as reserved. So Xen may
>> allocate the area for his own purpose before the secondary CPU has boot.
> 
> Could you please clarify how the same issue resolved with the piece of
> bootloader code which implements secondary processors wfe in HYP mode
> and jump to hypervisor?

I'm not sure to understand the question.

Currently, Xen add a bootmodule  (see start_xen) to cover Xen. The base
address is updated once Xen is relocated.

You may want to add a bootmodule for the non-relocated Xen.

I hope it answers to your question.

Regards,

-- 
Julien Grall

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Ian Campbell]

On Fri, 2015-04-10 at 15:25 +0100, Julien Grall wrote:
> On 10/04/15 14:58, Iurii Konovalenko wrote:
> > Hi, Julien!
> 
> Hi Iurii,
> 
> > On Wed, Apr 8, 2015 at 7:05 PM, Julien Grall <julien.grall@citrix.com> wrote:
> > 
> >> The virtualization extension requires LPAE. Any reason to make this
> >> optional?
> > I agree with you - there is no real reason to make it optional. I will
> > rework patch
> > to include functionality without any flags by default.
> 
> Before doing a such change, I'd like the point of view of Stefano or Ian
> on this patch.

Such things shouldn't be compile time optional. I don't think there's
any reason why it can't be done for every arm32 platform.

Normally I'd ask for a command line option which can overrides (i.e.
disable) this behaviour -- just in case. But I suspect that the relevant
code runs too early?

(I'll read the rest of this thread as I catch up with my email backlog).

Ian

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Ian Campbell]

On Wed, 2015-04-08 at 17:05 +0100, Julien Grall wrote:
> > -        /* Setup boot_pgtable: */
> > +        /*Skip constructing TLBs for secondary CPUs, use constructed by CPU0*/
> 
> /* ... */
> 
> TLBs is confusing. I first though you were talking to Translation
> Lookaside Buffer.

Was "tbls" (i.e. shortened "tables") what was meant? In which case I'd
prefer spelling it out..

Ian.

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Ian Campbell]

On Wed, 2015-04-08 at 15:36 +0300, Iurii Konovalenko wrote:
> From: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
> 
> Primary CPU relocate Xen in over 4GB space and wake up seondary CPUs.
> Secondary CPUs run on unrelocated copy of Xen until turning on MMU.

Which is a significant difference I think, and requires careful checking
that we do not clobber that copy while brining up the primary cpu.

Which would mean it needs to be reserved before we start allocating
heaps and relocating Xen. Ideally only the portion of head.S which we
need would be reserved, not the whole thing. And it should be freed once
everyone is up and running.

Alternatively perhaps the relevant portion of head.S could be copied to
a freshly allocated trampoline page.

> After turning on MMU secondary CPUs run on relocated copy of Xen.
> 
> To add ability to relocate Xen in over 4GB space add following to
> compilation command: ARM32_RELOCATE_OVER_4GB=y
> 
> Signed-off-by: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
> Signed-off-by: Andrii Anisov <andrii.anisov@globallogic.com>
> ---
>  xen/Rules.mk                   |  1 +
>  xen/arch/arm/arm32/head.S      | 21 ++++++++++++++++++++-
>  xen/arch/arm/platforms/rcar2.c |  9 ++++++++-
>  xen/arch/arm/setup.c           | 17 ++++++++++++++++-
>  xen/arch/arm/smpboot.c         | 33 +++++++++++++++++++++++++++++----
>  5 files changed, 74 insertions(+), 7 deletions(-)
> 
> diff --git a/xen/Rules.mk b/xen/Rules.mk
> index feb08d6..fbd34a5 100644
> --- a/xen/Rules.mk
> +++ b/xen/Rules.mk
> @@ -64,6 +64,7 @@ CFLAGS-$(HAS_PCI)       += -DHAS_PCI
>  CFLAGS-$(HAS_IOPORTS)   += -DHAS_IOPORTS
>  CFLAGS-$(HAS_PDX)       += -DHAS_PDX
>  CFLAGS-$(frame_pointer) += -fno-omit-frame-pointer -DCONFIG_FRAME_POINTER
> +CFLAGS-$(ARM32_RELOCATE_OVER_4GB) += -DARM32_RELOCATE_OVER_4GB
>  
>  ifneq ($(max_phys_cpus),)
>  CFLAGS-y                += -DMAX_PHYS_CPUS=$(max_phys_cpus)
> diff --git a/xen/arch/arm/arm32/head.S b/xen/arch/arm/arm32/head.S
> index 5c0263e..26be1d0 100644
> --- a/xen/arch/arm/arm32/head.S
> +++ b/xen/arch/arm/arm32/head.S
> @@ -262,8 +262,21 @@ cpu_init_done:
>          add   r4, r4, r10            /* r4 := paddr (boot_pagetable) */
>          mov   r5, #0                 /* r4:r5 is paddr (boot_pagetable) */
>          mcrr  CP64(r4, r5, HTTBR)
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +        teq   r7, #0
> +        beq   1f                     /* construct pagetable if CPU0 */
>  
> -        /* Setup boot_pgtable: */
> +        /*Skip constructing TLBs for secondary CPUs, use constructed by CPU0*/

Is this necessary due to the relocation for some reason?

Otherwise, iff its correct, then it should be done as a separate change
before this one.

Also please pay attention to the style used for comments in this file.

> @@ -427,6 +441,11 @@ paging:
>           * setup in init_secondary_pagetables. */
>  
>          ldr   r4, =init_ttbr         /* VA of HTTBR value stashed by CPU 0 */
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +        ldr   r1, =_start
> +        sub r4, r1
> +        add r4, #BOOT_RELOC_VIRT_START

Please follow the existing convention of thoroughly documenting what
each line is doing. (i.e. the "r1 := foo" comments)

> +#endif
>          ldrd  r4, r5, [r4]           /* Actual value */
>          dsb
>          mcrr  CP64(r4, r5, HTTBR)
> diff --git a/xen/arch/arm/platforms/rcar2.c b/xen/arch/arm/platforms/rcar2.c
> index aef544c..4365166 100644
> --- a/xen/arch/arm/platforms/rcar2.c
> +++ b/xen/arch/arm/platforms/rcar2.c
> @@ -25,6 +25,9 @@
>  #define RCAR2_RAM_SIZE                         0x1000
>  #define RCAR2_SMP_START_OFFSET                 0xFFC
>  
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +extern paddr_t xen_relocation_offset;

This certainly doesn't belong here, such things belong in headers.


> @@ -740,8 +743,20 @@ void __init start_xen(unsigned long boot_phys_offset,
>                               (paddr_t)(uintptr_t)(_end - _start + 1), NULL);
>      BUG_ON(!xen_bootmodule);
>  
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +    //save physical address of init_secondary

This is not the Xen style for comments.

> +    xen_relocation_offset = __pa(init_secondary);

How does __pa(init_secondary) end up being an offset? Shouldn't this be
some calculation involving boot_phys_offset?

> +#endif
>      xen_paddr = get_xen_paddr();
>      setup_pagetables(boot_phys_offset, xen_paddr);
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +    //Now Xen is relocated
> +    //Calculate offset of Xen relocation
> +    //It is difference between new physical address of init_secondary an old one
> +    //This offset is needed in several places when we have to write to old Xen location
> +    //(secondary CPUs run on old-located Xen and rely on some variables from CPU0)
> +    xen_relocation_offset = __pa(init_secondary) - xen_relocation_offset;

Oh I see, ewww. I think you should be able to calculate this directly
when you do the relocation, which is much nicer than relying on tricks
of __pa changing.

And rather than adding all of the uses of xen_relocatio_offset I think a
__boot_pa construct which does the adjustment should be used.

Ian.

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Ian Campbell]

On Fri, 2015-04-10 at 15:49 +0100, Julien Grall wrote:
> On 10/04/15 15:38, Andrii Anisov wrote:
> > Julien,
> > 
> >>> Secondary CPUs run on unrelocated copy of Xen until turning on MMU.
> >>> After turning on MMU secondary CPUs run on relocated copy of Xen.
> >>
> >> The non-relocated copy of Xen is not marked as reserved. So Xen may
> >> allocate the area for his own purpose before the secondary CPU has boot.
> > 
> > Could you please clarify how the same issue resolved with the piece of
> > bootloader code which implements secondary processors wfe in HYP mode
> > and jump to hypervisor?
> 
> I'm not sure to understand the question.
> 
> Currently, Xen add a bootmodule  (see start_xen) to cover Xen. The base
> address is updated once Xen is relocated.
> 
> You may want to add a bootmodule for the non-relocated Xen.
> 
> I hope it answers to your question.

And if the question is about bootloader code which holds a spinning
secondary CPU then the answer is that this region had better be either
in Secure World only RAM or reserved in the firmware description (i.e.
FDT or ACPI memory map). In FDT that would usually be done via
the /memreserve/ directive, although one cold also imagine fudging the
memory nodes.

Ian.

Re: [PATCH v1 1/2] arm: Add ability to relocate Xen in over 4GB space

[Ian Campbell]

On Wed, 2015-04-08 at 20:24 +0300, Andrii Anisov wrote:

I was thinking about this a little more...

> The context here is pretty simple: just a workaround of the real
> system limitations:
>      1. Renesas R-Car H2 evaluation board lager has 4GB of RAM, 2GB
>         mapped under 4GB boundary and 2GB over. 
>      2. The R-Car2 chip has IOMMU's for almost all active devices
>         except GPU. 
>      3. Within our system we need one user domain with 2GB of RAM and
>         dedicated GPU.
>      4. While doing PoC we have no access to the GPU driver(and
>         firmware) sources to introduce swiotlb here. 
> Having no option to handle GPU driver addressing with iommu (2) or
> swiotlb(4), it was decided to place the user domain described in 3
> within RAM below 4GB with 1:1 mapping. Therefore we needed to move all
> the rest (XEN, rest of domains) under 4GB.

Xen itself (i.e. .text and .data, no the heap etc) is at most 2MB. Are
your constraints such that at a 2GB-2MB (i.e. 2046M) domain would not be
acceptable?

Ian.