at91sam9g45: Issues while working with RAM that is separated on physical address space

at91sam9g45: Issues while working with RAM that is separated on physical address space

[P J]

Hello,

I'm using Linux 2.6.30 with a custom board design based on AT91SAM9G45-EKES.

The memory map (required) is as follows:
    phys           =>          virt
0x70000000    =>    0xC0000000  (128 MB)
0x20000000    =>    0xC8000000  (128 MB)

While booting I see this message: Ignoring RAM at 0x20000000 -
0x27FFFFFF (vmalloc region overlap).
The default and only option in the Kernel configuration is FLAT MEMORY MODEL.


Now I use the 2 patches put by Nicu at the following website:
http://blog.linuxconsulting.ro/2010/05/using-both-memory-controller-with.html

The patches basically do two things:
1) enable SPARSE memory &
2) Change the __virt_to_phys() and __phys_to_virt() in
arch/arm/include/asm/memory.h

After applying the patches
Linux does accept both the memory banks and the total memory is 256 MB
but crashes immediately after that.

The kernel dump is as follows:


Starting kernel ...

Uncompressing Linux................................................................................................................
done, booting the kernel.
Linux version 2.6.30 (prasant at prasant-laptop) (gcc version 4.2.0
20070413 (prerelease) (CodeSourcery Sourcery G++ Lite 2007q1-10)) #4
Fri Jul 22 14:46:45 IST 201
CPU: ARM926EJ-S [41069265] revision 5 (ARMv5TEJ), cr=00053177
CPU: VIVT data cache, VIVT instruction cache
Machine: Atmel AT91SAM9G45-EKES
Ignoring unrecognised tag 0x54410008
Memory policy: ECC disabled, Data cache writeback
Clocks: CPU 400 MHz, master 133 MHz, main 12.000 MHz
Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 62720
Kernel command line: console=ttyS0,115200 rootfstype=ext3
root=/dev/mmcblk0p2 rootdelay=3 ip=dhcp
mtdparts=atmel_nand:128k(bootstrap)ro,256k(uboot)ro,128k(env1))
NR_IRQS:192
AT91: 160 gpio irqs in 5 banks
PID hash table entries: 1024 (order: 10, 4096 bytes)
Console: colour dummy device 80x30
console [ttyS0] enabled
Dentry cache hash table entries: 32768 (order: 5, 131072 bytes)
Inode-cache hash table entries: 16384 (order: 4, 65536 bytes)
MEM: unordered memory banks.  Not freeing memmap.
Memory: 128MB 128MB = 256MB total
Memory: 255932KB available (3108K code, 236K data, 116K init, 0K highmem)
Unable to handle kernel NULL pointer dereference at virtual address 00000000
pgd = c0004000
[00000000] *pgd=00000000
Internal error: Oops: 5 [#1]
Modules linked in:
CPU: 0    Not tainted  (2.6.30 #4)
PC is at kmem_cache_alloc+0x1c/0x88
LR is at setup_cpu_cache+0x50/0x110
pc : [<c007f76c>]    lr : [<c0270288>]    psr: a00000d3
sp : c0349f40  ip : 0000001b  fp : ffffffe0
r10: 00000020  r9 : 00000000  r8 : 00000000
r7 : 000000d0  r6 : 00000000  r5 : c78000c0  r4 : a0000053
r3 : a00000d3  r2 : c02e9fe0  r1 : 000000d0  r0 : 00000000
Flags: NzCv  IRQs off  FIQs off  Mode SVC_32  ISA ARM  Segment kernel
Control: 0005317f  Table: 70004000  DAC: 00000017
Process swapper (pid: 0, stack limit = 0xc0348268)
Stack: (0xc0349f40 to 0xc034a000)
9f40: c03762dc c0354d54 c78000c0 c03762d8 00042000 c0270288 c78000c0 000001e0
9f60: 00000020 c00805e8 c0349f80 c0349f84 c04961e0 00000001 c02e9fe0 00000001
9f80: 00000000 00000071 41069265 c0023854 00000000 c0349fbc c0354d54 c0354d80
9fa0: 41069265 7002091c 00000000 c00128f0 00000000 c0354d80 00000074 00000040
9fc0: c0022eec c036e9a0 c0022ee8 c034bc90 70020950 c0008b74 c00084ec 00000000
9fe0: 00000000 c0022eec 00000000 00053175 c036ea98 70008034 00000000 00000000
[<c007f76c>] (kmem_cache_alloc+0x1c/0x88) from [<c0270288>]
(setup_cpu_cache+0x50/0x110)
[<c0270288>] (setup_cpu_cache+0x50/0x110) from [<c00805e8>]
(kmem_cache_create+0x3b8/0x460)
[<c00805e8>] (kmem_cache_create+0x3b8/0x460) from [<c00128f0>]
(kmem_cache_init+0x144/0x344)
[<c00128f0>] (kmem_cache_init+0x144/0x344) from [<c0008b74>]
(start_kernel+0x2a0/0x364)
[<c0008b74>] (start_kernel+0x2a0/0x364) from [<70008034>] (0x70008034)
Code: e1a07001 e10f4000 e3843080 e121f003 (e590c000)
---[ end trace 1b75b31a2719ed1c ]---
Kernel panic - not syncing: Attempted to kill the idle task!
[<c002b574>] (unwind_backtrace+0x0/0xdc) from [<c003c93c>] (panic+0x44/0x114)
[<c003c93c>] (panic+0x44/0x114) from [<c003f354>] (do_exit+0x5c/0x598)
[<c003f354>] (do_exit+0x5c/0x598) from [<c0029b24>] (die+0x138/0x158)
[<c0029b24>] (die+0x138/0x158) from [<c002c650>] (__do_kernel_fault+0x68/0x80)
[<c002c650>] (__do_kernel_fault+0x68/0x80) from [<c002c870>]
(do_page_fault+0x208/0x228)
[<c002c870>] (do_page_fault+0x208/0x228) from [<c0025224>]
(do_DataAbort+0x34/0x98)
[<c0025224>] (do_DataAbort+0x34/0x98) from [<c002594c>] (__dabt_svc+0x4c/0x60)
Exception stack(0xc0349ef8 to 0xc0349f40)
9ee0:                                                       00000000 000000d0
9f00: c02e9fe0 a00000d3 a0000053 c78000c0 00000000 000000d0 00000000 00000000
9f20: 00000020 ffffffe0 0000001b c0349f40 c0270288 c007f76c a00000d3 ffffffff
[<c002594c>] (__dabt_svc+0x4c/0x60) from [<c0270288>]
(setup_cpu_cache+0x50/0x110)
[<c0270288>] (setup_cpu_cache+0x50/0x110) from [<c03762d8>] (0xc03762d8


These patches are not working for me.

I want to know how to work with memories that are separate on physical
address space.

-Prasant J

at91sam9g45: Issues while working with RAM that is separated on physical address space

[Russell King - ARM Linux]

On Fri, Jul 22, 2011 at 07:06:40PM +0530, P J wrote:
> The memory map (required) is as follows:
>     phys           =>          virt
> 0x70000000    =>    0xC0000000  (128 MB)
> 0x20000000    =>    0xC8000000  (128 MB)

There is no way that the kernel will support this kind of hair-brained
memory layout.  This is even more true than it once was since we've moved
to using memblock to manage the physical memory layout in later kernels.

What the kernel will support is using 0x20000000 as the first bank and
0x70000000 as the second bank of memory.  I suggest trying with it mapped
that way around, and failing that then using highmem support without
sparse.

at91sam9g45: Issues while working with RAM that is separated on physical address space

[P J]

Hi Russell,

On Fri, Jul 22, 2011 at 7:25 PM, Russell King - ARM Linux
<linux@arm.linux.org.uk> wrote:
> On Fri, Jul 22, 2011 at 07:06:40PM +0530, P J wrote:
>> The memory map (required) is as follows:
>> ? ? phys ? ? ? ? ? => ? ? ? ? ?virt
>> 0x70000000 ? ?=> ? ?0xC0000000 ?(128 MB)
>> 0x20000000 ? ?=> ? ?0xC8000000 ?(128 MB)
>
> There is no way that the kernel will support this kind of hair-brained
> memory layout. ?This is even more true than it once was since we've moved
> to using memblock to manage the physical memory layout in later kernels.
>
> What the kernel will support is using 0x20000000 as the first bank and
> 0x70000000 as the second bank of memory. ?I suggest trying with it mapped
> that way around, and failing that then using highmem support without
> sparse.
>

I do not completely understand why this layout is not possible.

Anyways, my one memory bank (128 MB DDR2) is located at physical
address 0x70000000 to 0x77FFFFFF and is virtually mapped to 0xC0000000
to 0xC7FFFFFF.

If I have to continue to use this bank as per the above mapping then
how can I use the other memory bank (128MB DDR2) which is located at
physical address 0x20000000 to 0x27FFFFFF ?


-Prasant J

at91sam9g45: Issues while working with RAM that is separated on physical address space

[P J]

On Mon, Jul 25, 2011 at 7:36 PM, P J <pj0585@gmail.com> wrote:
> Hi Russell,
>
> On Fri, Jul 22, 2011 at 7:25 PM, Russell King - ARM Linux
> <linux@arm.linux.org.uk> wrote:
>> On Fri, Jul 22, 2011 at 07:06:40PM +0530, P J wrote:
>>> The memory map (required) is as follows:
>>> ? ? phys ? ? ? ? ? => ? ? ? ? ?virt
>>> 0x70000000 ? ?=> ? ?0xC0000000 ?(128 MB)
>>> 0x20000000 ? ?=> ? ?0xC8000000 ?(128 MB)
>>
>> There is no way that the kernel will support this kind of hair-brained
>> memory layout. ?This is even more true than it once was since we've moved
>> to using memblock to manage the physical memory layout in later kernels.
>>
>> What the kernel will support is using 0x20000000 as the first bank and
>> 0x70000000 as the second bank of memory. ?I suggest trying with it mapped
>> that way around, and failing that then using highmem support without
>> sparse.
>>
>
> I do not completely understand why this layout is not possible.
>
> Anyways, my one memory bank (128 MB DDR2) is located at physical
> address 0x70000000 to 0x77FFFFFF and is virtually mapped to 0xC0000000
> to 0xC7FFFFFF.
>
> If I have to continue to use this bank as per the above mapping then
> how can I use the other memory bank (128MB DDR2) which is located at
> physical address 0x20000000 to 0x27FFFFFF ?
>
>
> -Prasant J
>


I think that if I change my memory banks (like I use the memory bank
at 0x20000000 as the first one and the memory bank at 0x70000000 as
the second one) then also I will have issues.

The issue is because the memory banks are not getting correctly mapped
because the __virt_to_phys() and __phys_to_virt() are defined as
linear:

#define __virt_to_phys(x)	((x) - PAGE_OFFSET + PHYS_OFFSET)
#define __phys_to_virt(x)	((x) - PHYS_OFFSET + PAGE_OFFSET)

where PHYS_OFFSET is 0x70000000 and PAGE_OFFSET is 0xC0000000


Now the first bank which is at 0x70000000 gets correctly mapped
virtually but the second bank which is at 0x20000000 gets mapped
incorrectly. Even if I change the order of banks, (also I change the
PHYS_OFFSET to 0x20000000) then, always, only one bank gets mapped
correctly and the other one gets incorrectly mapped.

If I change the above defines to map both my memories correctly then
Linux does not boot.


Any suggestions, as to how I can get the mappings correct.

-Prasant J

at91sam9g45: Issues while working with RAM that is separated on physical address space

[Christian Glindkamp]

On 2011-07-26 16:38, P J wrote:
> On Mon, Jul 25, 2011 at 7:36 PM, P J <pj0585@gmail.com> wrote:
> > Hi Russell,
> >
> > On Fri, Jul 22, 2011 at 7:25 PM, Russell King - ARM Linux
> > <linux@arm.linux.org.uk> wrote:
> >> On Fri, Jul 22, 2011 at 07:06:40PM +0530, P J wrote:
> >>> The memory map (required) is as follows:
> >>> ? ? phys ? ? ? ? ? => ? ? ? ? ?virt
> >>> 0x70000000 ? ?=> ? ?0xC0000000 ?(128 MB)
> >>> 0x20000000 ? ?=> ? ?0xC8000000 ?(128 MB)
> >>
> >> There is no way that the kernel will support this kind of hair-brained
> >> memory layout. ?This is even more true than it once was since we've moved
> >> to using memblock to manage the physical memory layout in later kernels.
> >>
> >> What the kernel will support is using 0x20000000 as the first bank and
> >> 0x70000000 as the second bank of memory. ?I suggest trying with it mapped
> >> that way around, and failing that then using highmem support without
> >> sparse.
> >>

Actually, for me it is the other way around. I have an AT91SAM9G45 based
system with 256MB at 0x20000000 and 0x70000000, making a total of 512MB.
If I use mapping like this
 phys  ? ? ? ? => ? ?virt
 0x70000000 ? ?=> ? ?0xC0000000 ?(256 MB)
 0x20000000 ? ?=> ? ?0xD0000000 ?(256 MB)
if seems to work without any problems. This is with Linux 3.0.

If I switch the mapping around and load the zImage to 0x21000000 instead
of 0x71000000, the boot stops with the following message:

 Truncating RAM at 70000000-7fffffff to -96dfffff (vmalloc region overlap).
 Memory policy: ECC disabled, Data cache writeback

The high memory version isn't stable either, but boots at least.
For more see below.

> >
> > I do not completely understand why this layout is not possible.
> >
> > Anyways, my one memory bank (128 MB DDR2) is located at physical
> > address 0x70000000 to 0x77FFFFFF and is virtually mapped to 0xC0000000
> > to 0xC7FFFFFF.
> >
> > If I have to continue to use this bank as per the above mapping then
> > how can I use the other memory bank (128MB DDR2) which is located at
> > physical address 0x20000000 to 0x27FFFFFF ?
> >
> >
> > -Prasant J
> >
> 
> 
> I think that if I change my memory banks (like I use the memory bank
> at 0x20000000 as the first one and the memory bank at 0x70000000 as
> the second one) then also I will have issues.
> 
> The issue is because the memory banks are not getting correctly mapped
> because the __virt_to_phys() and __phys_to_virt() are defined as
> linear:
> 
> #define __virt_to_phys(x)	((x) - PAGE_OFFSET + PHYS_OFFSET)
> #define __phys_to_virt(x)	((x) - PHYS_OFFSET + PAGE_OFFSET)
> 
> where PHYS_OFFSET is 0x70000000 and PAGE_OFFSET is 0xC0000000
> 
> 
> Now the first bank which is at 0x70000000 gets correctly mapped
> virtually but the second bank which is at 0x20000000 gets mapped
> incorrectly. Even if I change the order of banks, (also I change the
> PHYS_OFFSET to 0x20000000) then, always, only one bank gets mapped
> correctly and the other one gets incorrectly mapped.
> 
> If I change the above defines to map both my memories correctly then
> Linux does not boot.
> 
> 
> Any suggestions, as to how I can get the mappings correct.

Have you also switched the order of the atags, so that 0x20000000 comes
first? And you do not have to edit these macros if you enable
CONFIG_AUTO_ZRELADDR. At least that worked for me with high memory.
You should also load the kernel into the RAM bank at 0x20000000.

The only problem that still exits with highmem for me is the following:
Even on non-highmem/non-sparsemem systems I get the following warning
when using an mmc as the rootfs:

------------[ cut here ]------------
WARNING: at kernel/irq/handle.c:130 handle_irq_event_percpu+0x70/0x194()
irq 29 handler atmci_interrupt+0x0/0x64c enabled interrupts
Backtrace: 
[<c018ed00>] (dump_backtrace+0x0/0x110) from [<c018f274>] (dump_stack+0x18/0x1c)
 r6:c01c9254 r5:c04831ac r4:00000082
[<c018f25c>] (dump_stack+0x0/0x1c) from [<c019c73c>] (warn_slowpath_common+0x54/0x6c)
[<c019c6e8>] (warn_slowpath_common+0x0/0x6c) from [<c019c7f8>] (warn_slowpath_fmt+0x38/0x40)
 r8:00000000 r7:c04ba500 r6:00000001 r5:0000001d r4:cf8f7b40
[<c019c7c0>] (warn_slowpath_fmt+0x0/0x40) from [<c01c9254>] (handle_irq_event_percpu+0x70/0x194)
 r3:0000001d r2:c04831c0
[<c01c91e4>] (handle_irq_event_percpu+0x0/0x194) from [<c01c93ac>] (handle_irq_event+0x34/0x44)
[<c01c9378>] (handle_irq_event+0x0/0x44) from [<c01cb6c0>] (handle_level_irq+0xd4/0xe8)
 r4:c04ba500
[<c01cb5ec>] (handle_level_irq+0x0/0xe8) from [<c01c8f5c>] (generic_handle_irq+0x34/0x3c)
 r4:0000001d
[<c01c8f28>] (generic_handle_irq+0x0/0x3c) from [<c018b068>] (asm_do_IRQ+0x68/0x98)
 r4:0000001d
[<c018b000>] (asm_do_IRQ+0x0/0x98) from [<c018ba54>] (__irq_svc+0x34/0x60)
Exception stack(0xc04b1f40 to 0xc04b1f88)
1f40: 00000000 0005317f 0005217f 60000013 c04b0000 c04b4b80 c04d32b0 c04b2000
1f60: 70004000 41069265 70022940 c04b1f94 600000d3 c04b1f88 c018cdb0 c018cdbc
1f80: 60000013 ffffffff
 r5:fefff000 r4:ffffffff
[<c018cd80>] (default_idle+0x0/0x40) from [<c018cbcc>] (cpu_idle+0x68/0xb0)
[<c018cb64>] (cpu_idle+0x0/0xb0) from [<c041db78>] (rest_init+0x5c/0x74)
 r6:c018730c r5:c04d3200 r4:c04b20e4
[<c041db1c>] (rest_init+0x0/0x74) from [<c0008d08>] (start_kernel+0x358/0x3c8)
[<c00089b0>] (start_kernel+0x0/0x3c8) from [<70008040>] (0x70008040)
---[ end trace ac1fb351351e0957 ]---

The system is still stable, but if switch to highmem, the kernel crashes
completely when doing this (using and USB drive as rootfs still works
flawlessly):

Unable to handle kernel NULL pointer dereference at virtual address 00000002
pgd = c0004000
[00000002] *pgd=00000000
Internal error: Oops: 817 [#1]
CPU: 0    Not tainted  (3.0.0+ #678)
PC is at atmci_interrupt+0x198/0x64c
LR is at 0x46
pc : [<c038ba4c>]    lr : [<00000046>]    psr: 80000093
sp : c04b1e58  ip : 00000000  fp : c04b1e9c
r10: 00464c45  r9 : cf999e94  r8 : 00000000
r7 : 00000004  r6 : 00000000  r5 : cf96a500  r4 : cf97ac00
r3 : 0000464c  r2 : 00001000  r1 : 464c457f  r0 : 00000000
Flags: Nzcv  IRQs off  FIQs on  Mode SVC_32  ISA ARM  Segment kernel
Control: 0005317f  Table: 2f9c8000  DAC: 00000017
Process swapper (pid: 0, stack limit = 0xc04b0270)
Stack: (0xc04b1e58 to 0xc04b2000)
1e40:                                                       cf96a52c 00000017
1e60: 00000002 00000000 c038aad8 464c457f 00000004 cf87cfc0 0000001d 0000001d
1e80: c04ba500 00000000 c04dc040 00000000 c04b1ed4 c04b1ea0 c01c8cb8 c038b8c4
1ea0: 90848946 00000003 c04b2000 c04ba500 00000000 0000001d c018693c c04b2000
1ec0: 41069265 20021cc4 c04b1eec c04b1ed8 c01c8e48 c01c8c90 c01c1d04 c04ba500
1ee0: c04b1f04 c04b1ef0 c01cb15c c01c8e24 c01a190c 0000001d c04b1f1c c04b1f08
1f00: c01c89f8 c01cb098 c0196f68 0000001d c04b1f34 c04b1f20 c018b068 c01c89d4
1f20: ffffffff fefff000 c04b1f8c c04b1f38 c018ba54 c018b010 00000000 0005317f
1f40: 0005217f 60000013 c04b0000 c04b4b80 c04d32d0 c018693c c04b2000 41069265
1f60: 20021cc4 c04b1f8c 600000d3 c04b1f80 c018cdb0 c018cdbc 60000013 ffffffff
1f80: c04b1fac c04b1f90 c018cbcc c018cd90 c018693c c04b20e4 c04d3288 c04d3220
1fa0: c04b1fbc c04b1fb0 c041c52c c018cb74 c04b1ff4 c04b1fc0 c0008c4c c041c4e0
1fc0: c0008578 00000000 00000000 c018693c 00000000 00053175 c04b200c c0186d40
1fe0: c04b4b74 20004000 00000000 c04b1ff8 20008040 c00089c0 00000000 00000000
Backtrace: 
[<c038b8b4>] (atmci_interrupt+0x0/0x64c) from [<c01c8cb8>] (handle_irq_event_percpu+0x38/0x194)
[<c01c8c80>] (handle_irq_event_percpu+0x0/0x194) from [<c01c8e48>] (handle_irq_event+0x34/0x44)
[<c01c8e14>] (handle_irq_event+0x0/0x44) from [<c01cb15c>] (handle_level_irq+0xd4/0xe8)
 r4:c04ba500
[<c01cb088>] (handle_level_irq+0x0/0xe8) from [<c01c89f8>] (generic_handle_irq+0x34/0x3c)
 r4:0000001d
[<c01c89c4>] (generic_handle_irq+0x0/0x3c) from [<c018b068>] (asm_do_IRQ+0x68/0x98)
 r4:0000001d
[<c018b000>] (asm_do_IRQ+0x0/0x98) from [<c018ba54>] (__irq_svc+0x34/0x60)
Exception stack(0xc04b1f38 to 0xc04b1f80)
1f20:                                                       00000000 0005317f
1f40: 0005217f 60000013 c04b0000 c04b4b80 c04d32d0 c018693c c04b2000 41069265
1f60: 20021cc4 c04b1f8c 600000d3 c04b1f80 c018cdb0 c018cdbc 60000013 ffffffff
 r5:fefff000 r4:ffffffff
[<c018cd80>] (default_idle+0x0/0x40) from [<c018cbcc>] (cpu_idle+0x68/0xb0)
[<c018cb64>] (cpu_idle+0x0/0xb0) from [<c041c52c>] (rest_init+0x5c/0x74)
 r6:c04d3220 r5:c04d3288 r4:c04b20e4
[<c041c4d0>] (rest_init+0x0/0x74) from [<c0008c4c>] (start_kernel+0x29c/0x308)
[<c00089b0>] (start_kernel+0x0/0x308) from [<20008040>] (0x20008040)
 r8:20004000 r7:c04b4b74 r6:c0186d40 r5:c04b200c r4:00053175
Code: e1a0ec23 e1a0a421 e1a03823 8a000017 (e5c03002) 
---[ end trace b6fce2ac8e8c707d ]---
Kernel panic - not syncing: Fatal exception in interrupt
Backtrace: 
[<c018ed00>] (dump_backtrace+0x0/0x110) from [<c018f274>] (dump_stack+0x18/0x1c)
 r6:c04b0000 r5:00000000 r4:c04d3654
[<c018f25c>] (dump_stack+0x0/0x1c) from [<c019c440>] (panic+0x60/0x1a4)
[<c019c3e0>] (panic+0x0/0x1a4) from [<c018f0c4>] (die+0x2b4/0x30c)
 r3:00010000 r2:c0484cf4 r1:00000000 r0:c047bc4c
[<c018ee10>] (die+0x0/0x30c) from [<c01908ac>] (__do_kernel_fault+0x70/0x90)
[<c019083c>] (__do_kernel_fault+0x0/0x90) from [<c0190a8c>] (do_page_fault+0x1c0/0x1d8)
 r8:00000817 r7:00000000 r6:c04b3f28 r5:00000002 r4:ffffffff
[<c01908cc>] (do_page_fault+0x0/0x1d8) from [<c018b2a8>] (do_DataAbort+0x40/0xa4)
[<c018b268>] (do_DataAbort+0x0/0xa4) from [<c018ba0c>] (__dabt_svc+0x4c/0x60)
Exception stack(0xc04b1e10 to 0xc04b1e58)
1e00:                                     00000000 464c457f 00001000 0000464c
1e20: cf97ac00 cf96a500 00000000 00000004 00000000 cf999e94 00464c45 c04b1e9c
1e40: 00000000 c04b1e58 00000046 c038ba4c 80000093 ffffffff
 r8:00000000 r7:00000004 r6:00000000 r5:c04b1e44 r4:ffffffff
[<c038b8b4>] (atmci_interrupt+0x0/0x64c) from [<c01c8cb8>] (handle_irq_event_percpu+0x38/0x194)
[<c01c8c80>] (handle_irq_event_percpu+0x0/0x194) from [<c01c8e48>] (handle_irq_event+0x34/0x44)
[<c01c8e14>] (handle_irq_event+0x0/0x44) from [<c01cb15c>] (handle_level_irq+0xd4/0xe8)
 r4:c04ba500
[<c01cb088>] (handle_level_irq+0x0/0xe8) from [<c01c89f8>] (generic_handle_irq+0x34/0x3c)
 r4:0000001d
[<c01c89c4>] (generic_handle_irq+0x0/0x3c) from [<c018b068>] (asm_do_IRQ+0x68/0x98)
 r4:0000001d
[<c018b000>] (asm_do_IRQ+0x0/0x98) from [<c018ba54>] (__irq_svc+0x34/0x60)
Exception stack(0xc04b1f38 to 0xc04b1f80)
1f20:                                                       00000000 0005317f
1f40: 0005217f 60000013 c04b0000 c04b4b80 c04d32d0 c018693c c04b2000 41069265
1f60: 20021cc4 c04b1f8c 600000d3 c04b1f80 c018cdb0 c018cdbc 60000013 ffffffff
 r5:fefff000 r4:ffffffff
[<c018cd80>] (default_idle+0x0/0x40) from [<c018cbcc>] (cpu_idle+0x68/0xb0)
[<c018cb64>] (cpu_idle+0x0/0xb0) from [<c041c52c>] (rest_init+0x5c/0x74)
 r6:c04d3220 r5:c04d3288 r4:c04b20e4
[<c041c4d0>] (rest_init+0x0/0x74) from [<c0008c4c>] (start_kernel+0x29c/0x308)
[<c00089b0>] (start_kernel+0x0/0x308) from [<20008040>] (0x20008040)
 r8:20004000 r7:c04b4b74 r6:c0186d40 r5:c04b200c r4:00053175

> 
> -Prasant J
> 
> _______________________________________________
> linux-arm-kernel mailing list
> linux-arm-kernel at lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

at91sam9g45: Issues while working with RAM that is separated on physical address space

[Russell King - ARM Linux]

On Tue, Jul 26, 2011 at 03:37:02PM +0200, Christian Glindkamp wrote:
> The only problem that still exits with highmem for me is the following:
> Even on non-highmem/non-sparsemem systems I get the following warning
> when using an mmc as the rootfs:
> 
> ------------[ cut here ]------------
> WARNING: at kernel/irq/handle.c:130 handle_irq_event_percpu+0x70/0x194()
> irq 29 handler atmci_interrupt+0x0/0x64c enabled interrupts
> Backtrace: 
> [<c018ed00>] (dump_backtrace+0x0/0x110) from [<c018f274>] (dump_stack+0x18/0x1c)
>  r6:c01c9254 r5:c04831ac r4:00000082
> [<c018f25c>] (dump_stack+0x0/0x1c) from [<c019c73c>] (warn_slowpath_common+0x54/0x6c)
> [<c019c6e8>] (warn_slowpath_common+0x0/0x6c) from [<c019c7f8>] (warn_slowpath_fmt+0x38/0x40)
>  r8:00000000 r7:c04ba500 r6:00000001 r5:0000001d r4:cf8f7b40
> [<c019c7c0>] (warn_slowpath_fmt+0x0/0x40) from [<c01c9254>] (handle_irq_event_percpu+0x70/0x194)
>  r3:0000001d r2:c04831c0
> [<c01c91e4>] (handle_irq_event_percpu+0x0/0x194) from [<c01c93ac>] (handle_irq_event+0x34/0x44)
> [<c01c9378>] (handle_irq_event+0x0/0x44) from [<c01cb6c0>] (handle_level_irq+0xd4/0xe8)
>  r4:c04ba500
> [<c01cb5ec>] (handle_level_irq+0x0/0xe8) from [<c01c8f5c>] (generic_handle_irq+0x34/0x3c)
>  r4:0000001d
> [<c01c8f28>] (generic_handle_irq+0x0/0x3c) from [<c018b068>] (asm_do_IRQ+0x68/0x98)
>  r4:0000001d
> [<c018b000>] (asm_do_IRQ+0x0/0x98) from [<c018ba54>] (__irq_svc+0x34/0x60)
> Exception stack(0xc04b1f40 to 0xc04b1f88)
> 1f40: 00000000 0005317f 0005217f 60000013 c04b0000 c04b4b80 c04d32b0 c04b2000
> 1f60: 70004000 41069265 70022940 c04b1f94 600000d3 c04b1f88 c018cdb0 c018cdbc
> 1f80: 60000013 ffffffff
>  r5:fefff000 r4:ffffffff
> [<c018cd80>] (default_idle+0x0/0x40) from [<c018cbcc>] (cpu_idle+0x68/0xb0)
> [<c018cb64>] (cpu_idle+0x0/0xb0) from [<c041db78>] (rest_init+0x5c/0x74)
>  r6:c018730c r5:c04d3200 r4:c04b20e4
> [<c041db1c>] (rest_init+0x0/0x74) from [<c0008d08>] (start_kernel+0x358/0x3c8)
> [<c00089b0>] (start_kernel+0x0/0x3c8) from [<70008040>] (0x70008040)
> ---[ end trace ac1fb351351e0957 ]---

That'll be because the driver is still using flush_dcache_page(), whereas
it should be using flush_kernel_dcache_page().  flush_dcache_page()
unfortunately results in IRQs being enabled due to the
flush_dcache_mmap_lock().

> The system is still stable, but if switch to highmem, the kernel crashes
> completely when doing this (using and USB drive as rootfs still works
> flawlessly):

That's because the driver is basically broken:

        void                    *buf = sg_virt(sg);
        unsigned int            offset = host->pio_offset;

                        memcpy(buf + offset, &value, remaining);

Highmem pages have a NULL sg_virt(sg) because they're by definition not
mapped.  Drivers really should not be using sg_virt() directly - who
knows how this got through the review process...

There's a well defined API for walking scatterlists in drivers - see
the sg_miter_* API in linux/scatterlist.h.  This takes care of the
highmem issues automatically, as well as using flush_kernel_dcache_page().

In short: the Atmel MCI driver is buggy and needs fixing.

at91sam9g45: Issues while working with RAM that is separated on physical address space

[P J]

On Tue, Jul 26, 2011 at 7:24 PM, Russell King - ARM Linux
<linux@arm.linux.org.uk> wrote:
> On Tue, Jul 26, 2011 at 03:37:02PM +0200, Christian Glindkamp wrote:
>> The only problem that still exits with highmem for me is the following:
>> Even on non-highmem/non-sparsemem systems I get the following warning
>> when using an mmc as the rootfs:
>>
>> ------------[ cut here ]------------
>> WARNING: at kernel/irq/handle.c:130 handle_irq_event_percpu+0x70/0x194()
>> irq 29 handler atmci_interrupt+0x0/0x64c enabled interrupts
>> Backtrace:
>> [<c018ed00>] (dump_backtrace+0x0/0x110) from [<c018f274>] (dump_stack+0x18/0x1c)
>> ?r6:c01c9254 r5:c04831ac r4:00000082
>> [<c018f25c>] (dump_stack+0x0/0x1c) from [<c019c73c>] (warn_slowpath_common+0x54/0x6c)
>> [<c019c6e8>] (warn_slowpath_common+0x0/0x6c) from [<c019c7f8>] (warn_slowpath_fmt+0x38/0x40)
>> ?r8:00000000 r7:c04ba500 r6:00000001 r5:0000001d r4:cf8f7b40
>> [<c019c7c0>] (warn_slowpath_fmt+0x0/0x40) from [<c01c9254>] (handle_irq_event_percpu+0x70/0x194)
>> ?r3:0000001d r2:c04831c0
>> [<c01c91e4>] (handle_irq_event_percpu+0x0/0x194) from [<c01c93ac>] (handle_irq_event+0x34/0x44)
>> [<c01c9378>] (handle_irq_event+0x0/0x44) from [<c01cb6c0>] (handle_level_irq+0xd4/0xe8)
>> ?r4:c04ba500
>> [<c01cb5ec>] (handle_level_irq+0x0/0xe8) from [<c01c8f5c>] (generic_handle_irq+0x34/0x3c)
>> ?r4:0000001d
>> [<c01c8f28>] (generic_handle_irq+0x0/0x3c) from [<c018b068>] (asm_do_IRQ+0x68/0x98)
>> ?r4:0000001d
>> [<c018b000>] (asm_do_IRQ+0x0/0x98) from [<c018ba54>] (__irq_svc+0x34/0x60)
>> Exception stack(0xc04b1f40 to 0xc04b1f88)
>> 1f40: 00000000 0005317f 0005217f 60000013 c04b0000 c04b4b80 c04d32b0 c04b2000
>> 1f60: 70004000 41069265 70022940 c04b1f94 600000d3 c04b1f88 c018cdb0 c018cdbc
>> 1f80: 60000013 ffffffff
>> ?r5:fefff000 r4:ffffffff
>> [<c018cd80>] (default_idle+0x0/0x40) from [<c018cbcc>] (cpu_idle+0x68/0xb0)
>> [<c018cb64>] (cpu_idle+0x0/0xb0) from [<c041db78>] (rest_init+0x5c/0x74)
>> ?r6:c018730c r5:c04d3200 r4:c04b20e4
>> [<c041db1c>] (rest_init+0x0/0x74) from [<c0008d08>] (start_kernel+0x358/0x3c8)
>> [<c00089b0>] (start_kernel+0x0/0x3c8) from [<70008040>] (0x70008040)
>> ---[ end trace ac1fb351351e0957 ]---
>
> That'll be because the driver is still using flush_dcache_page(), whereas
> it should be using flush_kernel_dcache_page(). ?flush_dcache_page()
> unfortunately results in IRQs being enabled due to the
> flush_dcache_mmap_lock().
>
>> The system is still stable, but if switch to highmem, the kernel crashes
>> completely when doing this (using and USB drive as rootfs still works
>> flawlessly):
>
> That's because the driver is basically broken:
>
> ? ? ? ?void ? ? ? ? ? ? ? ? ? ?*buf = sg_virt(sg);
> ? ? ? ?unsigned int ? ? ? ? ? ?offset = host->pio_offset;
>
> ? ? ? ? ? ? ? ? ? ? ? ?memcpy(buf + offset, &value, remaining);
>
> Highmem pages have a NULL sg_virt(sg) because they're by definition not
> mapped. ?Drivers really should not be using sg_virt() directly - who
> knows how this got through the review process...
>
> There's a well defined API for walking scatterlists in drivers - see
> the sg_miter_* API in linux/scatterlist.h. ?This takes care of the
> highmem issues automatically, as well as using flush_kernel_dcache_page().
>
> In short: the Atmel MCI driver is buggy and needs fixing.
>

@Christian:
I have not tested any change of memory order. I had just written my
concerns because I feel that it will be an identical situation.

For me the layout is as follows:
                  Phys                       =>        virt
0x70000000 - 0x77FFFFFF    => 0xC0000000 - 0xC7FFFFFF (128MB)

the other memory bank should ideally be mapped as
0x20000000 - 0x27FFFFFF    => 0xC8000000 - 0xCFFFFFFF (128MB)

I'm using DDR2, AT91SAM9G45 supports max of 256MB with DDR2 memories.

The mapping of second memory bank is going wrong because of the
reasons I have said earlier, so my second memory bank gets overlapped
with the vmalloc region and is ignored. With highemem seems like the
problem should have been solved but my kernel does not boot with
highmem enabled.

Any suggestions ? I'm using Linux 2.6.30..

-Prasant J

at91sam9g45: Issues while working with RAM that is separated on physical address space

[Nicolas Pitre]

On Tue, 26 Jul 2011, P J wrote:

> The mapping of second memory bank is going wrong because of the
> reasons I have said earlier, so my second memory bank gets overlapped
> with the vmalloc region and is ignored. With highemem seems like the
> problem should have been solved but my kernel does not boot with
> highmem enabled.
> 
> Any suggestions ? I'm using Linux 2.6.30..

Highmem support was introduced on ARM in v2.6.29 and became somewhat 
"reliable" on v2.6.31.  In any case, v2.6.30 is more than 2 years old. 
Please try to upgrade your kernel.


Nicolas

at91sam9g45: Issues while working with RAM that is separated on physical address space

[Prasant J]

On Wed, Jul 27, 2011 at 3:18 AM, Nicolas Pitre <nico@fluxnic.net> wrote:
> On Tue, 26 Jul 2011, P J wrote:
>
>> The mapping of second memory bank is going wrong because of the
>> reasons I have said earlier, so my second memory bank gets overlapped
>> with the vmalloc region and is ignored. With highemem seems like the
>> problem should have been solved but my kernel does not boot with
>> highmem enabled.
>>
>> Any suggestions ? I'm using Linux 2.6.30..
>
> Highmem support was introduced on ARM in v2.6.29 and became somewhat
> "reliable" on v2.6.31. ?In any case, v2.6.30 is more than 2 years old.
> Please try to upgrade your kernel.
>
>
> Nicolas
>

Thanks to everyone. I got 256 MB working. Its been running quite
stable for many days now.

1) I have used Nicu Pavels patch as reference:
    (http://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=6143/1)
    with minor changes

2) I switched to linux 3.0.0 (and also have tested with linux 3.0.3).
Applied the above patch (manually) with little changes: patch is as
follows:


diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 92622eb..c1773e1 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -275,6 +275,8 @@ config ARCH_AT91
 	select GENERIC_GPIO
 	select ARCH_REQUIRE_GPIOLIB
 	select HAVE_CLK
+	select ARCH_SPARSEMEM_ENABLE
+	select ARCH_SELECT_MEMORY_MODEL
+	select ARM_PATCH_PHYS_VIRT if MMU
 	help
 	  This enables support for systems based on the Atmel AT91RM9200,
 	  AT91SAM9 and AT91CAP9 processors.
diff --git a/arch/arm/mach-at91/include/mach/memory.h
b/arch/arm/mach-at91/include/mach/memory.h
index 14f4ef4..89aed9b 100644
--- a/arch/arm/mach-at91/include/mach/memory.h
+++ b/arch/arm/mach-at91/include/mach/memory.h
@@ -25,4 +25,24 @@

+ #define PHYS_OFFSET	0x70000000     /* DDRSDRC0 */
+/*#define PHYS_OFFSET	0x20000000 */ /* DDRSDRC1 - EBI */

+#if defined(CONFIG_ARCH_AT91SAM9G45) || defined(CONFIG_ARCH_AT91SAM9M10)
+/*
+ * Non-linear mapping like so:
+ * phys       => virt
+ * 0x70000000 => 0xc0000000
+ * 0x20000000 => 0xc8000000
+ */
+
+#define __phys_to_virt(p)   \
+            (((p) & 0x07ffffff) + (((p) & 0x40000000) ? 0xc0000000 :
0xc8000000))
+
+#define __virt_to_phys(v)   \
+            (((v) & 0x07ffffff) + (((v) & 0x08000000) ? 0x20000000 :
0x70000000 ))
+
+#define NODE_MEM_SIZE_BITS	27
+#define MAX_PHYSMEM_BITS	32
+#define SECTION_SIZE_BITS	27 /*128 Mb */
+#define HIGH_MEMORY_VIRT	0xd0000000
+#endif
+
 #endif
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 0ed29bf..38b7526 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -461,7 +461,11 @@ void __init bootmem_init(void)
 	for_each_node(node)
 		bootmem_free_node(node, mi);

+#ifdef HIGH_MEMORY_VIRT
+	high_memory = HIGH_MEMORY_VIRT;
+#else
 	high_memory = __va(((phys_addr_t)max_low << PAGE_SHIFT) - 1) + 1;
+#endif

 	/*
 	 * This doesn't seem to be used by the Linux memory manager any

3) High Memory is disabled.

4) Followed Russell Kings advice by changing the function in
atmel_mci.c to flush_kernel_dcache_page()

Its works pretty good for me. Now I have 256 MB working on my custom
board based on AT91SAM9G45-EKES

-Prasant Jalan