[Bug 1885332] Re: Error in user-mode calculation of ELF aux vector's AT_PHDR

[Dmitry <1885332@bugs.launchpad.net>]

When I switch to armv7 the issue goes away

$ cat Dockerfile.armv7 
FROM arm32v7/ubuntu
RUN apt-get update && \
    apt-get install -y \
    gcc make libpcre3-dev libreadline-dev git

RUN cd /home && git clone https://github.com/nginx/njs

RUN cd /home/njs && ./configure --cc-opt='-O0 -static -lm -lrt -pthread
-Wl,--whole-archive -lpthread -ltinfo -Wl,--no-whole-archive' && make
njs

$ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
$ docker build -t njs/armv7 -f Dockerfile.armv7 .
$ docker run -v `pwd`:/m -ti njs/armv7 cp /home/njs/build/njs /m/njs-armv7

$ readelf -l ./njs-armv7

Elf file type is EXEC (Executable file)
Entry point 0x12fb9
There are 7 program headers, starting at offset 52

Program Headers:
  Type           Offset   VirtAddr   PhysAddr   FileSiz MemSiz  Flg Align
  EXIDX          0x1be338 0x001ce338 0x001ce338 0x009b8 0x009b8 R   0x4
  LOAD           0x000000 0x00010000 0x00010000 0x1becf4 0x1becf4 R E 0x10000
  LOAD           0x1bedfc 0x001dedfc 0x001dedfc 0x17674 0x1c2cc RW  0x10000
  NOTE           0x000114 0x00010114 0x00010114 0x00044 0x00044 R   0x4
  TLS            0x1bedfc 0x001dedfc 0x001dedfc 0x00038 0x00060 R   0x4
  GNU_STACK      0x000000 0x00000000 0x00000000 0x00000 0x00000 RW  0x10
  GNU_RELRO      0x1bedfc 0x001dedfc 0x001dedfc 0x0e204 0x0e204 R   0x1

 Section to Segment mapping:
  Segment Sections...
   00     .ARM.exidx 
   01     .note.ABI-tag .note.gnu.build-id .rel.dyn .init .iplt .text __libc_freeres_fn __libc_thread_freeres_fn .fini .rodata .stapsdt.base __libc_subfreeres __libc_IO_vtables __libc_atexit __libc_thread_subfreeres .ARM.extab .ARM.exidx .eh_frame 
   02     .tdata .init_array .fini_array .data.rel.ro .got .data .bss __libc_freeres_ptrs 
   03     .note.ABI-tag .note.gnu.build-id 
   04     .tdata .tbss 
   05     
   06     .tdata .init_array .fini_array .data.rel.ro 

$ readelf -h ./njs-armv7
ELF Header:
  Magic:   7f 45 4c 46 01 01 01 03 00 00 00 00 00 00 00 00 
  Class:                             ELF32
  Data:                              2's complement, little endian
  Version:                           1 (current)
  OS/ABI:                            UNIX - GNU
  ABI Version:                       0
  Type:                              EXEC (Executable file)
  Machine:                           ARM
  Version:                           0x1
  Entry point address:               0x12fb9
  Start of program headers:          52 (bytes into file)
  Start of section headers:          5696248 (bytes into file)
  Flags:                             0x5000400, Version5 EABI, hard-float ABI
  Size of this header:               52 (bytes)
  Size of program headers:           32 (bytes)
  Number of program headers:         7
  Size of section headers:           40 (bytes)
  Number of section headers:         42
  Section header string table index: 41

$ qemu-arm -g 12345 ./njs-armv7 -c 'console.log("HH")'

$ gdb-multiarch ./njs-armv7 -ex 'source showstack.py'
ARGUMENTS
---------
argc = 3
arg 0 = ./njs-armv7
arg 1 = -c
arg 2 = console.log("HH")

...

AUX VECTOR
----------
AT_PHDR = 10034
AT_PHENT = 20
AT_PHNUM = 7
AT_PAGESZ = 1000
AT_BASE = 0
AT_FLAGS = 0
AT_ENTRY = 12fb9
AT_UID = 3e9
AT_EUID = 3e9
AT_GID = 3e9
AT_EGID = 3e9
AT_HWCAP = 1fb8d7
AT_CLKTCK = 64
AT_RANDOM = -104a0
AT_HWCAP2 = 1f
AT_NULL = 0

$ qemu-arm --version
qemu-arm version 2.11.1(Debian 1:2.11+dfsg-1ubuntu7.28)
Copyright (c) 2003-2017 Fabrice Bellard and the QEMU Project developers

-- 
You received this bug notification because you are a member of qemu-
devel-ml, which is subscribed to QEMU.
https://bugs.launchpad.net/bugs/1885332

Title:
  Error in user-mode calculation of ELF aux vector's AT_PHDR

Status in QEMU:
  New

Bug description:
  
  I have an (admittedly strange) statically-linked ELF binary for Linux that runs just fine on top of the Linux kernel in QEMU full-system emulation, but crashes before main in user-mode emulation. Specifically, it crashes when initializing thread-local storage in glibc's _dl_aux_init, because it reads out a strange value from the AT_PHDR entry of the ELF aux vector.

  The binary has these program headers:

    Program Headers:
      Type           Offset   VirtAddr   PhysAddr   FileSiz MemSiz  Flg Align
      EXIDX          0x065874 0x00075874 0x00075874 0x00570 0x00570 R   0x4
      PHDR           0x0a3000 0x00900000 0x00900000 0x00160 0x00160 R   0x1000
      LOAD           0x0a3000 0x00900000 0x00900000 0x00160 0x00160 R   0x1000
      LOAD           0x000000 0x00010000 0x00010000 0x65de8 0x65de8 R E 0x10000
      LOAD           0x066b7c 0x00086b7c 0x00086b7c 0x02384 0x02384 RW  0x10000
      NOTE           0x000114 0x00010114 0x00010114 0x00044 0x00044 R   0x4
      TLS            0x066b7c 0x00086b7c 0x00086b7c 0x00010 0x00030 R   0x4
      GNU_STACK      0x000000 0x00000000 0x00000000 0x00000 0x00000 RW  0x8
      GNU_RELRO      0x066b7c 0x00086b7c 0x00086b7c 0x00484 0x00484 R   0x1
      LOAD           0x07e000 0x00089000 0x00089000 0x03f44 0x03f44 R E 0x1000
      LOAD           0x098000 0x00030000 0x00030000 0x01000 0x01000 RW  0x1000

  If I build the Linux kernel with the following patch to the very end
  of create_elf_tables in fs/binfmt_elf.c

    /* Put the elf_info on the stack in the right place.  */
    elf_addr_t *my_auxv = (elf_addr_t *) mm->saved_auxv;
    int i;
    for (i = 0; i < 15; i++) {
      printk("0x%x = 0x%x", my_auxv[2*i], my_auxv[(2*i)+ 1]);
    }
    if (copy_to_user(sp, mm->saved_auxv, ei_index * sizeof(elf_addr_t)))
        return -EFAULT;
    return 0;

  and run it like this:

    qemu-system-arm \
      -M versatilepb \
      -nographic \
      -dtb ./dts/versatile-pb.dtb \
      -kernel zImage \
      -M versatilepb \
      -m 128M \
      -append "earlyprintk=vga,keep" \
      -initrd initramfs

  after I've built the kernel initramfs like this (where "init" is the
  binary in question):

    make ARCH=arm versatile_defconfig
    make ARCH=arm CROSS_COMPILE=arm-linux-gnueabi- all -j10
    cp "$1" arch/arm/boot/init
    cd arch/arm/boot
    echo init | cpio -o --format=newc > initramfs

  then I get the following output. This is the kernel's view of the aux
  vector for this binary:

    0x10 = 0x1d7
    0x6 = 0x1000
    0x11 = 0x64
    0x3 = 0x900000
    0x4 = 0x20
    0x5 = 0xb
    0x7 = 0x0
    0x8 = 0x0
    0x9 = 0x101b8
    0xb = 0x0
    0xc = 0x0
    0xd = 0x0
    0xe = 0x0
    0x17 = 0x0
    0x19 = 0xbec62fb5

  However, if I run "qemu-arm -g 12345 binary" and use GDB to peek at
  the aux vector at the beginning of __libc_start_init (for example,
  using this Python GDB API script: https://gist.github.com/langston-
  barrett/5573d64ae0c9953e2fa0fe26847a5e1e), then I see the following
  values:

    AT_PHDR = 0xae000
    AT_PHENT = 0x20
    AT_PHNUM = 0xb
    AT_PAGESZ = 0x1000
    AT_BASE = 0x0
    AT_FLAGS = 0x0
    AT_ENTRY = 0x10230
    AT_UID = 0x3e9
    AT_EUID = 0x3e9
    AT_GID = 0x3e9
    AT_EGID = 0x3e9
    AT_HWCAP = 0x1fb8d7
    AT_CLKTCK = 0x64
    AT_RANDOM = -0x103c0
    AT_HWCAP2 = 0x1f
    AT_NULL = 0x0

  The crucial difference is in AT_PHDR (0x3), which is indeed the
  virtual address of the PHDR segment when the kernel calculates it, but
  is not when QEMU calculates it.

  qemu-arm --version
  qemu-arm version 2.11.1(Debian 1:2.11+dfsg-1ubuntu7.26)

To manage notifications about this bug go to:
https://bugs.launchpad.net/qemu/+bug/1885332/+subscriptions