All of lore.kernel.org
 help / color / mirror / Atom feed
* XDP invalid memory access
@ 2020-01-15 22:11 Vincent Li
  2020-01-15 22:21 ` Toke Høiland-Jørgensen
  2020-01-16  2:24 ` Maciej Fijalkowski
  0 siblings, 2 replies; 16+ messages in thread
From: Vincent Li @ 2020-01-15 22:11 UTC (permalink / raw)
  To: xdp-newbies

Hi,

I am writing a sample XDP program to parse tcp packet options, code
below, compiled ok, when I attach it to a network interface

#clang -O2 -emit-llvm -c tcp_option.c -o - |llc -march=bpf
-filetype=obj -o tcp_option.o


# ip link set dev enp3s0 xdpgeneric object tcp_option.o verbose


Prog section 'prog' rejected: Permission denied (13)!

 - Type:         6

 - Instructions: 42 (0 over limit)

 - License:      GPL


Verifier analysis:


0: (61) r2 = *(u32 *)(r1 +4)

1: (61) r1 = *(u32 *)(r1 +0)

2: (bf) r3 = r1

3: (07) r3 += 54

4: (b7) r0 = 2

5: (2d) if r3 > r2 goto pc+35

 R0_w=inv2 R1_w=pkt(id=0,off=0,r=54,imm=0)
R2_w=pkt_end(id=0,off=0,imm=0) R3_w=pkt(id=0,off=54,r=54,imm=0)
R10=fp0

6: (71) r2 = *(u8 *)(r1 +12)

7: (71) r3 = *(u8 *)(r1 +13)

8: (67) r3 <<= 8

9: (4f) r3 |= r2

10: (b7) r0 = 2

11: (55) if r3 != 0x8 goto pc+29

 R0_w=inv2 R1_w=pkt(id=0,off=0,r=54,imm=0)
R2_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R3_w=inv8 R10=fp0

12: (71) r2 = *(u8 *)(r1 +23)

13: (b7) r0 = 2

14: (55) if r2 != 0x6 goto pc+26

 R0=inv2 R1=pkt(id=0,off=0,r=54,imm=0) R2=inv6 R3=inv8 R10=fp0

15: (69) r1 = *(u16 *)(r1 +46)

16: (bf) r2 = r1

17: (57) r2 &= 7936

18: (b7) r0 = 2

19: (55) if r2 != 0x200 goto pc+21

 R0_w=inv2 R1_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))
R2_w=inv512 R3=inv8 R10=fp0

20: (77) r1 >>= 2

21: (57) r1 &= 60

22: (07) r1 += -20

23: (18) r2 = 0x0

25: (63) *(u32 *)(r2 +0) = r1

R2 invalid mem access 'inv' <----------------

processed 25 insns (limit 1000000) max_states_per_insn 0 total_states
1 peak_states 1 mark_read 1


it appears optlen = tcphdr->doff*4 - sizeof(*tcphdr); is invalid ? if
I comment out lines between 60 and 73, no problem with invalid mem
access





     1 #include <stdint.h>

     2 #include <arpa/inet.h>

     3 #include <asm/byteorder.h>

     4 #include <linux/bpf.h>

     5 #include <linux/if_ether.h>

     6 #include <linux/ip.h>

     7 #include <linux/tcp.h>

     8 #include <linux/pkt_cls.h>

     9

    10 /*

    11 * Sample XDP to parse tcp option.

    12 * compile it with:

    13 *      clang -O2 -emit-llvm -c tcp_option.c -o - |llc
-march=bpf -filetype=obj -o tcp_option.o

    14  * attach it to a device with XDP as:

    15  *  ip link set dev lo xdp object tcp_option.o verbose

    16 */

    17

    18 #define SEC(NAME) __attribute__((section(NAME), used))

    19

    20 #define TCPOPT_EOL        0       /* End of options (1)              */

    21 #define TCPOPT_NOP        1       /* No-op (1)                       */

    22 #define TCPOPT_MAXSEG     2       /* Maximum segment size (4)        */

    23 #define TCPOPT_WSCALE     3       /* Window scaling (3)              */

    24 #define TCPOPT_SACKOK     4       /* Selective ACK permitted (2)     */

    25 #define TCPOPT_SACK       5       /* Actual selective ACK (10-34)    */

    26 #define TCPOPT_TSTAMP     8       /* Timestamp (10)                  */

    27

    28

    29 /* from bpf_helpers.h */

    30

    31 static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) =

    32         (void *) BPF_FUNC_trace_printk;

    33

    34 static unsigned long long (*bpf_get_prandom_u32)(void) =

    35 (void *) BPF_FUNC_get_prandom_u32;

    36

    37 const __u8 *op;

    38 int i, optlen;

    39

    40 static int tcp_option(void *data, void *data_end)

    41 {

    42 struct ethhdr *eth = (struct ethhdr *)data;

    43 struct iphdr *iph = (struct iphdr *)(eth + 1);

    44 struct tcphdr *tcphdr = (struct tcphdr *)(iph + 1);

    45 int tcplen;

    46

    47 /* sanity check needed by the eBPF verifier */

    48 if ((void *)(tcphdr + 1) > data_end)

    49 return 0;

    50

    51 /* skip non TCP packets */

    52 if (eth->h_proto != __constant_htons(ETH_P_IP) || iph->protocol
!= IPPROTO_TCP)

    53 return 0;

    54

    55 /* incompatible flags, or PSH already set */

    56 if (tcphdr->ack || tcphdr->fin || tcphdr->rst || tcphdr->psh)

    57 return 0;

    58

    59 if (tcphdr->syn) {

    60 optlen = tcphdr->doff*4 - sizeof(*tcphdr);

    61                 if (!optlen)

    62                       return -1;

    63 /*

    64                 for (i = 0; i < optlen; ) {

    65                         if (op[i] == TCPOPT_EOL) {

    66                  char fmt[] = "XDP: tcp opt eol kind seen %d \n";

    67                  bpf_trace_printk(fmt, sizeof(fmt), op[i]);

    68 }

    69                         if (op[i] < 2)

    70                                 i++;

    71                         else

    72                                 i += op[i+1] ? : 1;

    73                 }

    74 */

    75

    76                char fmt[] = "XDP: tcp syn seen \n";

    77                bpf_trace_printk(fmt, sizeof(fmt));

    78

    79 }

    80

    81 return 0;

    82 }

    83

    84 SEC("prog")

    85 int xdp_main(struct xdp_md *ctx)

    86 {

    87 void *data_end = (void *)(uintptr_t)ctx->data_end;

    88 void *data = (void *)(uintptr_t)ctx->data;

    89

    90 if (tcp_option(data, data_end))

    91 return XDP_DROP;

    92

    93 return XDP_PASS;

    94 }

    95

    96

    97 char _license[] SEC("license") = "GPL";

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-15 22:11 XDP invalid memory access Vincent Li
@ 2020-01-15 22:21 ` Toke Høiland-Jørgensen
  2020-01-15 22:31   ` Vincent Li
  2020-01-16  2:24 ` Maciej Fijalkowski
  1 sibling, 1 reply; 16+ messages in thread
From: Toke Høiland-Jørgensen @ 2020-01-15 22:21 UTC (permalink / raw)
  To: Vincent Li, xdp-newbies

Vincent Li <mchun.li@gmail.com> writes:

> Hi,
>
> I am writing a sample XDP program to parse tcp packet options, code
> below, compiled ok, when I attach it to a network interface
>
> #clang -O2 -emit-llvm -c tcp_option.c -o - |llc -march=bpf
> -filetype=obj -o tcp_option.o
>
>
> # ip link set dev enp3s0 xdpgeneric object tcp_option.o verbose
>
>
> Prog section 'prog' rejected: Permission denied (13)!
>
>  - Type:         6
>
>  - Instructions: 42 (0 over limit)
>
>  - License:      GPL
>
>
> Verifier analysis:
>
>
> 0: (61) r2 = *(u32 *)(r1 +4)
>
> 1: (61) r1 = *(u32 *)(r1 +0)
>
> 2: (bf) r3 = r1
>
> 3: (07) r3 += 54
>
> 4: (b7) r0 = 2
>
> 5: (2d) if r3 > r2 goto pc+35
>
>  R0_w=inv2 R1_w=pkt(id=0,off=0,r=54,imm=0)
> R2_w=pkt_end(id=0,off=0,imm=0) R3_w=pkt(id=0,off=54,r=54,imm=0)
> R10=fp0
>
> 6: (71) r2 = *(u8 *)(r1 +12)
>
> 7: (71) r3 = *(u8 *)(r1 +13)
>
> 8: (67) r3 <<= 8
>
> 9: (4f) r3 |= r2
>
> 10: (b7) r0 = 2
>
> 11: (55) if r3 != 0x8 goto pc+29
>
>  R0_w=inv2 R1_w=pkt(id=0,off=0,r=54,imm=0)
> R2_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R3_w=inv8 R10=fp0
>
> 12: (71) r2 = *(u8 *)(r1 +23)
>
> 13: (b7) r0 = 2
>
> 14: (55) if r2 != 0x6 goto pc+26
>
>  R0=inv2 R1=pkt(id=0,off=0,r=54,imm=0) R2=inv6 R3=inv8 R10=fp0
>
> 15: (69) r1 = *(u16 *)(r1 +46)
>
> 16: (bf) r2 = r1
>
> 17: (57) r2 &= 7936
>
> 18: (b7) r0 = 2
>
> 19: (55) if r2 != 0x200 goto pc+21
>
>  R0_w=inv2 R1_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))
> R2_w=inv512 R3=inv8 R10=fp0
>
> 20: (77) r1 >>= 2
>
> 21: (57) r1 &= 60
>
> 22: (07) r1 += -20
>
> 23: (18) r2 = 0x0
>
> 25: (63) *(u32 *)(r2 +0) = r1
>
> R2 invalid mem access 'inv' <----------------
>
> processed 25 insns (limit 1000000) max_states_per_insn 0 total_states
> 1 peak_states 1 mark_read 1
>
>
> it appears optlen = tcphdr->doff*4 - sizeof(*tcphdr); is invalid ? if
> I comment out lines between 60 and 73, no problem with invalid mem
> access

You have to check that you're not reading out of bounds before
dereferencing the bytes in the TCP header...

-Toke

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-15 22:21 ` Toke Høiland-Jørgensen
@ 2020-01-15 22:31   ` Vincent Li
  2020-01-16  0:40     ` David Ahern
  0 siblings, 1 reply; 16+ messages in thread
From: Vincent Li @ 2020-01-15 22:31 UTC (permalink / raw)
  To: Toke Høiland-Jørgensen; +Cc: xdp-newbies

On Wed, Jan 15, 2020 at 2:21 PM Toke Høiland-Jørgensen <toke@redhat.com> wrote:

> You have to check that you're not reading out of bounds before
> dereferencing the bytes in the TCP header...
>

I have below before the optlen

   47 /* sanity check needed by the eBPF verifier */

    48 if ((void *)(tcphdr + 1) > data_end)

    49 return 0;

this is not enough, how do I check the out of bounds properly?

Regards,

Vincent

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-15 22:31   ` Vincent Li
@ 2020-01-16  0:40     ` David Ahern
  2020-01-16  1:34       ` Vincent Li
  0 siblings, 1 reply; 16+ messages in thread
From: David Ahern @ 2020-01-16  0:40 UTC (permalink / raw)
  To: Vincent Li, Toke Høiland-Jørgensen; +Cc: xdp-newbies

On 1/15/20 3:31 PM, Vincent Li wrote:
> On Wed, Jan 15, 2020 at 2:21 PM Toke Høiland-Jørgensen <toke@redhat.com> wrote:
> 
>> You have to check that you're not reading out of bounds before
>> dereferencing the bytes in the TCP header...
>>
> 
> I have below before the optlen
> 
>    47 /* sanity check needed by the eBPF verifier */
> 
>     48 if ((void *)(tcphdr + 1) > data_end)
> 
>     49 return 0;
> 
> this is not enough, how do I check the out of bounds properly?
> 

options are optional and after 'struct tcphdr' you need to do something
like:

   if ((void *)(tcphdr + 1) + tcphdr->doff > data_end)
       return XDP_....

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-16  0:40     ` David Ahern
@ 2020-01-16  1:34       ` Vincent Li
  2020-01-16  3:19         ` Vincent Li
  0 siblings, 1 reply; 16+ messages in thread
From: Vincent Li @ 2020-01-16  1:34 UTC (permalink / raw)
  To: David Ahern; +Cc: Toke Høiland-Jørgensen, xdp-newbies

thank you! I will try that

On Wed, Jan 15, 2020 at 4:40 PM David Ahern <dsahern@gmail.com> wrote:
>
> On 1/15/20 3:31 PM, Vincent Li wrote:
> > On Wed, Jan 15, 2020 at 2:21 PM Toke Høiland-Jørgensen <toke@redhat.com> wrote:
> >
> >> You have to check that you're not reading out of bounds before
> >> dereferencing the bytes in the TCP header...
> >>
> >
> > I have below before the optlen
> >
> >    47 /* sanity check needed by the eBPF verifier */
> >
> >     48 if ((void *)(tcphdr + 1) > data_end)
> >
> >     49 return 0;
> >
> > this is not enough, how do I check the out of bounds properly?
> >
>
> options are optional and after 'struct tcphdr' you need to do something
> like:
>
>    if ((void *)(tcphdr + 1) + tcphdr->doff > data_end)
>        return XDP_....

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-15 22:11 XDP invalid memory access Vincent Li
  2020-01-15 22:21 ` Toke Høiland-Jørgensen
@ 2020-01-16  2:24 ` Maciej Fijalkowski
  2020-01-16  9:45   ` Toke Høiland-Jørgensen
                     ` (2 more replies)
  1 sibling, 3 replies; 16+ messages in thread
From: Maciej Fijalkowski @ 2020-01-16  2:24 UTC (permalink / raw)
  To: Vincent Li; +Cc: xdp-newbies, daniel, andriin, dsahern

On Wed, Jan 15, 2020 at 02:11:04PM -0800, Vincent Li wrote:
> Hi,
> 
> I am writing a sample XDP program to parse tcp packet options, code
> below, compiled ok, when I attach it to a network interface
> 
> #clang -O2 -emit-llvm -c tcp_option.c -o - |llc -march=bpf
> -filetype=obj -o tcp_option.o
> 
> 
> # ip link set dev enp3s0 xdpgeneric object tcp_option.o verbose
> 
> 
> Prog section 'prog' rejected: Permission denied (13)!
> 
>  - Type:         6
> 
>  - Instructions: 42 (0 over limit)
> 
>  - License:      GPL
> 
> 
> Verifier analysis:
> 
> 
> 0: (61) r2 = *(u32 *)(r1 +4)
> 
> 1: (61) r1 = *(u32 *)(r1 +0)
> 
> 2: (bf) r3 = r1
> 
> 3: (07) r3 += 54
> 
> 4: (b7) r0 = 2
> 
> 5: (2d) if r3 > r2 goto pc+35
> 
>  R0_w=inv2 R1_w=pkt(id=0,off=0,r=54,imm=0)
> R2_w=pkt_end(id=0,off=0,imm=0) R3_w=pkt(id=0,off=54,r=54,imm=0)
> R10=fp0
> 
> 6: (71) r2 = *(u8 *)(r1 +12)
> 
> 7: (71) r3 = *(u8 *)(r1 +13)
> 
> 8: (67) r3 <<= 8
> 
> 9: (4f) r3 |= r2
> 
> 10: (b7) r0 = 2
> 
> 11: (55) if r3 != 0x8 goto pc+29
> 
>  R0_w=inv2 R1_w=pkt(id=0,off=0,r=54,imm=0)
> R2_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R3_w=inv8 R10=fp0
> 
> 12: (71) r2 = *(u8 *)(r1 +23)
> 
> 13: (b7) r0 = 2
> 
> 14: (55) if r2 != 0x6 goto pc+26
> 
>  R0=inv2 R1=pkt(id=0,off=0,r=54,imm=0) R2=inv6 R3=inv8 R10=fp0
> 
> 15: (69) r1 = *(u16 *)(r1 +46)
> 
> 16: (bf) r2 = r1
> 
> 17: (57) r2 &= 7936
> 
> 18: (b7) r0 = 2
> 
> 19: (55) if r2 != 0x200 goto pc+21
> 
>  R0_w=inv2 R1_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))
> R2_w=inv512 R3=inv8 R10=fp0
> 
> 20: (77) r1 >>= 2
> 
> 21: (57) r1 &= 60
> 
> 22: (07) r1 += -20
> 
> 23: (18) r2 = 0x0
> 
> 25: (63) *(u32 *)(r2 +0) = r1
> 
> R2 invalid mem access 'inv' <----------------

So it's not out of the bounds access but rather null dereference since r2
has been loaded with 0 on previous insn.

> 
> processed 25 insns (limit 1000000) max_states_per_insn 0 total_states
> 1 peak_states 1 mark_read 1
> 
> 
> it appears optlen = tcphdr->doff*4 - sizeof(*tcphdr); is invalid ? if
> I comment out lines between 60 and 73, no problem with invalid mem
> access

I see that optlen is a global variable. This line might be valid but
you're using iproute2's loader for your XDP program, right? AFAIK it
doesn't have support for BPF global variables, only libbpf does (Daniel,
Andrii? is that true?).

So you have to either make the optlen a local variable or go with writing
the loader part that is based on libbpf usage (see samples/bpf directory
in kernel tree, for example xdp1_user.c).

Could you next time try a little better with the format of data you're
providing? I'm pretty sure other folks would have spotted earlier that
optlen is global if the code was properly formatted :)

To wrap this rant up, making optlen a local var does the job for me.

> 
> 
> 
> 
> 
>      1 #include <stdint.h>
> 
>      2 #include <arpa/inet.h>
> 
>      3 #include <asm/byteorder.h>
> 
>      4 #include <linux/bpf.h>
> 
>      5 #include <linux/if_ether.h>
> 
>      6 #include <linux/ip.h>
> 
>      7 #include <linux/tcp.h>
> 
>      8 #include <linux/pkt_cls.h>
> 
>      9
> 
>     10 /*
> 
>     11 * Sample XDP to parse tcp option.
> 
>     12 * compile it with:
> 
>     13 *      clang -O2 -emit-llvm -c tcp_option.c -o - |llc
> -march=bpf -filetype=obj -o tcp_option.o
> 
>     14  * attach it to a device with XDP as:
> 
>     15  *  ip link set dev lo xdp object tcp_option.o verbose
> 
>     16 */
> 
>     17
> 
>     18 #define SEC(NAME) __attribute__((section(NAME), used))
> 
>     19
> 
>     20 #define TCPOPT_EOL        0       /* End of options (1)              */
> 
>     21 #define TCPOPT_NOP        1       /* No-op (1)                       */
> 
>     22 #define TCPOPT_MAXSEG     2       /* Maximum segment size (4)        */
> 
>     23 #define TCPOPT_WSCALE     3       /* Window scaling (3)              */
> 
>     24 #define TCPOPT_SACKOK     4       /* Selective ACK permitted (2)     */
> 
>     25 #define TCPOPT_SACK       5       /* Actual selective ACK (10-34)    */
> 
>     26 #define TCPOPT_TSTAMP     8       /* Timestamp (10)                  */
> 
>     27
> 
>     28
> 
>     29 /* from bpf_helpers.h */
> 
>     30
> 
>     31 static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) =
> 
>     32         (void *) BPF_FUNC_trace_printk;
> 
>     33
> 
>     34 static unsigned long long (*bpf_get_prandom_u32)(void) =
> 
>     35 (void *) BPF_FUNC_get_prandom_u32;
> 
>     36
> 
>     37 const __u8 *op;
> 
>     38 int i, optlen;
> 
>     39
> 
>     40 static int tcp_option(void *data, void *data_end)
> 
>     41 {
> 
>     42 struct ethhdr *eth = (struct ethhdr *)data;
> 
>     43 struct iphdr *iph = (struct iphdr *)(eth + 1);
> 
>     44 struct tcphdr *tcphdr = (struct tcphdr *)(iph + 1);
> 
>     45 int tcplen;
> 
>     46
> 
>     47 /* sanity check needed by the eBPF verifier */
> 
>     48 if ((void *)(tcphdr + 1) > data_end)
> 
>     49 return 0;
> 
>     50
> 
>     51 /* skip non TCP packets */
> 
>     52 if (eth->h_proto != __constant_htons(ETH_P_IP) || iph->protocol
> != IPPROTO_TCP)
> 
>     53 return 0;
> 
>     54
> 
>     55 /* incompatible flags, or PSH already set */
> 
>     56 if (tcphdr->ack || tcphdr->fin || tcphdr->rst || tcphdr->psh)
> 
>     57 return 0;
> 
>     58
> 
>     59 if (tcphdr->syn) {
> 
>     60 optlen = tcphdr->doff*4 - sizeof(*tcphdr);
> 
>     61                 if (!optlen)
> 
>     62                       return -1;
> 
>     63 /*
> 
>     64                 for (i = 0; i < optlen; ) {
> 
>     65                         if (op[i] == TCPOPT_EOL) {
> 
>     66                  char fmt[] = "XDP: tcp opt eol kind seen %d \n";
> 
>     67                  bpf_trace_printk(fmt, sizeof(fmt), op[i]);
> 
>     68 }
> 
>     69                         if (op[i] < 2)
> 
>     70                                 i++;
> 
>     71                         else
> 
>     72                                 i += op[i+1] ? : 1;
> 
>     73                 }
> 
>     74 */
> 
>     75
> 
>     76                char fmt[] = "XDP: tcp syn seen \n";
> 
>     77                bpf_trace_printk(fmt, sizeof(fmt));
> 
>     78
> 
>     79 }
> 
>     80
> 
>     81 return 0;
> 
>     82 }
> 
>     83
> 
>     84 SEC("prog")
> 
>     85 int xdp_main(struct xdp_md *ctx)
> 
>     86 {
> 
>     87 void *data_end = (void *)(uintptr_t)ctx->data_end;
> 
>     88 void *data = (void *)(uintptr_t)ctx->data;
> 
>     89
> 
>     90 if (tcp_option(data, data_end))
> 
>     91 return XDP_DROP;
> 
>     92
> 
>     93 return XDP_PASS;
> 
>     94 }
> 
>     95
> 
>     96
> 
>     97 char _license[] SEC("license") = "GPL";

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-16  1:34       ` Vincent Li
@ 2020-01-16  3:19         ` Vincent Li
  2020-01-16  4:22           ` David Ahern
  0 siblings, 1 reply; 16+ messages in thread
From: Vincent Li @ 2020-01-16  3:19 UTC (permalink / raw)
  To: David Ahern; +Cc: Toke Høiland-Jørgensen, xdp-newbies

I changed the code below, not sure if I got it right from you, I still
got memory access error


 40 static int tcp_option(void *data, void *data_end)

 41 {

 42         struct ethhdr *eth = (struct ethhdr *)data;

 43         struct iphdr *iph = (struct iphdr *)(eth + 1);

 44         struct tcphdr *tcphdr = (struct tcphdr *)(iph + 1);

 45         int tcplen;

 46

 47         /* sanity check needed by the eBPF verifier */

 48         if ((void *)(tcphdr + 1)  > data_end)

 49                 return 0;

 50

 51         /* skip non TCP packets */

 52         if (eth->h_proto != __constant_htons(ETH_P_IP) ||
iph->protocol != IPPROTO_TCP)

 53                 return 0;

 54

 55         /* incompatible flags, or PSH already set */

 56         if (tcphdr->ack || tcphdr->fin || tcphdr->rst || tcphdr->psh)

 57                 return 0;

 58

 59         if (tcphdr->syn) {

 60                 if (((void *)(tcphdr + 1) + tcphdr->doff*4) > data_end)

 61                         return 0;

 62                 optlen = tcphdr->doff*4 - sizeof(*tcphdr);

 63                 for (i = 0; i < optlen; ) {

 64                         if (op[i] == TCPOPT_EOL ) {

 65                                 char fmt[] = "XDP: tcp source : %d
tcp option eol :%d\n";

 66                                 bpf_trace_printk(fmt, sizeof(fmt),
(int)tcphdr->source, TCPOPT_EOL);

 67                                 return 1;

 68                         }

 69                         if (op[i] < 2)

 70                                 i++;

 71                         else

 72                                 i += op[i+1] ? : 1;

 73                 }

 74                 /*

 75                if (tcphdr->doff*4 == 44 || tcphdr->doff*4 == 28) {

 76                         char fmt[] = "XDP: tcp source : %d data
offset :%d\n";

 77                         bpf_trace_printk(fmt, sizeof(fmt),
(int)tcphdr->source, (int)tcphdr->doff*4);

 78                        return 1;

 79                }

 80                */

 81         }

 82         return 0;

 83 }

# ip link set dev enp4s0 xdpgeneric object tcp_option.o verbose


Prog section 'prog' rejected: Permission denied (13)!

 - Type:         6

 - Instructions: 86 (0 over limit)

 - License:      GPL


Verifier analysis:


0: (b7) r0 = 2

1: (61) r2 = *(u32 *)(r1 +4)

2: (61) r1 = *(u32 *)(r1 +0)

3: (bf) r3 = r1

4: (07) r3 += 54

5: (2d) if r3 > r2 goto pc+79

 R0_w=inv2 R1_w=pkt(id=0,off=0,r=54,imm=0)
R2_w=pkt_end(id=0,off=0,imm=0) R3_w=pkt(id=0,off=54,r=54,imm=0)
R10=fp0

6: (71) r4 = *(u8 *)(r1 +12)

7: (71) r5 = *(u8 *)(r1 +13)

8: (67) r5 <<= 8

9: (4f) r5 |= r4

10: (55) if r5 != 0x8 goto pc+74

 R0_w=inv2 R1_w=pkt(id=0,off=0,r=54,imm=0)
R2_w=pkt_end(id=0,off=0,imm=0) R3_w=pkt(id=0,off=54,r=54,imm=0)
R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R5_w=inv8 R10=fp0

11: (71) r4 = *(u8 *)(r1 +23)

12: (55) if r4 != 0x6 goto pc+72

 R0=inv2 R1=pkt(id=0,off=0,r=54,imm=0) R2=pkt_end(id=0,off=0,imm=0)
R3=pkt(id=0,off=54,r=54,imm=0) R4=inv6 R5=inv8 R10=fp0

13: (69) r5 = *(u16 *)(r1 +46)

14: (bf) r4 = r5

15: (57) r4 &= 7936

16: (55) if r4 != 0x200 goto pc+68

 R0=inv2 R1=pkt(id=0,off=0,r=54,imm=0) R2=pkt_end(id=0,off=0,imm=0)
R3=pkt(id=0,off=54,r=54,imm=0) R4_w=inv512
R5_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff)) R10=fp0

17: (77) r5 >>= 2

18: (57) r5 &= 60

19: (0f) r3 += r5

last_idx 19 first_idx 12

regs=20 stack=0 before 18: (57) r5 &= 60

regs=20 stack=0 before 17: (77) r5 >>= 2

regs=20 stack=0 before 16: (55) if r4 != 0x200 goto pc+68

regs=20 stack=0 before 15: (57) r4 &= 7936

regs=20 stack=0 before 14: (bf) r4 = r5

regs=20 stack=0 before 13: (69) r5 = *(u16 *)(r1 +46)

20: (2d) if r3 > r2 goto pc+64

 R0=inv2 R1=pkt(id=0,off=0,r=54,imm=0) R2=pkt_end(id=0,off=0,imm=0)
R3=pkt(id=1,off=54,r=54,umax_value=60,var_off=(0x0; 0x3c)) R4=inv512
R5=invP(id=0,umax_value=60,var_off=(0x0; 0x3c)) R10=fp0

21: (18) r2 = 0x0

23: (b7) r4 = 0

24: (63) *(u32 *)(r2 +0) = r4

R2 invalid mem access 'inv'

processed 24 insns (limit 1000000) max_states_per_insn 0 total_states
2 peak_states 2 mark_read 2


Error fetching program/map!

On Wed, Jan 15, 2020 at 5:34 PM Vincent Li <mchun.li@gmail.com> wrote:
>
> thank you! I will try that
>
> On Wed, Jan 15, 2020 at 4:40 PM David Ahern <dsahern@gmail.com> wrote:
> >
> > On 1/15/20 3:31 PM, Vincent Li wrote:
> > > On Wed, Jan 15, 2020 at 2:21 PM Toke Høiland-Jørgensen <toke@redhat.com> wrote:
> > >
> > >> You have to check that you're not reading out of bounds before
> > >> dereferencing the bytes in the TCP header...
> > >>
> > >
> > > I have below before the optlen
> > >
> > >    47 /* sanity check needed by the eBPF verifier */
> > >
> > >     48 if ((void *)(tcphdr + 1) > data_end)
> > >
> > >     49 return 0;
> > >
> > > this is not enough, how do I check the out of bounds properly?
> > >
> >
> > options are optional and after 'struct tcphdr' you need to do something
> > like:
> >
> >    if ((void *)(tcphdr + 1) + tcphdr->doff > data_end)
> >        return XDP_....

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-16  3:19         ` Vincent Li
@ 2020-01-16  4:22           ` David Ahern
  0 siblings, 0 replies; 16+ messages in thread
From: David Ahern @ 2020-01-16  4:22 UTC (permalink / raw)
  To: Vincent Li; +Cc: Toke Høiland-Jørgensen, xdp-newbies

On 1/15/20 8:19 PM, Vincent Li wrote:
> 
>  59         if (tcphdr->syn) {
> 
>  60                 if (((void *)(tcphdr + 1) + tcphdr->doff*4) > data_end)
> 
>  61                         return 0;
> 
>  62                 optlen = tcphdr->doff*4 - sizeof(*tcphdr);
> 
>  63                 for (i = 0; i < optlen; ) {

A variable length loop counter. Change this to just look at i = 0 (no
loop) and see if it loads.

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-16  2:24 ` Maciej Fijalkowski
@ 2020-01-16  9:45   ` Toke Høiland-Jørgensen
  2020-01-16 19:06     ` Vincent Li
  2020-01-16 17:44   ` Vincent Li
  2020-01-16 18:58   ` Vincent Li
  2 siblings, 1 reply; 16+ messages in thread
From: Toke Høiland-Jørgensen @ 2020-01-16  9:45 UTC (permalink / raw)
  To: Maciej Fijalkowski, Vincent Li; +Cc: xdp-newbies, daniel, andriin, dsahern

Maciej Fijalkowski <maciej.fijalkowski@intel.com> writes:

> On Wed, Jan 15, 2020 at 02:11:04PM -0800, Vincent Li wrote:
>> Hi,
>> 
>> I am writing a sample XDP program to parse tcp packet options, code
>> below, compiled ok, when I attach it to a network interface
>> 
>> #clang -O2 -emit-llvm -c tcp_option.c -o - |llc -march=bpf
>> -filetype=obj -o tcp_option.o
>> 
>> 
>> # ip link set dev enp3s0 xdpgeneric object tcp_option.o verbose
>> 
>> 
>> Prog section 'prog' rejected: Permission denied (13)!
>> 
>>  - Type:         6
>> 
>>  - Instructions: 42 (0 over limit)
>> 
>>  - License:      GPL
>> 
>> 
>> Verifier analysis:
>> 
>> 
>> 0: (61) r2 = *(u32 *)(r1 +4)
>> 
>> 1: (61) r1 = *(u32 *)(r1 +0)
>> 
>> 2: (bf) r3 = r1
>> 
>> 3: (07) r3 += 54
>> 
>> 4: (b7) r0 = 2
>> 
>> 5: (2d) if r3 > r2 goto pc+35
>> 
>>  R0_w=inv2 R1_w=pkt(id=0,off=0,r=54,imm=0)
>> R2_w=pkt_end(id=0,off=0,imm=0) R3_w=pkt(id=0,off=54,r=54,imm=0)
>> R10=fp0
>> 
>> 6: (71) r2 = *(u8 *)(r1 +12)
>> 
>> 7: (71) r3 = *(u8 *)(r1 +13)
>> 
>> 8: (67) r3 <<= 8
>> 
>> 9: (4f) r3 |= r2
>> 
>> 10: (b7) r0 = 2
>> 
>> 11: (55) if r3 != 0x8 goto pc+29
>> 
>>  R0_w=inv2 R1_w=pkt(id=0,off=0,r=54,imm=0)
>> R2_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff)) R3_w=inv8 R10=fp0
>> 
>> 12: (71) r2 = *(u8 *)(r1 +23)
>> 
>> 13: (b7) r0 = 2
>> 
>> 14: (55) if r2 != 0x6 goto pc+26
>> 
>>  R0=inv2 R1=pkt(id=0,off=0,r=54,imm=0) R2=inv6 R3=inv8 R10=fp0
>> 
>> 15: (69) r1 = *(u16 *)(r1 +46)
>> 
>> 16: (bf) r2 = r1
>> 
>> 17: (57) r2 &= 7936
>> 
>> 18: (b7) r0 = 2
>> 
>> 19: (55) if r2 != 0x200 goto pc+21
>> 
>>  R0_w=inv2 R1_w=inv(id=0,umax_value=65535,var_off=(0x0; 0xffff))
>> R2_w=inv512 R3=inv8 R10=fp0
>> 
>> 20: (77) r1 >>= 2
>> 
>> 21: (57) r1 &= 60
>> 
>> 22: (07) r1 += -20
>> 
>> 23: (18) r2 = 0x0
>> 
>> 25: (63) *(u32 *)(r2 +0) = r1
>> 
>> R2 invalid mem access 'inv' <----------------
>
> So it's not out of the bounds access but rather null dereference since r2
> has been loaded with 0 on previous insn.
>
>> 
>> processed 25 insns (limit 1000000) max_states_per_insn 0 total_states
>> 1 peak_states 1 mark_read 1
>> 
>> 
>> it appears optlen = tcphdr->doff*4 - sizeof(*tcphdr); is invalid ? if
>> I comment out lines between 60 and 73, no problem with invalid mem
>> access
>
> I see that optlen is a global variable. This line might be valid but
> you're using iproute2's loader for your XDP program, right? AFAIK it
> doesn't have support for BPF global variables, only libbpf does (Daniel,
> Andrii? is that true?).

Yes, this is true. Converting the iproute2 loader is still on my todo list...

> So you have to either make the optlen a local variable or go with writing
> the loader part that is based on libbpf usage (see samples/bpf directory
> in kernel tree, for example xdp1_user.c).

You could also try the xdp-loader in xdp-tools:
https://github.com/xdp-project/xdp-tools

It's somewhat basic still, but should be able to at least load a basic
program - please file a bug report if it fails.

-Toke

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-16  2:24 ` Maciej Fijalkowski
  2020-01-16  9:45   ` Toke Høiland-Jørgensen
@ 2020-01-16 17:44   ` Vincent Li
  2020-01-16 18:58   ` Vincent Li
  2 siblings, 0 replies; 16+ messages in thread
From: Vincent Li @ 2020-01-16 17:44 UTC (permalink / raw)
  To: Maciej Fijalkowski; +Cc: xdp-newbies, daniel, andriin, dsahern



On Thu, 16 Jan 2020, Maciej Fijalkowski wrote:

> 
> I see that optlen is a global variable. This line might be valid but
> you're using iproute2's loader for your XDP program, right? AFAIK it
> doesn't have support for BPF global variables, only libbpf does (Daniel,
> Andrii? is that true?).
> 
> So you have to either make the optlen a local variable or go with writing
> the loader part that is based on libbpf usage (see samples/bpf directory
> in kernel tree, for example xdp1_user.c).

ok I will try use optlen as local variable or use libbpf

> 
> Could you next time try a little better with the format of data you're
> providing? I'm pretty sure other folks would have spotted earlier that
> optlen is global if the code was properly formatted :)

I am using gmail from chrome browser in plain text mode, seems still not 
good with the text format. I switched to alpine now, not sure I have the 
proper alpine setting, let me know next time if you still see not properly 
format :)

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-16  2:24 ` Maciej Fijalkowski
  2020-01-16  9:45   ` Toke Høiland-Jørgensen
  2020-01-16 17:44   ` Vincent Li
@ 2020-01-16 18:58   ` Vincent Li
  2 siblings, 0 replies; 16+ messages in thread
From: Vincent Li @ 2020-01-16 18:58 UTC (permalink / raw)
  To: Maciej Fijalkowski; +Cc: Vincent Li, xdp-newbies, daniel, andriin, dsahern



On Thu, 16 Jan 2020, Maciej Fijalkowski wrote:

> 
> I see that optlen is a global variable. This line might be valid but
> you're using iproute2's loader for your XDP program, right? AFAIK it
> doesn't have support for BPF global variables, only libbpf does (Daniel,
> Andrii? is that true?).
> 

move optlen to local variable resolves the iproute2 loading problem. I 
tried libbpf way (xdp-loader load enp3s0 tcp_option.o) from 
https://github.com/xdp-project/xdp-tools to load the program, but it 
failed with "Couldn't load BPF program: Relocation failed". I will have 
more detail in email respond to Toke.  

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-16  9:45   ` Toke Høiland-Jørgensen
@ 2020-01-16 19:06     ` Vincent Li
  2020-01-16 19:20       ` Andrii Nakryiko
  2020-01-17 11:07       ` Toke Høiland-Jørgensen
  0 siblings, 2 replies; 16+ messages in thread
From: Vincent Li @ 2020-01-16 19:06 UTC (permalink / raw)
  To: Toke Høiland-Jørgensen
  Cc: Maciej Fijalkowski, Vincent Li, xdp-newbies, daniel, andriin, dsahern

[-- Attachment #1: Type: text/plain, Size: 3323 bytes --]



On Thu, 16 Jan 2020, Toke Høiland-Jørgensen wrote:

Hi Toke,

> 
> You could also try the xdp-loader in xdp-tools:
> https://github.com/xdp-project/xdp-tools
> 
> It's somewhat basic still, but should be able to at least load a basic
> program - please file a bug report if it fails.

I tried the xdp-tools xdp-loader, when the optlen is global variable, I 
got:
# xdp-loader load enp3s0 tcp_option.o
Couldn't load BPF program: Relocation failed

if I move the optlen, i variable to local variable, I got:

# xdp-loader load enp3s0 tcp_option.o
Couldn't load eBPF object: Invalid argument(-22)

Here is the complete code:

#include <stdint.h>
#include <arpa/inet.h>
#include <asm/byteorder.h>
#include <linux/bpf.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/pkt_cls.h>

/*
 * Sample XDP to parse tcp option.
 * compile it with:
 * clang -O2 -emit-llvm -c tcp_option.c -o - |llc -march=bpf -filetype=obj -o tcp_option.o
 * attach it to a device with XDP as:
 * 	ip link set dev lo xdp object tcp_option.o verbose
 */

#define SEC(NAME) __attribute__((section(NAME), used))

#define TCPOPT_EOL        0       /* End of options (1)              */
#define TCPOPT_NOP        1       /* No-op (1)                       */
#define TCPOPT_MAXSEG     2       /* Maximum segment size (4)        */
#define TCPOPT_WSCALE     3       /* Window scaling (3)              */
#define TCPOPT_SACKOK     4       /* Selective ACK permitted (2)     */
#define TCPOPT_SACK       5       /* Actual selective ACK (10-34)    */
#define TCPOPT_TSTAMP     8       /* Timestamp (10)                  */


/* from bpf_helpers.h */

static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) =
(void *) BPF_FUNC_trace_printk;

static unsigned long long (*bpf_get_prandom_u32)(void) =
(void *) BPF_FUNC_get_prandom_u32;


static int tcp_option(void *data, void *data_end)
{
	struct ethhdr *eth = (struct ethhdr *)data;
	struct iphdr *iph = (struct iphdr *)(eth + 1);
	struct tcphdr *tcphdr = (struct tcphdr *)(iph + 1);
	const __u8 *op;
	int i, optlen;

	/* sanity check needed by the eBPF verifier */
	if ((void *)(tcphdr + 1)  > data_end)
		return 0;

	/* skip non TCP packets */
	if (eth->h_proto != __constant_htons(ETH_P_IP) || iph->protocol != IPPROTO_TCP)
		return 0;

	/* incompatible flags, or PSH already set */
	if (tcphdr->ack || tcphdr->fin || tcphdr->rst || tcphdr->psh)
		return 0;

	if (tcphdr->syn) {
		if (((void *)(tcphdr + 1) + tcphdr->doff*4) > data_end)
			return 0;

		optlen = tcphdr->doff*4 - sizeof(*tcphdr);
		for (i = 0; i < optlen; ) {
			if (op[i] == TCPOPT_EOL ) {
				char fmt[] = "XDP: tcp source : %d tcp option eol\n";
				bpf_trace_printk(fmt, sizeof(fmt), (int)tcphdr->source);
				return 1;
			}
			if (op[i] < 2)
				i++;
			else
				i += op[i+1] ? : 1;
		}
#if 0
		if (tcphdr->doff*4 == 44 || tcphdr->doff*4 == 28) {
			char fmt[] = "XDP: tcp source : %d data offset :%d\n";
			bpf_trace_printk(fmt, sizeof(fmt), (int)tcphdr->source, (int)tcphdr->doff*4);
			return 1;
		}
#endif
	}
	return 0;
}

SEC("prog")
int xdp_main(struct xdp_md *ctx)
{
	void *data_end = (void *)(uintptr_t)ctx->data_end;
	void *data = (void *)(uintptr_t)ctx->data;

	if (tcp_option(data, data_end))
		return XDP_DROP;

	return XDP_PASS;
}


char _license[] SEC("license") = "GPL";

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-16 19:06     ` Vincent Li
@ 2020-01-16 19:20       ` Andrii Nakryiko
  2020-01-16 22:10         ` Vincent Li
  2020-01-17 11:07       ` Toke Høiland-Jørgensen
  1 sibling, 1 reply; 16+ messages in thread
From: Andrii Nakryiko @ 2020-01-16 19:20 UTC (permalink / raw)
  To: Vincent Li, Toke Høiland-Jørgensen
  Cc: Maciej Fijalkowski, xdp-newbies, daniel, dsahern

On 1/16/20 11:06 AM, Vincent Li wrote:
> 
> 
> On Thu, 16 Jan 2020, Toke Høiland-Jørgensen wrote:
> 
> Hi Toke,
> 
>>
>> You could also try the xdp-loader in xdp-tools:
>> https://github.com/xdp-project/xdp-tools
>>
>> It's somewhat basic still, but should be able to at least load a basic
>> program - please file a bug report if it fails.
> 
> I tried the xdp-tools xdp-loader, when the optlen is global variable, I
> got:
> # xdp-loader load enp3s0 tcp_option.o
> Couldn't load BPF program: Relocation failed
> 
> if I move the optlen, i variable to local variable, I got:
> 
> # xdp-loader load enp3s0 tcp_option.o
> Couldn't load eBPF object: Invalid argument(-22)
> 
> Here is the complete code:
> 
> #include <stdint.h>
> #include <arpa/inet.h>
> #include <asm/byteorder.h>
> #include <linux/bpf.h>
> #include <linux/if_ether.h>
> #include <linux/ip.h>
> #include <linux/tcp.h>
> #include <linux/pkt_cls.h>
> 
> /*
>   * Sample XDP to parse tcp option.
>   * compile it with:
>   * clang -O2 -emit-llvm -c tcp_option.c -o - |llc -march=bpf -filetype=obj -o tcp_option.o
>   * attach it to a device with XDP as:
>   * 	ip link set dev lo xdp object tcp_option.o verbose
>   */
> 
> #define SEC(NAME) __attribute__((section(NAME), used))
> 
> #define TCPOPT_EOL        0       /* End of options (1)              */
> #define TCPOPT_NOP        1       /* No-op (1)                       */
> #define TCPOPT_MAXSEG     2       /* Maximum segment size (4)        */
> #define TCPOPT_WSCALE     3       /* Window scaling (3)              */
> #define TCPOPT_SACKOK     4       /* Selective ACK permitted (2)     */
> #define TCPOPT_SACK       5       /* Actual selective ACK (10-34)    */
> #define TCPOPT_TSTAMP     8       /* Timestamp (10)                  */
> 
> 
> /* from bpf_helpers.h */
> 
> static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) =
> (void *) BPF_FUNC_trace_printk;
> 
> static unsigned long long (*bpf_get_prandom_u32)(void) =
> (void *) BPF_FUNC_get_prandom_u32;
> 
> 
> static int tcp_option(void *data, void *data_end)

I don't know if xdp-loader handles subprogram calls and relocations. 
Mark it as __always_inline.

> {
> 	struct ethhdr *eth = (struct ethhdr *)data;
> 	struct iphdr *iph = (struct iphdr *)(eth + 1);
> 	struct tcphdr *tcphdr = (struct tcphdr *)(iph + 1);
> 	const __u8 *op;
> 	int i, optlen;
> 
> 	/* sanity check needed by the eBPF verifier */
> 	if ((void *)(tcphdr + 1)  > data_end)
> 		return 0;
> 
> 	/* skip non TCP packets */
> 	if (eth->h_proto != __constant_htons(ETH_P_IP) || iph->protocol != IPPROTO_TCP)
> 		return 0;
> 
> 	/* incompatible flags, or PSH already set */
> 	if (tcphdr->ack || tcphdr->fin || tcphdr->rst || tcphdr->psh)
> 		return 0;
> 
> 	if (tcphdr->syn) {
> 		if (((void *)(tcphdr + 1) + tcphdr->doff*4) > data_end)
> 			return 0;
> 
> 		optlen = tcphdr->doff*4 - sizeof(*tcphdr);
> 		for (i = 0; i < optlen; ) {
> 			if (op[i] == TCPOPT_EOL ) {
> 				char fmt[] = "XDP: tcp source : %d tcp option eol\n";
> 				bpf_trace_printk(fmt, sizeof(fmt), (int)tcphdr->source);
> 				return 1;
> 			}
> 			if (op[i] < 2)
> 				i++;
> 			else
> 				i += op[i+1] ? : 1;
> 		}
> #if 0
> 		if (tcphdr->doff*4 == 44 || tcphdr->doff*4 == 28) {
> 			char fmt[] = "XDP: tcp source : %d data offset :%d\n";
> 			bpf_trace_printk(fmt, sizeof(fmt), (int)tcphdr->source, (int)tcphdr->doff*4);
> 			return 1;
> 		}
> #endif
> 	}
> 	return 0;
> }
> 
> SEC("prog")
> int xdp_main(struct xdp_md *ctx)
> {
> 	void *data_end = (void *)(uintptr_t)ctx->data_end;
> 	void *data = (void *)(uintptr_t)ctx->data;
> 
> 	if (tcp_option(data, data_end))
> 		return XDP_DROP;
> 
> 	return XDP_PASS;
> }
> 
> 
> char _license[] SEC("license") = "GPL";
> 

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-16 19:20       ` Andrii Nakryiko
@ 2020-01-16 22:10         ` Vincent Li
  0 siblings, 0 replies; 16+ messages in thread
From: Vincent Li @ 2020-01-16 22:10 UTC (permalink / raw)
  To: Andrii Nakryiko
  Cc: Vincent Li, Toke Høiland-Jørgensen, Maciej Fijalkowski,
	xdp-newbies, daniel, dsahern

[-- Attachment #1: Type: text/plain, Size: 3095 bytes --]



On Thu, 16 Jan 2020, Andrii Nakryiko wrote:

> On 1/16/20 11:06 AM, Vincent Li wrote:
> > 
> > 
> > On Thu, 16 Jan 2020, Toke Høiland-Jørgensen wrote:
> > 
> > Hi Toke,
> > 
> >>
> >> You could also try the xdp-loader in xdp-tools:
> >> https://github.com/xdp-project/xdp-tools
> >>
> >> It's somewhat basic still, but should be able to at least load a basic
> >> program - please file a bug report if it fails.
> > 
> > I tried the xdp-tools xdp-loader, when the optlen is global variable, I
> > got:
> > # xdp-loader load enp3s0 tcp_option.o
> > Couldn't load BPF program: Relocation failed
> > 
> > if I move the optlen, i variable to local variable, I got:
> > 
> > # xdp-loader load enp3s0 tcp_option.o
> > Couldn't load eBPF object: Invalid argument(-22)
> > 
> > Here is the complete code:
> > 
> > #include <stdint.h>
> > #include <arpa/inet.h>
> > #include <asm/byteorder.h>
> > #include <linux/bpf.h>
> > #include <linux/if_ether.h>
> > #include <linux/ip.h>
> > #include <linux/tcp.h>
> > #include <linux/pkt_cls.h>
> > 
> > /*
> >   * Sample XDP to parse tcp option.
> >   * compile it with:
> >   * clang -O2 -emit-llvm -c tcp_option.c -o - |llc -march=bpf -filetype=obj -o tcp_option.o
> >   * attach it to a device with XDP as:
> >   * 	ip link set dev lo xdp object tcp_option.o verbose
> >   */
> > 
> > #define SEC(NAME) __attribute__((section(NAME), used))
> > 
> > #define TCPOPT_EOL        0       /* End of options (1)              */
> > #define TCPOPT_NOP        1       /* No-op (1)                       */
> > #define TCPOPT_MAXSEG     2       /* Maximum segment size (4)        */
> > #define TCPOPT_WSCALE     3       /* Window scaling (3)              */
> > #define TCPOPT_SACKOK     4       /* Selective ACK permitted (2)     */
> > #define TCPOPT_SACK       5       /* Actual selective ACK (10-34)    */
> > #define TCPOPT_TSTAMP     8       /* Timestamp (10)                  */
> > 
> > 
> > /* from bpf_helpers.h */
> > 
> > static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) =
> > (void *) BPF_FUNC_trace_printk;
> > 
> > static unsigned long long (*bpf_get_prandom_u32)(void) =
> > (void *) BPF_FUNC_get_prandom_u32;
> > 
> > 
> > static int tcp_option(void *data, void *data_end)
> 
> I don't know if xdp-loader handles subprogram calls and relocations. 
> Mark it as __always_inline.

I tried __always_inline the tcp_option function, same relocation error, 
the relocation error only occurs when optlen is global variable, if I put 
optlen as local variable, it complains different error:

xdp-loader load enp3s0 tcp_option.o
Couldn't load eBPF object: Invalid argument(-22)

I suspect this may not be specific to xdp-loader. In the beginning, I made 
some code changes in suricata/ebpf/xdp-filter.c to parse tcp options and 
I got same relocation error, this leads me to write a standalone sample 
xdp program  and use iproutes to load the sample program. now we know the 
global variable cause iproutes to fail. and I am back to the 
original problem of suricata/ebpf/xdp-filter.c  or xdp-loader  
libbpf way to load xdp program :).

 

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-16 19:06     ` Vincent Li
  2020-01-16 19:20       ` Andrii Nakryiko
@ 2020-01-17 11:07       ` Toke Høiland-Jørgensen
  2020-01-17 18:14         ` Vincent Li
  1 sibling, 1 reply; 16+ messages in thread
From: Toke Høiland-Jørgensen @ 2020-01-17 11:07 UTC (permalink / raw)
  To: Vincent Li; +Cc: Maciej Fijalkowski, xdp-newbies, daniel, andriin, dsahern

Vincent Li <mchun.li@gmail.com> writes:

> On Thu, 16 Jan 2020, Toke Høiland-Jørgensen wrote:
>
> Hi Toke,
>
>> 
>> You could also try the xdp-loader in xdp-tools:
>> https://github.com/xdp-project/xdp-tools
>> 
>> It's somewhat basic still, but should be able to at least load a basic
>> program - please file a bug report if it fails.
>
> I tried the xdp-tools xdp-loader, when the optlen is global variable, I 
> got:
> # xdp-loader load enp3s0 tcp_option.o
> Couldn't load BPF program: Relocation failed
>
> if I move the optlen, i variable to local variable, I got:
>
> # xdp-loader load enp3s0 tcp_option.o
> Couldn't load eBPF object: Invalid argument(-22)

OK, I tried this, and there were a couple of issues:

- The xdp-loader didn't set the BPF program type to XDP, and since your
  section name doesn't have an xdp_ prefix libbpf couldn't auto-detect
  it. I've pushed a fix for this to the xdp-tools repo so the loader
  will always set the program type to XDP now.

- There are a couple of bugs in your program:

First, when compiling with warnings turned on, I get this:

tcp_options.c:64:29: error: variable 'op' is uninitialized when used here [-Werror,-Wuninitialized]
                        if (op[i] == TCPOPT_EOL ) {
                            ^~
tcp_options.c:43:23: note: initialize the variable 'op' to silence this warning
        const __u8 *op;
                      ^
                       = 0

after fixing that (adding this line after the optlen = assignment):

                op = (const __u8 *)(tcphdr + 1);

the verifier then complains about out-of-bounds reading of the packet
data (pass -vv to xdp-loader to get the full debug output from libbpf).
You are not checking that the op pointer doesn't read out-of-bounds.

I fixed that by adding a couple of bounds checks inside the for loop.
The whole thing now looks like this:

                optlen = tcphdr->doff*4 - sizeof(*tcphdr);
                op = (const __u8 *)(tcphdr + 1);
                for (i = 0; i < optlen; ) {
                        if ((void *)op + i + 1 > data_end)
                                return 0;
                        if (op[i] == TCPOPT_EOL ) {
                                char fmt[] = "XDP: tcp source : %d tcp option eol\n";
                                bpf_trace_printk(fmt, sizeof(fmt), (int)tcphdr->source);
                                return 1;
                        }
                        if (op[i] < 2)
                                i++;
                        else
                                i += ((void *)op + 2 < data_end && op[i+1]) ? : 1;
                }


With this, I can successfully load the program using xdp-loader. Turning
the variables back into globals still doesn't work, but I think that
error message should be fairly obvious:

libbpf: invalid relo for 'op' in special section 0xfff2; forgot to initialize global var?..

-Toke

^ permalink raw reply	[flat|nested] 16+ messages in thread

* Re: XDP invalid memory access
  2020-01-17 11:07       ` Toke Høiland-Jørgensen
@ 2020-01-17 18:14         ` Vincent Li
  0 siblings, 0 replies; 16+ messages in thread
From: Vincent Li @ 2020-01-17 18:14 UTC (permalink / raw)
  To: Toke Høiland-Jørgensen
  Cc: Vincent Li, Maciej Fijalkowski, xdp-newbies, daniel, andriin, dsahern

[-- Attachment #1: Type: text/plain, Size: 3472 bytes --]



On Fri, 17 Jan 2020, Toke Høiland-Jørgensen wrote:

> Vincent Li <mchun.li@gmail.com> writes:
> 
> > On Thu, 16 Jan 2020, Toke Høiland-Jørgensen wrote:
> >
> > Hi Toke,
> >
> >> 
> >> You could also try the xdp-loader in xdp-tools:
> >> https://github.com/xdp-project/xdp-tools
> >> 
> >> It's somewhat basic still, but should be able to at least load a basic
> >> program - please file a bug report if it fails.
> >
> > I tried the xdp-tools xdp-loader, when the optlen is global variable, I 
> > got:
> > # xdp-loader load enp3s0 tcp_option.o
> > Couldn't load BPF program: Relocation failed
> >
> > if I move the optlen, i variable to local variable, I got:
> >
> > # xdp-loader load enp3s0 tcp_option.o
> > Couldn't load eBPF object: Invalid argument(-22)
> 
> OK, I tried this, and there were a couple of issues:
> 
> - The xdp-loader didn't set the BPF program type to XDP, and since your
>   section name doesn't have an xdp_ prefix libbpf couldn't auto-detect
>   it. I've pushed a fix for this to the xdp-tools repo so the loader
>   will always set the program type to XDP now.
> 
> - There are a couple of bugs in your program:
> 
> First, when compiling with warnings turned on, I get this:
> 
> tcp_options.c:64:29: error: variable 'op' is uninitialized when used here [-Werror,-Wuninitialized]
>                         if (op[i] == TCPOPT_EOL ) {
>                             ^~
> tcp_options.c:43:23: note: initialize the variable 'op' to silence this warning
>         const __u8 *op;
>                       ^
>                        = 0
> 
> after fixing that (adding this line after the optlen = assignment):
> 
>                 op = (const __u8 *)(tcphdr + 1);

Thank you for mentioning the warning flags to clang, I used clang -Wall 
and see the same warning

> 
> the verifier then complains about out-of-bounds reading of the packet
> data (pass -vv to xdp-loader to get the full debug output from libbpf).
> You are not checking that the op pointer doesn't read out-of-bounds.

-vv is also good tip, missed it

> 
> I fixed that by adding a couple of bounds checks inside the for loop.
> The whole thing now looks like this:
> 
>                 optlen = tcphdr->doff*4 - sizeof(*tcphdr);
>                 op = (const __u8 *)(tcphdr + 1);
>                 for (i = 0; i < optlen; ) {
>                         if ((void *)op + i + 1 > data_end)
>                                 return 0;
>                         if (op[i] == TCPOPT_EOL ) {
>                                 char fmt[] = "XDP: tcp source : %d tcp option eol\n";
>                                 bpf_trace_printk(fmt, sizeof(fmt), (int)tcphdr->source);
>                                 return 1;
>                         }
>                         if (op[i] < 2)
>                                 i++;
>                         else
>                                 i += ((void *)op + 2 < data_end && op[i+1]) ? : 1;
>                 }
> 
Good to know now everytime to check the memory bound before accessing the 
next memory

> 
> With this, I can successfully load the program using xdp-loader. Turning
> the variables back into globals still doesn't work, but I think that
> error message should be fairly obvious:
> 
> libbpf: invalid relo for 'op' in special section 0xfff2; forgot to initialize global var?..
> 
> -Toke

I cloned your updated xdp-loader and I can load the object file fine now, 
thank you for taking your time, lesson learned :)

Vincent
 

^ permalink raw reply	[flat|nested] 16+ messages in thread

end of thread, other threads:[~2020-01-17 18:14 UTC | newest]

Thread overview: 16+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-01-15 22:11 XDP invalid memory access Vincent Li
2020-01-15 22:21 ` Toke Høiland-Jørgensen
2020-01-15 22:31   ` Vincent Li
2020-01-16  0:40     ` David Ahern
2020-01-16  1:34       ` Vincent Li
2020-01-16  3:19         ` Vincent Li
2020-01-16  4:22           ` David Ahern
2020-01-16  2:24 ` Maciej Fijalkowski
2020-01-16  9:45   ` Toke Høiland-Jørgensen
2020-01-16 19:06     ` Vincent Li
2020-01-16 19:20       ` Andrii Nakryiko
2020-01-16 22:10         ` Vincent Li
2020-01-17 11:07       ` Toke Høiland-Jørgensen
2020-01-17 18:14         ` Vincent Li
2020-01-16 17:44   ` Vincent Li
2020-01-16 18:58   ` Vincent Li

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.