Re: [PATCH bpf-next v2 18/20] selftests/bpf: Use AF_INET for TX in xdp_metadata

From: Stanislav Fomichev <sdf@google.com>
To: Larysa Zaremba <larysa.zaremba@intel.com>
Cc: bpf@vger.kernel.org, ast@kernel.org, daniel@iogearbox.net,
	 andrii@kernel.org, martin.lau@linux.dev, song@kernel.org,
	yhs@fb.com,  john.fastabend@gmail.com, kpsingh@kernel.org,
	haoluo@google.com,  jolsa@kernel.org,
	David Ahern <dsahern@gmail.com>, Jakub Kicinski <kuba@kernel.org>,
	 Willem de Bruijn <willemb@google.com>,
	Jesper Dangaard Brouer <brouer@redhat.com>,
	 Anatoly Burakov <anatoly.burakov@intel.com>,
	Alexander Lobakin <alexandr.lobakin@intel.com>,
	 Magnus Karlsson <magnus.karlsson@gmail.com>,
	Maryam Tahhan <mtahhan@redhat.com>,
	 xdp-hints@xdp-project.net, netdev@vger.kernel.org
Subject: Re: [PATCH bpf-next v2 18/20] selftests/bpf: Use AF_INET for TX in xdp_metadata
Date: Thu, 6 Jul 2023 10:25:04 -0700	[thread overview]
Message-ID: <CAKH8qBukk2fpJjSj2cghZxuwUB0Ey2qmE+R=Bg=0FUMdrHYG9w@mail.gmail.com> (raw)
In-Reply-To: <ZKbLd8brydTvSocG@lincoln>

On Thu, Jul 6, 2023 at 7:15 AM Larysa Zaremba <larysa.zaremba@intel.com> wrote:
>
> On Wed, Jul 05, 2023 at 10:39:35AM -0700, Stanislav Fomichev wrote:
> > On 07/03, Larysa Zaremba wrote:
> > > The easiest way to simulate stripped VLAN tag in veth is to send a packet
> > > from VLAN interface, attached to veth. Unfortunately, this approach is
> > > incompatible with AF_XDP on TX side, because VLAN interfaces do not have
> > > such feature.
> > >
> > > Replace AF_XDP packet generation with sending the same datagram via
> > > AF_INET socket.
> > >
> > > This does not change the packet contents or hints values with one notable
> > > exception: rx_hash_type, which previously was expected to be 0, now is
> > > expected be at least XDP_RSS_TYPE_L4.
> > >
> > > Also, usage of AF_INET requires a little more complicated namespace setup,
> > > therefore open_netns() helper function is divided into smaller reusable
> > > pieces.
> >
> > Ack, it's probably OK for now, but, FYI, I'm trying to extend this part
> > with TX metadata:
> > https://lore.kernel.org/bpf/20230621170244.1283336-10-sdf@google.com/
> >
> > So probably long-term I'll switch it back to AF_XDP but will add
> > support for requesting vlan TX "offload" from the veth.
> >
>
> My bad for not reading your series. Amazing work as always!
>
> So, 'requesting vlan TX "offload"' with new hints capabilities? This would be
> pretty neat.
>
> But you think AF_INET TX is worth keeping for now, until TX hints are mature?

It's fine to replace current af_xdp tx with whatever you're suggesting.
I can bring it back later for tx when it's ready.

> > > Signed-off-by: Larysa Zaremba <larysa.zaremba@intel.com>
> > > ---
> > >  tools/testing/selftests/bpf/network_helpers.c |  37 +++-
> > >  tools/testing/selftests/bpf/network_helpers.h |   3 +
> > >  .../selftests/bpf/prog_tests/xdp_metadata.c   | 175 +++++++-----------
> > >  3 files changed, 98 insertions(+), 117 deletions(-)
> > >
> > > diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c
> > > index a105c0cd008a..19463230ece5 100644
> > > --- a/tools/testing/selftests/bpf/network_helpers.c
> > > +++ b/tools/testing/selftests/bpf/network_helpers.c
> > > @@ -386,28 +386,51 @@ char *ping_command(int family)
> > >     return "ping";
> > >  }
> > >
> > > +int get_cur_netns(void)
> > > +{
> > > +   int nsfd;
> > > +
> > > +   nsfd = open("/proc/self/ns/net", O_RDONLY);
> > > +   ASSERT_GE(nsfd, 0, "open /proc/self/ns/net");
> > > +   return nsfd;
> > > +}
> > > +
> > > +int get_netns(const char *name)
> > > +{
> > > +   char nspath[PATH_MAX];
> > > +   int nsfd;
> > > +
> > > +   snprintf(nspath, sizeof(nspath), "%s/%s", "/var/run/netns", name);
> > > +   nsfd = open(nspath, O_RDONLY | O_CLOEXEC);
> > > +   ASSERT_GE(nsfd, 0, "open /proc/self/ns/net");
> > > +   return nsfd;
> > > +}
> > > +
> > > +int set_netns(int netns_fd)
> > > +{
> > > +   return setns(netns_fd, CLONE_NEWNET);
> > > +}
> >
> > We have open_netns/close_netns in network_helpers.h that provide similar
> > functionality, let's use them instead?
> >
>
> I have divided open_netns() into smaller pieces (see below), because the code I
> have added into xdp_metadata looked better with those smaller pieces (I had to
> switch namespace several times).
>
> > > +
> > >  struct nstoken {
> > >     int orig_netns_fd;
> > >  };
> > >
> > >  struct nstoken *open_netns(const char *name)
> > >  {
> > > +   struct nstoken *token;
> > >     int nsfd;
> > > -   char nspath[PATH_MAX];
> > >     int err;
> > > -   struct nstoken *token;
> > >
> > >     token = calloc(1, sizeof(struct nstoken));
> > >     if (!ASSERT_OK_PTR(token, "malloc token"))
> > >             return NULL;
> > >
> > > -   token->orig_netns_fd = open("/proc/self/ns/net", O_RDONLY);
> > > -   if (!ASSERT_GE(token->orig_netns_fd, 0, "open /proc/self/ns/net"))
> > > +   token->orig_netns_fd = get_cur_netns();
> > > +   if (token->orig_netns_fd < 0)
> > >             goto fail;
> > >
> > > -   snprintf(nspath, sizeof(nspath), "%s/%s", "/var/run/netns", name);
> > > -   nsfd = open(nspath, O_RDONLY | O_CLOEXEC);
> > > -   if (!ASSERT_GE(nsfd, 0, "open netns fd"))
> > > +   nsfd = get_netns(name);
> > > +   if (nsfd < 0)
> > >             goto fail;
> > >
> > >     err = setns(nsfd, CLONE_NEWNET);
> > > diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h
> > > index 694185644da6..b18b9619595c 100644
> > > --- a/tools/testing/selftests/bpf/network_helpers.h
> > > +++ b/tools/testing/selftests/bpf/network_helpers.h
> > > @@ -58,6 +58,8 @@ int make_sockaddr(int family, const char *addr_str, __u16 port,
> > >  char *ping_command(int family);
> > >  int get_socket_local_port(int sock_fd);
> > >
> > > +int get_cur_netns(void);
> > > +int get_netns(const char *name);
> > >  struct nstoken;
> > >  /**
> > >   * open_netns() - Switch to specified network namespace by name.
> > > @@ -67,4 +69,5 @@ struct nstoken;
> > >   */
> > >  struct nstoken *open_netns(const char *name);
> > >  void close_netns(struct nstoken *token);
> > > +int set_netns(int netns_fd);
> > >  #endif
> > > diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c
> > > index 626c461fa34d..53b32a641e8e 100644
> > > --- a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c
> > > +++ b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c
> > > @@ -20,7 +20,7 @@
> > >
> > >  #define UDP_PAYLOAD_BYTES 4
> > >
> > > -#define AF_XDP_SOURCE_PORT 1234
> > > +#define UDP_SOURCE_PORT 1234
> > >  #define AF_XDP_CONSUMER_PORT 8080
> > >
> > >  #define UMEM_NUM 16
> > > @@ -33,6 +33,12 @@
> > >  #define RX_ADDR "10.0.0.2"
> > >  #define PREFIX_LEN "8"
> > >  #define FAMILY AF_INET
> > > +#define TX_NETNS_NAME "xdp_metadata_tx"
> > > +#define RX_NETNS_NAME "xdp_metadata_rx"
> > > +#define TX_MAC "00:00:00:00:00:01"
> > > +#define RX_MAC "00:00:00:00:00:02"
> > > +
> > > +#define XDP_RSS_TYPE_L4 BIT(3)
> > >
> > >  struct xsk {
> > >     void *umem_area;
> > > @@ -119,90 +125,28 @@ static void close_xsk(struct xsk *xsk)
> > >     munmap(xsk->umem_area, UMEM_SIZE);
> > >  }
> > >
> > > -static void ip_csum(struct iphdr *iph)
> > > +static int generate_packet_udp(void)
> > >  {
> > > -   __u32 sum = 0;
> > > -   __u16 *p;
> > > -   int i;
> > > -
> > > -   iph->check = 0;
> > > -   p = (void *)iph;
> > > -   for (i = 0; i < sizeof(*iph) / sizeof(*p); i++)
> > > -           sum += p[i];
> > > -
> > > -   while (sum >> 16)
> > > -           sum = (sum & 0xffff) + (sum >> 16);
> > > -
> > > -   iph->check = ~sum;
> > > -}
> > > -
> > > -static int generate_packet(struct xsk *xsk, __u16 dst_port)
> > > -{
> > > -   struct xdp_desc *tx_desc;
> > > -   struct udphdr *udph;
> > > -   struct ethhdr *eth;
> > > -   struct iphdr *iph;
> > > -   void *data;
> > > -   __u32 idx;
> > > -   int ret;
> > > -
> > > -   ret = xsk_ring_prod__reserve(&xsk->tx, 1, &idx);
> > > -   if (!ASSERT_EQ(ret, 1, "xsk_ring_prod__reserve"))
> > > -           return -1;
> > > -
> > > -   tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx);
> > > -   tx_desc->addr = idx % (UMEM_NUM / 2) * UMEM_FRAME_SIZE;
> > > -   printf("%p: tx_desc[%u]->addr=%llx\n", xsk, idx, tx_desc->addr);
> > > -   data = xsk_umem__get_data(xsk->umem_area, tx_desc->addr);
> > > -
> > > -   eth = data;
> > > -   iph = (void *)(eth + 1);
> > > -   udph = (void *)(iph + 1);
> > > -
> > > -   memcpy(eth->h_dest, "\x00\x00\x00\x00\x00\x02", ETH_ALEN);
> > > -   memcpy(eth->h_source, "\x00\x00\x00\x00\x00\x01", ETH_ALEN);
> > > -   eth->h_proto = htons(ETH_P_IP);
> > > -
> > > -   iph->version = 0x4;
> > > -   iph->ihl = 0x5;
> > > -   iph->tos = 0x9;
> > > -   iph->tot_len = htons(sizeof(*iph) + sizeof(*udph) + UDP_PAYLOAD_BYTES);
> > > -   iph->id = 0;
> > > -   iph->frag_off = 0;
> > > -   iph->ttl = 0;
> > > -   iph->protocol = IPPROTO_UDP;
> > > -   ASSERT_EQ(inet_pton(FAMILY, TX_ADDR, &iph->saddr), 1, "inet_pton(TX_ADDR)");
> > > -   ASSERT_EQ(inet_pton(FAMILY, RX_ADDR, &iph->daddr), 1, "inet_pton(RX_ADDR)");
> > > -   ip_csum(iph);
> > > -
> > > -   udph->source = htons(AF_XDP_SOURCE_PORT);
> > > -   udph->dest = htons(dst_port);
> > > -   udph->len = htons(sizeof(*udph) + UDP_PAYLOAD_BYTES);
> > > -   udph->check = 0;
> > > -
> > > -   memset(udph + 1, 0xAA, UDP_PAYLOAD_BYTES);
> > > -
> > > -   tx_desc->len = sizeof(*eth) + sizeof(*iph) + sizeof(*udph) + UDP_PAYLOAD_BYTES;
> > > -   xsk_ring_prod__submit(&xsk->tx, 1);
> > > -
> > > -   ret = sendto(xsk_socket__fd(xsk->socket), NULL, 0, MSG_DONTWAIT, NULL, 0);
> > > -   if (!ASSERT_GE(ret, 0, "sendto"))
> > > -           return ret;
> > > -
> > > -   return 0;
> > > -}
> > > -
> > > -static void complete_tx(struct xsk *xsk)
> > > -{
> > > -   __u32 idx;
> > > -   __u64 addr;
> > > -
> > > -   if (ASSERT_EQ(xsk_ring_cons__peek(&xsk->comp, 1, &idx), 1, "xsk_ring_cons__peek")) {
> > > -           addr = *xsk_ring_cons__comp_addr(&xsk->comp, idx);
> > > -
> > > -           printf("%p: complete tx idx=%u addr=%llx\n", xsk, idx, addr);
> > > -           xsk_ring_cons__release(&xsk->comp, 1);
> > > -   }
> > > +   char udp_payload[UDP_PAYLOAD_BYTES];
> > > +   struct sockaddr_in rx_addr;
> > > +   int sock_fd, err = 0;
> > > +
> > > +   /* Build a packet */
> > > +   memset(udp_payload, 0xAA, UDP_PAYLOAD_BYTES);
> > > +   rx_addr.sin_addr.s_addr = inet_addr(RX_ADDR);
> > > +   rx_addr.sin_family = AF_INET;
> > > +   rx_addr.sin_port = htons(UDP_SOURCE_PORT);
> > > +
> > > +   sock_fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
> > > +   if (!ASSERT_GE(sock_fd, 0, "socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)"))
> > > +           return sock_fd;
> > > +
> > > +   err = sendto(sock_fd, udp_payload, UDP_PAYLOAD_BYTES, MSG_DONTWAIT,
> > > +                (void *)&rx_addr, sizeof(rx_addr));
> > > +   ASSERT_GE(err, 0, "sendto");
> > > +
> > > +   close(sock_fd);
> > > +   return err;
> > >  }
> > >
> > >  static void refill_rx(struct xsk *xsk, __u64 addr)
> > > @@ -268,7 +212,8 @@ static int verify_xsk_metadata(struct xsk *xsk)
> > >     if (!ASSERT_NEQ(meta->rx_hash, 0, "rx_hash"))
> > >             return -1;
> > >
> > > -   ASSERT_EQ(meta->rx_hash_type, 0, "rx_hash_type");
> > > +   if (!ASSERT_NEQ(meta->rx_hash_type & XDP_RSS_TYPE_L4, 0, "rx_hash_type"))
> > > +           return -1;
> > >
> > >     xsk_ring_cons__release(&xsk->rx, 1);
> > >     refill_rx(xsk, comp_addr);
> > > @@ -281,40 +226,46 @@ void test_xdp_metadata(void)
> > >     struct xdp_metadata2 *bpf_obj2 = NULL;
> > >     struct xdp_metadata *bpf_obj = NULL;
> > >     struct bpf_program *new_prog, *prog;
> > > -   struct nstoken *tok = NULL;
> > > +   int prev_netns, rx_netns, tx_netns;
> > >     __u32 queue_id = QUEUE_ID;
> > >     struct bpf_map *prog_arr;
> > > -   struct xsk tx_xsk = {};
> > >     struct xsk rx_xsk = {};
> > >     __u32 val, key = 0;
> > >     int retries = 10;
> > >     int rx_ifindex;
> > > -   int tx_ifindex;
> > >     int sock_fd;
> > >     int ret;
> > >
> > > -   /* Setup new networking namespace, with a veth pair. */
> > > +   /* Setup new networking namespaces, with a veth pair. */
> > >
> > > -   SYS(out, "ip netns add xdp_metadata");
> > > -   tok = open_netns("xdp_metadata");
> > > +   SYS(out, "ip netns add " TX_NETNS_NAME);
> > > +   SYS(out, "ip netns add " RX_NETNS_NAME);
> > > +   prev_netns = get_cur_netns();
> > > +   tx_netns = get_netns(TX_NETNS_NAME);
> > > +   rx_netns = get_netns(RX_NETNS_NAME);
> > > +   if (prev_netns < 0 || tx_netns < 0 || rx_netns < 0)
> > > +           goto close_ns;
> > > +
> > > +   set_netns(tx_netns);
> > >     SYS(out, "ip link add numtxqueues 1 numrxqueues 1 " TX_NAME
> > >         " type veth peer " RX_NAME " numtxqueues 1 numrxqueues 1");
> > > -   SYS(out, "ip link set dev " TX_NAME " address 00:00:00:00:00:01");
> > > -   SYS(out, "ip link set dev " RX_NAME " address 00:00:00:00:00:02");
> > > +   SYS(out, "ip link set " RX_NAME " netns " RX_NETNS_NAME);
> > > +
> > > +   SYS(out, "ip link set dev " TX_NAME " address " TX_MAC);
> > >     SYS(out, "ip link set dev " TX_NAME " up");
> > > -   SYS(out, "ip link set dev " RX_NAME " up");
> > >     SYS(out, "ip addr add " TX_ADDR "/" PREFIX_LEN " dev " TX_NAME);
> > > -   SYS(out, "ip addr add " RX_ADDR "/" PREFIX_LEN " dev " RX_NAME);
> > >
> > > +   /* Avoid ARP calls */
> > > +   SYS(out, "ip -4 neigh add " RX_ADDR " lladdr " RX_MAC " dev " TX_NAME);
> > > +
> > > +   set_netns(rx_netns);
> > > +   SYS(out, "ip link set dev " RX_NAME " address " RX_MAC);
> > > +   SYS(out, "ip link set dev " RX_NAME " up");
> > > +   SYS(out, "ip addr add " RX_ADDR "/" PREFIX_LEN " dev " RX_NAME);
> > >     rx_ifindex = if_nametoindex(RX_NAME);
> > > -   tx_ifindex = if_nametoindex(TX_NAME);
> > >
> > >     /* Setup separate AF_XDP for TX and RX interfaces. */
> > >
> > > -   ret = open_xsk(tx_ifindex, &tx_xsk);
> > > -   if (!ASSERT_OK(ret, "open_xsk(TX_NAME)"))
> > > -           goto out;
> > > -
> > >     ret = open_xsk(rx_ifindex, &rx_xsk);
> > >     if (!ASSERT_OK(ret, "open_xsk(RX_NAME)"))
> > >             goto out;
> > > @@ -355,17 +306,16 @@ void test_xdp_metadata(void)
> > >             goto out;
> > >
> > >     /* Send packet destined to RX AF_XDP socket. */
> > > -   if (!ASSERT_GE(generate_packet(&tx_xsk, AF_XDP_CONSUMER_PORT), 0,
> > > -                  "generate AF_XDP_CONSUMER_PORT"))
> > > +   set_netns(tx_netns);
> > > +   if (!ASSERT_GE(generate_packet_udp(), 0, "generate UDP packet"))
> > >             goto out;
> > >
> > >     /* Verify AF_XDP RX packet has proper metadata. */
> > > +   set_netns(rx_netns);
> > >     if (!ASSERT_GE(verify_xsk_metadata(&rx_xsk), 0,
> > >                    "verify_xsk_metadata"))
> > >             goto out;
> > >
> > > -   complete_tx(&tx_xsk);
> > > -
> > >     /* Make sure freplace correctly picks up original bound device
> > >      * and doesn't crash.
> > >      */
> > > @@ -384,10 +334,11 @@ void test_xdp_metadata(void)
> > >             goto out;
> > >
> > >     /* Send packet to trigger . */
> > > -   if (!ASSERT_GE(generate_packet(&tx_xsk, AF_XDP_CONSUMER_PORT), 0,
> > > -                  "generate freplace packet"))
> > > +   set_netns(tx_netns);
> > > +   if (!ASSERT_GE(generate_packet_udp(), 0, "generate freplace packet"))
> > >             goto out;
> > >
> > > +   set_netns(rx_netns);
> > >     while (!retries--) {
> > >             if (bpf_obj2->bss->called)
> > >                     break;
> > > @@ -397,10 +348,14 @@ void test_xdp_metadata(void)
> > >
> > >  out:
> > >     close_xsk(&rx_xsk);
> > > -   close_xsk(&tx_xsk);
> > >     xdp_metadata2__destroy(bpf_obj2);
> > >     xdp_metadata__destroy(bpf_obj);
> > > -   if (tok)
> > > -           close_netns(tok);
> > > -   SYS_NOFAIL("ip netns del xdp_metadata");
> > > +   set_netns(prev_netns);
> > > +close_ns:
> > > +   close(prev_netns);
> > > +   close(tx_netns);
> > > +   close(rx_netns);
> > > +
> > > +   SYS_NOFAIL("ip netns del " RX_NETNS_NAME);
> > > +   SYS_NOFAIL("ip netns del " TX_NETNS_NAME);
> > >  }
> > > --
> > > 2.41.0
> > >