Re: XDP-ZC RSS

* Re: XDP-ZC RSS - Mellanox
@ 2022-09-12 13:41 Dries De Winter
  2022-09-13  8:50 ` Alasdair McWilliam
  0 siblings, 1 reply; 4+ messages in thread
From: Dries De Winter @ 2022-09-12 13:41 UTC (permalink / raw)
  To: siska; +Cc: xdp-newbies

> i am trying to figure it out, how can i configure Mellanox mlx5 card to 
> use XSK(XDP)-ZeroCopy queues instead of normal RX queues and use RSS to 
> spread traffic.

Hi,

Did you find a solution for your problem?

I had the exact same issue. My application needs to handle far too many input
flows so I cannot use flow steering. To let RSS do the job automatically,
I applied a patch locally (see below) to enable queues in the range
[#channels .. #channels * 2) when configuring RSS via 'ethtool -X'. This upper
half of queue numbers refers to XSK queues, like it does for 'ethtool -N'.

This patch works for me, but:
1/ I'm not familiar with the driver code, so it is very well possible that
I made some stupid mistakes.
2/ I have doubts whether this patch is the right way forward. I started using 
AF_XDP under the impression that it would be give me a hardware independent
interface for fast packet I/O, but with this approach user space ends up
implementing driver-specific logic anyway (how is user space supposed 
to know that it should bind to the upper half of queue indexes?).

Thanks,
Dries De Winter

---
 drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c   | 14 ++++++++++++--
 drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h   |  3 ++-
 drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c |  2 +-
 drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c  |  5 +++++
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
index 0015a81eb9a1..1a519cc39325 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
@@ -533,8 +533,12 @@ void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_chann
        nch = mlx5e_channels_get_num(chs);

        for (ix = 0; ix < chs->num; ix++)
+       {
                mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]);
-       res->rss_nch = chs->num;
+               if (!mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[chs->num + ix]))
+                       res->rss_rqns[chs->num  + ix] = res->drop_rqn;
+       }
+       res->rss_nch = chs->num * 2;

        mlx5e_rx_res_rss_enable(res);

@@ -632,6 +636,8 @@ int mlx5e_rx_res_xsk_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *c
        if (!mlx5e_channels_get_xsk_rqn(chs, ix, &rqn))
                return -EINVAL;

+       res->rss_rqns[mlx5e_channels_get_num(chs) + ix] = rqn;
+
        err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, rqn);
        if (err)
                mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to XSK RQ %#x (channel %u): err = %d\n",
@@ -640,7 +646,8 @@ int mlx5e_rx_res_xsk_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *c
        return err;
 }

-int mlx5e_rx_res_xsk_deactivate(struct mlx5e_rx_res *res, unsigned int ix)
+int mlx5e_rx_res_xsk_deactivate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs,
+                               unsigned int ix)
 {
        int err;

@@ -649,6 +656,9 @@ int mlx5e_rx_res_xsk_deactivate(struct mlx5e_rx_res *res, unsigned int ix)
                mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to drop RQ %#x (channel %u): err = %d\n",
                               mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt),
                               res->drop_rqn, ix, err);
+
+       res->rss_rqns[mlx5e_channels_get_num(chs) + ix] = res->drop_rqn;
+
        return err;
 }

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
index b39b20a720e0..0991944f4adf 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
@@ -42,7 +42,8 @@ void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_chann
 void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res);
 int mlx5e_rx_res_xsk_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs,
                              unsigned int ix);
-int mlx5e_rx_res_xsk_deactivate(struct mlx5e_rx_res *res, unsigned int ix);
+int mlx5e_rx_res_xsk_deactivate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs,
+                               unsigned int ix);

 /* Configuration API */
 void mlx5e_rx_res_rss_set_indir_uniform(struct mlx5e_rx_res *res, unsigned int nch);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
index 279cd8f4e79f..0a9b46ad7976 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
@@ -169,7 +169,7 @@ static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix)
                goto remove_pool;

        c = priv->channels.c[ix];
-       mlx5e_rx_res_xsk_deactivate(priv->rx_res, ix);
+       mlx5e_rx_res_xsk_deactivate(priv->rx_res, &priv->channels, ix);
        mlx5e_deactivate_xsk(c);
        mlx5e_close_xsk(c);

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index 2d3cd237355a..e7fcfe2e3325 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -2136,6 +2136,11 @@ int mlx5e_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
         */
        if (info->cmd == ETHTOOL_GRXRINGS) {
                info->data = priv->channels.params.num_channels;
+               if (priv->xsk.refcnt) {
+                       /* The upper half are XSK queues. */
+                       info->data *= 2;
+               }
+
                return 0;
        }

--
2.12.2

^ permalink raw reply related	[flat|nested] 4+ messages in thread