* [PATCH net-next 1/4] net/mlx5: Split function_setup() to enable and open functions
2022-02-08 17:14 [PATCH net-next 0/4] net/mlx5: Introduce devlink param to disable SF aux dev probe Moshe Shemesh
@ 2022-02-08 17:14 ` Moshe Shemesh
2022-02-08 17:14 ` [PATCH net-next 2/4] net/mlx5: Delete redundant default assignment of runtime devlink params Moshe Shemesh
` (3 subsequent siblings)
4 siblings, 0 replies; 13+ messages in thread
From: Moshe Shemesh @ 2022-02-08 17:14 UTC (permalink / raw)
To: David S. Miller, Jakub Kicinski
Cc: Jiri Pirko, Saeed Mahameed, netdev, linux-kernel, Shay Drory
From: Shay Drory <shayd@nvidia.com>
mlx5_cmd_init_hca() is taking ~0.2 seconds. In case of a user who
desire to disable some of the SF aux devices, and with large scale-1K
SFs for example, this user will waste more than 3 minutes on
mlx5_cmd_init_hca() which isn't needed at this stage.
Therefore, split function_setup() in order to avoid executing
mlx5_cmd_init_hca() in case user disables SFs aux dev via the devlink
param which will be introduced in downstream patch.
Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Saeed Mahameed <saeedm@nvidia.com>
Reviewed-by: Moshe Shemesh <moshe@nvidia.com>
---
.../net/ethernet/mellanox/mlx5/core/main.c | 81 +++++++++++++------
1 file changed, 56 insertions(+), 25 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 2c774f367199..73d2cec01ead 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -1006,7 +1006,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
mlx5_devcom_unregister_device(dev->priv.devcom);
}
-static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot)
+static int mlx5_function_enable(struct mlx5_core_dev *dev)
{
int err;
@@ -1070,28 +1070,53 @@ static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot)
goto reclaim_boot_pages;
}
+ return 0;
+
+reclaim_boot_pages:
+ mlx5_reclaim_startup_pages(dev);
+err_disable_hca:
+ mlx5_core_disable_hca(dev, 0);
+err_cmd_cleanup:
+ mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
+ mlx5_cmd_cleanup(dev);
+
+ return err;
+}
+
+static void mlx5_function_disable(struct mlx5_core_dev *dev)
+{
+ mlx5_reclaim_startup_pages(dev);
+ mlx5_core_disable_hca(dev, 0);
+ mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
+ mlx5_cmd_cleanup(dev);
+}
+
+static int mlx5_function_open(struct mlx5_core_dev *dev, bool boot)
+{
+ int err;
+
err = set_hca_ctrl(dev);
if (err) {
mlx5_core_err(dev, "set_hca_ctrl failed\n");
- goto reclaim_boot_pages;
+ return err;
}
err = set_hca_cap(dev);
if (err) {
mlx5_core_err(dev, "set_hca_cap failed\n");
- goto reclaim_boot_pages;
+ return err;
}
err = mlx5_satisfy_startup_pages(dev, 0);
if (err) {
mlx5_core_err(dev, "failed to allocate init pages\n");
- goto reclaim_boot_pages;
+ return err;
}
err = mlx5_cmd_init_hca(dev, sw_owner_id);
if (err) {
mlx5_core_err(dev, "init hca failed\n");
- goto reclaim_boot_pages;
+ return err;
}
mlx5_set_driver_version(dev);
@@ -1099,25 +1124,15 @@ static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot)
err = mlx5_query_hca_caps(dev);
if (err) {
mlx5_core_err(dev, "query hca failed\n");
- goto reclaim_boot_pages;
+ return err;
}
mlx5_start_health_poll(dev);
return 0;
-
-reclaim_boot_pages:
- mlx5_reclaim_startup_pages(dev);
-err_disable_hca:
- mlx5_core_disable_hca(dev, 0);
-err_cmd_cleanup:
- mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
- mlx5_cmd_cleanup(dev);
-
- return err;
}
-static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot)
+static int mlx5_function_close(struct mlx5_core_dev *dev, bool boot)
{
int err;
@@ -1127,14 +1142,30 @@ static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot)
mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n");
return err;
}
- mlx5_reclaim_startup_pages(dev);
- mlx5_core_disable_hca(dev, 0);
- mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
- mlx5_cmd_cleanup(dev);
return 0;
}
+static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot)
+{
+ int err;
+
+ err = mlx5_function_enable(dev);
+ if (err)
+ return err;
+
+ err = mlx5_function_open(dev, boot);
+ if (err)
+ mlx5_function_disable(dev);
+ return err;
+}
+
+static void mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot)
+{
+ if (!mlx5_function_close(dev, boot))
+ mlx5_function_disable(dev);
+}
+
static int mlx5_load(struct mlx5_core_dev *dev)
{
int err;
@@ -1290,7 +1321,7 @@ int mlx5_init_one(struct mlx5_core_dev *dev)
err = mlx5_function_setup(dev, true);
if (err)
- goto err_function;
+ goto err_function_setup;
err = mlx5_init_once(dev);
if (err) {
@@ -1324,7 +1355,7 @@ int mlx5_init_one(struct mlx5_core_dev *dev)
mlx5_cleanup_once(dev);
function_teardown:
mlx5_function_teardown(dev, true);
-err_function:
+err_function_setup:
dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
mutex_unlock(&dev->intf_state_mutex);
return err;
@@ -1366,7 +1397,7 @@ int mlx5_load_one(struct mlx5_core_dev *dev)
err = mlx5_function_setup(dev, false);
if (err)
- goto err_function;
+ goto err_function_setup;
err = mlx5_load(dev);
if (err)
@@ -1386,7 +1417,7 @@ int mlx5_load_one(struct mlx5_core_dev *dev)
mlx5_unload(dev);
err_load:
mlx5_function_teardown(dev, false);
-err_function:
+err_function_setup:
dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
out:
mutex_unlock(&dev->intf_state_mutex);
--
2.26.3
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH net-next 3/4] devlink: Add new "enable_sfs_aux_devs" generic device param
2022-02-08 17:14 [PATCH net-next 0/4] net/mlx5: Introduce devlink param to disable SF aux dev probe Moshe Shemesh
2022-02-08 17:14 ` [PATCH net-next 1/4] net/mlx5: Split function_setup() to enable and open functions Moshe Shemesh
2022-02-08 17:14 ` [PATCH net-next 2/4] net/mlx5: Delete redundant default assignment of runtime devlink params Moshe Shemesh
@ 2022-02-08 17:14 ` Moshe Shemesh
2022-02-08 17:14 ` [PATCH net-next 4/4] net/mlx5: Support enable_sfs_aux_devs devlink param Moshe Shemesh
2022-02-09 5:23 ` [PATCH net-next 0/4] net/mlx5: Introduce devlink param to disable SF aux dev probe Jakub Kicinski
4 siblings, 0 replies; 13+ messages in thread
From: Moshe Shemesh @ 2022-02-08 17:14 UTC (permalink / raw)
To: David S. Miller, Jakub Kicinski
Cc: Jiri Pirko, Saeed Mahameed, netdev, linux-kernel, Shay Drory
From: Shay Drory <shayd@nvidia.com>
Add new device generic parameter to enable/disable creation of
Sub-Functions auxiliary devices of a certain Physical-Function.
User who wants to use specific SFs auxiliary devices, can disable
the creation of all SF auxiliary devices upon SF creation.
After the SF is created, the user can enable only the requested
auxiliary devices.
for example:
$ devlink dev param set pci/0000:08:00.0 \
name enable_sfs_aux_devs value false cmode driverinit
Create SF:
$ devlink port add pci/0000:08:00.0 flavour pcisf pfnum 0 sfnum 11
$ devlink port function set pci/0000:08:00.0/32768 \
hw_addr 00:00:00:00:00:11 state active
Enable ETH auxiliary device:
$ devlink dev param set auxiliary/mlx5_core.sf.1 \
name enable_eth value true cmode driverinit
$ devlink dev reload auxiliary/mlx5_core.sf.1
At this point the user have SF devlink instance with auxiliary device
for the Ethernet functionality only.
Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Jiri Pirko <jiri@nvidia.com>
Reviewed-by: Moshe Shemesh <moshe@nvidia.com>
---
Documentation/networking/devlink/devlink-params.rst | 5 +++++
include/net/devlink.h | 4 ++++
net/core/devlink.c | 5 +++++
3 files changed, 14 insertions(+)
diff --git a/Documentation/networking/devlink/devlink-params.rst b/Documentation/networking/devlink/devlink-params.rst
index 4e01dc32bc08..aa0edb915f88 100644
--- a/Documentation/networking/devlink/devlink-params.rst
+++ b/Documentation/networking/devlink/devlink-params.rst
@@ -137,3 +137,8 @@ own name.
* - ``event_eq_size``
- u32
- Control the size of asynchronous control events EQ.
+ * - ``enable_sfs_aux_devs``
+ - Boolean
+ - When enabled, the device driver will instantiate all auxiliary devices of
+ SFs of the devlink device. When clear, the SFs of the devlink device will
+ not instantiate any auxiliary devices.
diff --git a/include/net/devlink.h b/include/net/devlink.h
index 8d5349d2fb68..8013252790bf 100644
--- a/include/net/devlink.h
+++ b/include/net/devlink.h
@@ -461,6 +461,7 @@ enum devlink_param_generic_id {
DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP,
DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
+ DEVLINK_PARAM_GENERIC_ID_ENABLE_SFS_AUX_DEVS,
/* add new param generic ids above here*/
__DEVLINK_PARAM_GENERIC_ID_MAX,
@@ -519,6 +520,9 @@ enum devlink_param_generic_id {
#define DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_NAME "event_eq_size"
#define DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_TYPE DEVLINK_PARAM_TYPE_U32
+#define DEVLINK_PARAM_GENERIC_ENABLE_SFS_AUX_DEVS_NAME "enable_sfs_aux_devs"
+#define DEVLINK_PARAM_GENERIC_ENABLE_SFS_AUX_DEVS_TYPE DEVLINK_PARAM_TYPE_BOOL
+
#define DEVLINK_PARAM_GENERIC(_id, _cmodes, _get, _set, _validate) \
{ \
.id = DEVLINK_PARAM_GENERIC_ID_##_id, \
diff --git a/net/core/devlink.c b/net/core/devlink.c
index fcd9f6d85cf1..b5368024ac18 100644
--- a/net/core/devlink.c
+++ b/net/core/devlink.c
@@ -4477,6 +4477,11 @@ static const struct devlink_param devlink_param_generic[] = {
.name = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_NAME,
.type = DEVLINK_PARAM_GENERIC_EVENT_EQ_SIZE_TYPE,
},
+ {
+ .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_SFS_AUX_DEVS,
+ .name = DEVLINK_PARAM_GENERIC_ENABLE_SFS_AUX_DEVS_NAME,
+ .type = DEVLINK_PARAM_GENERIC_ENABLE_SFS_AUX_DEVS_TYPE,
+ },
};
static int devlink_param_generic_verify(const struct devlink_param *param)
--
2.26.3
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH net-next 4/4] net/mlx5: Support enable_sfs_aux_devs devlink param
2022-02-08 17:14 [PATCH net-next 0/4] net/mlx5: Introduce devlink param to disable SF aux dev probe Moshe Shemesh
` (2 preceding siblings ...)
2022-02-08 17:14 ` [PATCH net-next 3/4] devlink: Add new "enable_sfs_aux_devs" generic device param Moshe Shemesh
@ 2022-02-08 17:14 ` Moshe Shemesh
2022-02-09 5:23 ` [PATCH net-next 0/4] net/mlx5: Introduce devlink param to disable SF aux dev probe Jakub Kicinski
4 siblings, 0 replies; 13+ messages in thread
From: Moshe Shemesh @ 2022-02-08 17:14 UTC (permalink / raw)
To: David S. Miller, Jakub Kicinski
Cc: Jiri Pirko, Saeed Mahameed, netdev, linux-kernel, Shay Drory
From: Shay Drory <shayd@nvidia.com>
In case user wants to configure the SFs, for example: to use only vdpa
functionality, he needs to fully probe a SF, configure what he wants,
and afterward reload the SF.
In order to save the time of the reload, exposing a devlink runtime
param on the PF, that when set to off, the SFs won't create any auxiliary
sub-device, so that the SF can be configured prior to its full probe.
Usage example:
$ devlink dev param set pci/0000:00:0b.0 name enable_sfs_aux_devs \
value false cmode runtime
Create SF:
$ devlink port add pci/0000:08:00.0 flavour pcisf pfnum 0 sfnum 11
$ devlink port function set pci/0000:08:00.0/32768 \
hw_addr 00:00:00:00:00:11 state active
Enable ETH auxiliary device:
$ devlink dev param set auxiliary/mlx5_core.sf.1 \
name enable_eth value true cmode driverinit
Now, in order to fully probe the SF, use devlink reload:
$ devlink dev reload auxiliary/mlx5_core.sf.1
At this point the user have SF devlink instance with auxiliary device
for the Ethernet functionality only.
Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Moshe Shemesh <moshe@nvidia.com>
---
drivers/net/ethernet/mellanox/mlx5/core/dev.c | 16 +++
.../net/ethernet/mellanox/mlx5/core/devlink.c | 31 +++++-
.../net/ethernet/mellanox/mlx5/core/health.c | 5 +-
.../net/ethernet/mellanox/mlx5/core/main.c | 102 +++++++++++++++++-
.../ethernet/mellanox/mlx5/core/mlx5_core.h | 6 ++
.../mellanox/mlx5/core/sf/dev/driver.c | 13 ++-
.../ethernet/mellanox/mlx5/core/sf/devlink.c | 40 +++++++
.../ethernet/mellanox/mlx5/core/sf/hw_table.c | 7 ++
.../net/ethernet/mellanox/mlx5/core/sf/priv.h | 2 +
include/linux/mlx5/driver.h | 1 +
10 files changed, 211 insertions(+), 12 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
index ba6dad97e308..89ead37b98ad 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c
@@ -333,6 +333,18 @@ static void del_adev(struct auxiliary_device *adev)
auxiliary_device_uninit(adev);
}
+void mlx5_dev_mark_unregistered(struct mlx5_core_dev *dev)
+{
+ mutex_lock(&mlx5_intf_mutex);
+ dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV;
+ mutex_unlock(&mlx5_intf_mutex);
+}
+
+bool mlx5_dev_is_unregistered(struct mlx5_core_dev *dev)
+{
+ return dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV;
+}
+
int mlx5_attach_device(struct mlx5_core_dev *dev)
{
struct mlx5_priv *priv = &dev->priv;
@@ -473,6 +485,10 @@ static int add_drivers(struct mlx5_core_dev *dev)
if (!is_supported)
continue;
+ if (mlx5_adev_devices[i].is_enabled &&
+ !(mlx5_adev_devices[i].is_enabled(dev)))
+ continue;
+
priv->adev[i] = add_adev(dev, i);
if (IS_ERR(priv->adev[i])) {
mlx5_core_warn(dev, "Device[%d] (%s) failed to load\n",
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index e832a3f4c18a..1b769b5da577 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -139,6 +139,13 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change,
struct pci_dev *pdev = dev->pdev;
bool sf_dev_allocated;
+ if (mlx5_dev_is_unregistered(dev)) {
+ if (action != DEVLINK_RELOAD_ACTION_DRIVER_REINIT)
+ return -EOPNOTSUPP;
+ mlx5_unload_one_light(dev);
+ return 0;
+ }
+
sf_dev_allocated = mlx5_sf_dev_allocated(dev);
if (sf_dev_allocated) {
/* Reload results in deleting SF device which further results in
@@ -182,6 +189,10 @@ static int mlx5_devlink_reload_up(struct devlink *devlink, enum devlink_reload_a
*actions_performed = BIT(action);
switch (action) {
case DEVLINK_RELOAD_ACTION_DRIVER_REINIT:
+ if (mlx5_dev_is_unregistered(dev)) {
+ mlx5_fw_reporters_create(dev);
+ return mlx5_init_one(dev);
+ }
return mlx5_load_one(dev);
case DEVLINK_RELOAD_ACTION_FW_ACTIVATE:
if (limit == DEVLINK_RELOAD_LIMIT_NO_RESET)
@@ -258,6 +269,9 @@ static int mlx5_devlink_trap_action_set(struct devlink *devlink,
struct mlx5_devlink_trap *dl_trap;
int err = 0;
+ if (mlx5_dev_is_unregistered(dev))
+ return -EOPNOTSUPP;
+
if (is_mdev_switchdev_mode(dev)) {
NL_SET_ERR_MSG_MOD(extack, "Devlink traps can't be set in switchdev mode");
return -EOPNOTSUPP;
@@ -440,6 +454,9 @@ static int mlx5_devlink_fs_mode_get(struct devlink *devlink, u32 id,
{
struct mlx5_core_dev *dev = devlink_priv(devlink);
+ if (mlx5_dev_is_unregistered(dev))
+ return -EOPNOTSUPP;
+
if (dev->priv.steering->mode == MLX5_FLOW_STEERING_MODE_SMFS)
strcpy(ctx->val.vstr, "smfs");
else
@@ -499,6 +516,9 @@ static int mlx5_devlink_esw_port_metadata_get(struct devlink *devlink, u32 id,
{
struct mlx5_core_dev *dev = devlink_priv(devlink);
+ if (mlx5_dev_is_unregistered(dev))
+ return -EOPNOTSUPP;
+
if (!MLX5_ESWITCH_MANAGER(dev))
return -EOPNOTSUPP;
@@ -542,6 +562,9 @@ static int mlx5_devlink_enable_remote_dev_reset_get(struct devlink *devlink, u32
{
struct mlx5_core_dev *dev = devlink_priv(devlink);
+ if (mlx5_dev_is_unregistered(dev))
+ return -EOPNOTSUPP;
+
ctx->val.vbool = mlx5_fw_reset_enable_remote_dev_reset_get(dev);
return 0;
}
@@ -588,7 +611,7 @@ static void mlx5_devlink_set_params_init_values(struct devlink *devlink)
struct mlx5_core_dev *dev = devlink_priv(devlink);
union devlink_param_value value;
- value.vbool = MLX5_CAP_GEN(dev, roce);
+ value.vbool = MLX5_CAP_GEN(dev, roce) && !mlx5_dev_is_unregistered(dev);
devlink_param_driverinit_value_set(devlink,
DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
value);
@@ -628,7 +651,7 @@ static int mlx5_devlink_eth_param_register(struct devlink *devlink)
if (err)
return err;
- value.vbool = true;
+ value.vbool = !mlx5_dev_is_unregistered(dev);
devlink_param_driverinit_value_set(devlink,
DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
value);
@@ -673,7 +696,7 @@ static int mlx5_devlink_rdma_param_register(struct devlink *devlink)
if (err)
return err;
- value.vbool = true;
+ value.vbool = !mlx5_dev_is_unregistered(devlink_priv(devlink));
devlink_param_driverinit_value_set(devlink,
DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA,
value);
@@ -705,7 +728,7 @@ static int mlx5_devlink_vnet_param_register(struct devlink *devlink)
if (err)
return err;
- value.vbool = true;
+ value.vbool = !mlx5_dev_is_unregistered(dev);
devlink_param_driverinit_value_set(devlink,
DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET,
value);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
index 737df402c927..1b40ada0e9c2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
@@ -700,7 +700,7 @@ static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = {
};
#define MLX5_REPORTER_FW_GRACEFUL_PERIOD 1200000
-static void mlx5_fw_reporters_create(struct mlx5_core_dev *dev)
+void mlx5_fw_reporters_create(struct mlx5_core_dev *dev)
{
struct mlx5_core_health *health = &dev->priv.health;
struct devlink *devlink = priv_to_devlink(dev);
@@ -893,7 +893,8 @@ int mlx5_health_init(struct mlx5_core_dev *dev)
struct mlx5_core_health *health;
char *name;
- mlx5_fw_reporters_create(dev);
+ if (!mlx5_dev_is_unregistered(dev))
+ mlx5_fw_reporters_create(dev);
health = &dev->priv.health;
name = kmalloc(64, GFP_KERNEL);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 73d2cec01ead..b5e40ad0b5fb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -1314,6 +1314,7 @@ static void mlx5_unload(struct mlx5_core_dev *dev)
int mlx5_init_one(struct mlx5_core_dev *dev)
{
+ bool light_probe = mlx5_dev_is_unregistered(dev);
int err = 0;
mutex_lock(&dev->intf_state_mutex);
@@ -1335,9 +1336,14 @@ int mlx5_init_one(struct mlx5_core_dev *dev)
set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
- err = mlx5_devlink_register(priv_to_devlink(dev));
- if (err)
- goto err_devlink_reg;
+ /* In case of light_probe, mlx5_devlink is already registered.
+ * Hence, don't register devlink again.
+ */
+ if (!light_probe) {
+ err = mlx5_devlink_register(priv_to_devlink(dev));
+ if (err)
+ goto err_devlink_reg;
+ }
err = mlx5_register_device(dev);
if (err)
@@ -1347,7 +1353,8 @@ int mlx5_init_one(struct mlx5_core_dev *dev)
return 0;
err_register:
- mlx5_devlink_unregister(priv_to_devlink(dev));
+ if (!light_probe)
+ mlx5_devlink_unregister(priv_to_devlink(dev));
err_devlink_reg:
clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
mlx5_unload(dev);
@@ -1443,6 +1450,93 @@ void mlx5_unload_one(struct mlx5_core_dev *dev)
mutex_unlock(&dev->intf_state_mutex);
}
+/* In case of light probe, we don't need a full query of hca_caps, but only the bellow caps.
+ * A full query of hca_caps will be done when the device will reload.
+ */
+static int mlx5_query_hca_caps_light(struct mlx5_core_dev *dev)
+{
+ int err;
+
+ err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL);
+ if (err)
+ return err;
+
+ if (MLX5_CAP_GEN(dev, eth_net_offloads)) {
+ err = mlx5_core_get_caps(dev, MLX5_CAP_ETHERNET_OFFLOADS);
+ if (err)
+ return err;
+ }
+
+ if (MLX5_CAP_GEN(dev, nic_flow_table) ||
+ MLX5_CAP_GEN(dev, ipoib_enhanced_offloads)) {
+ err = mlx5_core_get_caps(dev, MLX5_CAP_FLOW_TABLE);
+ if (err)
+ return err;
+ }
+
+ if (MLX5_CAP_GEN_64(dev, general_obj_types) &
+ MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
+ err = mlx5_core_get_caps(dev, MLX5_CAP_VDPA_EMULATION);
+ if (err)
+ return err;
+ }
+
+ return 0;
+}
+
+int mlx5_init_one_light(struct mlx5_core_dev *dev)
+{
+ int err;
+
+ dev->state = MLX5_DEVICE_STATE_UP;
+ err = mlx5_function_enable(dev);
+ if (err) {
+ mlx5_core_warn(dev, "mlx5_function_enable err=%d\n", err);
+ goto out;
+ }
+
+ err = mlx5_query_hca_caps_light(dev);
+ if (err) {
+ mlx5_core_warn(dev, "mlx5_query_hca_caps_light err=%d\n", err);
+ goto query_hca_caps_err;
+ }
+
+ err = mlx5_devlink_register(priv_to_devlink(dev));
+ if (err) {
+ mlx5_core_warn(dev, "mlx5_devlink_reg err = %d\n", err);
+ goto query_hca_caps_err;
+ }
+
+ return 0;
+
+query_hca_caps_err:
+ mlx5_function_disable(dev);
+out:
+ dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
+ return err;
+}
+
+void mlx5_uninit_one_light(struct mlx5_core_dev *dev)
+{
+ mlx5_devlink_unregister(priv_to_devlink(dev));
+ if (dev->state != MLX5_DEVICE_STATE_UP)
+ return;
+ mlx5_function_disable(dev);
+}
+
+/* xxx_ligth() function are used in order to configure the device without full
+ * init (light init). e.g.: There isn't a point in reload a device to light state.
+ * Hence, mlx5_load_one_light() isn't needed.
+ */
+
+void mlx5_unload_one_light(struct mlx5_core_dev *dev)
+{
+ if (dev->state != MLX5_DEVICE_STATE_UP)
+ return;
+ mlx5_function_disable(dev);
+ dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
+}
+
static const int types[] = {
MLX5_CAP_GENERAL,
MLX5_CAP_GENERAL_2,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index 6f8baa0f2a73..e2d827474552 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -209,11 +209,14 @@ int mlx5_attach_device(struct mlx5_core_dev *dev);
void mlx5_detach_device(struct mlx5_core_dev *dev);
int mlx5_register_device(struct mlx5_core_dev *dev);
void mlx5_unregister_device(struct mlx5_core_dev *dev);
+void mlx5_dev_mark_unregistered(struct mlx5_core_dev *dev);
+bool mlx5_dev_is_unregistered(struct mlx5_core_dev *dev);
struct mlx5_core_dev *mlx5_get_next_phys_dev(struct mlx5_core_dev *dev);
void mlx5_dev_list_lock(void);
void mlx5_dev_list_unlock(void);
int mlx5_dev_list_trylock(void);
+void mlx5_fw_reporters_create(struct mlx5_core_dev *dev);
int mlx5_query_mtpps(struct mlx5_core_dev *dev, u32 *mtpps, u32 mtpps_size);
int mlx5_set_mtpps(struct mlx5_core_dev *mdev, u32 *mtpps, u32 mtpps_size);
int mlx5_query_mtppse(struct mlx5_core_dev *mdev, u8 pin, u8 *arm, u8 *mode);
@@ -291,6 +294,9 @@ int mlx5_init_one(struct mlx5_core_dev *dev);
void mlx5_uninit_one(struct mlx5_core_dev *dev);
void mlx5_unload_one(struct mlx5_core_dev *dev);
int mlx5_load_one(struct mlx5_core_dev *dev);
+int mlx5_init_one_light(struct mlx5_core_dev *dev);
+void mlx5_uninit_one_light(struct mlx5_core_dev *dev);
+void mlx5_unload_one_light(struct mlx5_core_dev *dev);
int mlx5_vport_get_other_func_cap(struct mlx5_core_dev *dev, u16 function_id, void *out);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
index 7b4783ce213e..8d6e0e44e614 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c
@@ -28,6 +28,9 @@ static int mlx5_sf_dev_probe(struct auxiliary_device *adev, const struct auxilia
mdev->priv.adev_idx = adev->id;
sf_dev->mdev = mdev;
+ if (sf_dev->parent_mdev->priv.sfs_light_probe)
+ mlx5_dev_mark_unregistered(mdev);
+
err = mlx5_mdev_init(mdev, MLX5_DEFAULT_PROF);
if (err) {
mlx5_core_warn(mdev, "mlx5_mdev_init on err=%d\n", err);
@@ -41,7 +44,10 @@ static int mlx5_sf_dev_probe(struct auxiliary_device *adev, const struct auxilia
goto remap_err;
}
- err = mlx5_init_one(mdev);
+ if (sf_dev->parent_mdev->priv.sfs_light_probe)
+ err = mlx5_init_one_light(mdev);
+ else
+ err = mlx5_init_one(mdev);
if (err) {
mlx5_core_warn(mdev, "mlx5_init_one err=%d\n", err);
goto init_one_err;
@@ -64,7 +70,10 @@ static void mlx5_sf_dev_remove(struct auxiliary_device *adev)
struct devlink *devlink = priv_to_devlink(sf_dev->mdev);
devlink_unregister(devlink);
- mlx5_uninit_one(sf_dev->mdev);
+ if (mlx5_dev_is_unregistered(sf_dev->mdev))
+ mlx5_uninit_one_light(sf_dev->mdev);
+ else
+ mlx5_uninit_one(sf_dev->mdev);
iounmap(sf_dev->mdev->iseg);
mlx5_mdev_uninit(sf_dev->mdev);
mlx5_devlink_free(devlink);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c
index 3be659cd91f1..e3f6066ab7af 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c
@@ -10,6 +10,7 @@
#include "ecpf.h"
#define CREATE_TRACE_POINTS
#include "diag/sf_tracepoint.h"
+#include "devlink.h"
struct mlx5_sf {
struct devlink_port dl_port;
@@ -456,6 +457,44 @@ static int mlx5_sf_vhca_event(struct notifier_block *nb, unsigned long opcode, v
return 0;
}
+static int mlx5_devlink_sfs_light_probe_get(struct devlink *devlink, u32 id,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct mlx5_core_dev *dev = devlink_priv(devlink);
+
+ ctx->val.vbool = !dev->priv.sfs_light_probe;
+ return 0;
+}
+
+static int mlx5_devlink_sfs_light_probe_set(struct devlink *devlink, u32 id,
+ struct devlink_param_gset_ctx *ctx)
+{
+ struct mlx5_core_dev *dev = devlink_priv(devlink);
+
+ dev->priv.sfs_light_probe = !ctx->val.vbool;
+ return 0;
+}
+
+static const struct devlink_param sfs_light_probe_param =
+ DEVLINK_PARAM_GENERIC(ENABLE_SFS_AUX_DEVS, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
+ mlx5_devlink_sfs_light_probe_get,
+ mlx5_devlink_sfs_light_probe_set, NULL);
+
+int mlx5_devlink_sfs_light_probe_param_register(struct devlink *devlink)
+{
+ struct mlx5_core_dev *dev = devlink_priv(devlink);
+
+ if (!mlx5_core_is_pf(dev) || !mlx5_sf_max_functions(dev))
+ return 0;
+
+ return devlink_param_register(devlink, &sfs_light_probe_param);
+}
+
+void mlx5_devlink_sfs_light_probe_param_unregister(struct devlink *devlink)
+{
+ devlink_param_unregister(devlink, &sfs_light_probe_param);
+}
+
static void mlx5_sf_table_enable(struct mlx5_sf_table *table)
{
init_completion(&table->disable_complete);
@@ -533,6 +572,7 @@ int mlx5_sf_table_init(struct mlx5_core_dev *dev)
table->dev = dev;
xa_init(&table->port_indices);
dev->priv.sf_table = table;
+ dev->priv.sfs_light_probe = false;
refcount_set(&table->refcount, 0);
table->esw_nb.notifier_call = mlx5_sf_esw_event;
err = mlx5_esw_event_notifier_register(dev->priv.eswitch, &table->esw_nb);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c
index 17aa348989cb..664704598b76 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c
@@ -283,9 +283,15 @@ int mlx5_sf_hw_table_init(struct mlx5_core_dev *dev)
if (err)
goto ext_err;
+ err = mlx5_devlink_sfs_light_probe_param_register(priv_to_devlink(dev));
+ if (err)
+ goto sfs_reg_err;
+
mlx5_core_dbg(dev, "SF HW table: max sfs = %d, ext sfs = %d\n", max_fn, max_ext_fn);
return 0;
+sfs_reg_err:
+ mlx5_sf_hw_table_hwc_cleanup(&table->hwc[MLX5_SF_HWC_EXTERNAL]);
ext_err:
mlx5_sf_hw_table_hwc_cleanup(&table->hwc[MLX5_SF_HWC_LOCAL]);
table_err:
@@ -301,6 +307,7 @@ void mlx5_sf_hw_table_cleanup(struct mlx5_core_dev *dev)
if (!table)
return;
+ mlx5_devlink_sfs_light_probe_param_unregister(priv_to_devlink(dev));
mutex_destroy(&table->table_lock);
mlx5_sf_hw_table_hwc_cleanup(&table->hwc[MLX5_SF_HWC_EXTERNAL]);
mlx5_sf_hw_table_hwc_cleanup(&table->hwc[MLX5_SF_HWC_LOCAL]);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/priv.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/priv.h
index 7114f3fc335f..c228edc8e43c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/priv.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/priv.h
@@ -19,4 +19,6 @@ void mlx5_sf_hw_table_sf_free(struct mlx5_core_dev *dev, u32 controller, u16 id)
void mlx5_sf_hw_table_sf_deferred_free(struct mlx5_core_dev *dev, u32 controller, u16 id);
bool mlx5_sf_hw_table_supported(const struct mlx5_core_dev *dev);
+int mlx5_devlink_sfs_light_probe_param_register(struct devlink *devlink);
+void mlx5_devlink_sfs_light_probe_param_unregister(struct devlink *devlink);
#endif
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 78655d8d13a7..fd07175896ff 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -604,6 +604,7 @@ struct mlx5_priv {
struct mlx5_vhca_state_notifier *vhca_state_notifier;
struct mlx5_sf_dev_table *sf_dev_table;
struct mlx5_core_dev *parent_mdev;
+ u8 sfs_light_probe:1;
#endif
#ifdef CONFIG_MLX5_SF_MANAGER
struct mlx5_sf_hw_table *sf_hw_table;
--
2.26.3
^ permalink raw reply related [flat|nested] 13+ messages in thread