From: Shiraz Saleem <shiraz.saleem@intel.com>
To: dledford@redhat.com, jgg@nvidia.com, kuba@kernel.org,
davem@davemloft.net
Cc: linux-rdma@vger.kernel.org, netdev@vger.kernel.org,
david.m.ertman@intel.com, anthony.l.nguyen@intel.com,
Shiraz Saleem <shiraz.saleem@intel.com>
Subject: [PATCH v6 04/22] ice: Register auxiliary device to provide RDMA
Date: Thu, 20 May 2021 09:37:51 -0500 [thread overview]
Message-ID: <20210520143809.819-5-shiraz.saleem@intel.com> (raw)
In-Reply-To: <20210520143809.819-1-shiraz.saleem@intel.com>
From: Dave Ertman <david.m.ertman@intel.com>
Register ice client auxiliary RDMA device on the auxiliary bus per
PCIe device function for the auxiliary driver (irdma) to attach to.
It allows to realize a single RDMA driver (irdma) capable of working with
multiple netdev drivers over multi-generation Intel HW supporting RDMA.
There is no load ordering dependencies between ice and irdma.
Signed-off-by: Dave Ertman <david.m.ertman@intel.com>
Signed-off-by: Tony Nguyen <anthony.l.nguyen@intel.com>
Signed-off-by: Shiraz Saleem <shiraz.saleem@intel.com>
---
drivers/net/ethernet/intel/Kconfig | 1 +
drivers/net/ethernet/intel/ice/ice.h | 8 +++-
drivers/net/ethernet/intel/ice/ice_idc.c | 71 ++++++++++++++++++++++++++++++-
drivers/net/ethernet/intel/ice/ice_main.c | 11 ++++-
4 files changed, 87 insertions(+), 4 deletions(-)
diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig
index c1d1556..d8a12da 100644
--- a/drivers/net/ethernet/intel/Kconfig
+++ b/drivers/net/ethernet/intel/Kconfig
@@ -294,6 +294,7 @@ config ICE
tristate "Intel(R) Ethernet Connection E800 Series Support"
default n
depends on PCI_MSI
+ select AUXILIARY_BUS
select DIMLIB
select NET_DEVLINK
select PLDMFW
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index 225f8a5..228055e 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -34,6 +34,7 @@
#include <linux/if_bridge.h>
#include <linux/ctype.h>
#include <linux/bpf.h>
+#include <linux/auxiliary_bus.h>
#include <linux/avf/virtchnl.h>
#include <linux/cpu_rmap.h>
#include <linux/dim.h>
@@ -647,6 +648,8 @@ static inline void ice_clear_sriov_cap(struct ice_pf *pf)
void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size);
int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset);
void ice_print_link_msg(struct ice_vsi *vsi, bool isup);
+int ice_plug_aux_dev(struct ice_pf *pf);
+void ice_unplug_aux_dev(struct ice_pf *pf);
int ice_init_rdma(struct ice_pf *pf);
const char *ice_stat_str(enum ice_status stat_err);
const char *ice_aq_str(enum ice_aq_err aq_err);
@@ -678,8 +681,10 @@ int ice_aq_wait_for_event(struct ice_pf *pf, u16 opcode, unsigned long timeout,
*/
static inline void ice_set_rdma_cap(struct ice_pf *pf)
{
- if (pf->hw.func_caps.common_cap.rdma && pf->num_rdma_msix)
+ if (pf->hw.func_caps.common_cap.rdma && pf->num_rdma_msix) {
set_bit(ICE_FLAG_RDMA_ENA, pf->flags);
+ ice_plug_aux_dev(pf);
+ }
}
/**
@@ -688,6 +693,7 @@ static inline void ice_set_rdma_cap(struct ice_pf *pf)
*/
static inline void ice_clear_rdma_cap(struct ice_pf *pf)
{
+ ice_unplug_aux_dev(pf);
clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
}
#endif /* _ICE_H_ */
diff --git a/drivers/net/ethernet/intel/ice/ice_idc.c b/drivers/net/ethernet/intel/ice/ice_idc.c
index ffca0d5..e7bb8f6 100644
--- a/drivers/net/ethernet/intel/ice/ice_idc.c
+++ b/drivers/net/ethernet/intel/ice/ice_idc.c
@@ -255,6 +255,71 @@ static int ice_reserve_rdma_qvector(struct ice_pf *pf)
}
/**
+ * ice_adev_release - function to be mapped to AUX dev's release op
+ * @dev: pointer to device to free
+ */
+static void ice_adev_release(struct device *dev)
+{
+ struct iidc_auxiliary_dev *iadev;
+
+ iadev = container_of(dev, struct iidc_auxiliary_dev, adev.dev);
+ kfree(iadev);
+}
+
+/**
+ * ice_plug_aux_dev - allocate and register AUX device
+ * @pf: pointer to pf struct
+ */
+int ice_plug_aux_dev(struct ice_pf *pf)
+{
+ struct iidc_auxiliary_dev *iadev;
+ struct auxiliary_device *adev;
+ int ret;
+
+ iadev = kzalloc(sizeof(*iadev), GFP_KERNEL);
+ if (!iadev)
+ return -ENOMEM;
+
+ adev = &iadev->adev;
+ pf->adev = adev;
+ iadev->pf = pf;
+
+ adev->id = pf->aux_idx;
+ adev->dev.release = ice_adev_release;
+ adev->dev.parent = &pf->pdev->dev;
+ adev->name = IIDC_RDMA_ROCE_NAME;
+
+ ret = auxiliary_device_init(adev);
+ if (ret) {
+ pf->adev = NULL;
+ kfree(iadev);
+ return ret;
+ }
+
+ ret = auxiliary_device_add(adev);
+ if (ret) {
+ pf->adev = NULL;
+ auxiliary_device_uninit(adev);
+ return ret;
+ }
+
+ return 0;
+}
+
+/* ice_unplug_aux_dev - unregister and free AUX device
+ * @pf: pointer to pf struct
+ */
+void ice_unplug_aux_dev(struct ice_pf *pf)
+{
+ if (!pf->adev)
+ return;
+
+ auxiliary_device_delete(pf->adev);
+ auxiliary_device_uninit(pf->adev);
+ pf->adev = NULL;
+}
+
+/**
* ice_init_rdma - initializes PF for RDMA use
* @pf: ptr to ice_pf
*/
@@ -265,8 +330,10 @@ int ice_init_rdma(struct ice_pf *pf)
/* Reserve vector resources */
ret = ice_reserve_rdma_qvector(pf);
- if (ret < 0)
+ if (ret < 0) {
dev_err(dev, "failed to reserve vectors for RDMA\n");
+ return ret;
+ }
- return ret;
+ return ice_plug_aux_dev(pf);
}
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index 9d4570b..254cfc1 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -456,6 +456,8 @@ static void ice_pf_dis_all_vsi(struct ice_pf *pf, bool locked)
if (test_bit(ICE_PREPARED_FOR_RESET, pf->state))
return;
+ ice_unplug_aux_dev(pf);
+
/* Notify VFs of impending reset */
if (ice_check_sq_alive(hw, &hw->mailboxq))
ice_vc_notify_reset(pf);
@@ -2120,6 +2122,8 @@ int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset)
return -EBUSY;
}
+ ice_unplug_aux_dev(pf);
+
switch (reset) {
case ICE_RESET_PFR:
set_bit(ICE_PFR_REQ, pf->state);
@@ -4456,11 +4460,12 @@ static void ice_remove(struct pci_dev *pdev)
ice_free_vfs(pf);
}
- set_bit(ICE_DOWN, pf->state);
ice_service_task_stop(pf);
ice_aq_cancel_waiting_tasks(pf);
+ ice_unplug_aux_dev(pf);
ida_free(&ice_aux_ida, pf->aux_idx);
+ set_bit(ICE_DOWN, pf->state);
mutex_destroy(&(&pf->hw)->fdir_fltr_lock);
ice_deinit_lag(pf);
@@ -4616,6 +4621,8 @@ static int __maybe_unused ice_suspend(struct device *dev)
*/
disabled = ice_service_task_stop(pf);
+ ice_unplug_aux_dev(pf);
+
/* Already suspended?, then there is nothing to do */
if (test_and_set_bit(ICE_SUSPENDED, pf->state)) {
if (!disabled)
@@ -6286,6 +6293,8 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type)
/* if we get here, reset flow is successful */
clear_bit(ICE_RESET_FAILED, pf->state);
+
+ ice_plug_aux_dev(pf);
return;
err_vsi_rebuild:
--
1.8.3.1
next prev parent reply other threads:[~2021-05-20 14:39 UTC|newest]
Thread overview: 26+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-20 14:37 [PATCH v6 00/22] Add Intel Ethernet Protocol Driver for RDMA (irdma) Shiraz Saleem
2021-05-20 14:37 ` [PATCH v6 01/22] iidc: Introduce iidc.h Shiraz Saleem
2021-05-20 14:37 ` [PATCH v6 02/22] ice: Initialize RDMA support Shiraz Saleem
2021-05-20 14:37 ` [PATCH v6 03/22] ice: Implement iidc operations Shiraz Saleem
2021-05-20 14:37 ` Shiraz Saleem [this message]
2021-05-20 14:37 ` [PATCH v6 05/22] i40e: Prep i40e header for aux bus conversion Shiraz Saleem
2021-05-20 14:37 ` [PATCH v6 06/22] i40e: Register auxiliary devices to provide RDMA Shiraz Saleem
2021-05-20 19:49 ` Jason Gunthorpe
2021-05-20 14:37 ` [PATCH v6 07/22] RDMA/irdma: Register auxiliary driver and implement private channel OPs Shiraz Saleem
2021-05-20 14:37 ` [PATCH v6 08/22] RDMA/irdma: Implement device initialization definitions Shiraz Saleem
2021-05-20 14:37 ` [PATCH v6 09/22] RDMA/irdma: Implement HW Admin Queue OPs Shiraz Saleem
2021-05-20 14:37 ` [PATCH v6 10/22] RDMA/irdma: Add HMC backing store setup functions Shiraz Saleem
2021-05-20 14:37 ` [PATCH v6 11/22] RDMA/irdma: Add privileged UDA queue implementation Shiraz Saleem
2021-05-20 14:37 ` [PATCH v6 12/22] RDMA/irdma: Add QoS definitions Shiraz Saleem
2021-05-20 14:38 ` [PATCH v6 13/22] RDMA/irdma: Add connection manager Shiraz Saleem
2021-05-20 14:38 ` [PATCH v6 14/22] RDMA/irdma: Add PBLE resource manager Shiraz Saleem
2021-05-20 14:38 ` [PATCH v6 15/22] RDMA/irdma: Implement device supported verb APIs Shiraz Saleem
2021-05-20 14:38 ` [PATCH v6 16/22] RDMA/irdma: Add RoCEv2 UD OP support Shiraz Saleem
2021-05-20 14:38 ` [PATCH v6 17/22] RDMA/irdma: Add user/kernel shared libraries Shiraz Saleem
2021-05-20 14:38 ` [PATCH v6 18/22] RDMA/irdma: Add miscellaneous utility definitions Shiraz Saleem
2021-05-20 14:38 ` [PATCH v6 19/22] RDMA/irdma: Add dynamic tracing for CM Shiraz Saleem
2021-05-20 14:38 ` [PATCH v6 20/22] RDMA/irdma: Add ABI definitions Shiraz Saleem
2021-05-20 14:38 ` [PATCH v6 21/22] RDMA/irdma: Add irdma Kconfig/Makefile and remove i40iw Shiraz Saleem
2021-05-20 14:38 ` [PATCH v6 22/22] RDMA/irdma: Update MAINTAINERS file Shiraz Saleem
2021-05-20 20:03 ` [PATCH v6 00/22] Add Intel Ethernet Protocol Driver for RDMA (irdma) Jason Gunthorpe
2021-05-20 21:17 ` Saleem, Shiraz
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210520143809.819-5-shiraz.saleem@intel.com \
--to=shiraz.saleem@intel.com \
--cc=anthony.l.nguyen@intel.com \
--cc=davem@davemloft.net \
--cc=david.m.ertman@intel.com \
--cc=dledford@redhat.com \
--cc=jgg@nvidia.com \
--cc=kuba@kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=netdev@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).