* [PATCH 1/5] rdma-core/i40iw: Remove provider i40iw
2021-04-06 21:17 [PATCH 0/5] Add irdma user space provider for Intel Ethernet RDMA Tatyana Nikolova
@ 2021-04-06 21:17 ` Tatyana Nikolova
2021-04-06 22:55 ` Jason Gunthorpe
2021-04-06 21:17 ` [PATCH 2/5] rdma-core/irdma: Add Makefile and ABI definitions Tatyana Nikolova
` (3 subsequent siblings)
4 siblings, 1 reply; 10+ messages in thread
From: Tatyana Nikolova @ 2021-04-06 21:17 UTC (permalink / raw)
To: jgg, dledford, leon; +Cc: linux-rdma, Tatyana Nikolova
Remove provider i40iw and set up the environment
for its replacement - provider irdma which
supports both X722 and E810 Intel devices.
Signed-off-by: Tatyana Nikolova <tatyana.e.nikolova@intel.com>
---
CMakeLists.txt | 2 +-
MAINTAINERS | 7 +-
debian/control | 2 +-
debian/copyright | 8 +-
| 4 +-
| 107 --
| 2 +-
libibverbs/verbs.h | 2 +-
providers/i40iw/CMakeLists.txt | 5 -
providers/i40iw/i40e_devids.h | 72 --
providers/i40iw/i40iw-abi.h | 55 -
providers/i40iw/i40iw_d.h | 1746 -----------------------------
providers/i40iw/i40iw_osdep.h | 108 --
providers/i40iw/i40iw_register.h | 1030 -----------------
providers/i40iw/i40iw_status.h | 101 --
providers/i40iw/i40iw_uk.c | 1266 ---------------------
providers/i40iw/i40iw_umain.c | 226 ----
providers/i40iw/i40iw_umain.h | 181 ---
providers/i40iw/i40iw_user.h | 456 --------
providers/i40iw/i40iw_uverbs.c | 983 ----------------
redhat/rdma-core.spec | 6 +-
suse/rdma-core.spec | 4 +-
22 files changed, 19 insertions(+), 6354 deletions(-)
delete mode 100644 kernel-headers/rdma/i40iw-abi.h
delete mode 100644 providers/i40iw/CMakeLists.txt
delete mode 100644 providers/i40iw/i40e_devids.h
delete mode 100644 providers/i40iw/i40iw-abi.h
delete mode 100644 providers/i40iw/i40iw_d.h
delete mode 100644 providers/i40iw/i40iw_osdep.h
delete mode 100644 providers/i40iw/i40iw_register.h
delete mode 100644 providers/i40iw/i40iw_status.h
delete mode 100644 providers/i40iw/i40iw_uk.c
delete mode 100644 providers/i40iw/i40iw_umain.c
delete mode 100644 providers/i40iw/i40iw_umain.h
delete mode 100644 providers/i40iw/i40iw_user.h
delete mode 100644 providers/i40iw/i40iw_uverbs.c
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e9a2f49..ed43f59 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -666,7 +666,7 @@ add_subdirectory(providers/cxgb4) # NO SPARSE
add_subdirectory(providers/efa)
add_subdirectory(providers/efa/man)
add_subdirectory(providers/hns)
-add_subdirectory(providers/i40iw) # NO SPARSE
+add_subdirectory(providers/irdma)
add_subdirectory(providers/mlx4)
add_subdirectory(providers/mlx4/man)
add_subdirectory(providers/mlx5)
diff --git a/MAINTAINERS b/MAINTAINERS
index 948c3a7..8a03e55 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -74,10 +74,11 @@ M: Wei Hu(Xavier) <xavier.huwei@huawei.com>
S: Supported
F: providers/hns/
-I40IW USERSPACE PROVIDER (for i40iw.ko)
-M: Tatyana Nikolova <Tatyana.E.Nikolova@intel.com>
+IRDMA USERSPACE PROVIDER (for i40iw.ko and irdma.ko)
+M: Sindhu Devale <sindhu.devale@intel.com>
+M: Tatyana Nikolova <tatyana.e.nikolova@intel.com>
S: Supported
-F: providers/i40iw/
+F: providers/irdma/
RDMA Communication Manager Assistant (for librdmacm.so)
M: Haakon Bugge <haakon.bugge@oracle.com>
diff --git a/debian/control b/debian/control
index b606a87..c7d953c 100644
--- a/debian/control
+++ b/debian/control
@@ -95,8 +95,8 @@ Description: User space provider drivers for libibverbs
- efa: Amazon Elastic Fabric Adapter
- hfi1verbs: Intel Omni-Path HFI
- hns: HiSilicon Hip06 SoC
- - i40iw: Intel Ethernet Connection X722 RDMA
- ipathverbs: QLogic InfiniPath HCAs
+ - irdma: Intel Ethernet Connection RDMA
- mlx4: Mellanox ConnectX-3 InfiniBand HCAs
- mlx5: Mellanox Connect-IB/X-4+ InfiniBand HCAs
- mthca: Mellanox InfiniBand HCAs
diff --git a/debian/copyright b/debian/copyright
index 3bf582b..780c87f 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -166,16 +166,16 @@ Files: providers/hns/*
Copyright: 2016, Hisilicon Limited.
License: BSD-MIT or GPL-2
-Files: providers/i40iw/*
-Copyright: 2015-2016, Intel Corporation.
-License: BSD-MIT or GPL-2
-
Files: providers/ipathverbs/*
Copyright: 2006-2010, QLogic Corp.
2005, PathScale, Inc.
2013, Intel Corporation
License: BSD-MIT or GPL-2
+Files: providers/irdma/*
+Copyright: 2015-2020, Intel Corporation.
+License: BSD-MIT or GPL-2
+
Files: providers/mlx4/*
Copyright: 2004-2005, Topspin Communications.
2005-2007, Cisco, Inc.
--git a/kernel-headers/CMakeLists.txt b/kernel-headers/CMakeLists.txt
index 2d0766d..b961892 100644
--- a/kernel-headers/CMakeLists.txt
+++ b/kernel-headers/CMakeLists.txt
@@ -3,12 +3,12 @@ publish_internal_headers(rdma
rdma/cxgb4-abi.h
rdma/efa-abi.h
rdma/hns-abi.h
- rdma/i40iw-abi.h
rdma/ib_user_ioctl_cmds.h
rdma/ib_user_ioctl_verbs.h
rdma/ib_user_mad.h
rdma/ib_user_sa.h
rdma/ib_user_verbs.h
+ rdma/irdma-abi.h
rdma/mlx4-abi.h
rdma/mlx5-abi.h
rdma/mlx5_user_ioctl_cmds.h
@@ -61,8 +61,8 @@ rdma_kernel_provider_abi(
rdma/cxgb4-abi.h
rdma/efa-abi.h
rdma/hns-abi.h
- rdma/i40iw-abi.h
rdma/ib_user_verbs.h
+ rdma/irdma-abi.h
rdma/mlx4-abi.h
rdma/mlx5-abi.h
rdma/mthca-abi.h
diff --git a/kernel-headers/rdma/i40iw-abi.h b/kernel-headers/rdma/i40iw-abi.h
deleted file mode 100644
index 79890ba..0000000
--- a/kernel-headers/rdma/i40iw-abi.h
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Copyright (c) 2006 - 2016 Intel Corporation. All rights reserved.
- * Copyright (c) 2005 Topspin Communications. All rights reserved.
- * Copyright (c) 2005 Cisco Systems. All rights reserved.
- * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- */
-
-#ifndef I40IW_ABI_H
-#define I40IW_ABI_H
-
-#include <linux/types.h>
-
-#define I40IW_ABI_VER 5
-
-struct i40iw_alloc_ucontext_req {
- __u32 reserved32;
- __u8 userspace_ver;
- __u8 reserved8[3];
-};
-
-struct i40iw_alloc_ucontext_resp {
- __u32 max_pds; /* maximum pds allowed for this user process */
- __u32 max_qps; /* maximum qps allowed for this user process */
- __u32 wq_size; /* size of the WQs (sq+rq) allocated to the mmaped area */
- __u8 kernel_ver;
- __u8 reserved[3];
-};
-
-struct i40iw_alloc_pd_resp {
- __u32 pd_id;
- __u8 reserved[4];
-};
-
-struct i40iw_create_cq_req {
- __aligned_u64 user_cq_buffer;
- __aligned_u64 user_shadow_area;
-};
-
-struct i40iw_create_qp_req {
- __aligned_u64 user_wqe_buffers;
- __aligned_u64 user_compl_ctx;
-
- /* UDA QP PHB */
- __aligned_u64 user_sq_phb; /* place for VA of the sq phb buff */
- __aligned_u64 user_rq_phb; /* place for VA of the rq phb buff */
-};
-
-enum i40iw_memreg_type {
- IW_MEMREG_TYPE_MEM = 0x0000,
- IW_MEMREG_TYPE_QP = 0x0001,
- IW_MEMREG_TYPE_CQ = 0x0002,
-};
-
-struct i40iw_mem_reg_req {
- __u16 reg_type; /* Memory, QP or CQ */
- __u16 cq_pages;
- __u16 rq_pages;
- __u16 sq_pages;
-};
-
-struct i40iw_create_cq_resp {
- __u32 cq_id;
- __u32 cq_size;
- __u32 mmap_db_index;
- __u32 reserved;
-};
-
-struct i40iw_create_qp_resp {
- __u32 qp_id;
- __u32 actual_sq_size;
- __u32 actual_rq_size;
- __u32 i40iw_drv_opt;
- __u16 push_idx;
- __u8 lsmm;
- __u8 rsvd2;
-};
-
-#endif
--git a/kernel-headers/rdma/ib_user_ioctl_verbs.h b/kernel-headers/rdma/ib_user_ioctl_verbs.h
index 2248379..d4c98ef 100644
--- a/kernel-headers/rdma/ib_user_ioctl_verbs.h
+++ b/kernel-headers/rdma/ib_user_ioctl_verbs.h
@@ -239,7 +239,7 @@ enum rdma_driver_id {
RDMA_DRIVER_BNXT_RE,
RDMA_DRIVER_OCRDMA,
RDMA_DRIVER_NES,
- RDMA_DRIVER_I40IW,
+ RDMA_DRIVER_IRDMA,
RDMA_DRIVER_VMW_PVRDMA,
RDMA_DRIVER_QEDR,
RDMA_DRIVER_HNS,
diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
index 8161c4f..e9eaa9e 100644
--- a/libibverbs/verbs.h
+++ b/libibverbs/verbs.h
@@ -2201,8 +2201,8 @@ extern const struct verbs_device_ops verbs_provider_cxgb4;
extern const struct verbs_device_ops verbs_provider_efa;
extern const struct verbs_device_ops verbs_provider_hfi1verbs;
extern const struct verbs_device_ops verbs_provider_hns;
-extern const struct verbs_device_ops verbs_provider_i40iw;
extern const struct verbs_device_ops verbs_provider_ipathverbs;
+extern const struct verbs_device_ops verbs_provider_irdma;
extern const struct verbs_device_ops verbs_provider_mlx4;
extern const struct verbs_device_ops verbs_provider_mlx5;
extern const struct verbs_device_ops verbs_provider_mthca;
diff --git a/providers/i40iw/CMakeLists.txt b/providers/i40iw/CMakeLists.txt
deleted file mode 100644
index d8a3a3c..0000000
--- a/providers/i40iw/CMakeLists.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-rdma_provider(i40iw
- i40iw_uk.c
- i40iw_umain.c
- i40iw_uverbs.c
-)
diff --git a/providers/i40iw/i40e_devids.h b/providers/i40iw/i40e_devids.h
deleted file mode 100644
index 6c09a66..0000000
--- a/providers/i40iw/i40e_devids.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef _I40E_DEVIDS_H_
-#define _I40E_DEVIDS_H_
-
-/* Vendor ID */
-#define I40E_INTEL_VENDOR_ID 0x8086
-
-/* Device IDs */
-#define I40E_DEV_ID_SFP_XL710 0x1572
-#define I40E_DEV_ID_QEMU 0x1574
-#define I40E_DEV_ID_KX_B 0x1580
-#define I40E_DEV_ID_KX_C 0x1581
-#define I40E_DEV_ID_QSFP_A 0x1583
-#define I40E_DEV_ID_QSFP_B 0x1584
-#define I40E_DEV_ID_QSFP_C 0x1585
-#define I40E_DEV_ID_10G_BASE_T 0x1586
-#define I40E_DEV_ID_20G_KR2 0x1587
-#define I40E_DEV_ID_20G_KR2_A 0x1588
-#define I40E_DEV_ID_10G_BASE_T4 0x1589
-#define I40E_DEV_ID_25G_B 0x158A
-#define I40E_DEV_ID_25G_SFP28 0x158B
-#define I40E_DEV_ID_VF 0x154C
-#define I40E_DEV_ID_VF_HV 0x1571
-#define I40E_DEV_ID_X722_A0 0x374C
-#define I40E_DEV_ID_X722_A0_VF 0x374D
-#define I40E_DEV_ID_KX_X722 0x37CE
-#define I40E_DEV_ID_QSFP_X722 0x37CF
-#define I40E_DEV_ID_SFP_X722 0x37D0
-#define I40E_DEV_ID_1G_BASE_T_X722 0x37D1
-#define I40E_DEV_ID_10G_BASE_T_X722 0x37D2
-#define I40E_DEV_ID_SFP_I_X722 0x37D3
-#define I40E_DEV_ID_X722_VF 0x37CD
-#define I40E_DEV_ID_X722_VF_HV 0x37D9
-
-#define i40e_is_40G_device(d) ((d) == I40E_DEV_ID_QSFP_A || \
- (d) == I40E_DEV_ID_QSFP_B || \
- (d) == I40E_DEV_ID_QSFP_C)
-
-#endif /* _I40E_DEVIDS_H_ */
diff --git a/providers/i40iw/i40iw-abi.h b/providers/i40iw/i40iw-abi.h
deleted file mode 100644
index 1fee917..0000000
--- a/providers/i40iw/i40iw-abi.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef PROVIDER_I40IW_ABI_H
-#define PROVIDER_I40IW_ABI_H
-
-#include <infiniband/kern-abi.h>
-#include <rdma/i40iw-abi.h>
-#include <kernel-abi/i40iw-abi.h>
-
-#define I40IW_ABI_VER 5
-
-DECLARE_DRV_CMD(i40iw_ualloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
- empty, i40iw_alloc_pd_resp);
-DECLARE_DRV_CMD(i40iw_ucreate_cq, IB_USER_VERBS_CMD_CREATE_CQ,
- i40iw_create_cq_req, i40iw_create_cq_resp);
-DECLARE_DRV_CMD(i40iw_ucreate_qp, IB_USER_VERBS_CMD_CREATE_QP,
- i40iw_create_qp_req, i40iw_create_qp_resp);
-DECLARE_DRV_CMD(i40iw_get_context, IB_USER_VERBS_CMD_GET_CONTEXT,
- i40iw_alloc_ucontext_req, i40iw_alloc_ucontext_resp);
-DECLARE_DRV_CMD(i40iw_ureg_mr, IB_USER_VERBS_CMD_REG_MR,
- i40iw_mem_reg_req, empty);
-
-#endif /* I40IW_ABI_H */
diff --git a/providers/i40iw/i40iw_d.h b/providers/i40iw/i40iw_d.h
deleted file mode 100644
index 4652dcb..0000000
--- a/providers/i40iw/i40iw_d.h
+++ /dev/null
@@ -1,1746 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_D_H
-#define I40IW_D_H
-
-#define I40IW_DB_ADDR_OFFSET (4 * 1024 * 1024 - 64 * 1024)
-#define I40IW_VF_DB_ADDR_OFFSET (64 * 1024)
-
-#define I40IW_PUSH_OFFSET (4 * 1024 * 1024)
-#define I40IW_PF_FIRST_PUSH_PAGE_INDEX 16
-#define I40IW_VF_PUSH_OFFSET ((8 + 64) * 1024)
-#define I40IW_VF_FIRST_PUSH_PAGE_INDEX 2
-
-#define I40IW_PE_DB_SIZE_4M 1
-#define I40IW_PE_DB_SIZE_8M 2
-
-#define I40IW_DDP_VER 1
-#define I40IW_RDMAP_VER 1
-
-#define I40IW_RDMA_MODE_RDMAC 0
-#define I40IW_RDMA_MODE_IETF 1
-
-#define I40IW_QP_STATE_INVALID 0
-#define I40IW_QP_STATE_IDLE 1
-#define I40IW_QP_STATE_RTS 2
-#define I40IW_QP_STATE_CLOSING 3
-#define I40IW_QP_STATE_RESERVED 4
-#define I40IW_QP_STATE_TERMINATE 5
-#define I40IW_QP_STATE_ERROR 6
-
-#define I40IW_STAG_STATE_INVALID 0
-#define I40IW_STAG_STATE_VALID 1
-
-#define I40IW_STAG_TYPE_SHARED 0
-#define I40IW_STAG_TYPE_NONSHARED 1
-
-#define I40IW_MAX_USER_PRIORITY 8
-
-#define LS_64_1(val, bits) ((u64)(uintptr_t)val << bits)
-#define RS_64_1(val, bits) ((u64)(uintptr_t)val >> bits)
-#define LS_32_1(val, bits) (u32)(val << bits)
-#define RS_32_1(val, bits) (u32)(val >> bits)
-#define I40E_HI_DWORD(x) ((u32)((((x) >> 16) >> 16) & 0xFFFFFFFF))
-
-#define LS_64(val, field) (((u64)val << field ## _SHIFT) & (field ## _MASK))
-
-#define RS_64(val, field) ((u64)(val & field ## _MASK) >> field ## _SHIFT)
-#define LS_32(val, field) ((val << field ## _SHIFT) & (field ## _MASK))
-#define RS_32(val, field) ((val & field ## _MASK) >> field ## _SHIFT)
-
-#define TERM_DDP_LEN_TAGGED 14
-#define TERM_DDP_LEN_UNTAGGED 18
-#define TERM_RDMA_LEN 28
-#define RDMA_OPCODE_MASK 0x0f
-#define RDMA_READ_REQ_OPCODE 1
-#define Q2_BAD_FRAME_OFFSET 72
-#define CQE_MAJOR_DRV 0x8000
-
-#define I40IW_TERM_SENT 0x01
-#define I40IW_TERM_RCVD 0x02
-#define I40IW_TERM_DONE 0x04
-#define I40IW_MAC_HLEN 14
-#define I40IW_BYTE_0 0
-#define I40IW_BYTE_8 8
-#define I40IW_BYTE_16 16
-#define I40IW_BYTE_24 24
-#define I40IW_BYTE_32 32
-#define I40IW_BYTE_40 40
-#define I40IW_BYTE_48 48
-#define I40IW_BYTE_56 56
-#define I40IW_BYTE_64 64
-#define I40IW_BYTE_72 72
-#define I40IW_BYTE_80 80
-#define I40IW_BYTE_88 88
-#define I40IW_BYTE_96 96
-#define I40IW_BYTE_104 104
-#define I40IW_BYTE_112 112
-#define I40IW_BYTE_120 120
-#define I40IW_BYTE_128 128
-#define I40IW_BYTE_136 136
-#define I40IW_BYTE_144 144
-#define I40IW_BYTE_152 152
-#define I40IW_BYTE_160 160
-#define I40IW_BYTE_168 168
-#define I40IW_BYTE_176 176
-#define I40IW_BYTE_184 184
-#define I40IW_BYTE_192 192
-#define I40IW_BYTE_200 200
-#define I40IW_BYTE_208 208
-
-#define I40IW_INVALID_WQE_INDEX 0xffffffff
-
-#define I40IW_CQP_WAIT_POLL_REGS 1
-#define I40IW_CQP_WAIT_POLL_CQ 2
-#define I40IW_CQP_WAIT_EVENT 3
-
-#define I40IW_CQP_INIT_WQE(wqe) memset(wqe, 0, 64)
-
-#define I40IW_GET_CURRENT_CQ_ELEMENT(_cq) \
- ( \
- &((_cq)->cq_base[I40IW_RING_GETCURRENT_HEAD((_cq)->cq_ring)]) \
- )
-#define I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(_cq) \
- ( \
- &(((struct i40iw_extended_cqe *) \
- ((_cq)->cq_base))[I40IW_RING_GETCURRENT_HEAD((_cq)->cq_ring)]) \
- )
-
-#define I40IW_GET_CURRENT_AEQ_ELEMENT(_aeq) \
- ( \
- &_aeq->aeqe_base[I40IW_RING_GETCURRENT_TAIL(_aeq->aeq_ring)] \
- )
-
-#define I40IW_GET_CURRENT_CEQ_ELEMENT(_ceq) \
- ( \
- &_ceq->ceqe_base[I40IW_RING_GETCURRENT_TAIL(_ceq->ceq_ring)] \
- )
-
-#define I40IW_AE_SOURCE_RQ 0x1
-#define I40IW_AE_SOURCE_RQ_0011 0x3
-
-#define I40IW_AE_SOURCE_CQ 0x2
-#define I40IW_AE_SOURCE_CQ_0110 0x6
-#define I40IW_AE_SOURCE_CQ_1010 0xA
-#define I40IW_AE_SOURCE_CQ_1110 0xE
-
-#define I40IW_AE_SOURCE_SQ 0x5
-#define I40IW_AE_SOURCE_SQ_0111 0x7
-
-#define I40IW_AE_SOURCE_IN_RR_WR 0x9
-#define I40IW_AE_SOURCE_IN_RR_WR_1011 0xB
-#define I40IW_AE_SOURCE_OUT_RR 0xD
-#define I40IW_AE_SOURCE_OUT_RR_1111 0xF
-
-#define I40IW_TCP_STATE_NON_EXISTENT 0
-#define I40IW_TCP_STATE_CLOSED 1
-#define I40IW_TCP_STATE_LISTEN 2
-#define I40IW_STATE_SYN_SEND 3
-#define I40IW_TCP_STATE_SYN_RECEIVED 4
-#define I40IW_TCP_STATE_ESTABLISHED 5
-#define I40IW_TCP_STATE_CLOSE_WAIT 6
-#define I40IW_TCP_STATE_FIN_WAIT_1 7
-#define I40IW_TCP_STATE_CLOSING 8
-#define I40IW_TCP_STATE_LAST_ACK 9
-#define I40IW_TCP_STATE_FIN_WAIT_2 10
-#define I40IW_TCP_STATE_TIME_WAIT 11
-#define I40IW_TCP_STATE_RESERVED_1 12
-#define I40IW_TCP_STATE_RESERVED_2 13
-#define I40IW_TCP_STATE_RESERVED_3 14
-#define I40IW_TCP_STATE_RESERVED_4 15
-
-/* ILQ CQP hash table fields */
-#define I40IW_CQPSQ_QHASH_VLANID_SHIFT 32
-#define I40IW_CQPSQ_QHASH_VLANID_MASK \
- ((u64)0xfff << I40IW_CQPSQ_QHASH_VLANID_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_QPN_SHIFT 32
-#define I40IW_CQPSQ_QHASH_QPN_MASK \
- ((u64)0x3ffff << I40IW_CQPSQ_QHASH_QPN_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_QS_HANDLE_SHIFT 0
-#define I40IW_CQPSQ_QHASH_QS_HANDLE_MASK ((u64)0x3ff << I40IW_CQPSQ_QHASH_QS_HANDLE_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_SRC_PORT_SHIFT 16
-#define I40IW_CQPSQ_QHASH_SRC_PORT_MASK \
- ((u64)0xffff << I40IW_CQPSQ_QHASH_SRC_PORT_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_DEST_PORT_SHIFT 0
-#define I40IW_CQPSQ_QHASH_DEST_PORT_MASK \
- ((u64)0xffff << I40IW_CQPSQ_QHASH_DEST_PORT_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_ADDR0_SHIFT 32
-#define I40IW_CQPSQ_QHASH_ADDR0_MASK \
- ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR0_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_ADDR1_SHIFT 0
-#define I40IW_CQPSQ_QHASH_ADDR1_MASK \
- ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR1_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_ADDR2_SHIFT 32
-#define I40IW_CQPSQ_QHASH_ADDR2_MASK \
- ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR2_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_ADDR3_SHIFT 0
-#define I40IW_CQPSQ_QHASH_ADDR3_MASK \
- ((u64)0xffffffff << I40IW_CQPSQ_QHASH_ADDR3_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_WQEVALID_SHIFT 63
-#define I40IW_CQPSQ_QHASH_WQEVALID_MASK \
- ((u64)0x1 << I40IW_CQPSQ_QHASH_WQEVALID_SHIFT)
-#define I40IW_CQPSQ_QHASH_OPCODE_SHIFT 32
-#define I40IW_CQPSQ_QHASH_OPCODE_MASK \
- ((u64)0x3f << I40IW_CQPSQ_QHASH_OPCODE_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_MANAGE_SHIFT 61
-#define I40IW_CQPSQ_QHASH_MANAGE_MASK \
- ((u64)0x3 << I40IW_CQPSQ_QHASH_MANAGE_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_IPV4VALID_SHIFT 60
-#define I40IW_CQPSQ_QHASH_IPV4VALID_MASK \
- ((u64)0x1 << I40IW_CQPSQ_QHASH_IPV4VALID_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_VLANVALID_SHIFT 59
-#define I40IW_CQPSQ_QHASH_VLANVALID_MASK \
- ((u64)0x1 << I40IW_CQPSQ_QHASH_VLANVALID_SHIFT)
-
-#define I40IW_CQPSQ_QHASH_ENTRYTYPE_SHIFT 42
-#define I40IW_CQPSQ_QHASH_ENTRYTYPE_MASK \
- ((u64)0x7 << I40IW_CQPSQ_QHASH_ENTRYTYPE_SHIFT)
-/* CQP Host Context */
-#define I40IW_CQPHC_EN_DC_TCP_SHIFT 0
-#define I40IW_CQPHC_EN_DC_TCP_MASK (1UL << I40IW_CQPHC_EN_DC_TCP_SHIFT)
-
-#define I40IW_CQPHC_SQSIZE_SHIFT 8
-#define I40IW_CQPHC_SQSIZE_MASK (0xfUL << I40IW_CQPHC_SQSIZE_SHIFT)
-
-#define I40IW_CQPHC_DISABLE_PFPDUS_SHIFT 1
-#define I40IW_CQPHC_DISABLE_PFPDUS_MASK (0x1UL << I40IW_CQPHC_DISABLE_PFPDUS_SHIFT)
-
-#define I40IW_CQPHC_ENABLED_VFS_SHIFT 32
-#define I40IW_CQPHC_ENABLED_VFS_MASK (0x3fULL << I40IW_CQPHC_ENABLED_VFS_SHIFT)
-
-#define I40IW_CQPHC_HMC_PROFILE_SHIFT 0
-#define I40IW_CQPHC_HMC_PROFILE_MASK (0x7ULL << I40IW_CQPHC_HMC_PROFILE_SHIFT)
-
-#define I40IW_CQPHC_SVER_SHIFT 24
-#define I40IW_CQPHC_SVER_MASK (0xffUL << I40IW_CQPHC_SVER_SHIFT)
-
-#define I40IW_CQPHC_SQBASE_SHIFT 9
-#define I40IW_CQPHC_SQBASE_MASK \
- (0xfffffffffffffeULL << I40IW_CQPHC_SQBASE_SHIFT)
-
-#define I40IW_CQPHC_QPCTX_SHIFT 0
-#define I40IW_CQPHC_QPCTX_MASK \
- (0xffffffffffffffffULL << I40IW_CQPHC_QPCTX_SHIFT)
-#define I40IW_CQPHC_SVER 1
-
-#define I40IW_CQP_SW_SQSIZE_4 4
-#define I40IW_CQP_SW_SQSIZE_2048 2048
-
-/* iWARP QP Doorbell shadow area */
-#define I40IW_QP_DBSA_HW_SQ_TAIL_SHIFT 0
-#define I40IW_QP_DBSA_HW_SQ_TAIL_MASK \
- (0x3fffUL << I40IW_QP_DBSA_HW_SQ_TAIL_SHIFT)
-
-/* Completion Queue Doorbell shadow area */
-#define I40IW_CQ_DBSA_CQEIDX_SHIFT 0
-#define I40IW_CQ_DBSA_CQEIDX_MASK (0xfffffUL << I40IW_CQ_DBSA_CQEIDX_SHIFT)
-
-#define I40IW_CQ_DBSA_SW_CQ_SELECT_SHIFT 0
-#define I40IW_CQ_DBSA_SW_CQ_SELECT_MASK \
- (0x3fffUL << I40IW_CQ_DBSA_SW_CQ_SELECT_SHIFT)
-
-#define I40IW_CQ_DBSA_ARM_NEXT_SHIFT 14
-#define I40IW_CQ_DBSA_ARM_NEXT_MASK (1UL << I40IW_CQ_DBSA_ARM_NEXT_SHIFT)
-
-#define I40IW_CQ_DBSA_ARM_NEXT_SE_SHIFT 15
-#define I40IW_CQ_DBSA_ARM_NEXT_SE_MASK (1UL << I40IW_CQ_DBSA_ARM_NEXT_SE_SHIFT)
-
-#define I40IW_CQ_DBSA_ARM_SEQ_NUM_SHIFT 16
-#define I40IW_CQ_DBSA_ARM_SEQ_NUM_MASK \
- (0x3UL << I40IW_CQ_DBSA_ARM_SEQ_NUM_SHIFT)
-
-/* CQP and iWARP Completion Queue */
-#define I40IW_CQ_QPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQ_QPCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CCQ_OPRETVAL_SHIFT 0
-#define I40IW_CCQ_OPRETVAL_MASK (0xffffffffUL << I40IW_CCQ_OPRETVAL_SHIFT)
-
-#define I40IW_CQ_MINERR_SHIFT 0
-#define I40IW_CQ_MINERR_MASK (0xffffUL << I40IW_CQ_MINERR_SHIFT)
-
-#define I40IW_CQ_MAJERR_SHIFT 16
-#define I40IW_CQ_MAJERR_MASK (0xffffUL << I40IW_CQ_MAJERR_SHIFT)
-
-#define I40IW_CQ_WQEIDX_SHIFT 32
-#define I40IW_CQ_WQEIDX_MASK (0x3fffULL << I40IW_CQ_WQEIDX_SHIFT)
-
-#define I40IW_CQ_ERROR_SHIFT 55
-#define I40IW_CQ_ERROR_MASK (1ULL << I40IW_CQ_ERROR_SHIFT)
-
-#define I40IW_CQ_SQ_SHIFT 62
-#define I40IW_CQ_SQ_MASK (1ULL << I40IW_CQ_SQ_SHIFT)
-
-#define I40IW_CQ_VALID_SHIFT 63
-#define I40IW_CQ_VALID_MASK (1ULL << I40IW_CQ_VALID_SHIFT)
-
-#define I40IWCQ_PAYLDLEN_SHIFT 0
-#define I40IWCQ_PAYLDLEN_MASK (0xffffffffUL << I40IWCQ_PAYLDLEN_SHIFT)
-
-#define I40IWCQ_TCPSEQNUM_SHIFT 32
-#define I40IWCQ_TCPSEQNUM_MASK (0xffffffffULL << I40IWCQ_TCPSEQNUM_SHIFT)
-
-#define I40IWCQ_INVSTAG_SHIFT 0
-#define I40IWCQ_INVSTAG_MASK (0xffffffffUL << I40IWCQ_INVSTAG_SHIFT)
-
-#define I40IWCQ_QPID_SHIFT 32
-#define I40IWCQ_QPID_MASK (0x3ffffULL << I40IWCQ_QPID_SHIFT)
-
-#define I40IWCQ_PSHDROP_SHIFT 51
-#define I40IWCQ_PSHDROP_MASK (1ULL << I40IWCQ_PSHDROP_SHIFT)
-
-#define I40IWCQ_SRQ_SHIFT 52
-#define I40IWCQ_SRQ_MASK (1ULL << I40IWCQ_SRQ_SHIFT)
-
-#define I40IWCQ_STAG_SHIFT 53
-#define I40IWCQ_STAG_MASK (1ULL << I40IWCQ_STAG_SHIFT)
-
-#define I40IWCQ_SOEVENT_SHIFT 54
-#define I40IWCQ_SOEVENT_MASK (1ULL << I40IWCQ_SOEVENT_SHIFT)
-
-#define I40IWCQ_OP_SHIFT 56
-#define I40IWCQ_OP_MASK (0x3fULL << I40IWCQ_OP_SHIFT)
-
-/* CEQE format */
-#define I40IW_CEQE_CQCTX_SHIFT 0
-#define I40IW_CEQE_CQCTX_MASK \
- (0x7fffffffffffffffULL << I40IW_CEQE_CQCTX_SHIFT)
-
-#define I40IW_CEQE_VALID_SHIFT 63
-#define I40IW_CEQE_VALID_MASK (1ULL << I40IW_CEQE_VALID_SHIFT)
-
-/* AEQE format */
-#define I40IW_AEQE_COMPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_AEQE_COMPCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_AEQE_QPCQID_SHIFT 0
-#define I40IW_AEQE_QPCQID_MASK (0x3ffffUL << I40IW_AEQE_QPCQID_SHIFT)
-
-#define I40IW_AEQE_WQDESCIDX_SHIFT 18
-#define I40IW_AEQE_WQDESCIDX_MASK (0x3fffULL << I40IW_AEQE_WQDESCIDX_SHIFT)
-
-#define I40IW_AEQE_OVERFLOW_SHIFT 33
-#define I40IW_AEQE_OVERFLOW_MASK (1ULL << I40IW_AEQE_OVERFLOW_SHIFT)
-
-#define I40IW_AEQE_AECODE_SHIFT 34
-#define I40IW_AEQE_AECODE_MASK (0xffffULL << I40IW_AEQE_AECODE_SHIFT)
-
-#define I40IW_AEQE_AESRC_SHIFT 50
-#define I40IW_AEQE_AESRC_MASK (0xfULL << I40IW_AEQE_AESRC_SHIFT)
-
-#define I40IW_AEQE_IWSTATE_SHIFT 54
-#define I40IW_AEQE_IWSTATE_MASK (0x7ULL << I40IW_AEQE_IWSTATE_SHIFT)
-
-#define I40IW_AEQE_TCPSTATE_SHIFT 57
-#define I40IW_AEQE_TCPSTATE_MASK (0xfULL << I40IW_AEQE_TCPSTATE_SHIFT)
-
-#define I40IW_AEQE_Q2DATA_SHIFT 61
-#define I40IW_AEQE_Q2DATA_MASK (0x3ULL << I40IW_AEQE_Q2DATA_SHIFT)
-
-#define I40IW_AEQE_VALID_SHIFT 63
-#define I40IW_AEQE_VALID_MASK (1ULL << I40IW_AEQE_VALID_SHIFT)
-
-/* CQP SQ WQES */
-#define I40IW_QP_TYPE_IWARP 1
-#define I40IW_QP_TYPE_UDA 2
-#define I40IW_QP_TYPE_CQP 4
-
-#define I40IW_CQ_TYPE_IWARP 1
-#define I40IW_CQ_TYPE_ILQ 2
-#define I40IW_CQ_TYPE_IEQ 3
-#define I40IW_CQ_TYPE_CQP 4
-
-#define I40IWQP_TERM_SEND_TERM_AND_FIN 0
-#define I40IWQP_TERM_SEND_TERM_ONLY 1
-#define I40IWQP_TERM_SEND_FIN_ONLY 2
-#define I40IWQP_TERM_DONOT_SEND_TERM_OR_FIN 3
-
-#define I40IW_CQP_OP_CREATE_QP 0
-#define I40IW_CQP_OP_MODIFY_QP 0x1
-#define I40IW_CQP_OP_DESTROY_QP 0x02
-#define I40IW_CQP_OP_CREATE_CQ 0x03
-#define I40IW_CQP_OP_MODIFY_CQ 0x04
-#define I40IW_CQP_OP_DESTROY_CQ 0x05
-#define I40IW_CQP_OP_CREATE_SRQ 0x06
-#define I40IW_CQP_OP_MODIFY_SRQ 0x07
-#define I40IW_CQP_OP_DESTROY_SRQ 0x08
-#define I40IW_CQP_OP_ALLOC_STAG 0x09
-#define I40IW_CQP_OP_REG_MR 0x0a
-#define I40IW_CQP_OP_QUERY_STAG 0x0b
-#define I40IW_CQP_OP_REG_SMR 0x0c
-#define I40IW_CQP_OP_DEALLOC_STAG 0x0d
-#define I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE 0x0e
-#define I40IW_CQP_OP_MANAGE_ARP 0x0f
-#define I40IW_CQP_OP_MANAGE_VF_PBLE_BP 0x10
-#define I40IW_CQP_OP_MANAGE_PUSH_PAGES 0x11
-#define I40IW_CQP_OP_MANAGE_PE_TEAM 0x12
-#define I40IW_CQP_OP_UPLOAD_CONTEXT 0x13
-#define I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY 0x14
-#define I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE 0x15
-#define I40IW_CQP_OP_CREATE_CEQ 0x16
-#define I40IW_CQP_OP_DESTROY_CEQ 0x18
-#define I40IW_CQP_OP_CREATE_AEQ 0x19
-#define I40IW_CQP_OP_DESTROY_AEQ 0x1b
-#define I40IW_CQP_OP_CREATE_ADDR_VECT 0x1c
-#define I40IW_CQP_OP_MODIFY_ADDR_VECT 0x1d
-#define I40IW_CQP_OP_DESTROY_ADDR_VECT 0x1e
-#define I40IW_CQP_OP_UPDATE_PE_SDS 0x1f
-#define I40IW_CQP_OP_QUERY_FPM_VALUES 0x20
-#define I40IW_CQP_OP_COMMIT_FPM_VALUES 0x21
-#define I40IW_CQP_OP_FLUSH_WQES 0x22
-#define I40IW_CQP_OP_MANAGE_APBVT 0x23
-#define I40IW_CQP_OP_NOP 0x24
-#define I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY 0x25
-#define I40IW_CQP_OP_CREATE_UDA_MCAST_GROUP 0x26
-#define I40IW_CQP_OP_MODIFY_UDA_MCAST_GROUP 0x27
-#define I40IW_CQP_OP_DESTROY_UDA_MCAST_GROUP 0x28
-#define I40IW_CQP_OP_SUSPEND_QP 0x29
-#define I40IW_CQP_OP_RESUME_QP 0x2a
-#define I40IW_CQP_OP_SHMC_PAGES_ALLOCATED 0x2b
-#define I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE 0x2d
-
-#define I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT 16
-#define I40IW_UDA_QPSQ_NEXT_HEADER_MASK ((u64)0xff << I40IW_UDA_QPSQ_NEXT_HEADER_SHIFT)
-
-#define I40IW_UDA_QPSQ_OPCODE_SHIFT 32
-#define I40IW_UDA_QPSQ_OPCODE_MASK ((u64)0x3f << I40IW_UDA_QPSQ_OPCODE_SHIFT)
-
-#define I40IW_UDA_QPSQ_MACLEN_SHIFT 56
-#define I40IW_UDA_QPSQ_MACLEN_MASK \
- ((u64)0x7f << I40IW_UDA_QPSQ_MACLEN_SHIFT)
-
-#define I40IW_UDA_QPSQ_IPLEN_SHIFT 48
-#define I40IW_UDA_QPSQ_IPLEN_MASK \
- ((u64)0x7f << I40IW_UDA_QPSQ_IPLEN_SHIFT)
-
-#define I40IW_UDA_QPSQ_L4T_SHIFT 30
-#define I40IW_UDA_QPSQ_L4T_MASK \
- ((u64)0x3 << I40IW_UDA_QPSQ_L4T_SHIFT)
-
-#define I40IW_UDA_QPSQ_IIPT_SHIFT 28
-#define I40IW_UDA_QPSQ_IIPT_MASK \
- ((u64)0x3 << I40IW_UDA_QPSQ_IIPT_SHIFT)
-
-#define I40IW_UDA_QPSQ_L4LEN_SHIFT 24
-#define I40IW_UDA_QPSQ_L4LEN_MASK ((u64)0xf << I40IW_UDA_QPSQ_L4LEN_SHIFT)
-
-#define I40IW_UDA_QPSQ_AVIDX_SHIFT 0
-#define I40IW_UDA_QPSQ_AVIDX_MASK ((u64)0xffff << I40IW_UDA_QPSQ_AVIDX_SHIFT)
-
-#define I40IW_UDA_QPSQ_VALID_SHIFT 63
-#define I40IW_UDA_QPSQ_VALID_MASK \
- ((u64)0x1 << I40IW_UDA_QPSQ_VALID_SHIFT)
-
-#define I40IW_UDA_QPSQ_SIGCOMPL_SHIFT 62
-#define I40IW_UDA_QPSQ_SIGCOMPL_MASK ((u64)0x1 << I40IW_UDA_QPSQ_SIGCOMPL_SHIFT)
-
-#define I40IW_UDA_PAYLOADLEN_SHIFT 0
-#define I40IW_UDA_PAYLOADLEN_MASK ((u64)0x3fff << I40IW_UDA_PAYLOADLEN_SHIFT)
-
-#define I40IW_UDA_HDRLEN_SHIFT 16
-#define I40IW_UDA_HDRLEN_MASK ((u64)0x1ff << I40IW_UDA_HDRLEN_SHIFT)
-
-#define I40IW_VLAN_TAG_VALID_SHIFT 50
-#define I40IW_VLAN_TAG_VALID_MASK ((u64)0x1 << I40IW_VLAN_TAG_VALID_SHIFT)
-
-#define I40IW_UDA_L3PROTO_SHIFT 0
-#define I40IW_UDA_L3PROTO_MASK ((u64)0x3 << I40IW_UDA_L3PROTO_SHIFT)
-
-#define I40IW_UDA_L4PROTO_SHIFT 16
-#define I40IW_UDA_L4PROTO_MASK ((u64)0x3 << I40IW_UDA_L4PROTO_SHIFT)
-
-#define I40IW_UDA_QPSQ_DOLOOPBACK_SHIFT 44
-#define I40IW_UDA_QPSQ_DOLOOPBACK_MASK \
- ((u64)0x1 << I40IW_UDA_QPSQ_DOLOOPBACK_SHIFT)
-
-/* CQP SQ WQE common fields */
-#define I40IW_CQPSQ_OPCODE_SHIFT 32
-#define I40IW_CQPSQ_OPCODE_MASK (0x3fULL << I40IW_CQPSQ_OPCODE_SHIFT)
-
-#define I40IW_CQPSQ_WQEVALID_SHIFT 63
-#define I40IW_CQPSQ_WQEVALID_MASK (1ULL << I40IW_CQPSQ_WQEVALID_SHIFT)
-
-#define I40IW_CQPSQ_TPHVAL_SHIFT 0
-#define I40IW_CQPSQ_TPHVAL_MASK (0xffUL << I40IW_CQPSQ_TPHVAL_SHIFT)
-
-#define I40IW_CQPSQ_TPHEN_SHIFT 60
-#define I40IW_CQPSQ_TPHEN_MASK (1ULL << I40IW_CQPSQ_TPHEN_SHIFT)
-
-#define I40IW_CQPSQ_PBUFADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_PBUFADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-/* Create/Modify/Destroy QP */
-
-#define I40IW_CQPSQ_QP_NEWMSS_SHIFT 32
-#define I40IW_CQPSQ_QP_NEWMSS_MASK (0x3fffULL << I40IW_CQPSQ_QP_NEWMSS_SHIFT)
-
-#define I40IW_CQPSQ_QP_TERMLEN_SHIFT 48
-#define I40IW_CQPSQ_QP_TERMLEN_MASK (0xfULL << I40IW_CQPSQ_QP_TERMLEN_SHIFT)
-
-#define I40IW_CQPSQ_QP_QPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_QP_QPCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_QP_QPID_SHIFT 0
-#define I40IW_CQPSQ_QP_QPID_MASK (0x3FFFFUL)
-/* I40IWCQ_QPID_MASK */
-
-#define I40IW_CQPSQ_QP_OP_SHIFT 32
-#define I40IW_CQPSQ_QP_OP_MASK I40IWCQ_OP_MASK
-
-#define I40IW_CQPSQ_QP_ORDVALID_SHIFT 42
-#define I40IW_CQPSQ_QP_ORDVALID_MASK (1ULL << I40IW_CQPSQ_QP_ORDVALID_SHIFT)
-
-#define I40IW_CQPSQ_QP_TOECTXVALID_SHIFT 43
-#define I40IW_CQPSQ_QP_TOECTXVALID_MASK \
- (1ULL << I40IW_CQPSQ_QP_TOECTXVALID_SHIFT)
-
-#define I40IW_CQPSQ_QP_CACHEDVARVALID_SHIFT 44
-#define I40IW_CQPSQ_QP_CACHEDVARVALID_MASK \
- (1ULL << I40IW_CQPSQ_QP_CACHEDVARVALID_SHIFT)
-
-#define I40IW_CQPSQ_QP_VQ_SHIFT 45
-#define I40IW_CQPSQ_QP_VQ_MASK (1ULL << I40IW_CQPSQ_QP_VQ_SHIFT)
-
-#define I40IW_CQPSQ_QP_FORCELOOPBACK_SHIFT 46
-#define I40IW_CQPSQ_QP_FORCELOOPBACK_MASK \
- (1ULL << I40IW_CQPSQ_QP_FORCELOOPBACK_SHIFT)
-
-#define I40IW_CQPSQ_QP_CQNUMVALID_SHIFT 47
-#define I40IW_CQPSQ_QP_CQNUMVALID_MASK \
- (1ULL << I40IW_CQPSQ_QP_CQNUMVALID_SHIFT)
-
-#define I40IW_CQPSQ_QP_QPTYPE_SHIFT 48
-#define I40IW_CQPSQ_QP_QPTYPE_MASK (0x3ULL << I40IW_CQPSQ_QP_QPTYPE_SHIFT)
-
-#define I40IW_CQPSQ_QP_MSSCHANGE_SHIFT 52
-#define I40IW_CQPSQ_QP_MSSCHANGE_MASK (1ULL << I40IW_CQPSQ_QP_MSSCHANGE_SHIFT)
-
-#define I40IW_CQPSQ_QP_STATRSRC_SHIFT 53
-#define I40IW_CQPSQ_QP_STATRSRC_MASK (1ULL << I40IW_CQPSQ_QP_STATRSRC_SHIFT)
-
-#define I40IW_CQPSQ_QP_IGNOREMWBOUND_SHIFT 54
-#define I40IW_CQPSQ_QP_IGNOREMWBOUND_MASK \
- (1ULL << I40IW_CQPSQ_QP_IGNOREMWBOUND_SHIFT)
-
-#define I40IW_CQPSQ_QP_REMOVEHASHENTRY_SHIFT 55
-#define I40IW_CQPSQ_QP_REMOVEHASHENTRY_MASK \
- (1ULL << I40IW_CQPSQ_QP_REMOVEHASHENTRY_SHIFT)
-
-#define I40IW_CQPSQ_QP_TERMACT_SHIFT 56
-#define I40IW_CQPSQ_QP_TERMACT_MASK (0x3ULL << I40IW_CQPSQ_QP_TERMACT_SHIFT)
-
-#define I40IW_CQPSQ_QP_RESETCON_SHIFT 58
-#define I40IW_CQPSQ_QP_RESETCON_MASK (1ULL << I40IW_CQPSQ_QP_RESETCON_SHIFT)
-
-#define I40IW_CQPSQ_QP_ARPTABIDXVALID_SHIFT 59
-#define I40IW_CQPSQ_QP_ARPTABIDXVALID_MASK \
- (1ULL << I40IW_CQPSQ_QP_ARPTABIDXVALID_SHIFT)
-
-#define I40IW_CQPSQ_QP_NEXTIWSTATE_SHIFT 60
-#define I40IW_CQPSQ_QP_NEXTIWSTATE_MASK \
- (0x7ULL << I40IW_CQPSQ_QP_NEXTIWSTATE_SHIFT)
-
-#define I40IW_CQPSQ_QP_DBSHADOWADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_QP_DBSHADOWADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-/* Create/Modify/Destroy CQ */
-#define I40IW_CQPSQ_CQ_CQSIZE_SHIFT 0
-#define I40IW_CQPSQ_CQ_CQSIZE_MASK (0x3ffffUL << I40IW_CQPSQ_CQ_CQSIZE_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CQCTX_SHIFT 0
-#define I40IW_CQPSQ_CQ_CQCTX_MASK \
- (0x7fffffffffffffffULL << I40IW_CQPSQ_CQ_CQCTX_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CQCTX_SHIFT 0
-#define I40IW_CQPSQ_CQ_CQCTX_MASK \
- (0x7fffffffffffffffULL << I40IW_CQPSQ_CQ_CQCTX_SHIFT)
-
-#define I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_SHIFT 0
-#define I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_MASK \
- (0x3ffff << I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CEQID_SHIFT 24
-#define I40IW_CQPSQ_CQ_CEQID_MASK (0x7fUL << I40IW_CQPSQ_CQ_CEQID_SHIFT)
-
-#define I40IW_CQPSQ_CQ_OP_SHIFT 32
-#define I40IW_CQPSQ_CQ_OP_MASK (0x3fULL << I40IW_CQPSQ_CQ_OP_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CQRESIZE_SHIFT 43
-#define I40IW_CQPSQ_CQ_CQRESIZE_MASK (1ULL << I40IW_CQPSQ_CQ_CQRESIZE_SHIFT)
-
-#define I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT 44
-#define I40IW_CQPSQ_CQ_LPBLSIZE_MASK (3ULL << I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CHKOVERFLOW_SHIFT 46
-#define I40IW_CQPSQ_CQ_CHKOVERFLOW_MASK \
- (1ULL << I40IW_CQPSQ_CQ_CHKOVERFLOW_SHIFT)
-
-#define I40IW_CQPSQ_CQ_VIRTMAP_SHIFT 47
-#define I40IW_CQPSQ_CQ_VIRTMAP_MASK (1ULL << I40IW_CQPSQ_CQ_VIRTMAP_SHIFT)
-
-#define I40IW_CQPSQ_CQ_ENCEQEMASK_SHIFT 48
-#define I40IW_CQPSQ_CQ_ENCEQEMASK_MASK \
- (1ULL << I40IW_CQPSQ_CQ_ENCEQEMASK_SHIFT)
-
-#define I40IW_CQPSQ_CQ_CEQIDVALID_SHIFT 49
-#define I40IW_CQPSQ_CQ_CEQIDVALID_MASK \
- (1ULL << I40IW_CQPSQ_CQ_CEQIDVALID_SHIFT)
-
-#define I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_SHIFT 61
-#define I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_MASK \
- (1ULL << I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT_SHIFT)
-
-/* Create/Modify/Destroy Shared Receive Queue */
-
-#define I40IW_CQPSQ_SRQ_RQSIZE_SHIFT 0
-#define I40IW_CQPSQ_SRQ_RQSIZE_MASK (0xfUL << I40IW_CQPSQ_SRQ_RQSIZE_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_RQWQESIZE_SHIFT 4
-#define I40IW_CQPSQ_SRQ_RQWQESIZE_MASK \
- (0x7UL << I40IW_CQPSQ_SRQ_RQWQESIZE_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_SRQLIMIT_SHIFT 32
-#define I40IW_CQPSQ_SRQ_SRQLIMIT_MASK \
- (0xfffULL << I40IW_CQPSQ_SRQ_SRQLIMIT_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_SRQCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_SRQ_SRQCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_SRQ_PDID_SHIFT 16
-#define I40IW_CQPSQ_SRQ_PDID_MASK \
- (0x7fffULL << I40IW_CQPSQ_SRQ_PDID_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_SRQID_SHIFT 0
-#define I40IW_CQPSQ_SRQ_SRQID_MASK (0x7fffUL << I40IW_CQPSQ_SRQ_SRQID_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
-#define I40IW_CQPSQ_SRQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
-
-#define I40IW_CQPSQ_SRQ_VIRTMAP_SHIFT I40IW_CQPSQ_CQ_VIRTMAP_SHIFT
-#define I40IW_CQPSQ_SRQ_VIRTMAP_MASK I40IW_CQPSQ_CQ_VIRTMAP_MASK
-
-#define I40IW_CQPSQ_SRQ_TPHEN_SHIFT I40IW_CQPSQ_TPHEN_SHIFT
-#define I40IW_CQPSQ_SRQ_TPHEN_MASK I40IW_CQPSQ_TPHEN_MASK
-
-#define I40IW_CQPSQ_SRQ_ARMLIMITEVENT_SHIFT 61
-#define I40IW_CQPSQ_SRQ_ARMLIMITEVENT_MASK \
- (1ULL << I40IW_CQPSQ_SRQ_ARMLIMITEVENT_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_DBSHADOWAREA_SHIFT 6
-#define I40IW_CQPSQ_SRQ_DBSHADOWAREA_MASK \
- (0x3ffffffffffffffULL << I40IW_CQPSQ_SRQ_DBSHADOWAREA_SHIFT)
-
-#define I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_SHIFT 0
-#define I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_MASK \
- (0xfffffffUL << I40IW_CQPSQ_SRQ_FIRSTPMPBLIDX_SHIFT)
-
-/* Allocate/Register/Register Shared/Deallocate Stag */
-#define I40IW_CQPSQ_STAG_VA_FBO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_STAG_VA_FBO_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_STAG_STAGLEN_SHIFT 0
-#define I40IW_CQPSQ_STAG_STAGLEN_MASK \
- (0x3fffffffffffULL << I40IW_CQPSQ_STAG_STAGLEN_SHIFT)
-
-#define I40IW_CQPSQ_STAG_PDID_SHIFT 48
-#define I40IW_CQPSQ_STAG_PDID_MASK (0x7fffULL << I40IW_CQPSQ_STAG_PDID_SHIFT)
-
-#define I40IW_CQPSQ_STAG_KEY_SHIFT 0
-#define I40IW_CQPSQ_STAG_KEY_MASK (0xffUL << I40IW_CQPSQ_STAG_KEY_SHIFT)
-
-#define I40IW_CQPSQ_STAG_IDX_SHIFT 8
-#define I40IW_CQPSQ_STAG_IDX_MASK (0xffffffUL << I40IW_CQPSQ_STAG_IDX_SHIFT)
-
-#define I40IW_CQPSQ_STAG_PARENTSTAGIDX_SHIFT 32
-#define I40IW_CQPSQ_STAG_PARENTSTAGIDX_MASK \
- (0xffffffULL << I40IW_CQPSQ_STAG_PARENTSTAGIDX_SHIFT)
-
-#define I40IW_CQPSQ_STAG_MR_SHIFT 43
-#define I40IW_CQPSQ_STAG_MR_MASK (1ULL << I40IW_CQPSQ_STAG_MR_SHIFT)
-
-#define I40IW_CQPSQ_STAG_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
-#define I40IW_CQPSQ_STAG_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
-
-#define I40IW_CQPSQ_STAG_HPAGESIZE_SHIFT 46
-#define I40IW_CQPSQ_STAG_HPAGESIZE_MASK \
- (1ULL << I40IW_CQPSQ_STAG_HPAGESIZE_SHIFT)
-
-#define I40IW_CQPSQ_STAG_ARIGHTS_SHIFT 48
-#define I40IW_CQPSQ_STAG_ARIGHTS_MASK \
- (0x1fULL << I40IW_CQPSQ_STAG_ARIGHTS_SHIFT)
-
-#define I40IW_CQPSQ_STAG_REMACCENABLED_SHIFT 53
-#define I40IW_CQPSQ_STAG_REMACCENABLED_MASK \
- (1ULL << I40IW_CQPSQ_STAG_REMACCENABLED_SHIFT)
-
-#define I40IW_CQPSQ_STAG_VABASEDTO_SHIFT 59
-#define I40IW_CQPSQ_STAG_VABASEDTO_MASK \
- (1ULL << I40IW_CQPSQ_STAG_VABASEDTO_SHIFT)
-
-#define I40IW_CQPSQ_STAG_USEHMCFNIDX_SHIFT 60
-#define I40IW_CQPSQ_STAG_USEHMCFNIDX_MASK \
- (1ULL << I40IW_CQPSQ_STAG_USEHMCFNIDX_SHIFT)
-
-#define I40IW_CQPSQ_STAG_USEPFRID_SHIFT 61
-#define I40IW_CQPSQ_STAG_USEPFRID_MASK \
- (1ULL << I40IW_CQPSQ_STAG_USEPFRID_SHIFT)
-
-#define I40IW_CQPSQ_STAG_PBA_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_STAG_PBA_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_STAG_HMCFNIDX_SHIFT 0
-#define I40IW_CQPSQ_STAG_HMCFNIDX_MASK \
- (0x3fUL << I40IW_CQPSQ_STAG_HMCFNIDX_SHIFT)
-
-#define I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_SHIFT 0
-#define I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_MASK \
- (0xfffffffUL << I40IW_CQPSQ_STAG_FIRSTPMPBLIDX_SHIFT)
-
-/* Query stag */
-#define I40IW_CQPSQ_QUERYSTAG_IDX_SHIFT I40IW_CQPSQ_STAG_IDX_SHIFT
-#define I40IW_CQPSQ_QUERYSTAG_IDX_MASK I40IW_CQPSQ_STAG_IDX_MASK
-
-/* Allocate Local IP Address Entry */
-
-/* Manage Local IP Address Table - MLIPA */
-#define I40IW_CQPSQ_MLIPA_IPV6LO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_MLIPA_IPV6LO_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_MLIPA_IPV6HI_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_MLIPA_IPV6HI_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_MLIPA_IPV4_SHIFT 0
-#define I40IW_CQPSQ_MLIPA_IPV4_MASK \
- (0xffffffffUL << I40IW_CQPSQ_MLIPA_IPV4_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_IPTABLEIDX_SHIFT 0
-#define I40IW_CQPSQ_MLIPA_IPTABLEIDX_MASK \
- (0x3fUL << I40IW_CQPSQ_MLIPA_IPTABLEIDX_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_IPV4VALID_SHIFT 42
-#define I40IW_CQPSQ_MLIPA_IPV4VALID_MASK \
- (1ULL << I40IW_CQPSQ_MLIPA_IPV4VALID_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_IPV6VALID_SHIFT 43
-#define I40IW_CQPSQ_MLIPA_IPV6VALID_MASK \
- (1ULL << I40IW_CQPSQ_MLIPA_IPV6VALID_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_FREEENTRY_SHIFT 62
-#define I40IW_CQPSQ_MLIPA_FREEENTRY_MASK \
- (1ULL << I40IW_CQPSQ_MLIPA_FREEENTRY_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_SHIFT 61
-#define I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_MASK \
- (1ULL << I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC0_SHIFT 0
-#define I40IW_CQPSQ_MLIPA_MAC0_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC0_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC1_SHIFT 8
-#define I40IW_CQPSQ_MLIPA_MAC1_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC1_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC2_SHIFT 16
-#define I40IW_CQPSQ_MLIPA_MAC2_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC2_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC3_SHIFT 24
-#define I40IW_CQPSQ_MLIPA_MAC3_MASK (0xffUL << I40IW_CQPSQ_MLIPA_MAC3_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC4_SHIFT 32
-#define I40IW_CQPSQ_MLIPA_MAC4_MASK (0xffULL << I40IW_CQPSQ_MLIPA_MAC4_SHIFT)
-
-#define I40IW_CQPSQ_MLIPA_MAC5_SHIFT 40
-#define I40IW_CQPSQ_MLIPA_MAC5_MASK (0xffULL << I40IW_CQPSQ_MLIPA_MAC5_SHIFT)
-
-/* Manage ARP Table - MAT */
-#define I40IW_CQPSQ_MAT_REACHMAX_SHIFT 0
-#define I40IW_CQPSQ_MAT_REACHMAX_MASK \
- (0xffffffffUL << I40IW_CQPSQ_MAT_REACHMAX_SHIFT)
-
-#define I40IW_CQPSQ_MAT_MACADDR_SHIFT 0
-#define I40IW_CQPSQ_MAT_MACADDR_MASK \
- (0xffffffffffffULL << I40IW_CQPSQ_MAT_MACADDR_SHIFT)
-
-#define I40IW_CQPSQ_MAT_ARPENTRYIDX_SHIFT 0
-#define I40IW_CQPSQ_MAT_ARPENTRYIDX_MASK \
- (0xfffUL << I40IW_CQPSQ_MAT_ARPENTRYIDX_SHIFT)
-
-#define I40IW_CQPSQ_MAT_ENTRYVALID_SHIFT 42
-#define I40IW_CQPSQ_MAT_ENTRYVALID_MASK \
- (1ULL << I40IW_CQPSQ_MAT_ENTRYVALID_SHIFT)
-
-#define I40IW_CQPSQ_MAT_PERMANENT_SHIFT 43
-#define I40IW_CQPSQ_MAT_PERMANENT_MASK \
- (1ULL << I40IW_CQPSQ_MAT_PERMANENT_SHIFT)
-
-#define I40IW_CQPSQ_MAT_QUERY_SHIFT 44
-#define I40IW_CQPSQ_MAT_QUERY_MASK (1ULL << I40IW_CQPSQ_MAT_QUERY_SHIFT)
-
-/* Manage VF PBLE Backing Pages - MVPBP*/
-#define I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_SHIFT 0
-#define I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_MASK \
- (0x3ffULL << I40IW_CQPSQ_MVPBP_PD_ENTRY_CNT_SHIFT)
-
-#define I40IW_CQPSQ_MVPBP_FIRST_PD_INX_SHIFT 16
-#define I40IW_CQPSQ_MVPBP_FIRST_PD_INX_MASK \
- (0x1ffULL << I40IW_CQPSQ_MVPBP_FIRST_PD_INX_SHIFT)
-
-#define I40IW_CQPSQ_MVPBP_SD_INX_SHIFT 32
-#define I40IW_CQPSQ_MVPBP_SD_INX_MASK \
- (0xfffULL << I40IW_CQPSQ_MVPBP_SD_INX_SHIFT)
-
-#define I40IW_CQPSQ_MVPBP_INV_PD_ENT_SHIFT 62
-#define I40IW_CQPSQ_MVPBP_INV_PD_ENT_MASK \
- (0x1ULL << I40IW_CQPSQ_MVPBP_INV_PD_ENT_SHIFT)
-
-#define I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT 3
-#define I40IW_CQPSQ_MVPBP_PD_PLPBA_MASK \
- (0x1fffffffffffffffULL << I40IW_CQPSQ_MVPBP_PD_PLPBA_SHIFT)
-
-/* Manage Push Page - MPP */
-#define I40IW_INVALID_PUSH_PAGE_INDEX 0xffff
-
-#define I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT 0
-#define I40IW_CQPSQ_MPP_QS_HANDLE_MASK (0xffffUL << \
- I40IW_CQPSQ_MPP_QS_HANDLE_SHIFT)
-
-#define I40IW_CQPSQ_MPP_PPIDX_SHIFT 0
-#define I40IW_CQPSQ_MPP_PPIDX_MASK (0x3ffUL << I40IW_CQPSQ_MPP_PPIDX_SHIFT)
-
-#define I40IW_CQPSQ_MPP_FREE_PAGE_SHIFT 62
-#define I40IW_CQPSQ_MPP_FREE_PAGE_MASK (1ULL << I40IW_CQPSQ_MPP_FREE_PAGE_SHIFT)
-
-/* Upload Context - UCTX */
-#define I40IW_CQPSQ_UCTX_QPCTXADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IW_CQPSQ_UCTX_QPCTXADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IW_CQPSQ_UCTX_QPID_SHIFT 0
-#define I40IW_CQPSQ_UCTX_QPID_MASK (0x3ffffUL << I40IW_CQPSQ_UCTX_QPID_SHIFT)
-
-#define I40IW_CQPSQ_UCTX_QPTYPE_SHIFT 48
-#define I40IW_CQPSQ_UCTX_QPTYPE_MASK (0xfULL << I40IW_CQPSQ_UCTX_QPTYPE_SHIFT)
-
-#define I40IW_CQPSQ_UCTX_RAWFORMAT_SHIFT 61
-#define I40IW_CQPSQ_UCTX_RAWFORMAT_MASK \
- (1ULL << I40IW_CQPSQ_UCTX_RAWFORMAT_SHIFT)
-
-#define I40IW_CQPSQ_UCTX_FREEZEQP_SHIFT 62
-#define I40IW_CQPSQ_UCTX_FREEZEQP_MASK \
- (1ULL << I40IW_CQPSQ_UCTX_FREEZEQP_SHIFT)
-
-/* Manage HMC PM Function Table - MHMC */
-#define I40IW_CQPSQ_MHMC_VFIDX_SHIFT 0
-#define I40IW_CQPSQ_MHMC_VFIDX_MASK (0x7fUL << I40IW_CQPSQ_MHMC_VFIDX_SHIFT)
-
-#define I40IW_CQPSQ_MHMC_FREEPMFN_SHIFT 62
-#define I40IW_CQPSQ_MHMC_FREEPMFN_MASK \
- (1ULL << I40IW_CQPSQ_MHMC_FREEPMFN_SHIFT)
-
-/* Set HMC Resource Profile - SHMCRP */
-#define I40IW_CQPSQ_SHMCRP_HMC_PROFILE_SHIFT 0
-#define I40IW_CQPSQ_SHMCRP_HMC_PROFILE_MASK \
- (0x7ULL << I40IW_CQPSQ_SHMCRP_HMC_PROFILE_SHIFT)
-#define I40IW_CQPSQ_SHMCRP_VFNUM_SHIFT 32
-#define I40IW_CQPSQ_SHMCRP_VFNUM_MASK (0x3fULL << I40IW_CQPSQ_SHMCRP_VFNUM_SHIFT)
-
-/* Create/Destroy CEQ */
-#define I40IW_CQPSQ_CEQ_CEQSIZE_SHIFT 0
-#define I40IW_CQPSQ_CEQ_CEQSIZE_MASK \
- (0x1ffffUL << I40IW_CQPSQ_CEQ_CEQSIZE_SHIFT)
-
-#define I40IW_CQPSQ_CEQ_CEQID_SHIFT 0
-#define I40IW_CQPSQ_CEQ_CEQID_MASK (0x7fUL << I40IW_CQPSQ_CEQ_CEQID_SHIFT)
-
-#define I40IW_CQPSQ_CEQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
-#define I40IW_CQPSQ_CEQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
-
-#define I40IW_CQPSQ_CEQ_VMAP_SHIFT 47
-#define I40IW_CQPSQ_CEQ_VMAP_MASK (1ULL << I40IW_CQPSQ_CEQ_VMAP_SHIFT)
-
-#define I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_SHIFT 0
-#define I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_MASK \
- (0xfffffffUL << I40IW_CQPSQ_CEQ_FIRSTPMPBLIDX_SHIFT)
-
-/* Create/Destroy AEQ */
-#define I40IW_CQPSQ_AEQ_AEQECNT_SHIFT 0
-#define I40IW_CQPSQ_AEQ_AEQECNT_MASK \
- (0x7ffffUL << I40IW_CQPSQ_AEQ_AEQECNT_SHIFT)
-
-#define I40IW_CQPSQ_AEQ_LPBLSIZE_SHIFT I40IW_CQPSQ_CQ_LPBLSIZE_SHIFT
-#define I40IW_CQPSQ_AEQ_LPBLSIZE_MASK I40IW_CQPSQ_CQ_LPBLSIZE_MASK
-
-#define I40IW_CQPSQ_AEQ_VMAP_SHIFT 47
-#define I40IW_CQPSQ_AEQ_VMAP_MASK (1ULL << I40IW_CQPSQ_AEQ_VMAP_SHIFT)
-
-#define I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_SHIFT 0
-#define I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_MASK \
- (0xfffffffUL << I40IW_CQPSQ_AEQ_FIRSTPMPBLIDX_SHIFT)
-
-/* Commit FPM Values - CFPM */
-#define I40IW_CQPSQ_CFPM_HMCFNID_SHIFT 0
-#define I40IW_CQPSQ_CFPM_HMCFNID_MASK (0x3fUL << I40IW_CQPSQ_CFPM_HMCFNID_SHIFT)
-
-/* Flush WQEs - FWQE */
-#define I40IW_CQPSQ_FWQE_AECODE_SHIFT 0
-#define I40IW_CQPSQ_FWQE_AECODE_MASK (0xffffUL << I40IW_CQPSQ_FWQE_AECODE_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_AESOURCE_SHIFT 16
-#define I40IW_CQPSQ_FWQE_AESOURCE_MASK \
- (0xfUL << I40IW_CQPSQ_FWQE_AESOURCE_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_RQMNERR_SHIFT 0
-#define I40IW_CQPSQ_FWQE_RQMNERR_MASK \
- (0xffffUL << I40IW_CQPSQ_FWQE_RQMNERR_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_RQMJERR_SHIFT 16
-#define I40IW_CQPSQ_FWQE_RQMJERR_MASK \
- (0xffffUL << I40IW_CQPSQ_FWQE_RQMJERR_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_SQMNERR_SHIFT 32
-#define I40IW_CQPSQ_FWQE_SQMNERR_MASK \
- (0xffffULL << I40IW_CQPSQ_FWQE_SQMNERR_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_SQMJERR_SHIFT 48
-#define I40IW_CQPSQ_FWQE_SQMJERR_MASK \
- (0xffffULL << I40IW_CQPSQ_FWQE_SQMJERR_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_QPID_SHIFT 0
-#define I40IW_CQPSQ_FWQE_QPID_MASK (0x3ffffULL << I40IW_CQPSQ_FWQE_QPID_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_GENERATE_AE_SHIFT 59
-#define I40IW_CQPSQ_FWQE_GENERATE_AE_MASK (1ULL << \
- I40IW_CQPSQ_FWQE_GENERATE_AE_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_USERFLCODE_SHIFT 60
-#define I40IW_CQPSQ_FWQE_USERFLCODE_MASK \
- (1ULL << I40IW_CQPSQ_FWQE_USERFLCODE_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_FLUSHSQ_SHIFT 61
-#define I40IW_CQPSQ_FWQE_FLUSHSQ_MASK (1ULL << I40IW_CQPSQ_FWQE_FLUSHSQ_SHIFT)
-
-#define I40IW_CQPSQ_FWQE_FLUSHRQ_SHIFT 62
-#define I40IW_CQPSQ_FWQE_FLUSHRQ_MASK (1ULL << I40IW_CQPSQ_FWQE_FLUSHRQ_SHIFT)
-
-/* Manage Accelerated Port Table - MAPT */
-#define I40IW_CQPSQ_MAPT_PORT_SHIFT 0
-#define I40IW_CQPSQ_MAPT_PORT_MASK (0xffffUL << I40IW_CQPSQ_MAPT_PORT_SHIFT)
-
-#define I40IW_CQPSQ_MAPT_ADDPORT_SHIFT 62
-#define I40IW_CQPSQ_MAPT_ADDPORT_MASK (1ULL << I40IW_CQPSQ_MAPT_ADDPORT_SHIFT)
-
-/* Update Protocol Engine SDs */
-#define I40IW_CQPSQ_UPESD_SDCMD_SHIFT 0
-#define I40IW_CQPSQ_UPESD_SDCMD_MASK (0xffffffffUL << I40IW_CQPSQ_UPESD_SDCMD_SHIFT)
-
-#define I40IW_CQPSQ_UPESD_SDDATALOW_SHIFT 0
-#define I40IW_CQPSQ_UPESD_SDDATALOW_MASK \
- (0xffffffffUL << I40IW_CQPSQ_UPESD_SDDATALOW_SHIFT)
-
-#define I40IW_CQPSQ_UPESD_SDDATAHI_SHIFT 32
-#define I40IW_CQPSQ_UPESD_SDDATAHI_MASK \
- (0xffffffffULL << I40IW_CQPSQ_UPESD_SDDATAHI_SHIFT)
-#define I40IW_CQPSQ_UPESD_HMCFNID_SHIFT 0
-#define I40IW_CQPSQ_UPESD_HMCFNID_MASK \
- (0x3fUL << I40IW_CQPSQ_UPESD_HMCFNID_SHIFT)
-
-#define I40IW_CQPSQ_UPESD_ENTRY_VALID_SHIFT 63
-#define I40IW_CQPSQ_UPESD_ENTRY_VALID_MASK \
- ((u64)1 << I40IW_CQPSQ_UPESD_ENTRY_VALID_SHIFT)
-
-#define I40IW_CQPSQ_UPESD_ENTRY_COUNT_SHIFT 0
-#define I40IW_CQPSQ_UPESD_ENTRY_COUNT_MASK \
- (0xfUL << I40IW_CQPSQ_UPESD_ENTRY_COUNT_SHIFT)
-
-#define I40IW_CQPSQ_UPESD_SKIP_ENTRY_SHIFT 7
-#define I40IW_CQPSQ_UPESD_SKIP_ENTRY_MASK \
- (0x1UL << I40IW_CQPSQ_UPESD_SKIP_ENTRY_SHIFT)
-
-/* Suspend QP */
-#define I40IW_CQPSQ_SUSPENDQP_QPID_SHIFT 0
-#define I40IW_CQPSQ_SUSPENDQP_QPID_MASK (0x3FFFFUL)
-/* I40IWCQ_QPID_MASK */
-
-/* Resume QP */
-#define I40IW_CQPSQ_RESUMEQP_QSHANDLE_SHIFT 0
-#define I40IW_CQPSQ_RESUMEQP_QSHANDLE_MASK \
- (0xffffffffUL << I40IW_CQPSQ_RESUMEQP_QSHANDLE_SHIFT)
-
-#define I40IW_CQPSQ_RESUMEQP_QPID_SHIFT 0
-#define I40IW_CQPSQ_RESUMEQP_QPID_MASK (0x3FFFFUL)
-/* I40IWCQ_QPID_MASK */
-
-/* IW QP Context */
-#define I40IWQPC_DDP_VER_SHIFT 0
-#define I40IWQPC_DDP_VER_MASK (3UL << I40IWQPC_DDP_VER_SHIFT)
-
-#define I40IWQPC_SNAP_SHIFT 2
-#define I40IWQPC_SNAP_MASK (1UL << I40IWQPC_SNAP_SHIFT)
-
-#define I40IWQPC_IPV4_SHIFT 3
-#define I40IWQPC_IPV4_MASK (1UL << I40IWQPC_IPV4_SHIFT)
-
-#define I40IWQPC_NONAGLE_SHIFT 4
-#define I40IWQPC_NONAGLE_MASK (1UL << I40IWQPC_NONAGLE_SHIFT)
-
-#define I40IWQPC_INSERTVLANTAG_SHIFT 5
-#define I40IWQPC_INSERTVLANTAG_MASK (1 << I40IWQPC_INSERTVLANTAG_SHIFT)
-
-#define I40IWQPC_USESRQ_SHIFT 6
-#define I40IWQPC_USESRQ_MASK (1UL << I40IWQPC_USESRQ_SHIFT)
-
-#define I40IWQPC_TIMESTAMP_SHIFT 7
-#define I40IWQPC_TIMESTAMP_MASK (1UL << I40IWQPC_TIMESTAMP_SHIFT)
-
-#define I40IWQPC_RQWQESIZE_SHIFT 8
-#define I40IWQPC_RQWQESIZE_MASK (3UL << I40IWQPC_RQWQESIZE_SHIFT)
-
-#define I40IWQPC_INSERTL2TAG2_SHIFT 11
-#define I40IWQPC_INSERTL2TAG2_MASK (1UL << I40IWQPC_INSERTL2TAG2_SHIFT)
-
-#define I40IWQPC_LIMIT_SHIFT 12
-#define I40IWQPC_LIMIT_MASK (3UL << I40IWQPC_LIMIT_SHIFT)
-
-#define I40IWQPC_DROPOOOSEG_SHIFT 15
-#define I40IWQPC_DROPOOOSEG_MASK (1UL << I40IWQPC_DROPOOOSEG_SHIFT)
-
-#define I40IWQPC_DUPACK_THRESH_SHIFT 16
-#define I40IWQPC_DUPACK_THRESH_MASK (7UL << I40IWQPC_DUPACK_THRESH_SHIFT)
-
-#define I40IWQPC_ERR_RQ_IDX_VALID_SHIFT 19
-#define I40IWQPC_ERR_RQ_IDX_VALID_MASK (1UL << I40IWQPC_ERR_RQ_IDX_VALID_SHIFT)
-
-#define I40IWQPC_DIS_VLAN_CHECKS_SHIFT 19
-#define I40IWQPC_DIS_VLAN_CHECKS_MASK (7UL << I40IWQPC_DIS_VLAN_CHECKS_SHIFT)
-
-#define I40IWQPC_RCVTPHEN_SHIFT 28
-#define I40IWQPC_RCVTPHEN_MASK (1UL << I40IWQPC_RCVTPHEN_SHIFT)
-
-#define I40IWQPC_XMITTPHEN_SHIFT 29
-#define I40IWQPC_XMITTPHEN_MASK (1ULL << I40IWQPC_XMITTPHEN_SHIFT)
-
-#define I40IWQPC_RQTPHEN_SHIFT 30
-#define I40IWQPC_RQTPHEN_MASK (1UL << I40IWQPC_RQTPHEN_SHIFT)
-
-#define I40IWQPC_SQTPHEN_SHIFT 31
-#define I40IWQPC_SQTPHEN_MASK (1ULL << I40IWQPC_SQTPHEN_SHIFT)
-
-#define I40IWQPC_PPIDX_SHIFT 32
-#define I40IWQPC_PPIDX_MASK (0x3ffULL << I40IWQPC_PPIDX_SHIFT)
-
-#define I40IWQPC_PMENA_SHIFT 47
-#define I40IWQPC_PMENA_MASK (1ULL << I40IWQPC_PMENA_SHIFT)
-
-#define I40IWQPC_RDMAP_VER_SHIFT 62
-#define I40IWQPC_RDMAP_VER_MASK (3ULL << I40IWQPC_RDMAP_VER_SHIFT)
-
-#define I40IWQPC_SQADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPC_SQADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPC_RQADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPC_RQADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPC_TTL_SHIFT 0
-#define I40IWQPC_TTL_MASK (0xffUL << I40IWQPC_TTL_SHIFT)
-
-#define I40IWQPC_RQSIZE_SHIFT 8
-#define I40IWQPC_RQSIZE_MASK (0xfUL << I40IWQPC_RQSIZE_SHIFT)
-
-#define I40IWQPC_SQSIZE_SHIFT 12
-#define I40IWQPC_SQSIZE_MASK (0xfUL << I40IWQPC_SQSIZE_SHIFT)
-
-#define I40IWQPC_SRCMACADDRIDX_SHIFT 16
-#define I40IWQPC_SRCMACADDRIDX_MASK (0x3fUL << I40IWQPC_SRCMACADDRIDX_SHIFT)
-
-#define I40IWQPC_AVOIDSTRETCHACK_SHIFT 23
-#define I40IWQPC_AVOIDSTRETCHACK_MASK (1UL << I40IWQPC_AVOIDSTRETCHACK_SHIFT)
-
-#define I40IWQPC_TOS_SHIFT 24
-#define I40IWQPC_TOS_MASK (0xffUL << I40IWQPC_TOS_SHIFT)
-
-#define I40IWQPC_SRCPORTNUM_SHIFT 32
-#define I40IWQPC_SRCPORTNUM_MASK (0xffffULL << I40IWQPC_SRCPORTNUM_SHIFT)
-
-#define I40IWQPC_DESTPORTNUM_SHIFT 48
-#define I40IWQPC_DESTPORTNUM_MASK (0xffffULL << I40IWQPC_DESTPORTNUM_SHIFT)
-
-#define I40IWQPC_DESTIPADDR0_SHIFT 32
-#define I40IWQPC_DESTIPADDR0_MASK \
- (0xffffffffULL << I40IWQPC_DESTIPADDR0_SHIFT)
-
-#define I40IWQPC_DESTIPADDR1_SHIFT 0
-#define I40IWQPC_DESTIPADDR1_MASK \
- (0xffffffffULL << I40IWQPC_DESTIPADDR1_SHIFT)
-
-#define I40IWQPC_DESTIPADDR2_SHIFT 32
-#define I40IWQPC_DESTIPADDR2_MASK \
- (0xffffffffULL << I40IWQPC_DESTIPADDR2_SHIFT)
-
-#define I40IWQPC_DESTIPADDR3_SHIFT 0
-#define I40IWQPC_DESTIPADDR3_MASK \
- (0xffffffffULL << I40IWQPC_DESTIPADDR3_SHIFT)
-
-#define I40IWQPC_SNDMSS_SHIFT 16
-#define I40IWQPC_SNDMSS_MASK (0x3fffUL << I40IWQPC_SNDMSS_SHIFT)
-
-#define I40IWQPC_VLANTAG_SHIFT 32
-#define I40IWQPC_VLANTAG_MASK (0xffffULL << I40IWQPC_VLANTAG_SHIFT)
-
-#define I40IWQPC_ARPIDX_SHIFT 48
-#define I40IWQPC_ARPIDX_MASK (0xfffULL << I40IWQPC_ARPIDX_SHIFT)
-
-#define I40IWQPC_FLOWLABEL_SHIFT 0
-#define I40IWQPC_FLOWLABEL_MASK (0xfffffUL << I40IWQPC_FLOWLABEL_SHIFT)
-
-#define I40IWQPC_WSCALE_SHIFT 20
-#define I40IWQPC_WSCALE_MASK (1UL << I40IWQPC_WSCALE_SHIFT)
-
-#define I40IWQPC_KEEPALIVE_SHIFT 21
-#define I40IWQPC_KEEPALIVE_MASK (1UL << I40IWQPC_KEEPALIVE_SHIFT)
-
-#define I40IWQPC_IGNORE_TCP_OPT_SHIFT 22
-#define I40IWQPC_IGNORE_TCP_OPT_MASK (1UL << I40IWQPC_IGNORE_TCP_OPT_SHIFT)
-
-#define I40IWQPC_IGNORE_TCP_UNS_OPT_SHIFT 23
-#define I40IWQPC_IGNORE_TCP_UNS_OPT_MASK \
- (1UL << I40IWQPC_IGNORE_TCP_UNS_OPT_SHIFT)
-
-#define I40IWQPC_TCPSTATE_SHIFT 28
-#define I40IWQPC_TCPSTATE_MASK (0xfUL << I40IWQPC_TCPSTATE_SHIFT)
-
-#define I40IWQPC_RCVSCALE_SHIFT 32
-#define I40IWQPC_RCVSCALE_MASK (0xfULL << I40IWQPC_RCVSCALE_SHIFT)
-
-#define I40IWQPC_SNDSCALE_SHIFT 40
-#define I40IWQPC_SNDSCALE_MASK (0xfULL << I40IWQPC_SNDSCALE_SHIFT)
-
-#define I40IWQPC_PDIDX_SHIFT 48
-#define I40IWQPC_PDIDX_MASK (0x7fffULL << I40IWQPC_PDIDX_SHIFT)
-
-#define I40IWQPC_KALIVE_TIMER_MAX_PROBES_SHIFT 16
-#define I40IWQPC_KALIVE_TIMER_MAX_PROBES_MASK \
- (0xffUL << I40IWQPC_KALIVE_TIMER_MAX_PROBES_SHIFT)
-
-#define I40IWQPC_KEEPALIVE_INTERVAL_SHIFT 24
-#define I40IWQPC_KEEPALIVE_INTERVAL_MASK \
- (0xffUL << I40IWQPC_KEEPALIVE_INTERVAL_SHIFT)
-
-#define I40IWQPC_TIMESTAMP_RECENT_SHIFT 0
-#define I40IWQPC_TIMESTAMP_RECENT_MASK \
- (0xffffffffUL << I40IWQPC_TIMESTAMP_RECENT_SHIFT)
-
-#define I40IWQPC_TIMESTAMP_AGE_SHIFT 32
-#define I40IWQPC_TIMESTAMP_AGE_MASK \
- (0xffffffffULL << I40IWQPC_TIMESTAMP_AGE_SHIFT)
-
-#define I40IWQPC_SNDNXT_SHIFT 0
-#define I40IWQPC_SNDNXT_MASK (0xffffffffUL << I40IWQPC_SNDNXT_SHIFT)
-
-#define I40IWQPC_SNDWND_SHIFT 32
-#define I40IWQPC_SNDWND_MASK (0xffffffffULL << I40IWQPC_SNDWND_SHIFT)
-
-#define I40IWQPC_RCVNXT_SHIFT 0
-#define I40IWQPC_RCVNXT_MASK (0xffffffffUL << I40IWQPC_RCVNXT_SHIFT)
-
-#define I40IWQPC_RCVWND_SHIFT 32
-#define I40IWQPC_RCVWND_MASK (0xffffffffULL << I40IWQPC_RCVWND_SHIFT)
-
-#define I40IWQPC_SNDMAX_SHIFT 0
-#define I40IWQPC_SNDMAX_MASK (0xffffffffUL << I40IWQPC_SNDMAX_SHIFT)
-
-#define I40IWQPC_SNDUNA_SHIFT 32
-#define I40IWQPC_SNDUNA_MASK (0xffffffffULL << I40IWQPC_SNDUNA_SHIFT)
-
-#define I40IWQPC_SRTT_SHIFT 0
-#define I40IWQPC_SRTT_MASK (0xffffffffUL << I40IWQPC_SRTT_SHIFT)
-
-#define I40IWQPC_RTTVAR_SHIFT 32
-#define I40IWQPC_RTTVAR_MASK (0xffffffffULL << I40IWQPC_RTTVAR_SHIFT)
-
-#define I40IWQPC_SSTHRESH_SHIFT 0
-#define I40IWQPC_SSTHRESH_MASK (0xffffffffUL << I40IWQPC_SSTHRESH_SHIFT)
-
-#define I40IWQPC_CWND_SHIFT 32
-#define I40IWQPC_CWND_MASK (0xffffffffULL << I40IWQPC_CWND_SHIFT)
-
-#define I40IWQPC_SNDWL1_SHIFT 0
-#define I40IWQPC_SNDWL1_MASK (0xffffffffUL << I40IWQPC_SNDWL1_SHIFT)
-
-#define I40IWQPC_SNDWL2_SHIFT 32
-#define I40IWQPC_SNDWL2_MASK (0xffffffffULL << I40IWQPC_SNDWL2_SHIFT)
-
-#define I40IWQPC_ERR_RQ_IDX_SHIFT 32
-#define I40IWQPC_ERR_RQ_IDX_MASK (0x3fffULL << I40IWQPC_ERR_RQ_IDX_SHIFT)
-
-#define I40IWQPC_MAXSNDWND_SHIFT 0
-#define I40IWQPC_MAXSNDWND_MASK (0xffffffffUL << I40IWQPC_MAXSNDWND_SHIFT)
-
-#define I40IWQPC_REXMIT_THRESH_SHIFT 48
-#define I40IWQPC_REXMIT_THRESH_MASK (0x3fULL << I40IWQPC_REXMIT_THRESH_SHIFT)
-
-#define I40IWQPC_TXCQNUM_SHIFT 0
-#define I40IWQPC_TXCQNUM_MASK (0x1ffffUL << I40IWQPC_TXCQNUM_SHIFT)
-
-#define I40IWQPC_RXCQNUM_SHIFT 32
-#define I40IWQPC_RXCQNUM_MASK (0x1ffffULL << I40IWQPC_RXCQNUM_SHIFT)
-
-#define I40IWQPC_Q2ADDR_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPC_Q2ADDR_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPC_LASTBYTESENT_SHIFT 0
-#define I40IWQPC_LASTBYTESENT_MASK (0xffUL << I40IWQPC_LASTBYTESENT_SHIFT)
-
-#define I40IWQPC_SRQID_SHIFT 32
-#define I40IWQPC_SRQID_MASK (0xffULL << I40IWQPC_SRQID_SHIFT)
-
-#define I40IWQPC_ORDSIZE_SHIFT 0
-#define I40IWQPC_ORDSIZE_MASK (0x7fUL << I40IWQPC_ORDSIZE_SHIFT)
-
-#define I40IWQPC_IRDSIZE_SHIFT 16
-#define I40IWQPC_IRDSIZE_MASK (0x3UL << I40IWQPC_IRDSIZE_SHIFT)
-
-#define I40IWQPC_WRRDRSPOK_SHIFT 20
-#define I40IWQPC_WRRDRSPOK_MASK (1UL << I40IWQPC_WRRDRSPOK_SHIFT)
-
-#define I40IWQPC_RDOK_SHIFT 21
-#define I40IWQPC_RDOK_MASK (1UL << I40IWQPC_RDOK_SHIFT)
-
-#define I40IWQPC_SNDMARKERS_SHIFT 22
-#define I40IWQPC_SNDMARKERS_MASK (1UL << I40IWQPC_SNDMARKERS_SHIFT)
-
-#define I40IWQPC_BINDEN_SHIFT 23
-#define I40IWQPC_BINDEN_MASK (1UL << I40IWQPC_BINDEN_SHIFT)
-
-#define I40IWQPC_FASTREGEN_SHIFT 24
-#define I40IWQPC_FASTREGEN_MASK (1UL << I40IWQPC_FASTREGEN_SHIFT)
-
-#define I40IWQPC_PRIVEN_SHIFT 25
-#define I40IWQPC_PRIVEN_MASK (1UL << I40IWQPC_PRIVEN_SHIFT)
-
-#define I40IWQPC_LSMMPRESENT_SHIFT 26
-#define I40IWQPC_LSMMPRESENT_MASK (1UL << I40IWQPC_LSMMPRESENT_SHIFT)
-
-#define I40IWQPC_ADJUSTFORLSMM_SHIFT 27
-#define I40IWQPC_ADJUSTFORLSMM_MASK (1UL << I40IWQPC_ADJUSTFORLSMM_SHIFT)
-
-#define I40IWQPC_IWARPMODE_SHIFT 28
-#define I40IWQPC_IWARPMODE_MASK (1UL << I40IWQPC_IWARPMODE_SHIFT)
-
-#define I40IWQPC_RCVMARKERS_SHIFT 29
-#define I40IWQPC_RCVMARKERS_MASK (1UL << I40IWQPC_RCVMARKERS_SHIFT)
-
-#define I40IWQPC_ALIGNHDRS_SHIFT 30
-#define I40IWQPC_ALIGNHDRS_MASK (1UL << I40IWQPC_ALIGNHDRS_SHIFT)
-
-#define I40IWQPC_RCVNOMPACRC_SHIFT 31
-#define I40IWQPC_RCVNOMPACRC_MASK (1UL << I40IWQPC_RCVNOMPACRC_SHIFT)
-
-#define I40IWQPC_RCVMARKOFFSET_SHIFT 33
-#define I40IWQPC_RCVMARKOFFSET_MASK (0x1ffULL << I40IWQPC_RCVMARKOFFSET_SHIFT)
-
-#define I40IWQPC_SNDMARKOFFSET_SHIFT 48
-#define I40IWQPC_SNDMARKOFFSET_MASK (0x1ffULL << I40IWQPC_SNDMARKOFFSET_SHIFT)
-
-#define I40IWQPC_QPCOMPCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPC_QPCOMPCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPC_SQTPHVAL_SHIFT 0
-#define I40IWQPC_SQTPHVAL_MASK (0xffUL << I40IWQPC_SQTPHVAL_SHIFT)
-
-#define I40IWQPC_RQTPHVAL_SHIFT 8
-#define I40IWQPC_RQTPHVAL_MASK (0xffUL << I40IWQPC_RQTPHVAL_SHIFT)
-
-#define I40IWQPC_QSHANDLE_SHIFT 16
-#define I40IWQPC_QSHANDLE_MASK (0x3ffUL << I40IWQPC_QSHANDLE_SHIFT)
-
-#define I40IWQPC_EXCEPTION_LAN_QUEUE_SHIFT 32
-#define I40IWQPC_EXCEPTION_LAN_QUEUE_MASK (0xfffULL << \
- I40IWQPC_EXCEPTION_LAN_QUEUE_SHIFT)
-
-#define I40IWQPC_LOCAL_IPADDR3_SHIFT 0
-#define I40IWQPC_LOCAL_IPADDR3_MASK \
- (0xffffffffUL << I40IWQPC_LOCAL_IPADDR3_SHIFT)
-
-#define I40IWQPC_LOCAL_IPADDR2_SHIFT 32
-#define I40IWQPC_LOCAL_IPADDR2_MASK \
- (0xffffffffULL << I40IWQPC_LOCAL_IPADDR2_SHIFT)
-
-#define I40IWQPC_LOCAL_IPADDR1_SHIFT 0
-#define I40IWQPC_LOCAL_IPADDR1_MASK \
- (0xffffffffUL << I40IWQPC_LOCAL_IPADDR1_SHIFT)
-
-#define I40IWQPC_LOCAL_IPADDR0_SHIFT 32
-#define I40IWQPC_LOCAL_IPADDR0_MASK \
- (0xffffffffULL << I40IWQPC_LOCAL_IPADDR0_SHIFT)
-
-#define I40IW_QP_SW_MIN_WQSIZE 4 /*in WRs*/
-#define I40IW_SQ_RSVD 2
-#define I40IW_RQ_RSVD 1
-#define I40IW_QP_SW_MAX_SQ_QUANTAS 2048
-#define I40IW_QP_SW_MAX_RQ_QUANTAS 16384
-#define I40IWQP_OP_RDMA_WRITE 0
-#define I40IWQP_OP_RDMA_READ 1
-#define I40IWQP_OP_RDMA_SEND 3
-#define I40IWQP_OP_RDMA_SEND_INV 4
-#define I40IWQP_OP_RDMA_SEND_SOL_EVENT 5
-#define I40IWQP_OP_RDMA_SEND_SOL_EVENT_INV 6
-#define I40IWQP_OP_BIND_MW 8
-#define I40IWQP_OP_FAST_REGISTER 9
-#define I40IWQP_OP_LOCAL_INVALIDATE 10
-#define I40IWQP_OP_RDMA_READ_LOC_INV 11
-#define I40IWQP_OP_NOP 12
-
-#define I40IW_RSVD_SHIFT 41
-#define I40IW_RSVD_MASK (0x7fffULL << I40IW_RSVD_SHIFT)
-
-/* iwarp QP SQ WQE common fields */
-#define I40IWQPSQ_OPCODE_SHIFT 32
-#define I40IWQPSQ_OPCODE_MASK (0x3fULL << I40IWQPSQ_OPCODE_SHIFT)
-
-#define I40IWQPSQ_ADDFRAGCNT_SHIFT 38
-#define I40IWQPSQ_ADDFRAGCNT_MASK (0x7ULL << I40IWQPSQ_ADDFRAGCNT_SHIFT)
-
-#define I40IWQPSQ_PUSHWQE_SHIFT 56
-#define I40IWQPSQ_PUSHWQE_MASK (1ULL << I40IWQPSQ_PUSHWQE_SHIFT)
-
-#define I40IWQPSQ_STREAMMODE_SHIFT 58
-#define I40IWQPSQ_STREAMMODE_MASK (1ULL << I40IWQPSQ_STREAMMODE_SHIFT)
-
-#define I40IWQPSQ_WAITFORRCVPDU_SHIFT 59
-#define I40IWQPSQ_WAITFORRCVPDU_MASK (1ULL << I40IWQPSQ_WAITFORRCVPDU_SHIFT)
-
-#define I40IWQPSQ_READFENCE_SHIFT 60
-#define I40IWQPSQ_READFENCE_MASK (1ULL << I40IWQPSQ_READFENCE_SHIFT)
-
-#define I40IWQPSQ_LOCALFENCE_SHIFT 61
-#define I40IWQPSQ_LOCALFENCE_MASK (1ULL << I40IWQPSQ_LOCALFENCE_SHIFT)
-
-#define I40IWQPSQ_SIGCOMPL_SHIFT 62
-#define I40IWQPSQ_SIGCOMPL_MASK (1ULL << I40IWQPSQ_SIGCOMPL_SHIFT)
-
-#define I40IWQPSQ_VALID_SHIFT 63
-#define I40IWQPSQ_VALID_MASK (1ULL << I40IWQPSQ_VALID_SHIFT)
-
-#define I40IWQPSQ_FRAG_TO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPSQ_FRAG_TO_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPSQ_FRAG_LEN_SHIFT 0
-#define I40IWQPSQ_FRAG_LEN_MASK (0xffffffffUL << I40IWQPSQ_FRAG_LEN_SHIFT)
-
-#define I40IWQPSQ_FRAG_STAG_SHIFT 32
-#define I40IWQPSQ_FRAG_STAG_MASK (0xffffffffULL << I40IWQPSQ_FRAG_STAG_SHIFT)
-
-#define I40IWQPSQ_REMSTAGINV_SHIFT 0
-#define I40IWQPSQ_REMSTAGINV_MASK (0xffffffffUL << I40IWQPSQ_REMSTAGINV_SHIFT)
-
-#define I40IWQPSQ_INLINEDATAFLAG_SHIFT 57
-#define I40IWQPSQ_INLINEDATAFLAG_MASK (1ULL << I40IWQPSQ_INLINEDATAFLAG_SHIFT)
-
-#define I40IWQPSQ_INLINEDATALEN_SHIFT 48
-#define I40IWQPSQ_INLINEDATALEN_MASK \
- (0x7fULL << I40IWQPSQ_INLINEDATALEN_SHIFT)
-
-/* iwarp send with push mode */
-#define I40IWQPSQ_WQDESCIDX_SHIFT 0
-#define I40IWQPSQ_WQDESCIDX_MASK (0x3fffUL << I40IWQPSQ_WQDESCIDX_SHIFT)
-
-/* rdma write */
-#define I40IWQPSQ_REMSTAG_SHIFT 0
-#define I40IWQPSQ_REMSTAG_MASK (0xffffffffUL << I40IWQPSQ_REMSTAG_SHIFT)
-
-#define I40IWQPSQ_REMTO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPSQ_REMTO_MASK I40IW_CQPHC_QPCTX_MASK
-
-/* memory window */
-#define I40IWQPSQ_STAGRIGHTS_SHIFT 48
-#define I40IWQPSQ_STAGRIGHTS_MASK (0x1fULL << I40IWQPSQ_STAGRIGHTS_SHIFT)
-
-#define I40IWQPSQ_VABASEDTO_SHIFT 53
-#define I40IWQPSQ_VABASEDTO_MASK (1ULL << I40IWQPSQ_VABASEDTO_SHIFT)
-
-#define I40IWQPSQ_MWLEN_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPSQ_MWLEN_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPSQ_PARENTMRSTAG_SHIFT 0
-#define I40IWQPSQ_PARENTMRSTAG_MASK \
- (0xffffffffUL << I40IWQPSQ_PARENTMRSTAG_SHIFT)
-
-#define I40IWQPSQ_MWSTAG_SHIFT 32
-#define I40IWQPSQ_MWSTAG_MASK (0xffffffffULL << I40IWQPSQ_MWSTAG_SHIFT)
-
-#define I40IWQPSQ_BASEVA_TO_FBO_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPSQ_BASEVA_TO_FBO_MASK I40IW_CQPHC_QPCTX_MASK
-
-/* Local Invalidate */
-#define I40IWQPSQ_LOCSTAG_SHIFT 32
-#define I40IWQPSQ_LOCSTAG_MASK (0xffffffffULL << I40IWQPSQ_LOCSTAG_SHIFT)
-
-/* Fast Register */
-#define I40IWQPSQ_STAGKEY_SHIFT 0
-#define I40IWQPSQ_STAGKEY_MASK (0xffUL << I40IWQPSQ_STAGKEY_SHIFT)
-
-#define I40IWQPSQ_STAGINDEX_SHIFT 8
-#define I40IWQPSQ_STAGINDEX_MASK (0xffffffUL << I40IWQPSQ_STAGINDEX_SHIFT)
-
-#define I40IWQPSQ_COPYHOSTPBLS_SHIFT 43
-#define I40IWQPSQ_COPYHOSTPBLS_MASK (1ULL << I40IWQPSQ_COPYHOSTPBLS_SHIFT)
-
-#define I40IWQPSQ_LPBLSIZE_SHIFT 44
-#define I40IWQPSQ_LPBLSIZE_MASK (3ULL << I40IWQPSQ_LPBLSIZE_SHIFT)
-
-#define I40IWQPSQ_HPAGESIZE_SHIFT 46
-#define I40IWQPSQ_HPAGESIZE_MASK (3ULL << I40IWQPSQ_HPAGESIZE_SHIFT)
-
-#define I40IWQPSQ_STAGLEN_SHIFT 0
-#define I40IWQPSQ_STAGLEN_MASK (0x1ffffffffffULL << I40IWQPSQ_STAGLEN_SHIFT)
-
-#define I40IWQPSQ_FIRSTPMPBLIDXLO_SHIFT 48
-#define I40IWQPSQ_FIRSTPMPBLIDXLO_MASK \
- (0xffffULL << I40IWQPSQ_FIRSTPMPBLIDXLO_SHIFT)
-
-#define I40IWQPSQ_FIRSTPMPBLIDXHI_SHIFT 0
-#define I40IWQPSQ_FIRSTPMPBLIDXHI_MASK \
- (0xfffUL << I40IWQPSQ_FIRSTPMPBLIDXHI_SHIFT)
-
-#define I40IWQPSQ_PBLADDR_SHIFT 12
-#define I40IWQPSQ_PBLADDR_MASK (0xfffffffffffffULL << I40IWQPSQ_PBLADDR_SHIFT)
-
-/* iwarp QP RQ WQE common fields */
-#define I40IWQPRQ_ADDFRAGCNT_SHIFT I40IWQPSQ_ADDFRAGCNT_SHIFT
-#define I40IWQPRQ_ADDFRAGCNT_MASK I40IWQPSQ_ADDFRAGCNT_MASK
-
-#define I40IWQPRQ_VALID_SHIFT I40IWQPSQ_VALID_SHIFT
-#define I40IWQPRQ_VALID_MASK I40IWQPSQ_VALID_MASK
-
-#define I40IWQPRQ_COMPLCTX_SHIFT I40IW_CQPHC_QPCTX_SHIFT
-#define I40IWQPRQ_COMPLCTX_MASK I40IW_CQPHC_QPCTX_MASK
-
-#define I40IWQPRQ_FRAG_LEN_SHIFT I40IWQPSQ_FRAG_LEN_SHIFT
-#define I40IWQPRQ_FRAG_LEN_MASK I40IWQPSQ_FRAG_LEN_MASK
-
-#define I40IWQPRQ_STAG_SHIFT I40IWQPSQ_FRAG_STAG_SHIFT
-#define I40IWQPRQ_STAG_MASK I40IWQPSQ_FRAG_STAG_MASK
-
-#define I40IWQPRQ_TO_SHIFT I40IWQPSQ_FRAG_TO_SHIFT
-#define I40IWQPRQ_TO_MASK I40IWQPSQ_FRAG_TO_MASK
-
-/* Query FPM CQP buf */
-#define I40IW_QUERY_FPM_MAX_QPS_SHIFT 0
-#define I40IW_QUERY_FPM_MAX_QPS_MASK \
- (0x7ffffUL << I40IW_QUERY_FPM_MAX_QPS_SHIFT)
-
-#define I40IW_QUERY_FPM_MAX_CQS_SHIFT 0
-#define I40IW_QUERY_FPM_MAX_CQS_MASK \
- (0x3ffffUL << I40IW_QUERY_FPM_MAX_CQS_SHIFT)
-
-#define I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_SHIFT 0
-#define I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_MASK \
- (0x3fffUL << I40IW_QUERY_FPM_FIRST_PE_SD_INDEX_SHIFT)
-
-#define I40IW_QUERY_FPM_MAX_PE_SDS_SHIFT 32
-#define I40IW_QUERY_FPM_MAX_PE_SDS_MASK \
- (0x3fffULL << I40IW_QUERY_FPM_MAX_PE_SDS_SHIFT)
-
-#define I40IW_QUERY_FPM_MAX_QPS_SHIFT 0
-#define I40IW_QUERY_FPM_MAX_QPS_MASK \
- (0x7ffffUL << I40IW_QUERY_FPM_MAX_QPS_SHIFT)
-
-#define I40IW_QUERY_FPM_MAX_CQS_SHIFT 0
-#define I40IW_QUERY_FPM_MAX_CQS_MASK \
- (0x3ffffUL << I40IW_QUERY_FPM_MAX_CQS_SHIFT)
-
-#define I40IW_QUERY_FPM_MAX_CEQS_SHIFT 0
-#define I40IW_QUERY_FPM_MAX_CEQS_MASK \
- (0xffUL << I40IW_QUERY_FPM_MAX_CEQS_SHIFT)
-
-#define I40IW_QUERY_FPM_XFBLOCKSIZE_SHIFT 32
-#define I40IW_QUERY_FPM_XFBLOCKSIZE_MASK \
- (0xffffffffULL << I40IW_QUERY_FPM_XFBLOCKSIZE_SHIFT)
-
-#define I40IW_QUERY_FPM_Q1BLOCKSIZE_SHIFT 32
-#define I40IW_QUERY_FPM_Q1BLOCKSIZE_MASK \
- (0xffffffffULL << I40IW_QUERY_FPM_Q1BLOCKSIZE_SHIFT)
-
-#define I40IW_QUERY_FPM_HTMULTIPLIER_SHIFT 16
-#define I40IW_QUERY_FPM_HTMULTIPLIER_MASK \
- (0xfUL << I40IW_QUERY_FPM_HTMULTIPLIER_SHIFT)
-
-#define I40IW_QUERY_FPM_TIMERBUCKET_SHIFT 32
-#define I40IW_QUERY_FPM_TIMERBUCKET_MASK \
- (0xffFFULL << I40IW_QUERY_FPM_TIMERBUCKET_SHIFT)
-
-/* Static HMC pages allocated buf */
-#define I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_SHIFT 0
-#define I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_MASK \
- (0x3fUL << I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID_SHIFT)
-
-#define I40IW_HW_PAGE_SIZE 4096
-#define I40IW_DONE_COUNT 1000
-#define I40IW_SLEEP_COUNT 10
-
-enum {
- I40IW_QUEUES_ALIGNMENT_MASK = (128 - 1),
- I40IW_AEQ_ALIGNMENT_MASK = (256 - 1),
- I40IW_Q2_ALIGNMENT_MASK = (256 - 1),
- I40IW_CEQ_ALIGNMENT_MASK = (256 - 1),
- I40IW_CQ0_ALIGNMENT_MASK = (256 - 1),
- I40IW_HOST_CTX_ALIGNMENT_MASK = (4 - 1),
- I40IW_SHADOWAREA_MASK = (128 - 1),
- I40IW_FPM_QUERY_BUF_ALIGNMENT_MASK = 0,
- I40IW_FPM_COMMIT_BUF_ALIGNMENT_MASK = 0
-};
-
-enum i40iw_alignment {
- I40IW_CQP_ALIGNMENT = 0x200,
- I40IW_AEQ_ALIGNMENT = 0x100,
- I40IW_CEQ_ALIGNMENT = 0x100,
- I40IW_CQ0_ALIGNMENT = 0x100,
- I40IW_SD_BUF_ALIGNMENT = 0x100
-};
-
-#define I40IW_WQE_SIZE_64 64
-
-#define I40IW_QP_WQE_MIN_SIZE 32
-#define I40IW_QP_WQE_MAX_SIZE 128
-
-#define I40IW_CQE_QTYPE_RQ 0
-#define I40IW_CQE_QTYPE_SQ 1
-
-#define I40IW_RING_INIT(_ring, _size) \
- { \
- (_ring).head = 0; \
- (_ring).tail = 0; \
- (_ring).size = (_size); \
- }
-#define I40IW_RING_GETSIZE(_ring) ((_ring).size)
-#define I40IW_RING_GETCURRENT_HEAD(_ring) ((_ring).head)
-#define I40IW_RING_GETCURRENT_TAIL(_ring) ((_ring).tail)
-
-#define I40IW_RING_MOVE_HEAD(_ring, _retcode) \
- { \
- register u32 size; \
- size = (_ring).size; \
- if (!I40IW_RING_FULL_ERR(_ring)) { \
- (_ring).head = ((_ring).head + 1) % size; \
- (_retcode) = 0; \
- } else { \
- (_retcode) = I40IW_ERR_RING_FULL; \
- } \
- }
-
-#define I40IW_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
- { \
- register u32 size; \
- size = (_ring).size; \
- if ((I40IW_RING_WORK_AVAILABLE(_ring) + (_count)) < size) { \
- (_ring).head = ((_ring).head + (_count)) % size; \
- (_retcode) = 0; \
- } else { \
- (_retcode) = I40IW_ERR_RING_FULL; \
- } \
- }
-
-#define I40IW_RING_MOVE_TAIL(_ring) \
- (_ring).tail = ((_ring).tail + 1) % (_ring).size
-
-#define I40IW_RING_MOVE_HEAD_NOCHECK(_ring) \
- (_ring).head = ((_ring).head + 1) % (_ring).size
-
-#define I40IW_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \
- (_ring).tail = ((_ring).tail + (_count)) % (_ring).size
-
-#define I40IW_RING_SET_TAIL(_ring, _pos) \
- (_ring).tail = (_pos) % (_ring).size
-
-#define I40IW_RING_FULL_ERR(_ring) \
- ( \
- (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 1)) \
- )
-
-#define I40IW_ERR_RING_FULL2(_ring) \
- ( \
- (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 2)) \
- )
-
-#define I40IW_ERR_RING_FULL3(_ring) \
- ( \
- (I40IW_RING_WORK_AVAILABLE(_ring) == ((_ring).size - 3)) \
- )
-
-#define I40IW_RING_MORE_WORK(_ring) \
- ( \
- (I40IW_RING_WORK_AVAILABLE(_ring) != 0) \
- )
-
-#define I40IW_RING_WORK_AVAILABLE(_ring) \
- ( \
- (((_ring).head + (_ring).size - (_ring).tail) % (_ring).size) \
- )
-
-#define I40IW_RING_GET_WQES_AVAILABLE(_ring) \
- ( \
- ((_ring).size - I40IW_RING_WORK_AVAILABLE(_ring) - 1) \
- )
-
-#define I40IW_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \
- { \
- index = I40IW_RING_GETCURRENT_HEAD(_ring); \
- I40IW_RING_MOVE_HEAD(_ring, _retcode); \
- }
-
-/* Async Events codes */
-#define I40IW_AE_AMP_UNALLOCATED_STAG 0x0102
-#define I40IW_AE_AMP_INVALID_STAG 0x0103
-#define I40IW_AE_AMP_BAD_QP 0x0104
-#define I40IW_AE_AMP_BAD_PD 0x0105
-#define I40IW_AE_AMP_BAD_STAG_KEY 0x0106
-#define I40IW_AE_AMP_BAD_STAG_INDEX 0x0107
-#define I40IW_AE_AMP_BOUNDS_VIOLATION 0x0108
-#define I40IW_AE_AMP_RIGHTS_VIOLATION 0x0109
-#define I40IW_AE_AMP_TO_WRAP 0x010a
-#define I40IW_AE_AMP_FASTREG_SHARED 0x010b
-#define I40IW_AE_AMP_FASTREG_VALID_STAG 0x010c
-#define I40IW_AE_AMP_FASTREG_MW_STAG 0x010d
-#define I40IW_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
-#define I40IW_AE_AMP_FASTREG_PBL_TABLE_OVERFLOW 0x010f
-#define I40IW_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
-#define I40IW_AE_AMP_INVALIDATE_SHARED 0x0111
-#define I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
-#define I40IW_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
-#define I40IW_AE_AMP_MWBIND_VALID_STAG 0x0114
-#define I40IW_AE_AMP_MWBIND_OF_MR_STAG 0x0115
-#define I40IW_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
-#define I40IW_AE_AMP_MWBIND_TO_MW_STAG 0x0117
-#define I40IW_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
-#define I40IW_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
-#define I40IW_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
-#define I40IW_AE_AMP_MWBIND_BIND_DISABLED 0x011b
-#define I40IW_AE_AMP_WQE_INVALID_PARAMETER 0x0130
-#define I40IW_AE_BAD_CLOSE 0x0201
-#define I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
-#define I40IW_AE_CQ_OPERATION_ERROR 0x0203
-#define I40IW_AE_PRIV_OPERATION_DENIED 0x011c
-#define I40IW_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
-#define I40IW_AE_STAG_ZERO_INVALID 0x0206
-#define I40IW_AE_IB_RREQ_AND_Q1_FULL 0x0207
-#define I40IW_AE_SRQ_LIMIT 0x0209
-#define I40IW_AE_WQE_UNEXPECTED_OPCODE 0x020a
-#define I40IW_AE_WQE_INVALID_PARAMETER 0x020b
-#define I40IW_AE_WQE_LSMM_TOO_LONG 0x0220
-#define I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
-#define I40IW_AE_DDP_INVALID_MSN_RANGE_IS_NOT_VALID 0x0302
-#define I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
-#define I40IW_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
-#define I40IW_AE_DDP_UBE_INVALID_MO 0x0305
-#define I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
-#define I40IW_AE_DDP_UBE_INVALID_QN 0x0307
-#define I40IW_AE_DDP_NO_L_BIT 0x0308
-#define I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
-#define I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
-#define I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
-#define I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
-#define I40IW_AE_INVALID_ARP_ENTRY 0x0401
-#define I40IW_AE_INVALID_TCP_OPTION_RCVD 0x0402
-#define I40IW_AE_STALE_ARP_ENTRY 0x0403
-#define I40IW_AE_INVALID_WQE_LENGTH 0x0404
-#define I40IW_AE_INVALID_MAC_ENTRY 0x0405
-#define I40IW_AE_LLP_CLOSE_COMPLETE 0x0501
-#define I40IW_AE_LLP_CONNECTION_RESET 0x0502
-#define I40IW_AE_LLP_FIN_RECEIVED 0x0503
-#define I40IW_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504
-#define I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
-#define I40IW_AE_LLP_SEGMENT_TOO_LARGE 0x0506
-#define I40IW_AE_LLP_SEGMENT_TOO_SMALL 0x0507
-#define I40IW_AE_LLP_SYN_RECEIVED 0x0508
-#define I40IW_AE_LLP_TERMINATE_RECEIVED 0x0509
-#define I40IW_AE_LLP_TOO_MANY_RETRIES 0x050a
-#define I40IW_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
-#define I40IW_AE_LLP_DOUBT_REACHABILITY 0x050c
-#define I40IW_AE_LLP_RX_VLAN_MISMATCH 0x050d
-#define I40IW_AE_RESOURCE_EXHAUSTION 0x0520
-#define I40IW_AE_RESET_SENT 0x0601
-#define I40IW_AE_TERMINATE_SENT 0x0602
-#define I40IW_AE_RESET_NOT_SENT 0x0603
-#define I40IW_AE_LCE_QP_CATASTROPHIC 0x0700
-#define I40IW_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
-#define I40IW_AE_LCE_CQ_CATASTROPHIC 0x0702
-#define I40IW_AE_UDA_XMIT_FRAG_SEQ 0x0800
-#define I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0801
-#define I40IW_AE_UDA_XMIT_IPADDR_MISMATCH 0x0802
-#define I40IW_AE_QP_SUSPEND_COMPLETE 0x0900
-
-#define OP_DELETE_LOCAL_MAC_IPADDR_ENTRY 1
-#define OP_CEQ_DESTROY 2
-#define OP_AEQ_DESTROY 3
-#define OP_DELETE_ARP_CACHE_ENTRY 4
-#define OP_MANAGE_APBVT_ENTRY 5
-#define OP_CEQ_CREATE 6
-#define OP_AEQ_CREATE 7
-#define OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY 8
-#define OP_ADD_LOCAL_MAC_IPADDR_ENTRY 9
-#define OP_MANAGE_QHASH_TABLE_ENTRY 10
-#define OP_QP_MODIFY 11
-#define OP_QP_UPLOAD_CONTEXT 12
-#define OP_CQ_CREATE 13
-#define OP_CQ_DESTROY 14
-#define OP_QP_CREATE 15
-#define OP_QP_DESTROY 16
-#define OP_ALLOC_STAG 17
-#define OP_MR_REG_NON_SHARED 18
-#define OP_DEALLOC_STAG 19
-#define OP_MW_ALLOC 20
-#define OP_QP_FLUSH_WQES 21
-#define OP_ADD_ARP_CACHE_ENTRY 22
-#define OP_MANAGE_PUSH_PAGE 23
-#define OP_UPDATE_PE_SDS 24
-#define OP_MANAGE_HMC_PM_FUNC_TABLE 25
-#define OP_SUSPEND 26
-#define OP_RESUME 27
-#define OP_MANAGE_VF_PBLE_BP 28
-#define OP_QUERY_FPM_VALUES 29
-#define OP_COMMIT_FPM_VALUES 30
-#define OP_SIZE_CQP_STAT_ARRAY 31
-
-#endif
diff --git a/providers/i40iw/i40iw_osdep.h b/providers/i40iw/i40iw_osdep.h
deleted file mode 100644
index 92bedd3..0000000
--- a/providers/i40iw/i40iw_osdep.h
+++ /dev/null
@@ -1,108 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_OSDEP_H
-#define I40IW_OSDEP_H
-
-#include <stdbool.h>
-#include <stdio.h>
-#include <string.h>
-#include <util/udma_barrier.h>
-#include <linux/types.h>
-typedef unsigned char u8;
-typedef unsigned long long u64;
-typedef unsigned int u32;
-typedef unsigned short u16;
-typedef unsigned long i40iw_uintptr;
-typedef unsigned long *i40iw_bits_t;
-typedef __be16 BE16;
-typedef __be32 BE32;
-typedef __be64 BE64;
-typedef __le16 LE16;
-typedef __le32 LE32;
-typedef __le64 LE64;
-
-#define STATS_TIMER_DELAY 1000
-#define INLINE inline
-
-static inline void set_64bit_val(u64 *wqe_words, u32 byte_index, u64 value)
-{
- wqe_words[byte_index >> 3] = value;
-}
-
-/**
- * set_32bit_val - set 32 value to hw wqe
- * @wqe_words: wqe addr to write
- * @byte_index: index in wqe
- * @value: value to write
- **/
-static inline void set_32bit_val(u32 *wqe_words, u32 byte_index, u32 value)
-{
- wqe_words[byte_index >> 2] = value;
-}
-
-/**
- * get_64bit_val - read 64 bit value from wqe
- * @wqe_words: wqe addr
- * @byte_index: index to read from
- * @value: read value
- **/
-static inline void get_64bit_val(u64 *wqe_words, u32 byte_index, u64 *value)
-{
- *value = wqe_words[byte_index >> 3];
-}
-
-/**
- * get_32bit_val - read 32 bit value from wqe
- * @wqe_words: wqe addr
- * @byte_index: index to reaad from
- * @value: return 32 bit value
- **/
-static inline void get_32bit_val(u32 *wqe_words, u32 byte_index, u32 *value)
-{
- *value = wqe_words[byte_index >> 2];
-}
-
-#define i40iw_get_virt_to_phy
-#define IOMEM
-
-static inline void db_wr32(u32 value, u32 *wqe_word)
-{
- *wqe_word = value;
-}
-
-#define ACQUIRE_LOCK()
-#define RELEASE_LOCK()
-
-#endif /* _I40IW_OSDEP_H_ */
diff --git a/providers/i40iw/i40iw_register.h b/providers/i40iw/i40iw_register.h
deleted file mode 100644
index 5776818..0000000
--- a/providers/i40iw/i40iw_register.h
+++ /dev/null
@@ -1,1030 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_REGISTER_H
-#define I40IW_REGISTER_H
-
-#define I40E_GLGEN_STAT 0x000B612C /* Reset: POR */
-
-#define I40E_PFHMC_PDINV 0x000C0300 /* Reset: PFR */
-#define I40E_PFHMC_PDINV_PMSDIDX_SHIFT 0
-#define I40E_PFHMC_PDINV_PMSDIDX_MASK (0xFFF << I40E_PFHMC_PDINV_PMSDIDX_SHIFT)
-#define I40E_PFHMC_PDINV_PMPDIDX_SHIFT 16
-#define I40E_PFHMC_PDINV_PMPDIDX_MASK (0x1FF << I40E_PFHMC_PDINV_PMPDIDX_SHIFT)
-#define I40E_PFHMC_SDCMD_PMSDWR_SHIFT 31
-#define I40E_PFHMC_SDCMD_PMSDWR_MASK (0x1 << I40E_PFHMC_SDCMD_PMSDWR_SHIFT)
-#define I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT 0
-#define I40E_PFHMC_SDDATALOW_PMSDVALID_MASK (0x1 << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT)
-#define I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT 1
-#define I40E_PFHMC_SDDATALOW_PMSDTYPE_MASK (0x1 << I40E_PFHMC_SDDATALOW_PMSDTYPE_SHIFT)
-#define I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT 2
-#define I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_MASK (0x3FF << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT)
-
-#define I40E_PFINT_DYN_CTLN(_INTPF) (0x00034800 + ((_INTPF) * 4)) /* _i=0...511 */ /* Reset: PFR */
-#define I40E_PFINT_DYN_CTLN_INTENA_SHIFT 0
-#define I40E_PFINT_DYN_CTLN_INTENA_MASK (0x1 << I40E_PFINT_DYN_CTLN_INTENA_SHIFT)
-#define I40E_PFINT_DYN_CTLN_CLEARPBA_SHIFT 1
-#define I40E_PFINT_DYN_CTLN_CLEARPBA_MASK (0x1 << I40E_PFINT_DYN_CTLN_CLEARPBA_SHIFT)
-#define I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT 3
-#define I40E_PFINT_DYN_CTLN_ITR_INDX_MASK (0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)
-
-#define I40E_VFINT_DYN_CTLN1(_INTVF) (0x00003800 + ((_INTVF) * 4)) /* _i=0...15 */ /* Reset: VFR */
-#define I40E_GLHMC_VFPDINV(_i) (0x000C8300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */
-
-#define I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT 15
-#define I40E_PFHMC_PDINV_PMSDPARTSEL_MASK (0x1 << I40E_PFHMC_PDINV_PMSDPARTSEL_SHIFT)
-#define I40E_GLPCI_LBARCTRL 0x000BE484 /* Reset: POR */
-#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT 4
-#define I40E_GLPCI_LBARCTRL_PE_DB_SIZE_MASK (0x3 << I40E_GLPCI_LBARCTRL_PE_DB_SIZE_SHIFT)
-#define I40E_GLPCI_DREVID 0x0009C480 /* Reset: PCIR */
-#define I40E_GLPCI_DREVID_DEFAULT_REVID_SHIFT 0
-#define I40E_GLPCI_DREVID_DEFAULT_REVID_MASK 0xFF
-
-#define I40E_PFPE_AEQALLOC 0x00131180 /* Reset: PFR */
-#define I40E_PFPE_AEQALLOC_AECOUNT_SHIFT 0
-#define I40E_PFPE_AEQALLOC_AECOUNT_MASK (0xFFFFFFFF << I40E_PFPE_AEQALLOC_AECOUNT_SHIFT)
-#define I40E_PFPE_CCQPHIGH 0x00008200 /* Reset: PFR */
-#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0
-#define I40E_PFPE_CCQPHIGH_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_PFPE_CCQPHIGH_PECCQPHIGH_SHIFT)
-#define I40E_PFPE_CCQPLOW 0x00008180 /* Reset: PFR */
-#define I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT 0
-#define I40E_PFPE_CCQPLOW_PECCQPLOW_MASK (0xFFFFFFFF << I40E_PFPE_CCQPLOW_PECCQPLOW_SHIFT)
-#define I40E_PFPE_CCQPSTATUS 0x00008100 /* Reset: PFR */
-#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0
-#define I40E_PFPE_CCQPSTATUS_CCQP_DONE_MASK (0x1 << I40E_PFPE_CCQPSTATUS_CCQP_DONE_SHIFT)
-#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4
-#define I40E_PFPE_CCQPSTATUS_HMC_PROFILE_MASK (0x7 << I40E_PFPE_CCQPSTATUS_HMC_PROFILE_SHIFT)
-#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16
-#define I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_MASK (0x3F << I40E_PFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT)
-#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31
-#define I40E_PFPE_CCQPSTATUS_CCQP_ERR_MASK (0x1 << I40E_PFPE_CCQPSTATUS_CCQP_ERR_SHIFT)
-#define I40E_PFPE_CQACK 0x00131100 /* Reset: PFR */
-#define I40E_PFPE_CQACK_PECQID_SHIFT 0
-#define I40E_PFPE_CQACK_PECQID_MASK (0x1FFFF << I40E_PFPE_CQACK_PECQID_SHIFT)
-#define I40E_PFPE_CQARM 0x00131080 /* Reset: PFR */
-#define I40E_PFPE_CQARM_PECQID_SHIFT 0
-#define I40E_PFPE_CQARM_PECQID_MASK (0x1FFFF << I40E_PFPE_CQARM_PECQID_SHIFT)
-#define I40E_PFPE_CQPDB 0x00008000 /* Reset: PFR */
-#define I40E_PFPE_CQPDB_WQHEAD_SHIFT 0
-#define I40E_PFPE_CQPDB_WQHEAD_MASK (0x7FF << I40E_PFPE_CQPDB_WQHEAD_SHIFT)
-#define I40E_PFPE_CQPERRCODES 0x00008880 /* Reset: PFR */
-#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0
-#define I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_MASK (0xFFFF << I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT)
-#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16
-#define I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT)
-#define I40E_PFPE_CQPTAIL 0x00008080 /* Reset: PFR */
-#define I40E_PFPE_CQPTAIL_WQTAIL_SHIFT 0
-#define I40E_PFPE_CQPTAIL_WQTAIL_MASK (0x7FF << I40E_PFPE_CQPTAIL_WQTAIL_SHIFT)
-#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31
-#define I40E_PFPE_CQPTAIL_CQP_OP_ERR_MASK (0x1 << I40E_PFPE_CQPTAIL_CQP_OP_ERR_SHIFT)
-#define I40E_PFPE_FLMQ1ALLOCERR 0x00008980 /* Reset: PFR */
-#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0
-#define I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_PFPE_FLMQ1ALLOCERR_ERROR_COUNT_SHIFT)
-#define I40E_PFPE_FLMXMITALLOCERR 0x00008900 /* Reset: PFR */
-#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT 0
-#define I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_PFPE_FLMXMITALLOCERR_ERROR_COUNT_SHIFT)
-#define I40E_PFPE_IPCONFIG0 0x00008280 /* Reset: PFR */
-#define I40E_PFPE_IPCONFIG0_PEIPID_SHIFT 0
-#define I40E_PFPE_IPCONFIG0_PEIPID_MASK (0xFFFF << I40E_PFPE_IPCONFIG0_PEIPID_SHIFT)
-#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16
-#define I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_MASK (0x1 << I40E_PFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT)
-#define I40E_PFPE_MRTEIDXMASK 0x00008600 /* Reset: PFR */
-#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0
-#define I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK (0x1F << I40E_PFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT)
-#define I40E_PFPE_RCVUNEXPECTEDERROR 0x00008680 /* Reset: PFR */
-#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0
-#define I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_PFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT)
-#define I40E_PFPE_TCPNOWTIMER 0x00008580 /* Reset: PFR */
-#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0
-#define I40E_PFPE_TCPNOWTIMER_TCP_NOW_MASK (0xFFFFFFFF << I40E_PFPE_TCPNOWTIMER_TCP_NOW_SHIFT)
-
-#define I40E_PFPE_WQEALLOC 0x00138C00 /* Reset: PFR */
-#define I40E_PFPE_WQEALLOC_PEQPID_SHIFT 0
-#define I40E_PFPE_WQEALLOC_PEQPID_MASK (0x3FFFF << I40E_PFPE_WQEALLOC_PEQPID_SHIFT)
-#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20
-#define I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_MASK (0xFFF << I40E_PFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT)
-
-#define I40E_VFPE_AEQALLOC(_VF) (0x00130C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_AEQALLOC_MAX_INDEX 127
-#define I40E_VFPE_AEQALLOC_AECOUNT_SHIFT 0
-#define I40E_VFPE_AEQALLOC_AECOUNT_MASK (0xFFFFFFFF << I40E_VFPE_AEQALLOC_AECOUNT_SHIFT)
-#define I40E_VFPE_CCQPHIGH(_VF) (0x00001000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CCQPHIGH_MAX_INDEX 127
-#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT 0
-#define I40E_VFPE_CCQPHIGH_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_VFPE_CCQPHIGH_PECCQPHIGH_SHIFT)
-#define I40E_VFPE_CCQPLOW(_VF) (0x00000C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CCQPLOW_MAX_INDEX 127
-#define I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT 0
-#define I40E_VFPE_CCQPLOW_PECCQPLOW_MASK (0xFFFFFFFF << I40E_VFPE_CCQPLOW_PECCQPLOW_SHIFT)
-#define I40E_VFPE_CCQPSTATUS(_VF) (0x00000800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CCQPSTATUS_MAX_INDEX 127
-#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT 0
-#define I40E_VFPE_CCQPSTATUS_CCQP_DONE_MASK (0x1 << I40E_VFPE_CCQPSTATUS_CCQP_DONE_SHIFT)
-#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT 4
-#define I40E_VFPE_CCQPSTATUS_HMC_PROFILE_MASK (0x7 << I40E_VFPE_CCQPSTATUS_HMC_PROFILE_SHIFT)
-#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT 16
-#define I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_MASK (0x3F << I40E_VFPE_CCQPSTATUS_RDMA_EN_VFS_SHIFT)
-#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT 31
-#define I40E_VFPE_CCQPSTATUS_CCQP_ERR_MASK (0x1 << I40E_VFPE_CCQPSTATUS_CCQP_ERR_SHIFT)
-#define I40E_VFPE_CQACK(_VF) (0x00130800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CQACK_MAX_INDEX 127
-#define I40E_VFPE_CQACK_PECQID_SHIFT 0
-#define I40E_VFPE_CQACK_PECQID_MASK (0x1FFFF << I40E_VFPE_CQACK_PECQID_SHIFT)
-#define I40E_VFPE_CQARM(_VF) (0x00130400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CQARM_MAX_INDEX 127
-#define I40E_VFPE_CQARM_PECQID_SHIFT 0
-#define I40E_VFPE_CQARM_PECQID_MASK (0x1FFFF << I40E_VFPE_CQARM_PECQID_SHIFT)
-#define I40E_VFPE_CQPDB(_VF) (0x00000000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CQPDB_MAX_INDEX 127
-#define I40E_VFPE_CQPDB_WQHEAD_SHIFT 0
-#define I40E_VFPE_CQPDB_WQHEAD_MASK (0x7FF << I40E_VFPE_CQPDB_WQHEAD_SHIFT)
-#define I40E_VFPE_CQPERRCODES(_VF) (0x00001800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CQPERRCODES_MAX_INDEX 127
-#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT 0
-#define I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES_CQP_MINOR_CODE_SHIFT)
-#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT 16
-#define I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES_CQP_MAJOR_CODE_SHIFT)
-#define I40E_VFPE_CQPTAIL(_VF) (0x00000400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_CQPTAIL_MAX_INDEX 127
-#define I40E_VFPE_CQPTAIL_WQTAIL_SHIFT 0
-#define I40E_VFPE_CQPTAIL_WQTAIL_MASK (0x7FF << I40E_VFPE_CQPTAIL_WQTAIL_SHIFT)
-#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT 31
-#define I40E_VFPE_CQPTAIL_CQP_OP_ERR_MASK (0x1 << I40E_VFPE_CQPTAIL_CQP_OP_ERR_SHIFT)
-#define I40E_VFPE_IPCONFIG0(_VF) (0x00001400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_IPCONFIG0_MAX_INDEX 127
-#define I40E_VFPE_IPCONFIG0_PEIPID_SHIFT 0
-#define I40E_VFPE_IPCONFIG0_PEIPID_MASK (0xFFFF << I40E_VFPE_IPCONFIG0_PEIPID_SHIFT)
-#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT 16
-#define I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_MASK (0x1 << I40E_VFPE_IPCONFIG0_USEENTIREIDRANGE_SHIFT)
-#define I40E_VFPE_MRTEIDXMASK(_VF) (0x00003000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_MRTEIDXMASK_MAX_INDEX 127
-#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT 0
-#define I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_MASK (0x1F << I40E_VFPE_MRTEIDXMASK_MRTEIDXMASKBITS_SHIFT)
-#define I40E_VFPE_RCVUNEXPECTEDERROR(_VF) (0x00003400 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_RCVUNEXPECTEDERROR_MAX_INDEX 127
-#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT 0
-#define I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_VFPE_RCVUNEXPECTEDERROR_TCP_RX_UNEXP_ERR_SHIFT)
-#define I40E_VFPE_TCPNOWTIMER(_VF) (0x00002C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_TCPNOWTIMER_MAX_INDEX 127
-#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT 0
-#define I40E_VFPE_TCPNOWTIMER_TCP_NOW_MASK (0xFFFFFFFF << I40E_VFPE_TCPNOWTIMER_TCP_NOW_SHIFT)
-#define I40E_VFPE_WQEALLOC(_VF) (0x00138000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: VFR */
-#define I40E_VFPE_WQEALLOC_MAX_INDEX 127
-#define I40E_VFPE_WQEALLOC_PEQPID_SHIFT 0
-#define I40E_VFPE_WQEALLOC_PEQPID_MASK (0x3FFFF << I40E_VFPE_WQEALLOC_PEQPID_SHIFT)
-#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT 20
-#define I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_MASK (0xFFF << I40E_VFPE_WQEALLOC_WQE_DESC_INDEX_SHIFT)
-
-#define I40E_GLPE_CPUSTATUS0 0x0000D040 /* Reset: PE_CORER */
-#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT 0
-#define I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS0_PECPUSTATUS0_SHIFT)
-#define I40E_GLPE_CPUSTATUS1 0x0000D044 /* Reset: PE_CORER */
-#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT 0
-#define I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS1_PECPUSTATUS1_SHIFT)
-#define I40E_GLPE_CPUSTATUS2 0x0000D048 /* Reset: PE_CORER */
-#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT 0
-#define I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_MASK (0xFFFFFFFF << I40E_GLPE_CPUSTATUS2_PECPUSTATUS2_SHIFT)
-#define I40E_GLPE_CPUTRIG0 0x0000D060 /* Reset: PE_CORER */
-#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT 0
-#define I40E_GLPE_CPUTRIG0_PECPUTRIG0_MASK (0xFFFF << I40E_GLPE_CPUTRIG0_PECPUTRIG0_SHIFT)
-#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT 17
-#define I40E_GLPE_CPUTRIG0_TEPREQUEST0_MASK (0x1 << I40E_GLPE_CPUTRIG0_TEPREQUEST0_SHIFT)
-#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT 18
-#define I40E_GLPE_CPUTRIG0_OOPREQUEST0_MASK (0x1 << I40E_GLPE_CPUTRIG0_OOPREQUEST0_SHIFT)
-#define I40E_GLPE_DUAL40_RUPM 0x0000DA04 /* Reset: PE_CORER */
-#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT 0
-#define I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_MASK (0x1 << I40E_GLPE_DUAL40_RUPM_DUAL_40G_MODE_SHIFT)
-#define I40E_GLPE_PFAEQEDROPCNT(_i) (0x00131440 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */
-#define I40E_GLPE_PFAEQEDROPCNT_MAX_INDEX 15
-#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0
-#define I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFAEQEDROPCNT_AEQEDROPCNT_SHIFT)
-#define I40E_GLPE_PFCEQEDROPCNT(_i) (0x001313C0 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */
-#define I40E_GLPE_PFCEQEDROPCNT_MAX_INDEX 15
-#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0
-#define I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFCEQEDROPCNT_CEQEDROPCNT_SHIFT)
-#define I40E_GLPE_PFCQEDROPCNT(_i) (0x00131340 + ((_i) * 4)) /* _i=0...15 */ /* Reset: CORER */
-#define I40E_GLPE_PFCQEDROPCNT_MAX_INDEX 15
-#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT 0
-#define I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_MASK (0xFFFF << I40E_GLPE_PFCQEDROPCNT_CQEDROPCNT_SHIFT)
-#define I40E_GLPE_RUPM_CQPPOOL 0x0000DACC /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT 0
-#define I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_MASK (0xFF << I40E_GLPE_RUPM_CQPPOOL_CQPSPADS_SHIFT)
-#define I40E_GLPE_RUPM_FLRPOOL 0x0000DAC4 /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT 0
-#define I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_MASK (0xFF << I40E_GLPE_RUPM_FLRPOOL_FLRSPADS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL 0x0000DA00 /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT 0
-#define I40E_GLPE_RUPM_GCTL_ALLOFFTH_MASK (0xFF << I40E_GLPE_RUPM_GCTL_ALLOFFTH_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT 26
-#define I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P0_DIS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT 27
-#define I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P1_DIS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT 28
-#define I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P2_DIS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT 29
-#define I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_P3_DIS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT 30
-#define I40E_GLPE_RUPM_GCTL_RUPM_DIS_MASK (0x1 << I40E_GLPE_RUPM_GCTL_RUPM_DIS_SHIFT)
-#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT 31
-#define I40E_GLPE_RUPM_GCTL_SWLB_MODE_MASK (0x1 << I40E_GLPE_RUPM_GCTL_SWLB_MODE_SHIFT)
-#define I40E_GLPE_RUPM_PTXPOOL 0x0000DAC8 /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT 0
-#define I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_MASK (0xFF << I40E_GLPE_RUPM_PTXPOOL_PTXSPADS_SHIFT)
-#define I40E_GLPE_RUPM_PUSHPOOL 0x0000DAC0 /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT 0
-#define I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_MASK (0xFF << I40E_GLPE_RUPM_PUSHPOOL_PUSHSPADS_SHIFT)
-#define I40E_GLPE_RUPM_TXHOST_EN 0x0000DA08 /* Reset: PE_CORER */
-#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT 0
-#define I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_MASK (0x1 << I40E_GLPE_RUPM_TXHOST_EN_TXHOST_EN_SHIFT)
-#define I40E_GLPE_VFAEQEDROPCNT(_i) (0x00132540 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */
-#define I40E_GLPE_VFAEQEDROPCNT_MAX_INDEX 31
-#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT 0
-#define I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFAEQEDROPCNT_AEQEDROPCNT_SHIFT)
-#define I40E_GLPE_VFCEQEDROPCNT(_i) (0x00132440 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */
-#define I40E_GLPE_VFCEQEDROPCNT_MAX_INDEX 31
-#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT 0
-#define I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFCEQEDROPCNT_CEQEDROPCNT_SHIFT)
-#define I40E_GLPE_VFCQEDROPCNT(_i) (0x00132340 + ((_i) * 4)) /* _i=0...31 */ /* Reset: CORER */
-#define I40E_GLPE_VFCQEDROPCNT_MAX_INDEX 31
-#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT 0
-#define I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_MASK (0xFFFF << I40E_GLPE_VFCQEDROPCNT_CQEDROPCNT_SHIFT)
-#define I40E_GLPE_VFFLMOBJCTRL(_i) (0x0000D400 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPE_VFFLMOBJCTRL_MAX_INDEX 31
-#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT 0
-#define I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_MASK (0x7 << I40E_GLPE_VFFLMOBJCTRL_XMIT_BLOCKSIZE_SHIFT)
-#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT 8
-#define I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_MASK (0x7 << I40E_GLPE_VFFLMOBJCTRL_Q1_BLOCKSIZE_SHIFT)
-#define I40E_GLPE_VFFLMQ1ALLOCERR(_i) (0x0000C700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPE_VFFLMQ1ALLOCERR_MAX_INDEX 31
-#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT 0
-#define I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_GLPE_VFFLMQ1ALLOCERR_ERROR_COUNT_SHIFT)
-#define I40E_GLPE_VFFLMXMITALLOCERR(_i) (0x0000C600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPE_VFFLMXMITALLOCERR_MAX_INDEX 31
-#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT 0
-#define I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_MASK (0xFFFF << I40E_GLPE_VFFLMXMITALLOCERR_ERROR_COUNT_SHIFT)
-#define I40E_GLPE_VFUDACTRL(_i) (0x0000C000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPE_VFUDACTRL_MAX_INDEX 31
-#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT 0
-#define I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV4MCFRAGRESBP_SHIFT)
-#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT 1
-#define I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV4UCFRAGRESBP_SHIFT)
-#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT 2
-#define I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV6MCFRAGRESBP_SHIFT)
-#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT 3
-#define I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_MASK (0x1 << I40E_GLPE_VFUDACTRL_IPV6UCFRAGRESBP_SHIFT)
-#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT 4
-#define I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_MASK (0x1 << I40E_GLPE_VFUDACTRL_UDPMCFRAGRESFAIL_SHIFT)
-#define I40E_GLPE_VFUDAUCFBQPN(_i) (0x0000C100 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPE_VFUDAUCFBQPN_MAX_INDEX 31
-#define I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT 0
-#define I40E_GLPE_VFUDAUCFBQPN_QPN_MASK (0x3FFFF << I40E_GLPE_VFUDAUCFBQPN_QPN_SHIFT)
-#define I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT 31
-#define I40E_GLPE_VFUDAUCFBQPN_VALID_MASK (0x1 << I40E_GLPE_VFUDAUCFBQPN_VALID_SHIFT)
-
-#define I40E_GLPES_PFIP4RXDISCARD(_i) (0x00010600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXDISCARD_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0
-#define I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXDISCARD_IP4RXDISCARD_SHIFT)
-#define I40E_GLPES_PFIP4RXFRAGSHI(_i) (0x00010804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXFRAGSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0
-#define I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT)
-#define I40E_GLPES_PFIP4RXFRAGSLO(_i) (0x00010800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXFRAGSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0
-#define I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT)
-#define I40E_GLPES_PFIP4RXMCOCTSHI(_i) (0x00010A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXMCOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP4RXMCOCTSLO(_i) (0x00010A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXMCOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP4RXMCPKTSHI(_i) (0x00010C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXMCPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP4RXMCPKTSLO(_i) (0x00010C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXMCPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP4RXOCTSHI(_i) (0x00010204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP4RXOCTSLO(_i) (0x00010200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP4RXPKTSHI(_i) (0x00010404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP4RXPKTSLO(_i) (0x00010400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP4RXTRUNC(_i) (0x00010700 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4RXTRUNC_MAX_INDEX 15
-#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0
-#define I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4RXTRUNC_IP4RXTRUNC_SHIFT)
-#define I40E_GLPES_PFIP4TXFRAGSHI(_i) (0x00011E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXFRAGSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0
-#define I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT)
-#define I40E_GLPES_PFIP4TXFRAGSLO(_i) (0x00011E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXFRAGSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0
-#define I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT)
-#define I40E_GLPES_PFIP4TXMCOCTSHI(_i) (0x00012004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXMCOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP4TXMCOCTSLO(_i) (0x00012000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXMCOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP4TXMCPKTSHI(_i) (0x00012204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXMCPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP4TXMCPKTSLO(_i) (0x00012200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXMCPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP4TXNOROUTE(_i) (0x00012E00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXNOROUTE_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0
-#define I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_PFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT)
-#define I40E_GLPES_PFIP4TXOCTSHI(_i) (0x00011A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP4TXOCTSLO(_i) (0x00011A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP4TXPKTSHI(_i) (0x00011C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP4TXPKTSLO(_i) (0x00011C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP4TXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXDISCARD(_i) (0x00011200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXDISCARD_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0
-#define I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXDISCARD_IP6RXDISCARD_SHIFT)
-#define I40E_GLPES_PFIP6RXFRAGSHI(_i) (0x00011404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXFRAGSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0
-#define I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT)
-#define I40E_GLPES_PFIP6RXFRAGSLO(_i) (0x00011400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXFRAGSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0
-#define I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXMCOCTSHI(_i) (0x00011604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXMCOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP6RXMCOCTSLO(_i) (0x00011600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXMCOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXMCPKTSHI(_i) (0x00011804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXMCPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP6RXMCPKTSLO(_i) (0x00011800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXMCPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXOCTSHI(_i) (0x00010E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP6RXOCTSLO(_i) (0x00010E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXPKTSHI(_i) (0x00011004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP6RXPKTSLO(_i) (0x00011000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP6RXTRUNC(_i) (0x00011300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6RXTRUNC_MAX_INDEX 15
-#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0
-#define I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6RXTRUNC_IP6RXTRUNC_SHIFT)
-#define I40E_GLPES_PFIP6TXFRAGSHI(_i) (0x00012804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXFRAGSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0
-#define I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT)
-#define I40E_GLPES_PFIP6TXFRAGSLO(_i) (0x00012800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXFRAGSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0
-#define I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT)
-#define I40E_GLPES_PFIP6TXMCOCTSHI(_i) (0x00012A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXMCOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP6TXMCOCTSLO(_i) (0x00012A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXMCOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP6TXMCPKTSHI(_i) (0x00012C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXMCPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP6TXMCPKTSLO(_i) (0x00012C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXMCPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT)
-#define I40E_GLPES_PFIP6TXNOROUTE(_i) (0x00012F00 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXNOROUTE_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0
-#define I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_PFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT)
-#define I40E_GLPES_PFIP6TXOCTSHI(_i) (0x00012404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXOCTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT)
-#define I40E_GLPES_PFIP6TXOCTSLO(_i) (0x00012400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXOCTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT)
-#define I40E_GLPES_PFIP6TXPKTSHI(_i) (0x00012604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT)
-#define I40E_GLPES_PFIP6TXPKTSLO(_i) (0x00012600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFIP6TXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT)
-#define I40E_GLPES_PFRDMARXRDSHI(_i) (0x00013E04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXRDSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0
-#define I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXRDSHI_RDMARXRDSHI_SHIFT)
-#define I40E_GLPES_PFRDMARXRDSLO(_i) (0x00013E00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXRDSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0
-#define I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXRDSLO_RDMARXRDSLO_SHIFT)
-#define I40E_GLPES_PFRDMARXSNDSHI(_i) (0x00014004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXSNDSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0
-#define I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT)
-#define I40E_GLPES_PFRDMARXSNDSLO(_i) (0x00014000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXSNDSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0
-#define I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT)
-#define I40E_GLPES_PFRDMARXWRSHI(_i) (0x00013C04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXWRSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0
-#define I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_PFRDMARXWRSHI_RDMARXWRSHI_SHIFT)
-#define I40E_GLPES_PFRDMARXWRSLO(_i) (0x00013C00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMARXWRSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0
-#define I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMARXWRSLO_RDMARXWRSLO_SHIFT)
-#define I40E_GLPES_PFRDMATXRDSHI(_i) (0x00014404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXRDSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0
-#define I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXRDSHI_RDMARXRDSHI_SHIFT)
-#define I40E_GLPES_PFRDMATXRDSLO(_i) (0x00014400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXRDSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0
-#define I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXRDSLO_RDMARXRDSLO_SHIFT)
-#define I40E_GLPES_PFRDMATXSNDSHI(_i) (0x00014604 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXSNDSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0
-#define I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT)
-#define I40E_GLPES_PFRDMATXSNDSLO(_i) (0x00014600 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXSNDSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0
-#define I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT)
-#define I40E_GLPES_PFRDMATXWRSHI(_i) (0x00014204 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXWRSHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0
-#define I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_PFRDMATXWRSHI_RDMARXWRSHI_SHIFT)
-#define I40E_GLPES_PFRDMATXWRSLO(_i) (0x00014200 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMATXWRSLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0
-#define I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMATXWRSLO_RDMARXWRSLO_SHIFT)
-#define I40E_GLPES_PFRDMAVBNDHI(_i) (0x00014804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMAVBNDHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0
-#define I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVBNDHI_RDMAVBNDHI_SHIFT)
-#define I40E_GLPES_PFRDMAVBNDLO(_i) (0x00014800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMAVBNDLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0
-#define I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVBNDLO_RDMAVBNDLO_SHIFT)
-#define I40E_GLPES_PFRDMAVINVHI(_i) (0x00014A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMAVINVHI_MAX_INDEX 15
-#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT 0
-#define I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVINVHI_RDMAVINVHI_SHIFT)
-#define I40E_GLPES_PFRDMAVINVLO(_i) (0x00014A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRDMAVINVLO_MAX_INDEX 15
-#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT 0
-#define I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_MASK (0xFFFFFFFF << I40E_GLPES_PFRDMAVINVLO_RDMAVINVLO_SHIFT)
-#define I40E_GLPES_PFRXVLANERR(_i) (0x00010000 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFRXVLANERR_MAX_INDEX 15
-#define I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT 0
-#define I40E_GLPES_PFRXVLANERR_RXVLANERR_MASK (0xFFFFFF << I40E_GLPES_PFRXVLANERR_RXVLANERR_SHIFT)
-#define I40E_GLPES_PFTCPRTXSEG(_i) (0x00013600 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPRTXSEG_MAX_INDEX 15
-#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT 0
-#define I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPRTXSEG_TCPRTXSEG_SHIFT)
-#define I40E_GLPES_PFTCPRXOPTERR(_i) (0x00013200 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPRXOPTERR_MAX_INDEX 15
-#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0
-#define I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_MASK (0xFFFFFF << I40E_GLPES_PFTCPRXOPTERR_TCPRXOPTERR_SHIFT)
-#define I40E_GLPES_PFTCPRXPROTOERR(_i) (0x00013300 + ((_i) * 4)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPRXPROTOERR_MAX_INDEX 15
-#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0
-#define I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_MASK (0xFFFFFF << I40E_GLPES_PFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT)
-#define I40E_GLPES_PFTCPRXSEGSHI(_i) (0x00013004 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPRXSEGSHI_MAX_INDEX 15
-#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0
-#define I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_MASK (0xFFFF << I40E_GLPES_PFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT)
-#define I40E_GLPES_PFTCPRXSEGSLO(_i) (0x00013000 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPRXSEGSLO_MAX_INDEX 15
-#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0
-#define I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT)
-#define I40E_GLPES_PFTCPTXSEGHI(_i) (0x00013404 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPTXSEGHI_MAX_INDEX 15
-#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0
-#define I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_MASK (0xFFFF << I40E_GLPES_PFTCPTXSEGHI_TCPTXSEGHI_SHIFT)
-#define I40E_GLPES_PFTCPTXSEGLO(_i) (0x00013400 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFTCPTXSEGLO_MAX_INDEX 15
-#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0
-#define I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_MASK (0xFFFFFFFF << I40E_GLPES_PFTCPTXSEGLO_TCPTXSEGLO_SHIFT)
-#define I40E_GLPES_PFUDPRXPKTSHI(_i) (0x00013804 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFUDPRXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT)
-#define I40E_GLPES_PFUDPRXPKTSLO(_i) (0x00013800 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFUDPRXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT)
-#define I40E_GLPES_PFUDPTXPKTSHI(_i) (0x00013A04 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFUDPTXPKTSHI_MAX_INDEX 15
-#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0
-#define I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_MASK (0xFFFF << I40E_GLPES_PFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT)
-#define I40E_GLPES_PFUDPTXPKTSLO(_i) (0x00013A00 + ((_i) * 8)) /* _i=0...15 */ /* Reset: PE_CORER */
-#define I40E_GLPES_PFUDPTXPKTSLO_MAX_INDEX 15
-#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0
-#define I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_PFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT)
-#define I40E_GLPES_RDMARXMULTFPDUSHI 0x0001E014 /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT 0
-#define I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_MASK (0xFFFFFF << I40E_GLPES_RDMARXMULTFPDUSHI_RDMARXMULTFPDUSHI_SHIFT)
-#define I40E_GLPES_RDMARXMULTFPDUSLO 0x0001E010 /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT 0
-#define I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXMULTFPDUSLO_RDMARXMULTFPDUSLO_SHIFT)
-#define I40E_GLPES_RDMARXOOODDPHI 0x0001E01C /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT 0
-#define I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_MASK (0xFFFFFF << I40E_GLPES_RDMARXOOODDPHI_RDMARXOOODDPHI_SHIFT)
-#define I40E_GLPES_RDMARXOOODDPLO 0x0001E018 /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT 0
-#define I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXOOODDPLO_RDMARXOOODDPLO_SHIFT)
-#define I40E_GLPES_RDMARXOOONOMARK 0x0001E004 /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT 0
-#define I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXOOONOMARK_RDMAOOONOMARK_SHIFT)
-#define I40E_GLPES_RDMARXUNALIGN 0x0001E000 /* Reset: PE_CORER */
-#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT 0
-#define I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_MASK (0xFFFFFFFF << I40E_GLPES_RDMARXUNALIGN_RDMRXAUNALIGN_SHIFT)
-#define I40E_GLPES_TCPRXFOURHOLEHI 0x0001E044 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT 0
-#define I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXFOURHOLEHI_TCPRXFOURHOLEHI_SHIFT)
-#define I40E_GLPES_TCPRXFOURHOLELO 0x0001E040 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT 0
-#define I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXFOURHOLELO_TCPRXFOURHOLELO_SHIFT)
-#define I40E_GLPES_TCPRXONEHOLEHI 0x0001E02C /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT 0
-#define I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXONEHOLEHI_TCPRXONEHOLEHI_SHIFT)
-#define I40E_GLPES_TCPRXONEHOLELO 0x0001E028 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT 0
-#define I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXONEHOLELO_TCPRXONEHOLELO_SHIFT)
-#define I40E_GLPES_TCPRXPUREACKHI 0x0001E024 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT 0
-#define I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXPUREACKHI_TCPRXPUREACKSHI_SHIFT)
-#define I40E_GLPES_TCPRXPUREACKSLO 0x0001E020 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT 0
-#define I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXPUREACKSLO_TCPRXPUREACKLO_SHIFT)
-#define I40E_GLPES_TCPRXTHREEHOLEHI 0x0001E03C /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT 0
-#define I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXTHREEHOLEHI_TCPRXTHREEHOLEHI_SHIFT)
-#define I40E_GLPES_TCPRXTHREEHOLELO 0x0001E038 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT 0
-#define I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXTHREEHOLELO_TCPRXTHREEHOLELO_SHIFT)
-#define I40E_GLPES_TCPRXTWOHOLEHI 0x0001E034 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT 0
-#define I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_MASK (0xFFFFFF << I40E_GLPES_TCPRXTWOHOLEHI_TCPRXTWOHOLEHI_SHIFT)
-#define I40E_GLPES_TCPRXTWOHOLELO 0x0001E030 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT 0
-#define I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_MASK (0xFFFFFFFF << I40E_GLPES_TCPRXTWOHOLELO_TCPRXTWOHOLELO_SHIFT)
-#define I40E_GLPES_TCPTXRETRANSFASTHI 0x0001E04C /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT 0
-#define I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXRETRANSFASTHI_TCPTXRETRANSFASTHI_SHIFT)
-#define I40E_GLPES_TCPTXRETRANSFASTLO 0x0001E048 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT 0
-#define I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXRETRANSFASTLO_TCPTXRETRANSFASTLO_SHIFT)
-#define I40E_GLPES_TCPTXTOUTSFASTHI 0x0001E054 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT 0
-#define I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXTOUTSFASTHI_TCPTXTOUTSFASTHI_SHIFT)
-#define I40E_GLPES_TCPTXTOUTSFASTLO 0x0001E050 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT 0
-#define I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXTOUTSFASTLO_TCPTXTOUTSFASTLO_SHIFT)
-#define I40E_GLPES_TCPTXTOUTSHI 0x0001E05C /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT 0
-#define I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_MASK (0xFFFFFF << I40E_GLPES_TCPTXTOUTSHI_TCPTXTOUTSHI_SHIFT)
-#define I40E_GLPES_TCPTXTOUTSLO 0x0001E058 /* Reset: PE_CORER */
-#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT 0
-#define I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_MASK (0xFFFFFFFF << I40E_GLPES_TCPTXTOUTSLO_TCPTXTOUTSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXDISCARD(_i) (0x00018600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXDISCARD_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT 0
-#define I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXDISCARD_IP4RXDISCARD_SHIFT)
-#define I40E_GLPES_VFIP4RXFRAGSHI(_i) (0x00018804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXFRAGSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT 0
-#define I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXFRAGSHI_IP4RXFRAGSHI_SHIFT)
-#define I40E_GLPES_VFIP4RXFRAGSLO(_i) (0x00018800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXFRAGSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT 0
-#define I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXFRAGSLO_IP4RXFRAGSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXMCOCTSHI(_i) (0x00018A04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXMCOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXMCOCTSHI_IP4RXMCOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP4RXMCOCTSLO(_i) (0x00018A00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXMCOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXMCOCTSLO_IP4RXMCOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXMCPKTSHI(_i) (0x00018C04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXMCPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXMCPKTSHI_IP4RXMCPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP4RXMCPKTSLO(_i) (0x00018C00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXMCPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXMCPKTSLO_IP4RXMCPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXOCTSHI(_i) (0x00018204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXOCTSHI_IP4RXOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP4RXOCTSLO(_i) (0x00018200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXOCTSLO_IP4RXOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXPKTSHI(_i) (0x00018404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4RXPKTSHI_IP4RXPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP4RXPKTSLO(_i) (0x00018400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXPKTSLO_IP4RXPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP4RXTRUNC(_i) (0x00018700 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4RXTRUNC_MAX_INDEX 31
-#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT 0
-#define I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4RXTRUNC_IP4RXTRUNC_SHIFT)
-#define I40E_GLPES_VFIP4TXFRAGSHI(_i) (0x00019E04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXFRAGSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT 0
-#define I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXFRAGSHI_IP4TXFRAGSHI_SHIFT)
-#define I40E_GLPES_VFIP4TXFRAGSLO(_i) (0x00019E00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXFRAGSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT 0
-#define I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXFRAGSLO_IP4TXFRAGSLO_SHIFT)
-#define I40E_GLPES_VFIP4TXMCOCTSHI(_i) (0x0001A004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXMCOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXMCOCTSHI_IP4TXMCOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP4TXMCOCTSLO(_i) (0x0001A000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXMCOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXMCOCTSLO_IP4TXMCOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP4TXMCPKTSHI(_i) (0x0001A204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXMCPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXMCPKTSHI_IP4TXMCPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP4TXMCPKTSLO(_i) (0x0001A200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXMCPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXMCPKTSLO_IP4TXMCPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP4TXNOROUTE(_i) (0x0001AE00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXNOROUTE_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT 0
-#define I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_VFIP4TXNOROUTE_IP4TXNOROUTE_SHIFT)
-#define I40E_GLPES_VFIP4TXOCTSHI(_i) (0x00019A04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXOCTSHI_IP4TXOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP4TXOCTSLO(_i) (0x00019A00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXOCTSLO_IP4TXOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP4TXPKTSHI(_i) (0x00019C04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP4TXPKTSHI_IP4TXPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP4TXPKTSLO(_i) (0x00019C00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP4TXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP4TXPKTSLO_IP4TXPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXDISCARD(_i) (0x00019200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXDISCARD_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT 0
-#define I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXDISCARD_IP6RXDISCARD_SHIFT)
-#define I40E_GLPES_VFIP6RXFRAGSHI(_i) (0x00019404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXFRAGSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT 0
-#define I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXFRAGSHI_IP6RXFRAGSHI_SHIFT)
-#define I40E_GLPES_VFIP6RXFRAGSLO(_i) (0x00019400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXFRAGSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT 0
-#define I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXFRAGSLO_IP6RXFRAGSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXMCOCTSHI(_i) (0x00019604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXMCOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXMCOCTSHI_IP6RXMCOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP6RXMCOCTSLO(_i) (0x00019600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXMCOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXMCOCTSLO_IP6RXMCOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXMCPKTSHI(_i) (0x00019804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXMCPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXMCPKTSHI_IP6RXMCPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP6RXMCPKTSLO(_i) (0x00019800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXMCPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXMCPKTSLO_IP6RXMCPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXOCTSHI(_i) (0x00018E04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXOCTSHI_IP6RXOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP6RXOCTSLO(_i) (0x00018E00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXOCTSLO_IP6RXOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXPKTSHI(_i) (0x00019004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6RXPKTSHI_IP6RXPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP6RXPKTSLO(_i) (0x00019000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXPKTSLO_IP6RXPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP6RXTRUNC(_i) (0x00019300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6RXTRUNC_MAX_INDEX 31
-#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT 0
-#define I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6RXTRUNC_IP6RXTRUNC_SHIFT)
-#define I40E_GLPES_VFIP6TXFRAGSHI(_i) (0x0001A804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXFRAGSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT 0
-#define I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXFRAGSHI_IP6TXFRAGSHI_SHIFT)
-#define I40E_GLPES_VFIP6TXFRAGSLO(_i) (0x0001A800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXFRAGSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT 0
-#define I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXFRAGSLO_IP6TXFRAGSLO_SHIFT)
-#define I40E_GLPES_VFIP6TXMCOCTSHI(_i) (0x0001AA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXMCOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXMCOCTSHI_IP6TXMCOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP6TXMCOCTSLO(_i) (0x0001AA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXMCOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXMCOCTSLO_IP6TXMCOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP6TXMCPKTSHI(_i) (0x0001AC04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXMCPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXMCPKTSHI_IP6TXMCPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP6TXMCPKTSLO(_i) (0x0001AC00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXMCPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXMCPKTSLO_IP6TXMCPKTSLO_SHIFT)
-#define I40E_GLPES_VFIP6TXNOROUTE(_i) (0x0001AF00 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXNOROUTE_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT 0
-#define I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_MASK (0xFFFFFF << I40E_GLPES_VFIP6TXNOROUTE_IP6TXNOROUTE_SHIFT)
-#define I40E_GLPES_VFIP6TXOCTSHI(_i) (0x0001A404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXOCTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXOCTSHI_IP6TXOCTSHI_SHIFT)
-#define I40E_GLPES_VFIP6TXOCTSLO(_i) (0x0001A400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXOCTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXOCTSLO_IP6TXOCTSLO_SHIFT)
-#define I40E_GLPES_VFIP6TXPKTSHI(_i) (0x0001A604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFIP6TXPKTSHI_IP6TXPKTSHI_SHIFT)
-#define I40E_GLPES_VFIP6TXPKTSLO(_i) (0x0001A600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFIP6TXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFIP6TXPKTSLO_IP6TXPKTSLO_SHIFT)
-#define I40E_GLPES_VFRDMARXRDSHI(_i) (0x0001BE04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXRDSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT 0
-#define I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXRDSHI_RDMARXRDSHI_SHIFT)
-#define I40E_GLPES_VFRDMARXRDSLO(_i) (0x0001BE00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXRDSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT 0
-#define I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXRDSLO_RDMARXRDSLO_SHIFT)
-#define I40E_GLPES_VFRDMARXSNDSHI(_i) (0x0001C004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXSNDSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT 0
-#define I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXSNDSHI_RDMARXSNDSHI_SHIFT)
-#define I40E_GLPES_VFRDMARXSNDSLO(_i) (0x0001C000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXSNDSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT 0
-#define I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXSNDSLO_RDMARXSNDSLO_SHIFT)
-#define I40E_GLPES_VFRDMARXWRSHI(_i) (0x0001BC04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXWRSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT 0
-#define I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_VFRDMARXWRSHI_RDMARXWRSHI_SHIFT)
-#define I40E_GLPES_VFRDMARXWRSLO(_i) (0x0001BC00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMARXWRSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT 0
-#define I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMARXWRSLO_RDMARXWRSLO_SHIFT)
-#define I40E_GLPES_VFRDMATXRDSHI(_i) (0x0001C404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXRDSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT 0
-#define I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXRDSHI_RDMARXRDSHI_SHIFT)
-#define I40E_GLPES_VFRDMATXRDSLO(_i) (0x0001C400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXRDSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT 0
-#define I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXRDSLO_RDMARXRDSLO_SHIFT)
-#define I40E_GLPES_VFRDMATXSNDSHI(_i) (0x0001C604 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXSNDSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT 0
-#define I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXSNDSHI_RDMARXSNDSHI_SHIFT)
-#define I40E_GLPES_VFRDMATXSNDSLO(_i) (0x0001C600 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXSNDSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT 0
-#define I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXSNDSLO_RDMARXSNDSLO_SHIFT)
-#define I40E_GLPES_VFRDMATXWRSHI(_i) (0x0001C204 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXWRSHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT 0
-#define I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_MASK (0xFFFF << I40E_GLPES_VFRDMATXWRSHI_RDMARXWRSHI_SHIFT)
-#define I40E_GLPES_VFRDMATXWRSLO(_i) (0x0001C200 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMATXWRSLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT 0
-#define I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMATXWRSLO_RDMARXWRSLO_SHIFT)
-#define I40E_GLPES_VFRDMAVBNDHI(_i) (0x0001C804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMAVBNDHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT 0
-#define I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVBNDHI_RDMAVBNDHI_SHIFT)
-#define I40E_GLPES_VFRDMAVBNDLO(_i) (0x0001C800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMAVBNDLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT 0
-#define I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVBNDLO_RDMAVBNDLO_SHIFT)
-#define I40E_GLPES_VFRDMAVINVHI(_i) (0x0001CA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMAVINVHI_MAX_INDEX 31
-#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT 0
-#define I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVINVHI_RDMAVINVHI_SHIFT)
-#define I40E_GLPES_VFRDMAVINVLO(_i) (0x0001CA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRDMAVINVLO_MAX_INDEX 31
-#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT 0
-#define I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_MASK (0xFFFFFFFF << I40E_GLPES_VFRDMAVINVLO_RDMAVINVLO_SHIFT)
-#define I40E_GLPES_VFRXVLANERR(_i) (0x00018000 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFRXVLANERR_MAX_INDEX 31
-#define I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT 0
-#define I40E_GLPES_VFRXVLANERR_RXVLANERR_MASK (0xFFFFFF << I40E_GLPES_VFRXVLANERR_RXVLANERR_SHIFT)
-#define I40E_GLPES_VFTCPRTXSEG(_i) (0x0001B600 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPRTXSEG_MAX_INDEX 31
-#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT 0
-#define I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPRTXSEG_TCPRTXSEG_SHIFT)
-#define I40E_GLPES_VFTCPRXOPTERR(_i) (0x0001B200 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPRXOPTERR_MAX_INDEX 31
-#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT 0
-#define I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_MASK (0xFFFFFF << I40E_GLPES_VFTCPRXOPTERR_TCPRXOPTERR_SHIFT)
-#define I40E_GLPES_VFTCPRXPROTOERR(_i) (0x0001B300 + ((_i) * 4)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPRXPROTOERR_MAX_INDEX 31
-#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT 0
-#define I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_MASK (0xFFFFFF << I40E_GLPES_VFTCPRXPROTOERR_TCPRXPROTOERR_SHIFT)
-#define I40E_GLPES_VFTCPRXSEGSHI(_i) (0x0001B004 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPRXSEGSHI_MAX_INDEX 31
-#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT 0
-#define I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_MASK (0xFFFF << I40E_GLPES_VFTCPRXSEGSHI_TCPRXSEGSHI_SHIFT)
-#define I40E_GLPES_VFTCPRXSEGSLO(_i) (0x0001B000 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPRXSEGSLO_MAX_INDEX 31
-#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT 0
-#define I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPRXSEGSLO_TCPRXSEGSLO_SHIFT)
-#define I40E_GLPES_VFTCPTXSEGHI(_i) (0x0001B404 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPTXSEGHI_MAX_INDEX 31
-#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT 0
-#define I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_MASK (0xFFFF << I40E_GLPES_VFTCPTXSEGHI_TCPTXSEGHI_SHIFT)
-#define I40E_GLPES_VFTCPTXSEGLO(_i) (0x0001B400 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFTCPTXSEGLO_MAX_INDEX 31
-#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT 0
-#define I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_MASK (0xFFFFFFFF << I40E_GLPES_VFTCPTXSEGLO_TCPTXSEGLO_SHIFT)
-#define I40E_GLPES_VFUDPRXPKTSHI(_i) (0x0001B804 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFUDPRXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFUDPRXPKTSHI_UDPRXPKTSHI_SHIFT)
-#define I40E_GLPES_VFUDPRXPKTSLO(_i) (0x0001B800 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFUDPRXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFUDPRXPKTSLO_UDPRXPKTSLO_SHIFT)
-#define I40E_GLPES_VFUDPTXPKTSHI(_i) (0x0001BA04 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFUDPTXPKTSHI_MAX_INDEX 31
-#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT 0
-#define I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_MASK (0xFFFF << I40E_GLPES_VFUDPTXPKTSHI_UDPTXPKTSHI_SHIFT)
-#define I40E_GLPES_VFUDPTXPKTSLO(_i) (0x0001BA00 + ((_i) * 8)) /* _i=0...31 */ /* Reset: PE_CORER */
-#define I40E_GLPES_VFUDPTXPKTSLO_MAX_INDEX 31
-#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT 0
-#define I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_MASK (0xFFFFFFFF << I40E_GLPES_VFUDPTXPKTSLO_UDPTXPKTSLO_SHIFT)
-
-#define I40E_VFPE_AEQALLOC1 0x0000A400 /* Reset: VFR */
-#define I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT 0
-#define I40E_VFPE_AEQALLOC1_AECOUNT_MASK (0xFFFFFFFF << I40E_VFPE_AEQALLOC1_AECOUNT_SHIFT)
-#define I40E_VFPE_CCQPHIGH1 0x00009800 /* Reset: VFR */
-#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT 0
-#define I40E_VFPE_CCQPHIGH1_PECCQPHIGH_MASK (0xFFFFFFFF << I40E_VFPE_CCQPHIGH1_PECCQPHIGH_SHIFT)
-#define I40E_VFPE_CCQPLOW1 0x0000AC00 /* Reset: VFR */
-#define I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT 0
-#define I40E_VFPE_CCQPLOW1_PECCQPLOW_MASK (0xFFFFFFFF << I40E_VFPE_CCQPLOW1_PECCQPLOW_SHIFT)
-#define I40E_VFPE_CCQPSTATUS1 0x0000B800 /* Reset: VFR */
-#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT 0
-#define I40E_VFPE_CCQPSTATUS1_CCQP_DONE_MASK (0x1 << I40E_VFPE_CCQPSTATUS1_CCQP_DONE_SHIFT)
-#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT 4
-#define I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_MASK (0x7 << I40E_VFPE_CCQPSTATUS1_HMC_PROFILE_SHIFT)
-#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT 16
-#define I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_MASK (0x3F << I40E_VFPE_CCQPSTATUS1_RDMA_EN_VFS_SHIFT)
-#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT 31
-#define I40E_VFPE_CCQPSTATUS1_CCQP_ERR_MASK (0x1 << I40E_VFPE_CCQPSTATUS1_CCQP_ERR_SHIFT)
-#define I40E_VFPE_CQACK1 0x0000B000 /* Reset: VFR */
-#define I40E_VFPE_CQACK1_PECQID_SHIFT 0
-#define I40E_VFPE_CQACK1_PECQID_MASK (0x1FFFF << I40E_VFPE_CQACK1_PECQID_SHIFT)
-#define I40E_VFPE_CQARM1 0x0000B400 /* Reset: VFR */
-#define I40E_VFPE_CQARM1_PECQID_SHIFT 0
-#define I40E_VFPE_CQARM1_PECQID_MASK (0x1FFFF << I40E_VFPE_CQARM1_PECQID_SHIFT)
-#define I40E_VFPE_CQPDB1 0x0000BC00 /* Reset: VFR */
-#define I40E_VFPE_CQPDB1_WQHEAD_SHIFT 0
-#define I40E_VFPE_CQPDB1_WQHEAD_MASK (0x7FF << I40E_VFPE_CQPDB1_WQHEAD_SHIFT)
-#define I40E_VFPE_CQPERRCODES1 0x00009C00 /* Reset: VFR */
-#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT 0
-#define I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES1_CQP_MINOR_CODE_SHIFT)
-#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT 16
-#define I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_MASK (0xFFFF << I40E_VFPE_CQPERRCODES1_CQP_MAJOR_CODE_SHIFT)
-#define I40E_VFPE_CQPTAIL1 0x0000A000 /* Reset: VFR */
-#define I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT 0
-#define I40E_VFPE_CQPTAIL1_WQTAIL_MASK (0x7FF << I40E_VFPE_CQPTAIL1_WQTAIL_SHIFT)
-#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT 31
-#define I40E_VFPE_CQPTAIL1_CQP_OP_ERR_MASK (0x1 << I40E_VFPE_CQPTAIL1_CQP_OP_ERR_SHIFT)
-#define I40E_VFPE_IPCONFIG01 0x00008C00 /* Reset: VFR */
-#define I40E_VFPE_IPCONFIG01_PEIPID_SHIFT 0
-#define I40E_VFPE_IPCONFIG01_PEIPID_MASK (0xFFFF << I40E_VFPE_IPCONFIG01_PEIPID_SHIFT)
-#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT 16
-#define I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_MASK (0x1 << I40E_VFPE_IPCONFIG01_USEENTIREIDRANGE_SHIFT)
-#define I40E_VFPE_MRTEIDXMASK1 0x00009000 /* Reset: VFR */
-#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT 0
-#define I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_MASK (0x1F << I40E_VFPE_MRTEIDXMASK1_MRTEIDXMASKBITS_SHIFT)
-#define I40E_VFPE_RCVUNEXPECTEDERROR1 0x00009400 /* Reset: VFR */
-#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT 0
-#define I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_MASK (0xFFFFFF << I40E_VFPE_RCVUNEXPECTEDERROR1_TCP_RX_UNEXP_ERR_SHIFT)
-#define I40E_VFPE_TCPNOWTIMER1 0x0000A800 /* Reset: VFR */
-#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT 0
-#define I40E_VFPE_TCPNOWTIMER1_TCP_NOW_MASK (0xFFFFFFFF << I40E_VFPE_TCPNOWTIMER1_TCP_NOW_SHIFT)
-#define I40E_VFPE_WQEALLOC1 0x0000C000 /* Reset: VFR */
-#define I40E_VFPE_WQEALLOC1_PEQPID_SHIFT 0
-#define I40E_VFPE_WQEALLOC1_PEQPID_MASK (0x3FFFF << I40E_VFPE_WQEALLOC1_PEQPID_SHIFT)
-#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT 20
-#define I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_MASK (0xFFF << I40E_VFPE_WQEALLOC1_WQE_DESC_INDEX_SHIFT)
-#endif /* I40IW_REGISTER_H */
diff --git a/providers/i40iw/i40iw_status.h b/providers/i40iw/i40iw_status.h
deleted file mode 100644
index c64ce54..0000000
--- a/providers/i40iw/i40iw_status.h
+++ /dev/null
@@ -1,101 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_STATUS_H
-#define I40IW_STATUS_H
-
-/* Error Codes */
-enum i40iw_status_code {
- I40IW_SUCCESS = 0,
- I40IW_ERR_NVM = -1,
- I40IW_ERR_NVM_CHECKSUM = -2,
- I40IW_ERR_CONFIG = -4,
- I40IW_ERR_PARAM = -5,
- I40IW_ERR_DEVICE_NOT_SUPPORTED = -6,
- I40IW_ERR_RESET_FAILED = -7,
- I40IW_ERR_SWFW_SYNC = -8,
- I40IW_ERR_NO_MEMORY = -9,
- I40IW_ERR_BAD_PTR = -10,
- I40IW_ERR_INVALID_PD_ID = -11,
- I40IW_ERR_INVALID_QP_ID = -12,
- I40IW_ERR_INVALID_CQ_ID = -13,
- I40IW_ERR_INVALID_CEQ_ID = -14,
- I40IW_ERR_INVALID_AEQ_ID = -15,
- I40IW_ERR_INVALID_SIZE = -16,
- I40IW_ERR_INVALID_ARP_INDEX = -17,
- I40IW_ERR_INVALID_FPM_FUNC_ID = -18,
- I40IW_ERR_QP_INVALID_MSG_SIZE = -19,
- I40IW_ERR_QP_TOOMANY_WRS_POSTED = -20,
- I40IW_ERR_INVALID_FRAG_COUNT = -21,
- I40IW_ERR_QUEUE_EMPTY = -22,
- I40IW_ERR_INVALID_ALIGNMENT = -23,
- I40IW_ERR_FLUSHED_QUEUE = -24,
- I40IW_ERR_INVALID_PUSH_PAGE_INDEX = -25,
- I40IW_ERR_INVALID_IMM_DATA_SIZE = -26,
- I40IW_ERR_TIMEOUT = -27,
- I40IW_ERR_OPCODE_MISMATCH = -28,
- I40IW_ERR_CQP_COMPL_ERROR = -29,
- I40IW_ERR_INVALID_VF_ID = -30,
- I40IW_ERR_INVALID_HMCFN_ID = -31,
- I40IW_ERR_BACKING_PAGE_ERROR = -32,
- I40IW_ERR_NO_PBLCHUNKS_AVAILABLE = -33,
- I40IW_ERR_INVALID_PBLE_INDEX = -34,
- I40IW_ERR_INVALID_SD_INDEX = -35,
- I40IW_ERR_INVALID_PAGE_DESC_INDEX = -36,
- I40IW_ERR_INVALID_SD_TYPE = -37,
- I40IW_ERR_MEMCPY_FAILED = -38,
- I40IW_ERR_INVALID_HMC_OBJ_INDEX = -39,
- I40IW_ERR_INVALID_HMC_OBJ_COUNT = -40,
- I40IW_ERR_INVALID_SRQ_ARM_LIMIT = -41,
- I40IW_ERR_SRQ_ENABLED = -42,
- I40IW_ERR_BUF_TOO_SHORT = -43,
- I40IW_ERR_BAD_IWARP_CQE = -44,
- I40IW_ERR_NVM_BLANK_MODE = -45,
- I40IW_ERR_NOT_IMPLEMENTED = -46,
- I40IW_ERR_PE_DOORBELL_NOT_ENABLED = -47,
- I40IW_ERR_NOT_READY = -48,
- I40IW_NOT_SUPPORTED = -49,
- I40IW_ERR_FIRMWARE_API_VERSION = -50,
- I40IW_ERR_RING_FULL = -51,
- I40IW_ERR_MPA_CRC = -61,
- I40IW_ERR_NO_TXBUFS = -62,
- I40IW_ERR_SEQ_NUM = -63,
- I40IW_ERR_LIST_EMPTY = -64,
- I40IW_ERR_INVALID_MAC_ADDR = -65,
- I40IW_ERR_BAD_STAG = -66,
- I40IW_ERR_CQ_COMPL_ERROR = -67,
- I40IW_ERR_QUEUE_DESTROYED = -68
-
-};
-#endif
diff --git a/providers/i40iw/i40iw_uk.c b/providers/i40iw/i40iw_uk.c
deleted file mode 100644
index 624b330..0000000
--- a/providers/i40iw/i40iw_uk.c
+++ /dev/null
@@ -1,1266 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include <stdint.h>
-#include <stdatomic.h>
-
-#include "i40iw_osdep.h"
-#include "i40iw_status.h"
-#include "i40iw_d.h"
-#include "i40iw_user.h"
-#include "i40iw_register.h"
-
-static u32 nop_signature = 0x55550000;
-
-/**
- * i40iw_nop_1 - insert a nop wqe and move head. no post work
- * @qp: hw qp ptr
- */
-static enum i40iw_status_code i40iw_nop_1(struct i40iw_qp_uk *qp)
-{
- u64 header, *wqe;
- u64 *wqe_0 = NULL;
- u32 wqe_idx, peek_head;
- bool signaled = false;
-
- if (!qp->sq_ring.head)
- return I40IW_ERR_PARAM;
-
- wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- wqe = qp->sq_base[wqe_idx].elem;
-
- qp->sq_wrtrk_array[wqe_idx].wqe_size = I40IW_QP_WQE_MIN_SIZE;
-
- peek_head = (qp->sq_ring.head + 1) % qp->sq_ring.size;
- wqe_0 = qp->sq_base[peek_head].elem;
- if (peek_head)
- wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID);
- else
- wqe_0[3] = LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
- set_64bit_val(wqe, I40IW_BYTE_0, 0);
- set_64bit_val(wqe, I40IW_BYTE_8, 0);
- set_64bit_val(wqe, I40IW_BYTE_16, 0);
-
- header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |
- LS_64(signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID) | nop_signature++;
-
- udma_to_device_barrier(); /* Memory barrier to ensure data is written before valid bit is set */
-
- set_64bit_val(wqe, I40IW_BYTE_24, header);
- return 0;
-}
-
-/**
- * i40iw_qp_post_wr - post wr to hrdware
- * @qp: hw qp ptr
- */
-void i40iw_qp_post_wr(struct i40iw_qp_uk *qp)
-{
- u64 temp;
- u32 hw_sq_tail;
- u32 sw_sq_head;
-
- /* valid bit is written and loads completed before reading shadow
- *
- * Whatever is happening here does not match our common macros for
- * producer/consumer DMA and may not be portable, however on x86-64
- * the required barrier is MFENCE, get a 'portable' version via C11
- * atomic.
- */
- atomic_thread_fence(memory_order_seq_cst);
-
- /* read the doorbell shadow area */
- get_64bit_val(qp->shadow_area, I40IW_BYTE_0, &temp);
-
- hw_sq_tail = (u32)RS_64(temp, I40IW_QP_DBSA_HW_SQ_TAIL);
- sw_sq_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- if (sw_sq_head != hw_sq_tail) {
- if (sw_sq_head > qp->initial_ring.head) {
- if ((hw_sq_tail >= qp->initial_ring.head) &&
- (hw_sq_tail < sw_sq_head)) {
- db_wr32(qp->qp_id, qp->wqe_alloc_reg);
- }
- } else if (sw_sq_head != qp->initial_ring.head) {
- if ((hw_sq_tail >= qp->initial_ring.head) ||
- (hw_sq_tail < sw_sq_head)) {
- db_wr32(qp->qp_id, qp->wqe_alloc_reg);
- }
- }
- }
-
- qp->initial_ring.head = qp->sq_ring.head;
-}
-
-/**
- * i40iw_qp_ring_push_db - ring qp doorbell
- * @qp: hw qp ptr
- * @wqe_idx: wqe index
- */
-static void i40iw_qp_ring_push_db(struct i40iw_qp_uk *qp, u32 wqe_idx)
-{
- set_32bit_val(qp->push_db, 0, LS_32((wqe_idx >> 2), I40E_PFPE_WQEALLOC_WQE_DESC_INDEX) | qp->qp_id);
- qp->initial_ring.head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
-}
-
-/**
- * i40iw_qp_get_next_send_wqe - return next wqe ptr
- * @qp: hw qp ptr
- * @wqe_idx: return wqe index
- * @wqe_size: size of sq wqe
- */
-u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp,
- u32 *wqe_idx,
- u8 wqe_size,
- u32 total_size,
- u64 wr_id
- )
-{
- u64 *wqe = NULL;
- u64 wqe_ptr;
- u32 peek_head = 0;
- u16 offset;
- enum i40iw_status_code ret_code = 0;
- u8 nop_wqe_cnt = 0, i;
- u64 *wqe_0 = NULL;
-
- *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
-
- if (!*wqe_idx)
- qp->swqe_polarity = !qp->swqe_polarity;
- wqe_ptr = (uintptr_t)qp->sq_base[*wqe_idx].elem;
- offset = (u16)(wqe_ptr) & 0x7F;
- if ((offset + wqe_size) > I40IW_QP_WQE_MAX_SIZE) {
- nop_wqe_cnt = (u8)(I40IW_QP_WQE_MAX_SIZE - offset) / I40IW_QP_WQE_MIN_SIZE;
- for (i = 0; i < nop_wqe_cnt; i++) {
- i40iw_nop_1(qp);
- I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
- if (ret_code)
- return NULL;
- }
-
- *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- if (!*wqe_idx)
- qp->swqe_polarity = !qp->swqe_polarity;
- }
-
- if (((*wqe_idx & 3) == 1) && (wqe_size == I40IW_WQE_SIZE_64)) {
- i40iw_nop_1(qp);
- I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
- if (ret_code)
- return NULL;
- *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- if (!*wqe_idx)
- qp->swqe_polarity = !qp->swqe_polarity;
- }
-
- I40IW_RING_MOVE_HEAD_BY_COUNT(qp->sq_ring,
- (wqe_size / I40IW_QP_WQE_MIN_SIZE), ret_code);
- if (ret_code)
- return NULL;
-
- wqe = qp->sq_base[*wqe_idx].elem;
-
- peek_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring);
- wqe_0 = qp->sq_base[peek_head].elem;
-
- if (((peek_head & 3) == 1) || ((peek_head & 3) == 3)) {
- if (RS_64(wqe_0[3], I40IWQPSQ_VALID) != !qp->swqe_polarity)
- wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID);
- }
-
- qp->sq_wrtrk_array[*wqe_idx].wrid = wr_id;
- qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
- qp->sq_wrtrk_array[*wqe_idx].wqe_size = wqe_size;
- return wqe;
-}
-
-/**
- * i40iw_set_fragment - set fragment in wqe
- * @wqe: wqe for setting fragment
- * @offset: offset value
- * @sge: sge length and stag
- */
-static void i40iw_set_fragment(u64 *wqe, u32 offset, struct i40iw_sge *sge)
-{
- if (sge) {
- set_64bit_val(wqe, offset, LS_64(sge->tag_off, I40IWQPSQ_FRAG_TO));
- set_64bit_val(wqe, (offset + I40IW_BYTE_8),
- (LS_64(sge->len, I40IWQPSQ_FRAG_LEN) |
- LS_64(sge->stag, I40IWQPSQ_FRAG_STAG)));
- }
-}
-
-/**
- * i40iw_qp_get_next_recv_wqe - get next qp's rcv wqe
- * @qp: hw qp ptr
- * @wqe_idx: return wqe index
- */
-u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx)
-{
- u64 *wqe = NULL;
- enum i40iw_status_code ret_code;
-
- if (I40IW_RING_FULL_ERR(qp->rq_ring))
- return NULL;
-
- I40IW_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code);
- if (ret_code)
- return NULL;
- if (!*wqe_idx)
- qp->rwqe_polarity = !qp->rwqe_polarity;
- /* rq_wqe_size_multiplier is no of qwords in one rq wqe */
- wqe = qp->rq_base[*wqe_idx * (qp->rq_wqe_size_multiplier >> 2)].elem;
-
- return wqe;
-}
-
-/**
- * i40iw_rdma_write - rdma write operation
- * @qp: hw qp ptr
- * @info: post sq information
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_rdma_write(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- bool post_sq)
-{
- u64 header;
- u64 *wqe;
- struct i40iw_rdma_write *op_info;
- u32 i, wqe_idx;
- u32 total_size = 0, byte_off;
- enum i40iw_status_code ret_code;
- bool read_fence = false;
- u8 wqe_size;
-
- op_info = &info->op.rdma_write;
- if (op_info->num_lo_sges > qp->max_sq_frag_cnt)
- return I40IW_ERR_INVALID_FRAG_COUNT;
-
- for (i = 0; i < op_info->num_lo_sges; i++)
- total_size += op_info->lo_sg_list[i].len;
-
- if (total_size > I40IW_MAX_OUTBOUND_MESSAGE_SIZE)
- return I40IW_ERR_QP_INVALID_MSG_SIZE;
-
- read_fence |= info->read_fence;
-
- ret_code = i40iw_fragcnt_to_wqesize_sq(op_info->num_lo_sges, &wqe_size);
- if (ret_code)
- return ret_code;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size,
- total_size,info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- set_64bit_val(wqe, I40IW_BYTE_16,
- LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO));
- if (!op_info->rem_addr.stag)
- return I40IW_ERR_BAD_STAG;
-
- header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) |
- LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |
- LS_64((op_info->num_lo_sges > 1 ? (op_info->num_lo_sges - 1) : 0), I40IWQPSQ_ADDFRAGCNT) |
- LS_64(read_fence, I40IWQPSQ_READFENCE) |
- LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_set_fragment(wqe, I40IW_BYTE_0, op_info->lo_sg_list);
-
- for (i = 1, byte_off = I40IW_BYTE_32; i < op_info->num_lo_sges; i++) {
- i40iw_set_fragment(wqe, byte_off, &op_info->lo_sg_list[i]);
- byte_off += 16;
- }
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, I40IW_BYTE_24, header);
-
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_rdma_read - rdma read command
- * @qp: hw qp ptr
- * @info: post sq information
- * @inv_stag: flag for inv_stag
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_rdma_read(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- bool inv_stag,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_rdma_read *op_info;
- u64 header;
- u32 wqe_idx;
- enum i40iw_status_code ret_code;
- u8 wqe_size;
- bool local_fence = false;
-
- op_info = &info->op.rdma_read;
- ret_code = i40iw_fragcnt_to_wqesize_sq(1, &wqe_size);
- if (ret_code)
- return ret_code;
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size,
- op_info->lo_addr.len,info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- local_fence |= info->local_fence;
-
- set_64bit_val(wqe, I40IW_BYTE_16, LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO));
- header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) |
- LS_64((inv_stag ? I40IWQP_OP_RDMA_READ_LOC_INV : I40IWQP_OP_RDMA_READ), I40IWQPSQ_OPCODE) |
- LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
- LS_64(local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_set_fragment(wqe, I40IW_BYTE_0, &op_info->lo_addr);
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, I40IW_BYTE_24, header);
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_send - rdma send command
- * @qp: hw qp ptr
- * @info: post sq information
- * @stag_to_inv: stag_to_inv value
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_send(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- u32 stag_to_inv,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_post_send *op_info;
- u64 header;
- u32 i, wqe_idx, total_size = 0, byte_off;
- enum i40iw_status_code ret_code;
- bool read_fence = false;
- u8 wqe_size;
-
- op_info = &info->op.send;
- if (qp->max_sq_frag_cnt < op_info->num_sges)
- return I40IW_ERR_INVALID_FRAG_COUNT;
-
- for (i = 0; i < op_info->num_sges; i++)
- total_size += op_info->sg_list[i].len;
- ret_code = i40iw_fragcnt_to_wqesize_sq(op_info->num_sges, &wqe_size);
- if (ret_code)
- return ret_code;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size,
- total_size,info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
-
- read_fence |= info->read_fence;
-
- set_64bit_val(wqe, I40IW_BYTE_16, 0);
- header = LS_64(stag_to_inv, I40IWQPSQ_REMSTAG) |
- LS_64(info->op_type, I40IWQPSQ_OPCODE) |
- LS_64((op_info->num_sges > 1 ? (op_info->num_sges - 1) : 0),
- I40IWQPSQ_ADDFRAGCNT) |
- LS_64(read_fence, I40IWQPSQ_READFENCE) |
- LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_set_fragment(wqe, I40IW_BYTE_0, op_info->sg_list);
-
- for (i = 1, byte_off = I40IW_BYTE_32; i < op_info->num_sges; i++) {
- i40iw_set_fragment(wqe, byte_off, &op_info->sg_list[i]);
- byte_off += 16;
- }
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, I40IW_BYTE_24, header);
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_inline_rdma_write - inline rdma write operation
- * @qp: hw qp ptr
- * @info: post sq information
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- bool post_sq)
-{
- u64 *wqe;
- u8 *dest, *src;
- struct i40iw_inline_rdma_write *op_info;
- u64 *push;
- u64 header = 0;
- u32 wqe_idx;
- enum i40iw_status_code ret_code;
- bool read_fence = false;
- u8 wqe_size;
-
- op_info = &info->op.inline_rdma_write;
- if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE)
- return I40IW_ERR_INVALID_IMM_DATA_SIZE;
-
- ret_code = i40iw_inline_data_size_to_wqesize(op_info->len, &wqe_size);
- if (ret_code)
- return ret_code;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size,
- op_info->len,info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
-
- read_fence |= info->read_fence;
- set_64bit_val(wqe, I40IW_BYTE_16,
- LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO));
-
- header = LS_64(op_info->rem_addr.stag, I40IWQPSQ_REMSTAG) |
- LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |
- LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) |
- LS_64(1, I40IWQPSQ_INLINEDATAFLAG) |
- LS_64((qp->push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) |
- LS_64(read_fence, I40IWQPSQ_READFENCE) |
- LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- dest = (u8 *)wqe;
- src = (u8 *)(op_info->data);
-
- if (op_info->len <= I40IW_BYTE_16) {
- memcpy(dest, src, op_info->len);
- } else {
- memcpy(dest, src, I40IW_BYTE_16);
- src += I40IW_BYTE_16;
- dest = (u8 *)wqe + I40IW_BYTE_32;
- memcpy(dest, src, op_info->len - I40IW_BYTE_16);
- }
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, I40IW_BYTE_24, header);
-
- if (qp->push_db) {
- push = (u64 *)((uintptr_t)qp->push_wqe + (wqe_idx & 0x3) * 0x20);
- memcpy(push, wqe, (op_info->len > 16) ? op_info->len + 16 : 32);
- i40iw_qp_ring_push_db(qp, wqe_idx);
- } else {
- if (post_sq)
- i40iw_qp_post_wr(qp);
- }
-
- return 0;
-}
-
-/**
- * i40iw_inline_send - inline send operation
- * @qp: hw qp ptr
- * @info: post sq information
- * @stag_to_inv: remote stag
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- u32 stag_to_inv,
- bool post_sq)
-{
- u64 *wqe;
- u8 *dest, *src;
- struct i40iw_post_inline_send *op_info;
- u64 header;
- u32 wqe_idx;
- enum i40iw_status_code ret_code;
- bool read_fence = false;
- u8 wqe_size;
- u64 *push;
-
- op_info = &info->op.inline_send;
- if (op_info->len > I40IW_MAX_INLINE_DATA_SIZE)
- return I40IW_ERR_INVALID_IMM_DATA_SIZE;
-
- ret_code = i40iw_inline_data_size_to_wqesize(op_info->len, &wqe_size);
- if (ret_code)
- return ret_code;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size,
- op_info->len,info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
-
- read_fence |= info->read_fence;
- header = LS_64(stag_to_inv, I40IWQPSQ_REMSTAG) |
- LS_64(info->op_type, I40IWQPSQ_OPCODE) |
- LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) |
- LS_64(1, I40IWQPSQ_INLINEDATAFLAG) |
- LS_64((qp->push_db ? 1 : 0), I40IWQPSQ_PUSHWQE) |
- LS_64(read_fence, I40IWQPSQ_READFENCE) |
- LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- dest = (u8 *)wqe;
- src = (u8 *)(op_info->data);
-
- if (op_info->len <= I40IW_BYTE_16) {
- memcpy(dest, src, op_info->len);
- } else {
- memcpy(dest, src, I40IW_BYTE_16);
- src += I40IW_BYTE_16;
- dest = (u8 *)wqe + I40IW_BYTE_32;
- memcpy(dest, src, op_info->len - I40IW_BYTE_16);
- }
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, I40IW_BYTE_24, header);
-
- if (qp->push_db) {
- push = (u64 *)((uintptr_t)qp->push_wqe + (wqe_idx & 0x3) * 0x20);
- memcpy(push, wqe, (op_info->len > 16) ? op_info->len + 16 : 32);
- i40iw_qp_ring_push_db(qp, wqe_idx);
- } else {
- if (post_sq)
- i40iw_qp_post_wr(qp);
- }
-
- return 0;
-}
-
-/**
- * i40iw_stag_local_invalidate - stag invalidate operation
- * @qp: hw qp ptr
- * @info: post sq information
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_stag_local_invalidate(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_inv_local_stag *op_info;
- u64 header;
- u32 wqe_idx;
- bool local_fence = false;
-
- op_info = &info->op.inv_local_stag;
- local_fence = info->local_fence;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
- 0,info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- set_64bit_val(wqe, I40IW_BYTE_0, 0);
- set_64bit_val(wqe, I40IW_BYTE_8,
- LS_64(op_info->target_stag, I40IWQPSQ_LOCSTAG));
- set_64bit_val(wqe, I40IW_BYTE_16, 0);
- header = LS_64(I40IW_OP_TYPE_INV_STAG, I40IWQPSQ_OPCODE) |
- LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
- LS_64(local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, I40IW_BYTE_24, header);
-
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_mw_bind - Memory Window bind operation
- * @qp: hw qp ptr
- * @info: post sq information
- * @post_sq: flag to post sq
- */
-static enum i40iw_status_code i40iw_mw_bind(struct i40iw_qp_uk *qp,
- struct i40iw_post_sq_info *info,
- bool post_sq)
-{
- u64 *wqe;
- struct i40iw_bind_window *op_info;
- u64 header;
- u32 wqe_idx;
- bool local_fence = false;
-
- op_info = &info->op.bind_window;
-
- local_fence |= info->local_fence;
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
- 0,info->wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- set_64bit_val(wqe, I40IW_BYTE_0, (uintptr_t)op_info->va);
- set_64bit_val(wqe, I40IW_BYTE_8,
- LS_64(op_info->mr_stag, I40IWQPSQ_PARENTMRSTAG) |
- LS_64(op_info->mw_stag, I40IWQPSQ_MWSTAG));
- set_64bit_val(wqe, I40IW_BYTE_16, op_info->bind_length);
- header = LS_64(I40IW_OP_TYPE_BIND_MW, I40IWQPSQ_OPCODE) |
- LS_64(((op_info->enable_reads << 2) |
- (op_info->enable_writes << 3)),
- I40IWQPSQ_STAGRIGHTS) |
- LS_64((op_info->addressing_type == I40IW_ADDR_TYPE_VA_BASED ? 1 : 0),
- I40IWQPSQ_VABASEDTO) |
- LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
- LS_64(local_fence, I40IWQPSQ_LOCALFENCE) |
- LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, I40IW_BYTE_24, header);
-
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_post_receive - post receive wqe
- * @qp: hw qp ptr
- * @info: post rq information
- */
-static enum i40iw_status_code i40iw_post_receive(struct i40iw_qp_uk *qp,
- struct i40iw_post_rq_info *info)
-{
- u64 *wqe;
- u64 header;
- u32 total_size = 0, wqe_idx, i, byte_off;
-
- if (qp->max_rq_frag_cnt < info->num_sges)
- return I40IW_ERR_INVALID_FRAG_COUNT;
- for (i = 0; i < info->num_sges; i++)
- total_size += info->sg_list[i].len;
- wqe = i40iw_qp_get_next_recv_wqe(qp, &wqe_idx);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
-
- qp->rq_wrid_array[wqe_idx] = info->wr_id;
- set_64bit_val(wqe, I40IW_BYTE_16, 0);
-
- header = LS_64((info->num_sges > 1 ? (info->num_sges - 1) : 0),
- I40IWQPSQ_ADDFRAGCNT) |
- LS_64(qp->rwqe_polarity, I40IWQPSQ_VALID);
-
- i40iw_set_fragment(wqe, I40IW_BYTE_0, info->sg_list);
-
- for (i = 1, byte_off = I40IW_BYTE_32; i < info->num_sges; i++) {
- i40iw_set_fragment(wqe, byte_off, &info->sg_list[i]);
- byte_off += 16;
- }
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, I40IW_BYTE_24, header);
-
- return 0;
-}
-
-/**
- * i40iw_cq_request_notification - cq notification request (door bell)
- * @cq: hw cq
- * @cq_notify: notification type
- */
-static void i40iw_cq_request_notification(struct i40iw_cq_uk *cq,
- enum i40iw_completion_notify cq_notify)
-{
- u64 temp_val;
- u16 sw_cq_sel;
- u8 arm_next_se = 0;
- u8 arm_next = 0;
- u8 arm_seq_num;
-
- get_64bit_val(cq->shadow_area, I40IW_BYTE_32, &temp_val);
- arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);
- arm_seq_num++;
-
- sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);
- arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);
- arm_next_se |= 1;
- if (cq_notify == IW_CQ_COMPL_EVENT)
- arm_next = 1;
- temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) |
- LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) |
- LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) |
- LS_64(arm_next, I40IW_CQ_DBSA_ARM_NEXT);
-
- set_64bit_val(cq->shadow_area, I40IW_BYTE_32, temp_val);
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- db_wr32(cq->cq_id, cq->cqe_alloc_reg);
-}
-
-/**
- * i40iw_cq_post_entries - update tail in shadow memory
- * @cq: hw cq
- * @count: # of entries processed
- */
-static enum i40iw_status_code i40iw_cq_post_entries(struct i40iw_cq_uk *cq,
- u8 count)
-{
- I40IW_RING_MOVE_TAIL_BY_COUNT(cq->cq_ring, count);
- set_64bit_val(cq->shadow_area, I40IW_BYTE_0,
- I40IW_RING_GETCURRENT_HEAD(cq->cq_ring));
- return 0;
-}
-
-/**
- * i40iw_cq_poll_completion - get cq completion info
- * @cq: hw cq
- * @info: cq poll information returned
- * @post_cq: update cq tail
- */
-static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq,
- struct i40iw_cq_poll_info *info)
-{
- u64 comp_ctx, qword0, qword2, qword3, wqe_qword;
- u64 *cqe, *sw_wqe;
- struct i40iw_qp_uk *qp;
- struct i40iw_ring *pring = NULL;
- u32 wqe_idx, q_type, array_idx = 0;
- enum i40iw_status_code ret_code = 0;
- bool move_cq_head = true;
- u8 polarity;
- u8 addl_wqes = 0;
-
- if (cq->avoid_mem_cflct)
- cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(cq);
- else
- cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(cq);
-
- get_64bit_val(cqe, I40IW_BYTE_24, &qword3);
- polarity = (u8)RS_64(qword3, I40IW_CQ_VALID);
-
- if (polarity != cq->polarity)
- return I40IW_ERR_QUEUE_EMPTY;
-
- udma_from_device_barrier();
-
- q_type = (u8)RS_64(qword3, I40IW_CQ_SQ);
- info->error = (bool)RS_64(qword3, I40IW_CQ_ERROR);
- info->push_dropped = (bool)RS_64(qword3, I40IWCQ_PSHDROP);
- if (info->error) {
- info->comp_status = I40IW_COMPL_STATUS_FLUSHED;
- info->major_err = (bool)RS_64(qword3, I40IW_CQ_MAJERR);
- info->minor_err = (bool)RS_64(qword3, I40IW_CQ_MINERR);
- } else {
- info->comp_status = I40IW_COMPL_STATUS_SUCCESS;
- }
-
- get_64bit_val(cqe, I40IW_BYTE_0, &qword0);
- get_64bit_val(cqe, I40IW_BYTE_16, &qword2);
-
- info->tcp_seq_num = (u8)RS_64(qword0, I40IWCQ_TCPSEQNUM);
-
- info->qp_id = (u32)RS_64(qword2, I40IWCQ_QPID);
-
- get_64bit_val(cqe, I40IW_BYTE_8, &comp_ctx);
-
- info->solicited_event = (bool)RS_64(qword3, I40IWCQ_SOEVENT);
- info->is_srq = (bool)RS_64(qword3, I40IWCQ_SRQ);
-
- qp = (struct i40iw_qp_uk *)(i40iw_uintptr)comp_ctx;
- if (!qp) {
- ret_code = I40IW_ERR_QUEUE_DESTROYED;
- goto exit;
- }
- wqe_idx = (u32)RS_64(qword3, I40IW_CQ_WQEIDX);
- info->qp_handle = (i40iw_qp_handle)(i40iw_uintptr)qp;
-
- if (q_type == I40IW_CQE_QTYPE_RQ) {
- array_idx = (wqe_idx * 4) / qp->rq_wqe_size_multiplier;
- if (info->comp_status == I40IW_COMPL_STATUS_FLUSHED) {
- info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail];
- array_idx = qp->rq_ring.tail;
- } else {
- info->wr_id = qp->rq_wrid_array[array_idx];
- }
-
- info->op_type = I40IW_OP_TYPE_REC;
- if (qword3 & I40IWCQ_STAG_MASK) {
- info->stag_invalid_set = true;
- info->inv_stag = (u32)RS_64(qword2, I40IWCQ_INVSTAG);
- } else {
- info->stag_invalid_set = false;
- }
- info->bytes_xfered = (u32)RS_64(qword0, I40IWCQ_PAYLDLEN);
- I40IW_RING_SET_TAIL(qp->rq_ring, array_idx + 1);
- pring = &qp->rq_ring;
- } else {
- if (qp->first_sq_wq) {
- qp->first_sq_wq = false;
- if (!wqe_idx && (qp->sq_ring.head == qp->sq_ring.tail)) {
- I40IW_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
- I40IW_RING_MOVE_TAIL(cq->cq_ring);
- set_64bit_val(cq->shadow_area, I40IW_BYTE_0,
- I40IW_RING_GETCURRENT_HEAD(cq->cq_ring));
- memset(info, 0, sizeof(struct i40iw_cq_poll_info));
- return i40iw_cq_poll_completion(cq, info);
- }
- }
-
- if (info->comp_status != I40IW_COMPL_STATUS_FLUSHED) {
- info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
- info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;
-
- info->op_type = (u8)RS_64(qword3, I40IWCQ_OP);
- sw_wqe = qp->sq_base[wqe_idx].elem;
- get_64bit_val(sw_wqe, I40IW_BYTE_24, &wqe_qword);
- addl_wqes = qp->sq_wrtrk_array[wqe_idx].wqe_size/I40IW_QP_WQE_MIN_SIZE;
- I40IW_RING_SET_TAIL(qp->sq_ring, (wqe_idx + addl_wqes));
- } else {
- do {
- u8 op_type;
- u32 tail;
-
- tail = qp->sq_ring.tail;
- sw_wqe = qp->sq_base[tail].elem;
- get_64bit_val(sw_wqe, I40IW_BYTE_24, &wqe_qword);
- op_type = (u8)RS_64(wqe_qword, I40IWQPSQ_OPCODE);
- info->op_type = op_type;
- addl_wqes = qp->sq_wrtrk_array[tail].wqe_size/I40IW_QP_WQE_MIN_SIZE;
- I40IW_RING_SET_TAIL(qp->sq_ring, (tail + addl_wqes));
- if (op_type != I40IWQP_OP_NOP) {
- info->wr_id = qp->sq_wrtrk_array[tail].wrid;
- info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
- break;
- }
- } while (1);
- }
- pring = &qp->sq_ring;
- }
-
- ret_code = 0;
-
-exit:
- if (!ret_code &&
- (info->comp_status == I40IW_COMPL_STATUS_FLUSHED))
- if (pring && (I40IW_RING_MORE_WORK(*pring)))
- move_cq_head = false;
-
- if (move_cq_head) {
- I40IW_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
-
- if (I40IW_RING_GETCURRENT_HEAD(cq->cq_ring) == 0)
- cq->polarity ^= 1;
-
- I40IW_RING_MOVE_TAIL(cq->cq_ring);
- set_64bit_val(cq->shadow_area, I40IW_BYTE_0,
- I40IW_RING_GETCURRENT_HEAD(cq->cq_ring));
- } else {
- if (info->is_srq)
- return ret_code;
- qword3 &= ~I40IW_CQ_WQEIDX_MASK;
- qword3 |= LS_64(pring->tail, I40IW_CQ_WQEIDX);
- set_64bit_val(cqe, I40IW_BYTE_24, qword3);
- }
-
- return ret_code;
-}
-
-/**
- * i40iw_qp_roundup - return round up QP WQ depth
- * @wqdepth: WQ depth in quantas to round up
- */
-static int i40iw_qp_round_up(u32 wqdepth)
-{
- int scount = 1;
-
- for (wqdepth--; scount <= 16; scount *= 2)
- wqdepth |= wqdepth >> scount;
-
- return ++wqdepth;
-}
-
-/**
- * i40iw_get_wqe_shift - get shift count for maximum wqe size
- * @sge: Maximum Scatter Gather Elements wqe
- * @inline_data: Maximum inline data size
- * @shift: Returns the shift needed based on sge
- *
- * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size.
- * For 1 SGE or inline data <= 16, shift = 0 (wqe size of 32 bytes).
- * For 2 or 3 SGEs or inline data <= 48, shift = 1 (wqe size of 64 bytes).
- * Shift of 2 otherwise (wqe size of 128 bytes).
- */
-void i40iw_get_wqe_shift(u32 sge, u32 inline_data, u8 *shift)
-{
- *shift = 0;
- if (sge > 1 || inline_data > 16)
- *shift = (sge < 4 && inline_data <= 48) ? 1 : 2;
-}
-
-/*
- * i40iw_get_sqdepth - get SQ depth (quantas)
- * @sq_size: SQ size
- * @shift: shift which determines size of WQE
- * @sqdepth: depth of SQ
- *
- */
-enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth)
-{
- *sqdepth = i40iw_qp_round_up((sq_size << shift) + I40IW_SQ_RSVD);
-
- if (*sqdepth < (I40IW_QP_SW_MIN_WQSIZE << shift))
- *sqdepth = I40IW_QP_SW_MIN_WQSIZE << shift;
- else if (*sqdepth > I40IW_QP_SW_MAX_SQ_QUANTAS)
- return I40IW_ERR_INVALID_SIZE;
-
- return 0;
-}
-
-/*
- * i40iw_get_rq_depth - get RQ depth (quantas)
- * @rq_size: RQ size
- * @shift: shift which determines size of WQE
- * @rqdepth: depth of RQ
- *
- */
-enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth)
-{
- *rqdepth = i40iw_qp_round_up((rq_size << shift) + I40IW_RQ_RSVD);
-
- if (*rqdepth < (I40IW_QP_SW_MIN_WQSIZE << shift))
- *rqdepth = I40IW_QP_SW_MIN_WQSIZE << shift;
- else if (*rqdepth > I40IW_QP_SW_MAX_RQ_QUANTAS)
- return I40IW_ERR_INVALID_SIZE;
-
- return 0;
-}
-
-static struct i40iw_qp_uk_ops iw_qp_uk_ops = {
- i40iw_qp_post_wr,
- i40iw_qp_ring_push_db,
- i40iw_rdma_write,
- i40iw_rdma_read,
- i40iw_send,
- i40iw_inline_rdma_write,
- i40iw_inline_send,
- i40iw_stag_local_invalidate,
- i40iw_mw_bind,
- i40iw_post_receive,
- i40iw_nop
-};
-
-static struct i40iw_cq_ops iw_cq_ops = {
- i40iw_cq_request_notification,
- i40iw_cq_poll_completion,
- i40iw_cq_post_entries,
- i40iw_clean_cq
-};
-
-static struct i40iw_device_uk_ops iw_device_uk_ops = {
- i40iw_cq_uk_init,
- i40iw_qp_uk_init,
-};
-
-/**
- * i40iw_qp_uk_init - initialize shared qp
- * @qp: hw qp (user and kernel)
- * @info: qp initialization info
- *
- * initializes the vars used in both user and kernel mode.
- * size of the wqe depends on numbers of max. fragements
- * allowed. Then size of wqe * the number of wqes should be the
- * amount of memory allocated for sq and rq. If srq is used,
- * then rq_base will point to one rq wqe only (not the whole
- * array of wqes)
- */
-enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp,
- struct i40iw_qp_uk_init_info *info)
-{
- enum i40iw_status_code ret_code = 0;
- u32 sq_ring_size;
- u8 sqshift, rqshift;
-
- if (info->max_sq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
- return I40IW_ERR_INVALID_FRAG_COUNT;
-
- if (info->max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
- return I40IW_ERR_INVALID_FRAG_COUNT;
- i40iw_get_wqe_shift(info->max_sq_frag_cnt, info->max_inline_data, &sqshift);
-
- qp->sq_base = info->sq;
- qp->rq_base = info->rq;
- qp->shadow_area = info->shadow_area;
- qp->sq_wrtrk_array = info->sq_wrtrk_array;
- qp->rq_wrid_array = info->rq_wrid_array;
-
- qp->wqe_alloc_reg = info->wqe_alloc_reg;
- qp->qp_id = info->qp_id;
-
- qp->sq_size = info->sq_size;
- qp->push_db = info->push_db;
- qp->push_wqe = info->push_wqe;
-
- qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
- sq_ring_size = qp->sq_size << sqshift;
-
- I40IW_RING_INIT(qp->sq_ring, sq_ring_size);
- I40IW_RING_INIT(qp->initial_ring, sq_ring_size);
- I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code);
- I40IW_RING_MOVE_TAIL(qp->sq_ring);
- I40IW_RING_MOVE_HEAD(qp->initial_ring, ret_code);
- qp->swqe_polarity = 1;
- qp->first_sq_wq = true;
- qp->swqe_polarity_deferred = 1;
- qp->rwqe_polarity = 0;
-
- if (!qp->use_srq) {
- qp->rq_size = info->rq_size;
- qp->max_rq_frag_cnt = info->max_rq_frag_cnt;
- I40IW_RING_INIT(qp->rq_ring, qp->rq_size);
- switch (info->abi_ver) {
- case 4:
- i40iw_get_wqe_shift(info->max_rq_frag_cnt, 0, &rqshift);
- break;
- case 5: /* fallthrough until next ABI version */
- default:
- rqshift = I40IW_MAX_RQ_WQE_SHIFT;
- break;
- }
- qp->rq_wqe_size = rqshift;
- qp->rq_wqe_size_multiplier = 4 << rqshift;
- }
- qp->ops = iw_qp_uk_ops;
-
- return ret_code;
-}
-
-/**
- * i40iw_cq_uk_init - initialize shared cq (user and kernel)
- * @cq: hw cq
- * @info: hw cq initialization info
- */
-enum i40iw_status_code i40iw_cq_uk_init(struct i40iw_cq_uk *cq,
- struct i40iw_cq_uk_init_info *info)
-{
- if ((info->cq_size < I40IW_MIN_CQ_SIZE) ||
- (info->cq_size > I40IW_MAX_CQ_SIZE))
- return I40IW_ERR_INVALID_SIZE;
- cq->cq_base = (struct i40iw_cqe *)info->cq_base;
- cq->cq_id = info->cq_id;
- cq->cq_size = info->cq_size;
- cq->cqe_alloc_reg = info->cqe_alloc_reg;
- cq->shadow_area = info->shadow_area;
- cq->avoid_mem_cflct = info->avoid_mem_cflct;
-
- I40IW_RING_INIT(cq->cq_ring, cq->cq_size);
- cq->polarity = 1;
- cq->ops = iw_cq_ops;
-
- return 0;
-}
-
-/**
- * i40iw_device_init_uk - setup routines for iwarp shared device
- * @dev: iwarp shared (user and kernel)
- */
-void i40iw_device_init_uk(struct i40iw_dev_uk *dev)
-{
- dev->ops_uk = iw_device_uk_ops;
-}
-
-/**
- * i40iw_clean_cq - clean cq entries
- * @ queue completion context
- * @cq: cq to clean
- */
-void i40iw_clean_cq(void *queue, struct i40iw_cq_uk *cq)
-{
- u64 *cqe;
- u64 qword3, comp_ctx;
- u32 cq_head;
- u8 polarity, temp;
-
- cq_head = cq->cq_ring.head;
- temp = cq->polarity;
- do {
- if (cq->avoid_mem_cflct)
- cqe = (u64 *)&(((struct i40iw_extended_cqe *)cq->cq_base)[cq_head]);
- else
- cqe = (u64 *)&cq->cq_base[cq_head];
- get_64bit_val(cqe, I40IW_BYTE_24, &qword3);
- polarity = (u8)RS_64(qword3, I40IW_CQ_VALID);
-
- if (polarity != temp)
- break;
-
- get_64bit_val(cqe, I40IW_BYTE_8, &comp_ctx);
- if ((void *)(i40iw_uintptr)comp_ctx == queue)
- set_64bit_val(cqe, I40IW_BYTE_8, 0);
-
- cq_head = (cq_head + 1) % cq->cq_ring.size;
- if (!cq_head)
- temp ^= 1;
- } while (true);
-}
-
-/**
- * i40iw_nop - send a nop
- * @qp: hw qp ptr
- * @wr_id: work request id
- * @signaled: flag if signaled for completion
- * @post_sq: flag to post sq
- */
-enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp,
- u64 wr_id,
- bool signaled,
- bool post_sq)
-{
- u64 header, *wqe;
- u32 wqe_idx;
-
- wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
- 0,wr_id);
- if (!wqe)
- return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
- set_64bit_val(wqe, I40IW_BYTE_0, 0);
- set_64bit_val(wqe, I40IW_BYTE_8, 0);
- set_64bit_val(wqe, I40IW_BYTE_16, 0);
-
- header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |
- LS_64(signaled, I40IWQPSQ_SIGCOMPL) |
- LS_64(qp->swqe_polarity, I40IWQPSQ_VALID);
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, I40IW_BYTE_24, header);
- if (post_sq)
- i40iw_qp_post_wr(qp);
-
- return 0;
-}
-
-/**
- * i40iw_fragcnt_to_wqesize_sq - calculate wqe size based on fragment count for SQ
- * @frag_cnt: number of fragments
- * @wqe_size: size of sq wqe returned
- */
-enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size)
-{
- switch (frag_cnt) {
- case 0:
- case 1:
- *wqe_size = I40IW_QP_WQE_MIN_SIZE;
- break;
- case 2:
- case 3:
- *wqe_size = 64;
- break;
- case 4:
- case 5:
- *wqe_size = 96;
- break;
- case 6:
- case 7:
- *wqe_size = 128;
- break;
- default:
- return I40IW_ERR_INVALID_FRAG_COUNT;
- }
-
- return 0;
-}
-
-/**
- * i40iw_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ
- * @frag_cnt: number of fragments
- * @wqe_size: size of rq wqe returned
- */
-enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size)
-{
- switch (frag_cnt) {
- case 0:
- case 1:
- *wqe_size = 32;
- break;
- case 2:
- case 3:
- *wqe_size = 64;
- break;
- case 4:
- case 5:
- case 6:
- case 7:
- *wqe_size = 128;
- break;
- default:
- return I40IW_ERR_INVALID_FRAG_COUNT;
- }
-
- return 0;
-}
-
-/**
- * i40iw_inline_data_size_to_wqesize - based on inline data, wqe size
- * @data_size: data size for inline
- * @wqe_size: size of sq wqe returned
- */
-enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size,
- u8 *wqe_size)
-{
- if (data_size > I40IW_MAX_INLINE_DATA_SIZE)
- return I40IW_ERR_INVALID_IMM_DATA_SIZE;
-
- if (data_size <= 16)
- *wqe_size = I40IW_QP_WQE_MIN_SIZE;
- else
- *wqe_size = 64;
-
- return 0;
-}
diff --git a/providers/i40iw/i40iw_umain.c b/providers/i40iw/i40iw_umain.c
deleted file mode 100644
index c5ac079..0000000
--- a/providers/i40iw/i40iw_umain.c
+++ /dev/null
@@ -1,226 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include <config.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <errno.h>
-#include <sys/mman.h>
-#include <pthread.h>
-
-#include "i40e_devids.h"
-#include "i40iw_umain.h"
-#include "i40iw-abi.h"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-static void i40iw_ufree_context(struct ibv_context *ibctx);
-
-#define INTEL_HCA(v, d) VERBS_PCI_MATCH(v, d, NULL)
-static const struct verbs_match_ent hca_table[] = {
- VERBS_DRIVER_ID(RDMA_DRIVER_I40IW),
-#ifdef I40E_DEV_ID_X722_A0
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0),
-#endif
-#ifdef I40E_DEV_ID_X722_A0_VF
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0_VF),
-#endif
-#ifdef I40E_DEV_ID_KX_X722
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_X722),
-#endif
-#ifdef I40E_DEV_ID_QSFP_X722
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_X722),
-#endif
-#ifdef I40E_DEV_ID_SFP_X722
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722),
-#endif
-#ifdef I40E_DEV_ID_1G_BASE_T_X722
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722),
-#endif
-#ifdef I40E_DEV_ID_10G_BASE_T_X722
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722),
-#endif
-#ifdef I40E_DEV_ID_SFP_I_X722
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_I_X722),
-#endif
-#ifdef I40E_DEV_ID_X722_VF
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF),
-#endif
-#ifdef I40E_DEV_ID_X722_VF_HV
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF_HV),
-#endif
-#ifdef I40E_DEV_ID_X722_FPGA
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_FPGA),
-#endif
-#ifdef I40E_DEV_ID_X722_FPGA_VF
- INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_FPGA_VF),
-#endif
- {}
-};
-
-static const struct verbs_context_ops i40iw_uctx_ops = {
- .query_device_ex = i40iw_uquery_device,
- .query_port = i40iw_uquery_port,
- .alloc_pd = i40iw_ualloc_pd,
- .dealloc_pd = i40iw_ufree_pd,
- .reg_mr = i40iw_ureg_mr,
- .dereg_mr = i40iw_udereg_mr,
- .create_cq = i40iw_ucreate_cq,
- .poll_cq = i40iw_upoll_cq,
- .req_notify_cq = i40iw_uarm_cq,
- .cq_event = i40iw_cq_event,
- .destroy_cq = i40iw_udestroy_cq,
- .create_qp = i40iw_ucreate_qp,
- .query_qp = i40iw_uquery_qp,
- .modify_qp = i40iw_umodify_qp,
- .destroy_qp = i40iw_udestroy_qp,
- .post_send = i40iw_upost_send,
- .post_recv = i40iw_upost_recv,
- .async_event = i40iw_async_event,
- .free_context = i40iw_ufree_context,
-};
-
-/**
- * i40iw_ualloc_context - allocate context for user app
- * @ibdev: pointer to device created during i40iw_driver_init
- * @cmd_fd: save fd for the device
- *
- * Returns callback routines table and calls driver for allocating
- * context and getting back resource information to return as ibv_context.
- */
-
-static struct verbs_context *i40iw_ualloc_context(struct ibv_device *ibdev,
- int cmd_fd,
- void *private_data)
-{
- struct ibv_pd *ibv_pd;
- struct i40iw_uvcontext *iwvctx;
- struct i40iw_get_context cmd;
- struct i40iw_get_context_resp resp;
-
- iwvctx = verbs_init_and_alloc_context(ibdev, cmd_fd, iwvctx, ibv_ctx,
- RDMA_DRIVER_I40IW);
- if (!iwvctx)
- return NULL;
-
- cmd.userspace_ver = I40IW_ABI_VER;
- memset(&resp, 0, sizeof(resp));
- if (ibv_cmd_get_context(&iwvctx->ibv_ctx, (struct ibv_get_context *)&cmd,
- sizeof(cmd), &resp.ibv_resp, sizeof(resp))) {
-
- cmd.userspace_ver = 4;
- if (ibv_cmd_get_context(&iwvctx->ibv_ctx, (struct ibv_get_context *)&cmd,
- sizeof(cmd), &resp.ibv_resp, sizeof(resp)))
- goto err_free;
-
- }
-
- if (resp.kernel_ver > I40IW_ABI_VER) {
- fprintf(stderr, PFX "%s: incompatible kernel driver version: %d. Need version %d\n",
- __func__, resp.kernel_ver, I40IW_ABI_VER);
- goto err_free;
- }
-
- verbs_set_ops(&iwvctx->ibv_ctx, &i40iw_uctx_ops);
- iwvctx->max_pds = resp.max_pds;
- iwvctx->max_qps = resp.max_qps;
- iwvctx->wq_size = resp.wq_size;
- iwvctx->abi_ver = resp.kernel_ver;
-
- i40iw_device_init_uk(&iwvctx->dev);
- ibv_pd = i40iw_ualloc_pd(&iwvctx->ibv_ctx.context);
- if (!ibv_pd)
- goto err_free;
- ibv_pd->context = &iwvctx->ibv_ctx.context;
- iwvctx->iwupd = to_i40iw_upd(ibv_pd);
-
- return &iwvctx->ibv_ctx;
-
-err_free:
- fprintf(stderr, PFX "%s: failed to allocate context for device.\n", __func__);
- verbs_uninit_context(&iwvctx->ibv_ctx);
- free(iwvctx);
-
- return NULL;
-}
-
-/**
- * i40iw_ufree_context - free context that was allocated
- * @ibctx: context allocated ptr
- */
-static void i40iw_ufree_context(struct ibv_context *ibctx)
-{
- struct i40iw_uvcontext *iwvctx = to_i40iw_uctx(ibctx);
-
- i40iw_ufree_pd(&iwvctx->iwupd->ibv_pd);
-
- verbs_uninit_context(&iwvctx->ibv_ctx);
- free(iwvctx);
-}
-
-static void i40iw_uninit_device(struct verbs_device *verbs_device)
-{
- struct i40iw_udevice *dev = to_i40iw_udev(&verbs_device->device);
-
- free(dev);
-}
-
-static struct verbs_device *
-i40iw_device_alloc(struct verbs_sysfs_dev *sysfs_dev)
-{
- struct i40iw_udevice *dev;
-
- dev = calloc(1, sizeof(*dev));
- if (!dev)
- return NULL;
-
- dev->page_size = I40IW_HW_PAGE_SIZE;
- return &dev->ibv_dev;
-}
-
-static const struct verbs_device_ops i40iw_udev_ops = {
- .name = "i40iw",
- .match_min_abi_version = 0,
- .match_max_abi_version = INT_MAX,
- .match_table = hca_table,
- .alloc_device = i40iw_device_alloc,
- .uninit_device = i40iw_uninit_device,
- .alloc_context = i40iw_ualloc_context,
-};
-PROVIDER_DRIVER(i40iw, i40iw_udev_ops);
diff --git a/providers/i40iw/i40iw_umain.h b/providers/i40iw/i40iw_umain.h
deleted file mode 100644
index fe643dd..0000000
--- a/providers/i40iw/i40iw_umain.h
+++ /dev/null
@@ -1,181 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_UMAIN_H
-#define I40IW_UMAIN_H
-
-#include <inttypes.h>
-#include <stddef.h>
-#include <endian.h>
-#include <util/compiler.h>
-
-#include <infiniband/driver.h>
-#include <util/udma_barrier.h>
-
-#include "i40iw_osdep.h"
-#include "i40iw_d.h"
-#include "i40iw_status.h"
-#include "i40iw_user.h"
-
-#define PFX "libi40iw-"
-
-#define I40IW_BASE_PUSH_PAGE 1
-#define I40IW_U_MINCQ_SIZE 4
-
-#define I40IW_WC_WITH_VLAN (1 << 3)
-#define I40IW_UD_RX_BATCH_SZ 64
-#define I40IW_UD_MAX_SG_LIST_SZ 1
-#define I40IW_CQ_BUF_OV_ERR 0x3
-
-#define MAX_WQ_DEPTH 16384
-#define MIN_WQ_DEPTH 4
-
-#define I40E_DB_SHADOW_AREA_SIZE 64
-#define I40E_DB_CQ_OFFSET 0x40
-
-struct i40iw_udevice {
- struct verbs_device ibv_dev;
- int page_size;
-};
-
-struct i40iw_upd {
- struct ibv_pd ibv_pd;
- void volatile *db;
- void volatile *arm_cq_page;
- void volatile *arm_cq;
- uint32_t pd_id;
-};
-
-struct i40iw_uvcontext {
- struct verbs_context ibv_ctx;
- struct i40iw_upd *iwupd;
- uint32_t max_pds; /* maximum pds allowed for this user process */
- uint32_t max_qps; /* maximum qps allowed for this user process */
- uint32_t wq_size; /* size of the WQs (sq+rq) + shadow allocated to the mmaped area */
- struct i40iw_dev_uk dev;
- int abi_ver;
-};
-
-struct i40iw_uqp;
-
-struct i40iw_ucq {
- struct ibv_cq ibv_cq;
- struct verbs_mr vmr;
- struct ibv_mr mr_shadow_area;
- pthread_spinlock_t lock;
- uint8_t is_armed;
- uint8_t skip_arm;
- int arm_sol;
- int skip_sol;
- int comp_vector;
- struct i40iw_uqp *udqp;
- struct i40iw_cq_uk cq;
-};
-
-struct i40iw_uqp {
- struct ibv_qp ibv_qp;
- struct i40iw_ucq *send_cq;
- struct i40iw_ucq *recv_cq;
- struct verbs_mr vmr;
- uint32_t i40iw_drv_opt;
- pthread_spinlock_t lock;
- u32 *push_db; /* mapped as uncached memory*/
- u64 *push_wqe; /* mapped as write combined memory*/
- uint16_t sq_sig_all;
- uint16_t qperr;
- uint16_t rsvd;
- uint32_t pending_rcvs;
- uint32_t wq_size;
- struct ibv_recv_wr *pend_rx_wr;
- struct i40iw_qp_uk qp;
-
-};
-
-#define to_i40iw_uxxx(xxx, type) \
- container_of(ib##xxx, struct i40iw_u##type, ibv_##xxx)
-
-static inline struct i40iw_udevice *to_i40iw_udev(struct ibv_device *ibdev)
-{
- return container_of(ibdev, struct i40iw_udevice, ibv_dev.device);
-}
-
-static inline struct i40iw_uvcontext *to_i40iw_uctx(struct ibv_context *ibctx)
-{
- return container_of(ibctx, struct i40iw_uvcontext, ibv_ctx.context);
-}
-
-static inline struct i40iw_upd *to_i40iw_upd(struct ibv_pd *ibpd)
-{
- return to_i40iw_uxxx(pd, pd);
-}
-
-static inline struct i40iw_ucq *to_i40iw_ucq(struct ibv_cq *ibcq)
-{
- return to_i40iw_uxxx(cq, cq);
-}
-
-static inline struct i40iw_uqp *to_i40iw_uqp(struct ibv_qp *ibqp)
-{
- return to_i40iw_uxxx(qp, qp);
-}
-
-/* i40iw_uverbs.c */
-int i40iw_uquery_device(struct ibv_context *context,
- const struct ibv_query_device_ex_input *input,
- struct ibv_device_attr_ex *attr, size_t attr_size);
-int i40iw_uquery_port(struct ibv_context *, uint8_t, struct ibv_port_attr *);
-struct ibv_pd *i40iw_ualloc_pd(struct ibv_context *);
-int i40iw_ufree_pd(struct ibv_pd *);
-struct ibv_mr *i40iw_ureg_mr(struct ibv_pd *pd, void *addr, size_t length,
- uint64_t hca_va, int access);
-int i40iw_udereg_mr(struct verbs_mr *vmr);
-struct ibv_cq *i40iw_ucreate_cq(struct ibv_context *, int, struct ibv_comp_channel *, int);
-int i40iw_udestroy_cq(struct ibv_cq *);
-int i40iw_upoll_cq(struct ibv_cq *, int, struct ibv_wc *);
-int i40iw_uarm_cq(struct ibv_cq *, int);
-void i40iw_cq_event(struct ibv_cq *);
-struct ibv_srq *i40iw_ucreate_srq(struct ibv_pd *, struct ibv_srq_init_attr *);
-int i40iw_umodify_srq(struct ibv_srq *, struct ibv_srq_attr *, int);
-int i40iw_udestroy_srq(struct ibv_srq *);
-int i40iw_upost_srq_recv(struct ibv_srq *, struct ibv_recv_wr *, struct ibv_recv_wr **);
-struct ibv_qp *i40iw_ucreate_qp(struct ibv_pd *, struct ibv_qp_init_attr *);
-int i40iw_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int, struct ibv_qp_init_attr *init_attr);
-int i40iw_umodify_qp(struct ibv_qp *, struct ibv_qp_attr *, int);
-int i40iw_udestroy_qp(struct ibv_qp *);
-int i40iw_upost_send(struct ibv_qp *, struct ibv_send_wr *, struct ibv_send_wr **);
-int i40iw_upost_recv(struct ibv_qp *, struct ibv_recv_wr *, struct ibv_recv_wr **);
-void i40iw_async_event(struct ibv_context *context,
- struct ibv_async_event *event);
-
-#endif /* i40iw_umain_H */
diff --git a/providers/i40iw/i40iw_user.h b/providers/i40iw/i40iw_user.h
deleted file mode 100644
index 921848c..0000000
--- a/providers/i40iw/i40iw_user.h
+++ /dev/null
@@ -1,456 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#ifndef I40IW_USER_H
-#define I40IW_USER_H
-
-enum i40iw_device_capabilities_const {
- I40IW_WQE_SIZE = 4,
- I40IW_CQP_WQE_SIZE = 8,
- I40IW_CQE_SIZE = 4,
- I40IW_EXTENDED_CQE_SIZE = 8,
- I40IW_AEQE_SIZE = 2,
- I40IW_CEQE_SIZE = 1,
- I40IW_CQP_CTX_SIZE = 8,
- I40IW_SHADOW_AREA_SIZE = 8,
- I40IW_CEQ_MAX_COUNT = 256,
- I40IW_QUERY_FPM_BUF_SIZE = 128,
- I40IW_COMMIT_FPM_BUF_SIZE = 128,
- I40IW_MIN_IW_QP_ID = 1,
- I40IW_MAX_IW_QP_ID = 262143,
- I40IW_MIN_CEQID = 0,
- I40IW_MAX_CEQID = 256,
- I40IW_MIN_CQID = 0,
- I40IW_MAX_CQID = 131071,
- I40IW_MIN_AEQ_ENTRIES = 1,
- I40IW_MAX_AEQ_ENTRIES = 524287,
- I40IW_MIN_CEQ_ENTRIES = 1,
- I40IW_MAX_CEQ_ENTRIES = 131071,
- I40IW_MIN_CQ_SIZE = 1,
- I40IW_MAX_CQ_SIZE = 1048575,
- I40IW_MAX_AEQ_ALLOCATE_COUNT = 255,
- I40IW_DB_ID_ZERO = 0,
- I40IW_MAX_WQ_FRAGMENT_COUNT = 3,
- I40IW_MAX_SGE_RD = 1,
- I40IW_MAX_OUTBOUND_MESSAGE_SIZE = 2147483647,
- I40IW_MAX_INBOUND_MESSAGE_SIZE = 2147483647,
- I40IW_MAX_PUSH_PAGE_COUNT = 4096,
- I40IW_MAX_PE_ENABLED_VF_COUNT = 32,
- I40IW_MAX_VF_FPM_ID = 47,
- I40IW_MAX_VF_PER_PF = 127,
- I40IW_MAX_SQ_PAYLOAD_SIZE = 2145386496,
- I40IW_MAX_INLINE_DATA_SIZE = 48,
- I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE = 48,
- I40IW_MAX_IRD_SIZE = 32,
- I40IW_QPCTX_ENCD_MAXIRD = 3,
- I40IW_MAX_WQ_ENTRIES = 2048,
- I40IW_MAX_ORD_SIZE = 32,
- I40IW_Q2_BUFFER_SIZE = (248 + 100),
- I40IW_MAX_WQE_SIZE_RQ = 128,
- I40IW_QP_CTX_SIZE = 248
-};
-
-#define i40iw_handle void *
-#define i40iw_adapter_handle i40iw_handle
-#define i40iw_qp_handle i40iw_handle
-#define i40iw_cq_handle i40iw_handle
-#define i40iw_srq_handle i40iw_handle
-#define i40iw_pd_id i40iw_handle
-#define i40iw_stag_handle i40iw_handle
-#define i40iw_stag_index u32
-#define i40iw_stag u32
-#define i40iw_stag_key u8
-
-#define i40iw_tagged_offset u64
-#define i40iw_access_privileges u32
-#define i40iw_physical_fragment u64
-#define i40iw_address_list u64 *
-
-#define I40IW_CREATE_STAG(index, key) (((index) << 8) + (key))
-
-#define I40IW_STAG_KEY_FROM_STAG(stag) ((stag) && 0x000000FF)
-
-#define I40IW_STAG_INDEX_FROM_STAG(stag) (((stag) && 0xFFFFFF00) >> 8)
-
-#define I40IW_MAX_MR_SIZE 0x10000000000L
-#define I40IW_MAX_RQ_WQE_SHIFT 2
-
-struct i40iw_qp_uk;
-struct i40iw_cq_uk;
-struct i40iw_srq_uk;
-struct i40iw_qp_uk_init_info;
-struct i40iw_cq_uk_init_info;
-struct i40iw_srq_uk_init_info;
-
-struct i40iw_sge {
- i40iw_tagged_offset tag_off;
- u32 len;
- i40iw_stag stag;
-};
-
-#define i40iw_sgl struct i40iw_sge *
-
-struct i40iw_ring {
- volatile u32 head;
- volatile u32 tail;
- u32 size;
-};
-
-struct i40iw_cqe {
- u64 buf[I40IW_CQE_SIZE];
-};
-
-struct i40iw_extended_cqe {
- u64 buf[I40IW_EXTENDED_CQE_SIZE];
-};
-
-struct i40iw_wqe {
- u64 buf[I40IW_WQE_SIZE];
-};
-
-struct i40iw_qp_uk_ops;
-
-enum i40iw_addressing_type {
- I40IW_ADDR_TYPE_ZERO_BASED = 0,
- I40IW_ADDR_TYPE_VA_BASED = 1,
-};
-
-#define I40IW_ACCESS_FLAGS_LOCALREAD 0x01
-#define I40IW_ACCESS_FLAGS_LOCALWRITE 0x02
-#define I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04
-#define I40IW_ACCESS_FLAGS_REMOTEREAD 0x05
-#define I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08
-#define I40IW_ACCESS_FLAGS_REMOTEWRITE 0x0a
-#define I40IW_ACCESS_FLAGS_BIND_WINDOW 0x10
-#define I40IW_ACCESS_FLAGS_ALL 0x1F
-
-#define I40IW_OP_TYPE_RDMA_WRITE 0
-#define I40IW_OP_TYPE_RDMA_READ 1
-#define I40IW_OP_TYPE_SEND 3
-#define I40IW_OP_TYPE_SEND_INV 4
-#define I40IW_OP_TYPE_SEND_SOL 5
-#define I40IW_OP_TYPE_SEND_SOL_INV 6
-#define I40IW_OP_TYPE_REC 7
-#define I40IW_OP_TYPE_BIND_MW 8
-#define I40IW_OP_TYPE_FAST_REG_NSMR 9
-#define I40IW_OP_TYPE_INV_STAG 10
-#define I40IW_OP_TYPE_RDMA_READ_INV_STAG 11
-#define I40IW_OP_TYPE_NOP 12
-
-enum i40iw_completion_status {
- I40IW_COMPL_STATUS_SUCCESS = 0,
- I40IW_COMPL_STATUS_FLUSHED,
- I40IW_COMPL_STATUS_INVALID_WQE,
- I40IW_COMPL_STATUS_QP_CATASTROPHIC,
- I40IW_COMPL_STATUS_REMOTE_TERMINATION,
- I40IW_COMPL_STATUS_INVALID_STAG,
- I40IW_COMPL_STATUS_BASE_BOUND_VIOLATION,
- I40IW_COMPL_STATUS_ACCESS_VIOLATION,
- I40IW_COMPL_STATUS_INVALID_PD_ID,
- I40IW_COMPL_STATUS_WRAP_ERROR,
- I40IW_COMPL_STATUS_STAG_INVALID_PDID,
- I40IW_COMPL_STATUS_RDMA_READ_ZERO_ORD,
- I40IW_COMPL_STATUS_QP_NOT_PRIVLEDGED,
- I40IW_COMPL_STATUS_STAG_NOT_INVALID,
- I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_SIZE,
- I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_ENTRY,
- I40IW_COMPL_STATUS_INVALID_FBO,
- I40IW_COMPL_STATUS_INVALID_LENGTH,
- I40IW_COMPL_STATUS_INVALID_ACCESS,
- I40IW_COMPL_STATUS_PHYS_BUFFER_LIST_TOO_LONG,
- I40IW_COMPL_STATUS_INVALID_VIRT_ADDRESS,
- I40IW_COMPL_STATUS_INVALID_REGION,
- I40IW_COMPL_STATUS_INVALID_WINDOW,
- I40IW_COMPL_STATUS_INVALID_TOTAL_LENGTH
-};
-
-enum i40iw_completion_notify {
- IW_CQ_COMPL_EVENT = 0,
- IW_CQ_COMPL_SOLICITED = 1
-};
-
-struct i40iw_post_send {
- i40iw_sgl sg_list;
- u32 num_sges;
-};
-
-struct i40iw_post_inline_send {
- void *data;
- u32 len;
-};
-
-struct i40iw_post_send_w_inv {
- i40iw_sgl sg_list;
- u32 num_sges;
- i40iw_stag remote_stag_to_inv;
-};
-
-struct i40iw_post_inline_send_w_inv {
- void *data;
- u32 len;
- i40iw_stag remote_stag_to_inv;
-};
-
-struct i40iw_rdma_write {
- i40iw_sgl lo_sg_list;
- u32 num_lo_sges;
- struct i40iw_sge rem_addr;
-};
-
-struct i40iw_inline_rdma_write {
- void *data;
- u32 len;
- struct i40iw_sge rem_addr;
-};
-
-struct i40iw_rdma_read {
- struct i40iw_sge lo_addr;
- struct i40iw_sge rem_addr;
-};
-
-struct i40iw_bind_window {
- i40iw_stag mr_stag;
- u64 bind_length;
- void *va;
- enum i40iw_addressing_type addressing_type;
- bool enable_reads;
- bool enable_writes;
- i40iw_stag mw_stag;
-};
-
-struct i40iw_inv_local_stag {
- i40iw_stag target_stag;
-};
-
-struct i40iw_post_sq_info {
- u64 wr_id;
- u8 op_type;
- bool signaled;
- bool read_fence;
- bool local_fence;
- bool inline_data;
- bool defer_flag;
- union {
- struct i40iw_post_send send;
- struct i40iw_post_send send_w_sol;
- struct i40iw_post_send_w_inv send_w_inv;
- struct i40iw_post_send_w_inv send_w_sol_inv;
- struct i40iw_rdma_write rdma_write;
- struct i40iw_rdma_read rdma_read;
- struct i40iw_rdma_read rdma_read_inv;
- struct i40iw_bind_window bind_window;
- struct i40iw_inv_local_stag inv_local_stag;
- struct i40iw_inline_rdma_write inline_rdma_write;
- struct i40iw_post_inline_send inline_send;
- struct i40iw_post_inline_send inline_send_w_sol;
- struct i40iw_post_inline_send_w_inv inline_send_w_inv;
- struct i40iw_post_inline_send_w_inv inline_send_w_sol_inv;
- } op;
-};
-
-struct i40iw_post_rq_info {
- u64 wr_id;
- i40iw_sgl sg_list;
- u32 num_sges;
-};
-
-struct i40iw_cq_poll_info {
- u64 wr_id;
- i40iw_qp_handle qp_handle;
- u32 bytes_xfered;
- u32 tcp_seq_num;
- u32 qp_id;
- i40iw_stag inv_stag;
- enum i40iw_completion_status comp_status;
- u16 major_err;
- u16 minor_err;
- u8 op_type;
- bool stag_invalid_set;
- bool push_dropped;
- bool error;
- bool is_srq;
- bool solicited_event;
-};
-
-struct i40iw_qp_uk_ops {
- void (*iw_qp_post_wr)(struct i40iw_qp_uk *);
- void (*iw_qp_ring_push_db)(struct i40iw_qp_uk *, u32);
- enum i40iw_status_code (*iw_rdma_write)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, bool);
- enum i40iw_status_code (*iw_rdma_read)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, bool, bool);
- enum i40iw_status_code (*iw_send)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, u32, bool);
- enum i40iw_status_code (*iw_inline_rdma_write)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, bool);
- enum i40iw_status_code (*iw_inline_send)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, u32, bool);
- enum i40iw_status_code (*iw_stag_local_invalidate)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, bool);
- enum i40iw_status_code (*iw_mw_bind)(struct i40iw_qp_uk *,
- struct i40iw_post_sq_info *, bool);
- enum i40iw_status_code (*iw_post_receive)(struct i40iw_qp_uk *,
- struct i40iw_post_rq_info *);
- enum i40iw_status_code (*iw_post_nop)(struct i40iw_qp_uk *, u64, bool, bool);
-};
-
-struct i40iw_cq_ops {
- void (*iw_cq_request_notification)(struct i40iw_cq_uk *,
- enum i40iw_completion_notify);
- enum i40iw_status_code (*iw_cq_poll_completion)(struct i40iw_cq_uk *,
- struct i40iw_cq_poll_info *);
- enum i40iw_status_code (*iw_cq_post_entries)(struct i40iw_cq_uk *, u8 count);
- void (*iw_cq_clean)(void *, struct i40iw_cq_uk *);
-};
-
-struct i40iw_dev_uk;
-
-struct i40iw_device_uk_ops {
- enum i40iw_status_code (*iwarp_cq_uk_init)(struct i40iw_cq_uk *,
- struct i40iw_cq_uk_init_info *);
- enum i40iw_status_code (*iwarp_qp_uk_init)(struct i40iw_qp_uk *,
- struct i40iw_qp_uk_init_info *);
-};
-
-struct i40iw_dev_uk {
- struct i40iw_device_uk_ops ops_uk;
-};
-
-struct i40iw_sq_uk_wr_trk_info {
- u64 wrid;
- u32 wr_len;
- u8 wqe_size;
- u8 reserved[3];
-};
-
-struct i40iw_qp_quanta {
- u64 elem[I40IW_WQE_SIZE];
-};
-
-struct i40iw_qp_uk {
- struct i40iw_qp_quanta *sq_base;
- struct i40iw_qp_quanta *rq_base;
- u32 IOMEM *wqe_alloc_reg;
- struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
- u64 *rq_wrid_array;
- u64 *shadow_area;
- u32 *push_db;
- u64 *push_wqe;
- struct i40iw_ring sq_ring;
- struct i40iw_ring rq_ring;
- struct i40iw_ring initial_ring;
- u32 qp_id;
- u32 sq_size;
- u32 rq_size;
- u32 max_sq_frag_cnt;
- u32 max_rq_frag_cnt;
- struct i40iw_qp_uk_ops ops;
- bool use_srq;
- u8 swqe_polarity;
- u8 swqe_polarity_deferred;
- u8 rwqe_polarity;
- u8 rq_wqe_size;
- u8 rq_wqe_size_multiplier;
- bool first_sq_wq;
- bool deferred_flag;
-};
-
-struct i40iw_cq_uk {
- struct i40iw_cqe *cq_base;
- u32 IOMEM *cqe_alloc_reg;
- u64 *shadow_area;
- u32 cq_id;
- u32 cq_size;
- struct i40iw_ring cq_ring;
- u8 polarity;
- bool avoid_mem_cflct;
-
- struct i40iw_cq_ops ops;
-};
-
-struct i40iw_qp_uk_init_info {
- struct i40iw_qp_quanta *sq;
- struct i40iw_qp_quanta *rq;
- u32 IOMEM *wqe_alloc_reg;
- u64 *shadow_area;
- struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
- u64 *rq_wrid_array;
- u32 *push_db;
- u64 *push_wqe;
- u32 qp_id;
- u32 sq_size;
- u32 rq_size;
- u32 max_sq_frag_cnt;
- u32 max_rq_frag_cnt;
- u32 max_inline_data;
- int abi_ver;
-};
-
-struct i40iw_cq_uk_init_info {
- u32 IOMEM *cqe_alloc_reg;
- struct i40iw_cqe *cq_base;
- u64 *shadow_area;
- u32 cq_size;
- u32 cq_id;
- bool avoid_mem_cflct;
-};
-
-void i40iw_device_init_uk(struct i40iw_dev_uk *dev);
-
-void i40iw_qp_post_wr(struct i40iw_qp_uk *qp);
-u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx,
- u8 wqe_size,
- u32 total_size,
- u64 wr_id
- );
-
-u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx);
-u64 *i40iw_qp_get_next_srq_wqe(struct i40iw_srq_uk *srq, u32 *wqe_idx);
-
-enum i40iw_status_code i40iw_cq_uk_init(struct i40iw_cq_uk *cq,
- struct i40iw_cq_uk_init_info *info);
-enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp,
- struct i40iw_qp_uk_init_info *info);
-
-void i40iw_clean_cq(void *queue, struct i40iw_cq_uk *cq);
-enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, u64 wr_id,
- bool signaled, bool post_sq);
-enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size);
-enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size);
-enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size,
- u8 *wqe_size);
-void i40iw_get_wqe_shift(u32 sge, u32 inline_data, u8 *shift);
-enum i40iw_status_code i40iw_get_sqdepth(u32 sq_size, u8 shift, u32 *sqdepth);
-enum i40iw_status_code i40iw_get_rqdepth(u32 rq_size, u8 shift, u32 *rqdepth);
-#endif
diff --git a/providers/i40iw/i40iw_uverbs.c b/providers/i40iw/i40iw_uverbs.c
deleted file mode 100644
index c170bb3..0000000
--- a/providers/i40iw/i40iw_uverbs.c
+++ /dev/null
@@ -1,983 +0,0 @@
-/*******************************************************************************
-*
-* Copyright (c) 2015-2016 Intel Corporation. All rights reserved.
-*
-* This software is available to you under a choice of one of two
-* licenses. You may choose to be licensed under the terms of the GNU
-* General Public License (GPL) Version 2, available from the file
-* COPYING in the main directory of this source tree, or the
-* OpenFabrics.org BSD license below:
-*
-* Redistribution and use in source and binary forms, with or
-* without modification, are permitted provided that the following
-* conditions are met:
-*
-* - Redistributions of source code must retain the above
-* copyright notice, this list of conditions and the following
-* disclaimer.
-*
-* - Redistributions in binary form must reproduce the above
-* copyright notice, this list of conditions and the following
-* disclaimer in the documentation and/or other materials
-* provided with the distribution.
-*
-* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
-* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-* SOFTWARE.
-*
-*******************************************************************************/
-
-#include <config.h>
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-#include <signal.h>
-#include <errno.h>
-#include <pthread.h>
-#include <malloc.h>
-#include <sys/mman.h>
-#include <linux/if_ether.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-#include "i40iw_umain.h"
-#include "i40iw-abi.h"
-
-/**
- * i40iw_uquery_device - call driver to query device for max resources
- * @context: user context for the device
- * @attr: where to save all the mx resources from the driver
- **/
-int i40iw_uquery_device(struct ibv_context *context,
- const struct ibv_query_device_ex_input *input,
- struct ibv_device_attr_ex *attr, size_t attr_size)
-{
- struct ib_uverbs_ex_query_device_resp resp;
- size_t resp_size = sizeof(resp);
- uint64_t i40iw_fw_ver;
- int ret;
- unsigned int minor, major;
-
- ret = ibv_cmd_query_device_any(context, input, attr, attr_size, &resp,
- &resp_size);
- if (ret)
- return ret;
-
- i40iw_fw_ver = resp.base.fw_ver;
- major = (i40iw_fw_ver >> 16) & 0xffff;
- minor = i40iw_fw_ver & 0xffff;
-
- snprintf(attr->orig_attr.fw_ver, sizeof(attr->orig_attr.fw_ver),
- "%d.%d", major, minor);
-
- return 0;
-}
-
-/**
- * i40iw_uquery_port - get port attributes (msg size, lnk, mtu...)
- * @context: user context of the device
- * @port: port for the attributes
- * @attr: to return port attributes
- **/
-int i40iw_uquery_port(struct ibv_context *context, uint8_t port, struct ibv_port_attr *attr)
-{
- struct ibv_query_port cmd;
-
- return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
-}
-
-/**
- * i40iw_ualloc_pd - allocates protection domain and return pd ptr
- * @context: user context of the device
- **/
-struct ibv_pd *i40iw_ualloc_pd(struct ibv_context *context)
-{
- struct ibv_alloc_pd cmd;
- struct i40iw_ualloc_pd_resp resp;
- struct i40iw_upd *iwupd;
- void *map;
-
- iwupd = malloc(sizeof(*iwupd));
- if (!iwupd)
- return NULL;
- memset(&resp, 0, sizeof(resp));
- if (ibv_cmd_alloc_pd(context, &iwupd->ibv_pd, &cmd, sizeof(cmd), &resp.ibv_resp, sizeof(resp)))
- goto err_free;
-
- iwupd->pd_id = resp.pd_id;
- map = mmap(NULL, I40IW_HW_PAGE_SIZE, PROT_WRITE | PROT_READ, MAP_SHARED, context->cmd_fd, 0);
- if (map == MAP_FAILED) {
- ibv_cmd_dealloc_pd(&iwupd->ibv_pd);
- goto err_free;
- }
- iwupd->db = map;
-
- return &iwupd->ibv_pd;
-
-err_free:
- free(iwupd);
- return NULL;
-}
-
-/**
- * i40iw_ufree_pd - free pd resources
- * @pd: pd to free resources
- */
-int i40iw_ufree_pd(struct ibv_pd *pd)
-{
- int ret;
- struct i40iw_upd *iwupd;
-
- iwupd = to_i40iw_upd(pd);
- ret = ibv_cmd_dealloc_pd(pd);
- if (ret)
- return ret;
-
- munmap((void *)iwupd->db, I40IW_HW_PAGE_SIZE);
- free(iwupd);
-
- return 0;
-}
-
-/**
- * i40iw_ureg_mr - register user memory region
- * @pd: pd for the mr
- * @addr: user address of the memory region
- * @length: length of the memory
- * @access: access allowed on this mr
- */
-struct ibv_mr *i40iw_ureg_mr(struct ibv_pd *pd, void *addr, size_t length,
- uint64_t hca_va, int access)
-{
- struct verbs_mr *vmr;
- struct i40iw_ureg_mr cmd;
- struct ib_uverbs_reg_mr_resp resp;
-
- vmr = malloc(sizeof(*vmr));
- if (!vmr)
- return NULL;
-
- cmd.reg_type = IW_MEMREG_TYPE_MEM;
-
- if (ibv_cmd_reg_mr(pd, addr, length, hca_va, access, vmr, &cmd.ibv_cmd,
- sizeof(cmd), &resp, sizeof(resp))) {
- fprintf(stderr, PFX "%s: Failed to register memory\n", __func__);
- free(vmr);
- return NULL;
- }
- return &vmr->ibv_mr;
-}
-
-/**
- * i40iw_udereg_mr - re-register memory region
- * @mr: mr that was allocated
- */
-int i40iw_udereg_mr(struct verbs_mr *vmr)
-{
- int ret;
-
- ret = ibv_cmd_dereg_mr(vmr);
- if (ret)
- return ret;
-
- free(vmr);
- return 0;
-}
-
-/**
- * i40iw_num_of_pages - number of pages needed
- * @size: size for number of pages
- */
-static inline u32 i40iw_num_of_pages(u32 size)
-{
- return (size + 4095) >> 12;
-}
-
-/**
- * i40iw_ucreate_cq - create completion queue for user app
- * @context: user context of the device
- * @cqe: number of cq entries in the cq ring
- * @channel: channel info (context, refcnt..)
- * @comp_vector: save in ucq struct
- */
-struct ibv_cq *i40iw_ucreate_cq(struct ibv_context *context, int cqe,
- struct ibv_comp_channel *channel, int comp_vector)
-{
- struct i40iw_ucq *iwucq;
- struct i40iw_ucreate_cq cmd;
- struct i40iw_ucreate_cq_resp resp;
- struct i40iw_cq_uk_init_info info;
- int ret;
- struct i40iw_uvcontext *iwvctx = to_i40iw_uctx(context);
- u32 cqe_struct_size;
- u32 totalsize;
- u32 cq_pages;
-
- struct i40iw_ureg_mr reg_mr_cmd;
-
- struct ib_uverbs_reg_mr_resp reg_mr_resp;
-
- if (cqe > I40IW_MAX_CQ_SIZE)
- return NULL;
-
- cqe++;
- memset(&cmd, 0, sizeof(cmd));
- memset(&resp, 0, sizeof(resp));
- memset(&info, 0, sizeof(info));
- memset(®_mr_cmd, 0, sizeof(reg_mr_cmd));
-
- iwucq = malloc(sizeof(*iwucq));
- if (!iwucq)
- return NULL;
- memset(iwucq, 0, sizeof(*iwucq));
-
- if (pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE)) {
- free(iwucq);
- return NULL;
- }
- if (cqe < I40IW_U_MINCQ_SIZE)
- cqe = I40IW_U_MINCQ_SIZE;
-
- info.cq_size = cqe;
- iwucq->comp_vector = comp_vector;
- cqe_struct_size = sizeof(struct i40iw_cqe);
- cq_pages = i40iw_num_of_pages(info.cq_size * cqe_struct_size);
- totalsize = (cq_pages << 12) + I40E_DB_SHADOW_AREA_SIZE;
-
- info.cq_base = memalign(I40IW_HW_PAGE_SIZE, totalsize);
-
- if (!info.cq_base)
- goto err;
-
- memset(info.cq_base, 0, totalsize);
- info.shadow_area = (u64 *)((u8 *)info.cq_base + (cq_pages << 12));
- reg_mr_cmd.reg_type = IW_MEMREG_TYPE_CQ;
-
- reg_mr_cmd.cq_pages = cq_pages;
-
- ret = ibv_cmd_reg_mr(&iwvctx->iwupd->ibv_pd, (void *)info.cq_base,
- totalsize, (uintptr_t)info.cq_base,
- IBV_ACCESS_LOCAL_WRITE, &iwucq->vmr,
- ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd),
- ®_mr_resp, sizeof(reg_mr_resp));
- if (ret) {
- fprintf(stderr, PFX "%s: failed to pin memory for CQ\n", __func__);
- goto err;
- }
-
- cmd.user_cq_buffer = (__u64)((uintptr_t)info.cq_base);
- ret = ibv_cmd_create_cq(context, info.cq_size, channel, comp_vector,
- &iwucq->ibv_cq, &cmd.ibv_cmd, sizeof(cmd),
- &resp.ibv_resp, sizeof(resp));
- if (ret) {
- ibv_cmd_dereg_mr(&iwucq->vmr);
- fprintf(stderr, PFX "%s: failed to create CQ\n", __func__);
- goto err;
- }
-
- info.cq_id = (uint16_t)resp.cq_id;
- info.shadow_area = (u64 *)((u8 *)info.shadow_area + resp.reserved);
-
- info.cqe_alloc_reg = (u32 *)((u8 *)iwvctx->iwupd->db + I40E_DB_CQ_OFFSET);
- ret = iwvctx->dev.ops_uk.iwarp_cq_uk_init(&iwucq->cq, &info);
- if (!ret)
- return &iwucq->ibv_cq;
- else
- fprintf(stderr, PFX "%s: failed to initialize CQ, status %d\n", __func__, ret);
-err:
- if (info.cq_base)
- free(info.cq_base);
- if (pthread_spin_destroy(&iwucq->lock))
- return NULL;
- free(iwucq);
- return NULL;
-}
-
-/**
- * i40iw_udestroy_cq - destroys cq
- * @cq: ptr to cq to be destroyed
- */
-int i40iw_udestroy_cq(struct ibv_cq *cq)
-{
- struct i40iw_ucq *iwucq = to_i40iw_ucq(cq);
- int ret;
-
- ret = pthread_spin_destroy(&iwucq->lock);
- if (ret)
- return ret;
-
- ret = ibv_cmd_destroy_cq(cq);
- if (ret)
- return ret;
-
- ibv_cmd_dereg_mr(&iwucq->vmr);
-
- free(iwucq->cq.cq_base);
- free(iwucq);
-
- return 0;
-}
-
-/**
- * i40iw_upoll_cq - user app to poll cq
- * @cq: cq to poll
- * @num_entries: max cq entries to poll
- * @entry: for each completion complete entry
- */
-int i40iw_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry)
-{
- struct i40iw_ucq *iwucq;
- int cqe_count = 0;
- struct i40iw_cq_poll_info cq_poll_info;
- int ret;
-
- iwucq = to_i40iw_ucq(cq);
-
- ret = pthread_spin_lock(&iwucq->lock);
- if (ret)
- return ret;
- while (cqe_count < num_entries) {
- ret = iwucq->cq.ops.iw_cq_poll_completion(&iwucq->cq, &cq_poll_info);
- if (ret == I40IW_ERR_QUEUE_EMPTY) {
- break;
- } else if (ret == I40IW_ERR_QUEUE_DESTROYED) {
- continue;
- } else if (ret) {
- fprintf(stderr, PFX "%s: Error polling CQ, status %d\n", __func__, ret);
- if (!cqe_count)
- /* Indicate error */
- cqe_count = -1;
- break;
- }
- entry->wc_flags = 0;
- entry->wr_id = cq_poll_info.wr_id;
-
- if (cq_poll_info.error) {
- entry->status = IBV_WC_WR_FLUSH_ERR;
- entry->vendor_err = cq_poll_info.major_err << 16 | cq_poll_info.minor_err;
- } else {
- entry->status = IBV_WC_SUCCESS;
- }
-
- switch (cq_poll_info.op_type) {
- case I40IW_OP_TYPE_RDMA_WRITE:
- entry->opcode = IBV_WC_RDMA_WRITE;
- break;
- case I40IW_OP_TYPE_RDMA_READ_INV_STAG:
- case I40IW_OP_TYPE_RDMA_READ:
- entry->opcode = IBV_WC_RDMA_READ;
- break;
- case I40IW_OP_TYPE_SEND_SOL:
- case I40IW_OP_TYPE_SEND_SOL_INV:
- case I40IW_OP_TYPE_SEND_INV:
- case I40IW_OP_TYPE_SEND:
- entry->opcode = IBV_WC_SEND;
- break;
- case I40IW_OP_TYPE_REC:
- entry->opcode = IBV_WC_RECV;
- break;
- default:
- entry->opcode = IBV_WC_RECV;
- break;
- }
-
- entry->imm_data = 0;
- entry->qp_num = cq_poll_info.qp_id;
- entry->src_qp = cq_poll_info.qp_id;
- entry->byte_len = cq_poll_info.bytes_xfered;
- entry++;
- cqe_count++;
- }
- pthread_spin_unlock(&iwucq->lock);
- return cqe_count;
-}
-
-/**
- * i40iw_arm_cq - arm of cq
- * @iwucq: cq to which arm
- * @cq_notify: notification params
- */
-static void i40iw_arm_cq(struct i40iw_ucq *iwucq, enum i40iw_completion_notify cq_notify)
-{
- iwucq->is_armed = 1;
- iwucq->arm_sol = 1;
- iwucq->skip_arm = 0;
- iwucq->skip_sol = 1;
-
- iwucq->cq.ops.iw_cq_request_notification(&iwucq->cq, cq_notify);
-}
-
-/**
- * i40iw_uarm_cq - callback for arm of cq
- * @cq: cq to arm
- * @solicited: to get notify params
- */
-int i40iw_uarm_cq(struct ibv_cq *cq, int solicited)
-{
- struct i40iw_ucq *iwucq;
- enum i40iw_completion_notify cq_notify = IW_CQ_COMPL_EVENT;
- int ret;
-
- iwucq = to_i40iw_ucq(cq);
- if (solicited)
- cq_notify = IW_CQ_COMPL_SOLICITED;
-
- ret = pthread_spin_lock(&iwucq->lock);
- if (ret)
- return ret;
-
- if (iwucq->is_armed) {
- if ((iwucq->arm_sol) && (!solicited)) {
- i40iw_arm_cq(iwucq, cq_notify);
- } else {
- iwucq->skip_arm = 1;
- iwucq->skip_sol &= solicited;
- }
- } else {
- i40iw_arm_cq(iwucq, cq_notify);
- }
-
- pthread_spin_unlock(&iwucq->lock);
-
- return 0;
-}
-
-/**
- * i40iw_cq_event - cq to do completion event
- * @cq: cq to arm
- */
-void i40iw_cq_event(struct ibv_cq *cq)
-{
- struct i40iw_ucq *iwucq;
-
- iwucq = to_i40iw_ucq(cq);
- if (pthread_spin_lock(&iwucq->lock))
- return;
-
- if (iwucq->skip_arm)
- i40iw_arm_cq(iwucq, IW_CQ_COMPL_EVENT);
- else
- iwucq->is_armed = 0;
-
- pthread_spin_unlock(&iwucq->lock);
-}
-
-static int i40iw_destroy_vmapped_qp(struct i40iw_uqp *iwuqp,
- struct i40iw_qp_quanta *sq_base)
-{
- int ret;
-
- ret = ibv_cmd_destroy_qp(&iwuqp->ibv_qp);
- if (ret)
- return ret;
-
- if (iwuqp->push_db)
- munmap(iwuqp->push_db, I40IW_HW_PAGE_SIZE);
- if (iwuqp->push_wqe)
- munmap(iwuqp->push_wqe, I40IW_HW_PAGE_SIZE);
-
- ibv_cmd_dereg_mr(&iwuqp->vmr);
- free((void *)sq_base);
-
- return 0;
-}
-
-/**
- * i40iw_vmapped_qp - create resources for qp
- * @iwuqp: qp struct for resources
- * @pd: pd for thes qp
- * @attr: atributes of qp passed
- * @resp: response back from create qp
- * @sqdepth: depth of sq
- * @rqdepth: depth of rq
- * @info: info for initializing user level qp
- */
-static int i40iw_vmapped_qp(struct i40iw_uqp *iwuqp, struct ibv_pd *pd,
- struct ibv_qp_init_attr *attr,
- struct i40iw_ucreate_qp_resp *resp, int sqdepth,
- int rqdepth, struct i40iw_qp_uk_init_info *info)
-{
- struct i40iw_ucreate_qp cmd;
- int sqsize, rqsize, totalqpsize;
- int ret;
- struct i40iw_ureg_mr reg_mr_cmd;
- u32 sq_pages, rq_pages;
- struct ib_uverbs_reg_mr_resp reg_mr_resp;
-
- memset(®_mr_cmd, 0, sizeof(reg_mr_cmd));
- sqsize = sqdepth * I40IW_QP_WQE_MIN_SIZE;
- rqsize = rqdepth * I40IW_QP_WQE_MIN_SIZE;
-
- sq_pages = i40iw_num_of_pages(sqsize);
- rq_pages = i40iw_num_of_pages(rqsize);
- sqsize = sq_pages << 12;
- rqsize = rq_pages << 12;
- totalqpsize = rqsize + sqsize + I40E_DB_SHADOW_AREA_SIZE;
- info->sq = memalign(I40IW_HW_PAGE_SIZE, totalqpsize);
-
- if (!info->sq) {
- fprintf(stderr, PFX "%s: failed to allocate memory for SQ\n", __func__);
- return 0;
- }
-
- memset(info->sq, 0, totalqpsize);
- info->rq = &info->sq[sqsize / I40IW_QP_WQE_MIN_SIZE];
- info->shadow_area = info->rq[rqsize / I40IW_QP_WQE_MIN_SIZE].elem;
-
- reg_mr_cmd.reg_type = IW_MEMREG_TYPE_QP;
- reg_mr_cmd.sq_pages = sq_pages;
- reg_mr_cmd.rq_pages = rq_pages;
-
- ret = ibv_cmd_reg_mr(pd, (void *)info->sq, totalqpsize,
- (uintptr_t)info->sq, IBV_ACCESS_LOCAL_WRITE,
- &iwuqp->vmr, ®_mr_cmd.ibv_cmd,
- sizeof(reg_mr_cmd), ®_mr_resp,
- sizeof(reg_mr_resp));
- if (ret) {
- fprintf(stderr, PFX "%s: failed to pin memory for SQ\n", __func__);
- free(info->sq);
- return 0;
- }
- cmd.user_wqe_buffers = (__u64)((uintptr_t)info->sq);
- cmd.user_compl_ctx = (uintptr_t)&iwuqp->qp;
-
- ret = ibv_cmd_create_qp(pd, &iwuqp->ibv_qp, attr, &cmd.ibv_cmd, sizeof(cmd),
- &resp->ibv_resp, sizeof(struct i40iw_ucreate_qp_resp));
- if (ret) {
- fprintf(stderr, PFX "%s: failed to create QP, status %d\n", __func__, ret);
- ibv_cmd_dereg_mr(&iwuqp->vmr);
- free(info->sq);
- return 0;
- }
-
- iwuqp->send_cq = to_i40iw_ucq(attr->send_cq);
- iwuqp->recv_cq = to_i40iw_ucq(attr->recv_cq);
- info->sq_size = resp->actual_sq_size;
- info->rq_size = resp->actual_rq_size;
-
- if (resp->push_idx != I40IW_INVALID_PUSH_PAGE_INDEX) {
- void *map;
- u64 offset;
-
- offset = (resp->push_idx + I40IW_BASE_PUSH_PAGE) * I40IW_HW_PAGE_SIZE;
-
- map = mmap(NULL, I40IW_HW_PAGE_SIZE, PROT_WRITE | PROT_READ, MAP_SHARED,
- pd->context->cmd_fd, offset);
- if (map == MAP_FAILED) {
- fprintf(stderr, PFX "%s: failed to map push page, errno %d\n", __func__, errno);
- info->push_wqe = NULL;
- info->push_db = NULL;
- } else {
- info->push_wqe = map;
-
- offset += I40IW_HW_PAGE_SIZE;
- map = mmap(NULL, I40IW_HW_PAGE_SIZE, PROT_WRITE | PROT_READ, MAP_SHARED,
- pd->context->cmd_fd, offset);
- if (map == MAP_FAILED) {
- fprintf(stderr, PFX "%s: failed to map push doorbell, errno %d\n", __func__, errno);
- munmap(info->push_wqe, I40IW_HW_PAGE_SIZE);
- info->push_wqe = NULL;
- info->push_db = NULL;
- } else {
- info->push_db = map;
- }
- iwuqp->push_db = info->push_db;
- iwuqp->push_wqe = info->push_wqe;
- }
- }
- return 1;
-}
-
-/**
- * i40iw_ucreate_qp - create qp on user app
- * @pd: pd for the qp
- * @attr: attributes of the qp to be created (sizes, sge, cq)
- */
-struct ibv_qp *i40iw_ucreate_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
-{
- struct i40iw_ucreate_qp_resp resp;
- struct i40iw_uvcontext *iwvctx = to_i40iw_uctx(pd->context);
- struct i40iw_uqp *iwuqp;
- struct i40iw_qp_uk_init_info info;
- u32 sqdepth, rqdepth;
- u8 sqshift, rqshift;
-
- if (attr->qp_type != IBV_QPT_RC) {
- fprintf(stderr, PFX "%s: failed to create QP, unsupported QP type: 0x%x\n", __func__, attr->qp_type);
- return NULL;
- }
-
- if (attr->cap.max_send_sge > I40IW_MAX_WQ_FRAGMENT_COUNT)
- attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
-
- if (attr->cap.max_recv_sge > I40IW_MAX_WQ_FRAGMENT_COUNT)
- attr->cap.max_recv_sge = I40IW_MAX_WQ_FRAGMENT_COUNT;
-
- if (attr->cap.max_inline_data > I40IW_MAX_INLINE_DATA_SIZE)
- attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE;
-
- i40iw_get_wqe_shift(attr->cap.max_send_sge, attr->cap.max_inline_data, &sqshift);
- if (i40iw_get_sqdepth(attr->cap.max_send_wr, sqshift, &sqdepth)) {
- fprintf(stderr, PFX "invalid SQ attributes, max_send_wr=%d max_send_sge=%d max_inline=%d\n",
- attr->cap.max_send_wr, attr->cap.max_send_sge, attr->cap.max_inline_data);
- return NULL;
- }
-
- switch (iwvctx->abi_ver) {
- case 4:
- i40iw_get_wqe_shift(attr->cap.max_recv_sge, 0, &rqshift);
- break;
- case 5: /* fallthrough until next ABI version */
- default:
- rqshift = I40IW_MAX_RQ_WQE_SHIFT;
- break;
- }
-
- if (i40iw_get_rqdepth(attr->cap.max_recv_wr, rqshift, &rqdepth)) {
- fprintf(stderr, PFX "invalid RQ attributes, max_recv_wr=%d max_recv_sge=%d\n",
- attr->cap.max_recv_wr, attr->cap.max_recv_sge);
- return NULL;
- }
-
- iwuqp = memalign(1024, sizeof(*iwuqp));
- if (!iwuqp)
- return NULL;
- memset(iwuqp, 0, sizeof(*iwuqp));
-
- if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE))
- goto err_free_qp;
-
- memset(&info, 0, sizeof(info));
-
- info.sq_size = sqdepth >> sqshift;
- info.rq_size = rqdepth >> rqshift;
- attr->cap.max_send_wr = info.sq_size;
- attr->cap.max_recv_wr = info.rq_size;
-
- info.max_sq_frag_cnt = attr->cap.max_send_sge;
- info.max_rq_frag_cnt = attr->cap.max_recv_sge;
-
- info.wqe_alloc_reg = (u32 *)iwvctx->iwupd->db;
- info.sq_wrtrk_array = calloc(sqdepth, sizeof(*info.sq_wrtrk_array));
- info.abi_ver = iwvctx->abi_ver;
-
- if (!info.sq_wrtrk_array) {
- fprintf(stderr, PFX "%s: failed to allocate memory for SQ work array\n", __func__);
- goto err_destroy_lock;
- }
-
- info.rq_wrid_array = calloc(rqdepth, sizeof(*info.rq_wrid_array));
- if (!info.rq_wrid_array) {
- fprintf(stderr, PFX "%s: failed to allocate memory for RQ work array\n", __func__);
- goto err_free_sq_wrtrk;
- }
-
- iwuqp->sq_sig_all = attr->sq_sig_all;
- memset(&resp, 0, sizeof(resp));
- if (!i40iw_vmapped_qp(iwuqp, pd, attr, &resp, sqdepth, rqdepth, &info)) {
- fprintf(stderr, PFX "%s: failed to map QP\n", __func__);
- goto err_free_rq_wrid;
- }
- info.qp_id = resp.qp_id;
- iwuqp->i40iw_drv_opt = resp.i40iw_drv_opt;
- iwuqp->ibv_qp.qp_num = resp.qp_id;
-
- info.max_sq_frag_cnt = attr->cap.max_send_sge;
- info.max_rq_frag_cnt = attr->cap.max_recv_sge;
- info.max_inline_data = attr->cap.max_inline_data;
-
- if (!iwvctx->dev.ops_uk.iwarp_qp_uk_init(&iwuqp->qp, &info)) {
- attr->cap.max_send_wr = (sqdepth - I40IW_SQ_RSVD) >> sqshift;
- attr->cap.max_recv_wr = (rqdepth - I40IW_RQ_RSVD) >> rqshift;
- return &iwuqp->ibv_qp;
- }
-
- i40iw_destroy_vmapped_qp(iwuqp, info.sq);
-err_free_rq_wrid:
- free(info.rq_wrid_array);
-err_free_sq_wrtrk:
- free(info.sq_wrtrk_array);
-err_destroy_lock:
- pthread_spin_destroy(&iwuqp->lock);
-err_free_qp:
- free(iwuqp);
- return NULL;
-}
-
-/**
- * i40iw_uquery_qp - query qp for some attribute
- * @qp: qp for the attributes query
- * @attr: to return the attributes
- * @attr_mask: mask of what is query for
- * @init_attr: initial attributes during create_qp
- */
-int i40iw_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask,
- struct ibv_qp_init_attr *init_attr)
-{
- struct ibv_query_qp cmd;
-
- return ibv_cmd_query_qp(qp, attr, attr_mask, init_attr, &cmd, sizeof(cmd));
-}
-
-/**
- * i40iw_umodify_qp - send qp modify to driver
- * @qp: qp to modify
- * @attr: attribute to modify
- * @attr_mask: mask of the attribute
- */
-int i40iw_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
-{
- struct ibv_modify_qp cmd = {};
-
- return ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof(cmd));
-}
-
-/**
- * i40iw_udestroy_qp - destroy qp
- * @qp: qp to destroy
- */
-int i40iw_udestroy_qp(struct ibv_qp *qp)
-{
- struct i40iw_uqp *iwuqp = to_i40iw_uqp(qp);
- int ret;
-
- ret = pthread_spin_destroy(&iwuqp->lock);
- if (ret)
- return ret;
-
- ret = i40iw_destroy_vmapped_qp(iwuqp, iwuqp->qp.sq_base);
- if (ret)
- return ret;
-
- if (iwuqp->qp.sq_wrtrk_array)
- free(iwuqp->qp.sq_wrtrk_array);
- if (iwuqp->qp.rq_wrid_array)
- free(iwuqp->qp.rq_wrid_array);
- /* Clean any pending completions from the cq(s) */
- if (iwuqp->send_cq)
- i40iw_clean_cq((void *)&iwuqp->qp, &iwuqp->send_cq->cq);
-
- if ((iwuqp->recv_cq) && (iwuqp->recv_cq != iwuqp->send_cq))
- i40iw_clean_cq((void *)&iwuqp->qp, &iwuqp->recv_cq->cq);
-
- free(iwuqp);
-
- return 0;
-}
-
-/**
- * i40iw_copy_sg_list - copy sg list for qp
- * @sg_list: copied into sg_list
- * @sgl: copy from sgl
- * @num_sges: count of sg entries
- */
-static void i40iw_copy_sg_list(struct i40iw_sge *sg_list, struct ibv_sge *sgl,
- int num_sges)
-{
- unsigned int i;
-
- for (i = 0; (i < num_sges) && (i < I40IW_MAX_WQ_FRAGMENT_COUNT); i++) {
- sg_list[i].tag_off = sgl[i].addr;
- sg_list[i].len = sgl[i].length;
- sg_list[i].stag = sgl[i].lkey;
- }
-}
-
-/**
- * i40iw_post_send - post send wr for user application
- * @ib_qp: qp ptr for wr
- * @ib_wr: work request ptr
- * @bad_wr: return of bad wr if err
- */
-int i40iw_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr, struct ibv_send_wr **bad_wr)
-{
- struct i40iw_uqp *iwuqp;
- struct i40iw_post_sq_info info;
- enum i40iw_status_code ret = 0;
- int err = 0;
-
- iwuqp = (struct i40iw_uqp *)ib_qp;
-
- err = pthread_spin_lock(&iwuqp->lock);
- if (err)
- return err;
- while (ib_wr) {
- memset(&info, 0, sizeof(info));
- info.wr_id = (u64)(ib_wr->wr_id);
- if ((ib_wr->send_flags & IBV_SEND_SIGNALED) || iwuqp->sq_sig_all)
- info.signaled = true;
- if (ib_wr->send_flags & IBV_SEND_FENCE)
- info.read_fence = true;
-
- switch (ib_wr->opcode) {
- case IBV_WR_SEND:
- /* fall-through */
- case IBV_WR_SEND_WITH_INV:
- if (ib_wr->opcode == IBV_WR_SEND) {
- if (ib_wr->send_flags & IBV_SEND_SOLICITED)
- info.op_type = I40IW_OP_TYPE_SEND_SOL;
- else
- info.op_type = I40IW_OP_TYPE_SEND;
- } else {
- if (ib_wr->send_flags & IBV_SEND_SOLICITED)
- info.op_type = I40IW_OP_TYPE_SEND_SOL_INV;
- else
- info.op_type = I40IW_OP_TYPE_SEND_INV;
- }
-
- if (ib_wr->send_flags & IBV_SEND_INLINE) {
- info.op.inline_send.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr;
- info.op.inline_send.len = ib_wr->sg_list[0].length;
- ret = iwuqp->qp.ops.iw_inline_send(&iwuqp->qp, &info,
- ib_wr->invalidate_rkey, false);
- } else {
- info.op.send.num_sges = ib_wr->num_sge;
- info.op.send.sg_list = (struct i40iw_sge *)ib_wr->sg_list;
- ret = iwuqp->qp.ops.iw_send(&iwuqp->qp, &info,
- ib_wr->invalidate_rkey, false);
- }
-
- if (ret) {
- if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
- err = -ENOMEM;
- else
- err = -EINVAL;
- }
- break;
-
- case IBV_WR_RDMA_WRITE:
- info.op_type = I40IW_OP_TYPE_RDMA_WRITE;
-
- if (ib_wr->send_flags & IBV_SEND_INLINE) {
- info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr;
- info.op.inline_rdma_write.len = ib_wr->sg_list[0].length;
- info.op.inline_rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
- info.op.inline_rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
- ret = iwuqp->qp.ops.iw_inline_rdma_write(&iwuqp->qp, &info, false);
- } else {
- info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
- info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
- info.op.rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
- info.op.rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
- ret = iwuqp->qp.ops.iw_rdma_write(&iwuqp->qp, &info, false);
- }
-
- if (ret) {
- if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
- err = -ENOMEM;
- else
- err = -EINVAL;
- }
- break;
-
- case IBV_WR_RDMA_READ:
- if (ib_wr->num_sge > I40IW_MAX_SGE_RD) {
- err = -EINVAL;
- break;
- }
- info.op_type = I40IW_OP_TYPE_RDMA_READ;
- info.op.rdma_read.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
- info.op.rdma_read.rem_addr.stag = ib_wr->wr.rdma.rkey;
- info.op.rdma_read.lo_addr.tag_off = ib_wr->sg_list->addr;
- info.op.rdma_read.lo_addr.stag = ib_wr->sg_list->lkey;
- info.op.rdma_read.lo_addr.len = ib_wr->sg_list->length;
- ret = iwuqp->qp.ops.iw_rdma_read(&iwuqp->qp, &info, false, false);
- if (ret) {
- if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
- err = -ENOMEM;
- else
- err = -EINVAL;
- }
- break;
-
- default:
- /* error */
- err = -EINVAL;
- fprintf(stderr, PFX "%s: post work request failed, invalid opcode: 0x%x\n", __func__, ib_wr->opcode);
- break;
- }
-
- if (err)
- break;
-
- ib_wr = ib_wr->next;
- }
-
- if (err)
- *bad_wr = ib_wr;
- else
- iwuqp->qp.ops.iw_qp_post_wr(&iwuqp->qp);
-
- pthread_spin_unlock(&iwuqp->lock);
-
- return err;
-}
-
-/**
- * i40iw_post_recv - post receive wr for user application
- * @ib_wr: work request for receive
- * @bad_wr: bad wr caused an error
- */
-int i40iw_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr, struct ibv_recv_wr **bad_wr)
-{
- struct i40iw_uqp *iwuqp = to_i40iw_uqp(ib_qp);
- enum i40iw_status_code ret = 0;
- int err = 0;
- struct i40iw_post_rq_info post_recv;
- struct i40iw_sge sg_list[I40IW_MAX_WQ_FRAGMENT_COUNT];
-
- memset(&post_recv, 0, sizeof(post_recv));
- err = pthread_spin_lock(&iwuqp->lock);
- if (err)
- return err;
- while (ib_wr) {
- post_recv.num_sges = ib_wr->num_sge;
- post_recv.wr_id = ib_wr->wr_id;
- i40iw_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
- post_recv.sg_list = sg_list;
- ret = iwuqp->qp.ops.iw_post_receive(&iwuqp->qp, &post_recv);
- if (ret) {
- fprintf(stderr, PFX "%s: failed to post receives, status %d\n", __func__, ret);
- if (ret == I40IW_ERR_QP_TOOMANY_WRS_POSTED)
- err = -ENOMEM;
- else
- err = -EINVAL;
- *bad_wr = ib_wr;
- goto error;
- }
- ib_wr = ib_wr->next;
- }
-
-error:
- pthread_spin_unlock(&iwuqp->lock);
- return err;
-}
-
-/**
- * i40iw_async_event - handle async events from driver
- * @context: ibv_context
- * @event: event received
- */
-void i40iw_async_event(struct ibv_context *context,
- struct ibv_async_event *event)
-{
- struct i40iw_uqp *iwuqp;
-
- switch (event->event_type) {
- case IBV_EVENT_QP_FATAL:
- case IBV_EVENT_QP_ACCESS_ERR:
- iwuqp = to_i40iw_uqp(event->element.qp);
- iwuqp->qperr = 1;
- break;
-
- default:
- break;
- }
-}
diff --git a/redhat/rdma-core.spec b/redhat/rdma-core.spec
index 87735ef..7d75ed5 100644
--- a/redhat/rdma-core.spec
+++ b/redhat/rdma-core.spec
@@ -151,10 +151,10 @@ Provides: libefa = %{version}-%{release}
Obsoletes: libefa < %{version}-%{release}
Provides: libhfi1 = %{version}-%{release}
Obsoletes: libhfi1 < %{version}-%{release}
-Provides: libi40iw = %{version}-%{release}
-Obsoletes: libi40iw < %{version}-%{release}
Provides: libipathverbs = %{version}-%{release}
Obsoletes: libipathverbs < %{version}-%{release}
+Provides: libirdma = %{version}-%{release}
+Obsoletes: libirdma < %{version}-%{release}
Provides: libmlx4 = %{version}-%{release}
Obsoletes: libmlx4 < %{version}-%{release}
Provides: libmlx5 = %{version}-%{release}
@@ -179,8 +179,8 @@ Device-specific plug-in ibverbs userspace drivers are included:
- libefa: Amazon Elastic Fabric Adapter
- libhfi1: Intel Omni-Path HFI
- libhns: HiSilicon Hip06 SoC
-- libi40iw: Intel Ethernet Connection X722 RDMA
- libipathverbs: QLogic InfiniPath HCA
+- libirdma: Intel Ethernet Connection RDMA
- libmlx4: Mellanox ConnectX-3 InfiniBand HCA
- libmlx5: Mellanox Connect-IB/X-4+ InfiniBand HCA
- libmthca: Mellanox InfiniBand HCA
diff --git a/suse/rdma-core.spec b/suse/rdma-core.spec
index d3c20db..7383dc8 100644
--- a/suse/rdma-core.spec
+++ b/suse/rdma-core.spec
@@ -185,8 +185,8 @@ Requires: %{name}%{?_isa} = %{version}-%{release}
Obsoletes: libcxgb4-rdmav2 < %{version}-%{release}
Obsoletes: libefa-rdmav2 < %{version}-%{release}
Obsoletes: libhfi1verbs-rdmav2 < %{version}-%{release}
-Obsoletes: libi40iw-rdmav2 < %{version}-%{release}
Obsoletes: libipathverbs-rdmav2 < %{version}-%{release}
+Obsoletes: libirdma-rdmav2 < %{version}-%{release}
Obsoletes: libmlx4-rdmav2 < %{version}-%{release}
Obsoletes: libmlx5-rdmav2 < %{version}-%{release}
Obsoletes: libmthca-rdmav2 < %{version}-%{release}
@@ -213,8 +213,8 @@ Device-specific plug-in ibverbs userspace drivers are included:
- libefa: Amazon Elastic Fabric Adapter
- libhfi1: Intel Omni-Path HFI
- libhns: HiSilicon Hip06 SoC
-- libi40iw: Intel Ethernet Connection X722 RDMA
- libipathverbs: QLogic InfiniPath HCA
+- libirdma: Intel Ethernet Connection RDMA
- libmlx4: Mellanox ConnectX-3 InfiniBand HCA
- libmlx5: Mellanox Connect-IB/X-4+ InfiniBand HCA
- libmthca: Mellanox InfiniBand HCA
--
1.8.3.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 1/5] rdma-core/i40iw: Remove provider i40iw
2021-04-06 21:17 ` [PATCH 1/5] rdma-core/i40iw: Remove provider i40iw Tatyana Nikolova
@ 2021-04-06 22:55 ` Jason Gunthorpe
0 siblings, 0 replies; 10+ messages in thread
From: Jason Gunthorpe @ 2021-04-06 22:55 UTC (permalink / raw)
To: Tatyana Nikolova; +Cc: dledford, leon, linux-rdma
On Tue, Apr 06, 2021 at 04:17:24PM -0500, Tatyana Nikolova wrote:
> Remove provider i40iw and set up the environment
> for its replacement - provider irdma which
> supports both X722 and E810 Intel devices.
>
> Signed-off-by: Tatyana Nikolova <tatyana.e.nikolova@intel.com>
> CMakeLists.txt | 2 +-
> MAINTAINERS | 7 +-
> debian/control | 2 +-
> debian/copyright | 8 +-
> kernel-headers/CMakeLists.txt | 4 +-
> kernel-headers/rdma/i40iw-abi.h | 107 --
> kernel-headers/rdma/ib_user_ioctl_verbs.h | 2 +-
> libibverbs/verbs.h | 2 +-
> providers/i40iw/CMakeLists.txt | 5 -
> providers/i40iw/i40e_devids.h | 72 --
> providers/i40iw/i40iw-abi.h | 55 -
> providers/i40iw/i40iw_d.h | 1746 -----------------------------
> providers/i40iw/i40iw_osdep.h | 108 --
> providers/i40iw/i40iw_register.h | 1030 -----------------
> providers/i40iw/i40iw_status.h | 101 --
> providers/i40iw/i40iw_uk.c | 1266 ---------------------
> providers/i40iw/i40iw_umain.c | 226 ----
> providers/i40iw/i40iw_umain.h | 181 ---
> providers/i40iw/i40iw_user.h | 456 --------
> providers/i40iw/i40iw_uverbs.c | 983 ----------------
> redhat/rdma-core.spec | 6 +-
> suse/rdma-core.spec | 4 +-
> 22 files changed, 19 insertions(+), 6354 deletions(-)
> delete mode 100644 kernel-headers/rdma/i40iw-abi.h
> delete mode 100644 providers/i40iw/CMakeLists.txt
> delete mode 100644 providers/i40iw/i40e_devids.h
> delete mode 100644 providers/i40iw/i40iw-abi.h
> delete mode 100644 providers/i40iw/i40iw_d.h
> delete mode 100644 providers/i40iw/i40iw_osdep.h
> delete mode 100644 providers/i40iw/i40iw_register.h
> delete mode 100644 providers/i40iw/i40iw_status.h
> delete mode 100644 providers/i40iw/i40iw_uk.c
> delete mode 100644 providers/i40iw/i40iw_umain.c
> delete mode 100644 providers/i40iw/i40iw_umain.h
> delete mode 100644 providers/i40iw/i40iw_user.h
> delete mode 100644 providers/i40iw/i40iw_uverbs.c
>
> diff --git a/CMakeLists.txt b/CMakeLists.txt
> index e9a2f49..ed43f59 100644
> +++ b/CMakeLists.txt
> @@ -666,7 +666,7 @@ add_subdirectory(providers/cxgb4) # NO SPARSE
> add_subdirectory(providers/efa)
> add_subdirectory(providers/efa/man)
> add_subdirectory(providers/hns)
> -add_subdirectory(providers/i40iw) # NO SPARSE
> +add_subdirectory(providers/irdma)
This doesn't seem bisectable, why did you organize the patches like
this?
Jason
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 2/5] rdma-core/irdma: Add Makefile and ABI definitions
2021-04-06 21:17 [PATCH 0/5] Add irdma user space provider for Intel Ethernet RDMA Tatyana Nikolova
2021-04-06 21:17 ` [PATCH 1/5] rdma-core/i40iw: Remove provider i40iw Tatyana Nikolova
@ 2021-04-06 21:17 ` Tatyana Nikolova
2021-04-06 22:54 ` Jason Gunthorpe
2021-04-06 21:17 ` [PATCH 3/5] rdma-core/irdma: Add user/kernel shared libraries Tatyana Nikolova
` (2 subsequent siblings)
4 siblings, 1 reply; 10+ messages in thread
From: Tatyana Nikolova @ 2021-04-06 21:17 UTC (permalink / raw)
To: jgg, dledford, leon; +Cc: linux-rdma, Tatyana Nikolova
Add Makefile and ABI definitions for irdma provider.
Add utility macros to rdma-core util.h which are used by irdma.
Signed-off-by: Tatyana Nikolova <tatyana.e.nikolova@intel.com>
---
providers/irdma/CMakeLists.txt | 8 ++++++++
providers/irdma/abi.h | 39 +++++++++++++++++++++++++++++++++++++++
util/util.h | 8 ++++++--
3 files changed, 53 insertions(+), 2 deletions(-)
create mode 100644 providers/irdma/CMakeLists.txt
create mode 100644 providers/irdma/abi.h
diff --git a/providers/irdma/CMakeLists.txt b/providers/irdma/CMakeLists.txt
new file mode 100644
index 0000000..1542482
--- /dev/null
+++ b/providers/irdma/CMakeLists.txt
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Intel Corporation.
+
+rdma_provider(irdma
+ uk.c
+ umain.c
+ uverbs.c
+)
diff --git a/providers/irdma/abi.h b/providers/irdma/abi.h
new file mode 100644
index 0000000..bacce40
--- /dev/null
+++ b/providers/irdma/abi.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (C) 2019 - 2020 Intel Corporation */
+#ifndef PROVIDER_IRDMA_ABI_H
+#define PROVIDER_IRDMA_ABI_H
+
+#include "irdma.h"
+#include <infiniband/kern-abi.h>
+#include <rdma/irdma-abi.h>
+#include <kernel-abi/irdma-abi.h>
+
+#define IRDMA_MIN_ABI_VERSION 0
+#define IRDMA_MAX_ABI_VERSION 5
+
+DECLARE_DRV_CMD(irdma_ualloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
+ empty, irdma_alloc_pd_resp);
+DECLARE_DRV_CMD(irdma_ucreate_cq, IB_USER_VERBS_CMD_CREATE_CQ,
+ irdma_create_cq_req, irdma_create_cq_resp);
+DECLARE_DRV_CMD(irdma_ucreate_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ,
+ irdma_create_cq_req, irdma_create_cq_resp);
+DECLARE_DRV_CMD(irdma_uresize_cq, IB_USER_VERBS_CMD_RESIZE_CQ,
+ irdma_resize_cq_req, empty);
+DECLARE_DRV_CMD(irdma_ucreate_qp, IB_USER_VERBS_CMD_CREATE_QP,
+ irdma_create_qp_req, irdma_create_qp_resp);
+DECLARE_DRV_CMD(irdma_umodify_qp, IB_USER_VERBS_EX_CMD_MODIFY_QP,
+ empty, irdma_modify_qp_resp);
+DECLARE_DRV_CMD(irdma_get_context, IB_USER_VERBS_CMD_GET_CONTEXT,
+ irdma_alloc_ucontext_req, irdma_alloc_ucontext_resp);
+DECLARE_DRV_CMD(irdma_ureg_mr, IB_USER_VERBS_CMD_REG_MR,
+ irdma_mem_reg_req, empty);
+DECLARE_DRV_CMD(irdma_ucreate_ah, IB_USER_VERBS_CMD_CREATE_AH,
+ empty, irdma_create_ah_resp);
+
+struct irdma_modify_qp_cmd {
+ struct ibv_modify_qp_ex ibv_cmd;
+ __u8 sq_flush;
+ __u8 rq_flush;
+ __u8 rsvd[6];
+};
+#endif /* PROVIDER_IRDMA_ABI_H */
diff --git a/util/util.h b/util/util.h
index 6db46b1..2c05631 100644
--- a/util/util.h
+++ b/util/util.h
@@ -26,12 +26,16 @@ static inline bool __good_snprintf(size_t len, int rc)
#define offsetofend(_type, _member) \
(offsetof(_type, _member) + sizeof(((_type *)0)->_member))
-#define BITS_PER_LONG (8 * sizeof(long))
+#define BITS_PER_LONG (8 * sizeof(long))
+#define BITS_PER_LONG_LONG (8 * sizeof(long long))
#define GENMASK(h, l) \
(((~0UL) - (1UL << (l)) + 1) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
+#define GENMASK_ULL(h, l) \
+ (((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
-#define BIT(nr) (1UL << (nr))
+#define BIT(nr) (1UL << (nr))
+#define BIT_ULL(nr) (1ULL << (nr))
#define __bf_shf(x) (__builtin_ffsll(x) - 1)
--
1.8.3.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 2/5] rdma-core/irdma: Add Makefile and ABI definitions
2021-04-06 21:17 ` [PATCH 2/5] rdma-core/irdma: Add Makefile and ABI definitions Tatyana Nikolova
@ 2021-04-06 22:54 ` Jason Gunthorpe
0 siblings, 0 replies; 10+ messages in thread
From: Jason Gunthorpe @ 2021-04-06 22:54 UTC (permalink / raw)
To: Tatyana Nikolova; +Cc: dledford, leon, linux-rdma
On Tue, Apr 06, 2021 at 04:17:25PM -0500, Tatyana Nikolova wrote:
> Add Makefile and ABI definitions for irdma provider.
> Add utility macros to rdma-core util.h which are used by irdma.
>
> Signed-off-by: Tatyana Nikolova <tatyana.e.nikolova@intel.com>
> providers/irdma/CMakeLists.txt | 8 ++++++++
> providers/irdma/abi.h | 39 +++++++++++++++++++++++++++++++++++++++
> util/util.h | 8 ++++++--
> 3 files changed, 53 insertions(+), 2 deletions(-)
> create mode 100644 providers/irdma/CMakeLists.txt
> create mode 100644 providers/irdma/abi.h
>
> diff --git a/providers/irdma/CMakeLists.txt b/providers/irdma/CMakeLists.txt
> new file mode 100644
> index 0000000..1542482
> +++ b/providers/irdma/CMakeLists.txt
> @@ -0,0 +1,8 @@
> +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
> +# Copyright (c) 2019, Intel Corporation.
> +
> +rdma_provider(irdma
> + uk.c
> + umain.c
> + uverbs.c
> +)
> diff --git a/providers/irdma/abi.h b/providers/irdma/abi.h
> new file mode 100644
> index 0000000..bacce40
> +++ b/providers/irdma/abi.h
> @@ -0,0 +1,39 @@
> +/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
> +/* Copyright (C) 2019 - 2020 Intel Corporation */
> +#ifndef PROVIDER_IRDMA_ABI_H
> +#define PROVIDER_IRDMA_ABI_H
> +
> +#include "irdma.h"
> +#include <infiniband/kern-abi.h>
> +#include <rdma/irdma-abi.h>
> +#include <kernel-abi/irdma-abi.h>
> +
> +#define IRDMA_MIN_ABI_VERSION 0
> +#define IRDMA_MAX_ABI_VERSION 5
> +
> +DECLARE_DRV_CMD(irdma_ualloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
> + empty, irdma_alloc_pd_resp);
> +DECLARE_DRV_CMD(irdma_ucreate_cq, IB_USER_VERBS_CMD_CREATE_CQ,
> + irdma_create_cq_req, irdma_create_cq_resp);
> +DECLARE_DRV_CMD(irdma_ucreate_cq_ex, IB_USER_VERBS_EX_CMD_CREATE_CQ,
> + irdma_create_cq_req, irdma_create_cq_resp);
> +DECLARE_DRV_CMD(irdma_uresize_cq, IB_USER_VERBS_CMD_RESIZE_CQ,
> + irdma_resize_cq_req, empty);
> +DECLARE_DRV_CMD(irdma_ucreate_qp, IB_USER_VERBS_CMD_CREATE_QP,
> + irdma_create_qp_req, irdma_create_qp_resp);
> +DECLARE_DRV_CMD(irdma_umodify_qp, IB_USER_VERBS_EX_CMD_MODIFY_QP,
> + empty, irdma_modify_qp_resp);
> +DECLARE_DRV_CMD(irdma_get_context, IB_USER_VERBS_CMD_GET_CONTEXT,
> + irdma_alloc_ucontext_req, irdma_alloc_ucontext_resp);
> +DECLARE_DRV_CMD(irdma_ureg_mr, IB_USER_VERBS_CMD_REG_MR,
> + irdma_mem_reg_req, empty);
> +DECLARE_DRV_CMD(irdma_ucreate_ah, IB_USER_VERBS_CMD_CREATE_AH,
> + empty, irdma_create_ah_resp);
> +
> +struct irdma_modify_qp_cmd {
> + struct ibv_modify_qp_ex ibv_cmd;
> + __u8 sq_flush;
> + __u8 rq_flush;
> + __u8 rsvd[6];
> +};
Huh? What is this? Why is it here?
Jason
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 3/5] rdma-core/irdma: Add user/kernel shared libraries
2021-04-06 21:17 [PATCH 0/5] Add irdma user space provider for Intel Ethernet RDMA Tatyana Nikolova
2021-04-06 21:17 ` [PATCH 1/5] rdma-core/i40iw: Remove provider i40iw Tatyana Nikolova
2021-04-06 21:17 ` [PATCH 2/5] rdma-core/irdma: Add Makefile and ABI definitions Tatyana Nikolova
@ 2021-04-06 21:17 ` Tatyana Nikolova
2021-04-06 21:17 ` [PATCH 4/5] rdma-core/irdma: Add library setup and utility definitions Tatyana Nikolova
2021-04-06 21:17 ` [PATCH 5/5] rdma-core/irdma: Implement device supported verb APIs Tatyana Nikolova
4 siblings, 0 replies; 10+ messages in thread
From: Tatyana Nikolova @ 2021-04-06 21:17 UTC (permalink / raw)
To: jgg, dledford, leon; +Cc: linux-rdma, Tatyana Nikolova
Building the WQE descriptors for different verb
operations are similar in kernel and user-space.
Add these shared functions and headers.
Signed-off-by: Tatyana Nikolova <tatyana.e.nikolova@intel.com>
---
providers/irdma/irdma.h | 53 ++
providers/irdma/uk.c | 1737 +++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 1790 insertions(+)
create mode 100644 providers/irdma/irdma.h
create mode 100644 providers/irdma/uk.c
diff --git a/providers/irdma/irdma.h b/providers/irdma/irdma.h
new file mode 100644
index 0000000..3a6f270
--- /dev/null
+++ b/providers/irdma/irdma.h
@@ -0,0 +1,53 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2017 - 2021 Intel Corporation */
+#ifndef IRDMA_H
+#define IRDMA_H
+
+#define IRDMA_WQEALLOC_WQE_DESC_INDEX GENMASK(31, 20)
+
+enum irdma_vers {
+ IRDMA_GEN_RSVD,
+ IRDMA_GEN_1,
+ IRDMA_GEN_2,
+};
+
+struct irdma_uk_attrs {
+ __u64 feature_flags;
+ __u32 max_hw_wq_frags;
+ __u32 max_hw_read_sges;
+ __u32 max_hw_inline;
+ __u32 max_hw_rq_quanta;
+ __u32 max_hw_wq_quanta;
+ __u32 min_hw_cq_size;
+ __u32 max_hw_cq_size;
+ __u16 max_hw_sq_chunk;
+ __u8 hw_rev;
+};
+
+struct irdma_hw_attrs {
+ struct irdma_uk_attrs uk_attrs;
+ __u64 max_hw_outbound_msg_size;
+ __u64 max_hw_inbound_msg_size;
+ __u64 max_mr_size;
+ __u32 min_hw_qp_id;
+ __u32 min_hw_aeq_size;
+ __u32 max_hw_aeq_size;
+ __u32 min_hw_ceq_size;
+ __u32 max_hw_ceq_size;
+ __u32 max_hw_device_pages;
+ __u32 max_hw_vf_fpm_id;
+ __u32 first_hw_vf_fpm_id;
+ __u32 max_hw_ird;
+ __u32 max_hw_ord;
+ __u32 max_hw_wqes;
+ __u32 max_hw_pds;
+ __u32 max_hw_ena_vf_count;
+ __u32 max_qp_wr;
+ __u32 max_pe_ready_count;
+ __u32 max_done_count;
+ __u32 max_sleep_count;
+ __u32 max_cqp_compl_wait_time_ms;
+ __u16 max_stat_inst;
+};
+
+#endif /* IRDMA_H*/
diff --git a/providers/irdma/uk.c b/providers/irdma/uk.c
new file mode 100644
index 0000000..c6d6b43
--- /dev/null
+++ b/providers/irdma/uk.c
@@ -0,0 +1,1737 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (c) 2015 - 2021 Intel Corporation */
+#include "osdep.h"
+#include "status.h"
+#include "defs.h"
+#include "user.h"
+#include "irdma.h"
+
+/**
+ * irdma_set_fragment - set fragment in wqe
+ * @wqe: wqe for setting fragment
+ * @offset: offset value
+ * @sge: sge length and stag
+ * @valid: The wqe valid
+ */
+static void irdma_set_fragment(__le64 *wqe, __u32 offset, struct irdma_sge *sge,
+ __u8 valid)
+{
+ if (sge) {
+ set_64bit_val(wqe, offset,
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
+ set_64bit_val(wqe, offset + 8,
+ FIELD_PREP(IRDMAQPSQ_VALID, valid) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag));
+ } else {
+ set_64bit_val(wqe, offset, 0);
+ set_64bit_val(wqe, offset + 8,
+ FIELD_PREP(IRDMAQPSQ_VALID, valid));
+ }
+}
+
+/**
+ * irdma_set_fragment_gen_1 - set fragment in wqe
+ * @wqe: wqe for setting fragment
+ * @offset: offset value
+ * @sge: sge length and stag
+ * @valid: wqe valid flag
+ */
+static void irdma_set_fragment_gen_1(__le64 *wqe, __u32 offset,
+ struct irdma_sge *sge, __u8 valid)
+{
+ if (sge) {
+ set_64bit_val(wqe, offset,
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
+ set_64bit_val(wqe, offset + 8,
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) |
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag));
+ } else {
+ set_64bit_val(wqe, offset, 0);
+ set_64bit_val(wqe, offset + 8, 0);
+ }
+}
+
+/**
+ * irdma_nop_1 - insert a NOP wqe
+ * @qp: hw qp ptr
+ */
+static enum irdma_status_code irdma_nop_1(struct irdma_qp_uk *qp)
+{
+ __u64 hdr;
+ __le64 *wqe;
+ __u32 wqe_idx;
+ bool signaled = false;
+
+ if (!qp->sq_ring.head)
+ return IRDMA_ERR_PARAM;
+
+ wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
+ wqe = qp->sq_base[wqe_idx].elem;
+
+ qp->sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA;
+
+ set_64bit_val(wqe, 0, 0);
+ set_64bit_val(wqe, 8, 0);
+ set_64bit_val(wqe, 16, 0);
+
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+
+ /* make sure WQE is written before valid bit is set */
+ udma_to_device_barrier();
+
+ set_64bit_val(wqe, 24, hdr);
+
+ return 0;
+}
+
+/**
+ * irdma_clr_wqes - clear next 128 sq entries
+ * @qp: hw qp ptr
+ * @qp_wqe_idx: wqe_idx
+ */
+void irdma_clr_wqes(struct irdma_qp_uk *qp, __u32 qp_wqe_idx)
+{
+ __le64 *wqe;
+ __u32 wqe_idx;
+
+ if (!(qp_wqe_idx & 0x7F)) {
+ wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size;
+ wqe = qp->sq_base[wqe_idx].elem;
+ if (wqe_idx)
+ memset(wqe, qp->swqe_polarity ? 0 : 0xFF, 0x1000);
+ else
+ memset(wqe, qp->swqe_polarity ? 0xFF : 0, 0x1000);
+ }
+}
+
+/**
+ * irdma_qp_post_wr - ring doorbell
+ * @qp: hw qp ptr
+ */
+void irdma_qp_post_wr(struct irdma_qp_uk *qp)
+{
+ __u64 temp;
+ __u32 hw_sq_tail;
+ __u32 sw_sq_head;
+
+ /* valid bit is written and loads completed before reading shadow */
+ atomic_thread_fence(memory_order_seq_cst);
+
+ /* read the doorbell shadow area */
+ get_64bit_val(qp->shadow_area, 0, &temp);
+
+ hw_sq_tail = (__u32)FIELD_GET(IRDMA_QP_DBSA_HW_SQ_TAIL, temp);
+ sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
+ if (sw_sq_head != qp->initial_ring.head) {
+ if (qp->push_dropped) {
+ db_wr32(qp->qp_id, qp->wqe_alloc_db);
+ qp->push_dropped = false;
+ } else if (sw_sq_head != hw_sq_tail) {
+ if (sw_sq_head > qp->initial_ring.head) {
+ if (hw_sq_tail >= qp->initial_ring.head &&
+ hw_sq_tail < sw_sq_head)
+ db_wr32(qp->qp_id, qp->wqe_alloc_db);
+ } else {
+ if (hw_sq_tail >= qp->initial_ring.head ||
+ hw_sq_tail < sw_sq_head)
+ db_wr32(qp->qp_id, qp->wqe_alloc_db);
+ }
+ }
+ }
+
+ qp->initial_ring.head = qp->sq_ring.head;
+}
+
+/**
+ * irdma_qp_ring_push_db - ring qp doorbell
+ * @qp: hw qp ptr
+ * @wqe_idx: wqe index
+ */
+static void irdma_qp_ring_push_db(struct irdma_qp_uk *qp, __u32 wqe_idx)
+{
+ set_32bit_val(qp->push_db, 0,
+ FIELD_PREP(IRDMA_WQEALLOC_WQE_DESC_INDEX, wqe_idx >> 3) | qp->qp_id);
+ qp->initial_ring.head = qp->sq_ring.head;
+ qp->push_mode = true;
+ qp->push_dropped = false;
+}
+
+void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, __u16 quanta,
+ __u32 wqe_idx, bool post_sq)
+{
+ __le64 *push;
+
+ if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) !=
+ IRDMA_RING_CURRENT_TAIL(qp->sq_ring) &&
+ !qp->push_mode) {
+ if (post_sq)
+ irdma_qp_post_wr(qp);
+ } else {
+ push = (__le64 *)((uintptr_t)qp->push_wqe +
+ (wqe_idx & 0x7) * 0x20);
+ memcpy(push, wqe, quanta * IRDMA_QP_WQE_MIN_SIZE);
+ irdma_qp_ring_push_db(qp, wqe_idx);
+ }
+}
+
+/**
+ * irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go
+ * @qp: hw qp ptr
+ * @wqe_idx: return wqe index
+ * @quanta: size of WR in quanta
+ * @total_size: size of WR in bytes
+ * @info: info on WR
+ */
+__le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, __u32 *wqe_idx,
+ __u16 quanta, __u32 total_size,
+ struct irdma_post_sq_info *info)
+{
+ __le64 *wqe;
+ __le64 *wqe_0 = NULL;
+ __u32 nop_wqe_idx;
+ __u16 avail_quanta;
+ __u16 i;
+
+ avail_quanta = qp->uk_attrs->max_hw_sq_chunk -
+ (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) %
+ qp->uk_attrs->max_hw_sq_chunk);
+ if (quanta <= avail_quanta) {
+ /* WR fits in current chunk */
+ if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
+ return NULL;
+ } else {
+ /* Need to pad with NOP */
+ if (quanta + avail_quanta >
+ IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
+ return NULL;
+
+ nop_wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
+ for (i = 0; i < avail_quanta; i++) {
+ irdma_nop_1(qp);
+ IRDMA_RING_MOVE_HEAD_NOCHECK(qp->sq_ring);
+ }
+ if (qp->push_db && info->push_wqe)
+ irdma_qp_push_wqe(qp, qp->sq_base[nop_wqe_idx].elem,
+ avail_quanta, nop_wqe_idx, true);
+ }
+
+ *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
+ if (!*wqe_idx)
+ qp->swqe_polarity = !qp->swqe_polarity;
+
+ IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta);
+
+ wqe = qp->sq_base[*wqe_idx].elem;
+ if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 &&
+ (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
+ wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
+ wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity));
+ }
+ qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
+ qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
+ qp->sq_wrtrk_array[*wqe_idx].quanta = quanta;
+
+ return wqe;
+}
+
+/**
+ * irdma_qp_get_next_recv_wqe - get next qp's rcv wqe
+ * @qp: hw qp ptr
+ * @wqe_idx: return wqe index
+ */
+__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, __u32 *wqe_idx)
+{
+ __le64 *wqe;
+ enum irdma_status_code ret_code;
+
+ if (IRDMA_RING_FULL_ERR(qp->rq_ring))
+ return NULL;
+
+ IRDMA_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code);
+ if (ret_code)
+ return NULL;
+
+ if (!*wqe_idx)
+ qp->rwqe_polarity = !qp->rwqe_polarity;
+ /* rq_wqe_size_multiplier is no of 32 byte quanta in one rq wqe */
+ wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem;
+
+ return wqe;
+}
+
+/**
+ * irdma_rdma_write - rdma write operation
+ * @qp: hw qp ptr
+ * @info: post sq information
+ * @post_sq: flag to post sq
+ */
+static enum irdma_status_code irdma_rdma_write(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool post_sq)
+{
+ __u64 hdr;
+ __le64 *wqe;
+ struct irdma_rdma_write *op_info;
+ __u32 i, wqe_idx;
+ __u32 total_size = 0, byte_off;
+ enum irdma_status_code ret_code;
+ __u32 frag_cnt, addl_frag_cnt;
+ bool read_fence = false;
+ __u16 quanta;
+
+ info->push_wqe = qp->push_db ? true : false;
+
+ op_info = &info->op.rdma_write;
+ if (op_info->num_lo_sges > qp->max_sq_frag_cnt)
+ return IRDMA_ERR_INVALID_FRAG_COUNT;
+
+ for (i = 0; i < op_info->num_lo_sges; i++)
+ total_size += op_info->lo_sg_list[i].len;
+
+ read_fence |= info->read_fence;
+
+ if (info->imm_data_valid)
+ frag_cnt = op_info->num_lo_sges + 1;
+ else
+ frag_cnt = op_info->num_lo_sges;
+ addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0;
+ ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, &quanta);
+ if (ret_code)
+ return ret_code;
+
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
+ info);
+ if (!wqe)
+ return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+ irdma_clr_wqes(qp, wqe_idx);
+
+ set_64bit_val(wqe, 16,
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+
+ if (info->imm_data_valid) {
+ set_64bit_val(wqe, 0,
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
+ i = 0;
+ } else {
+ qp->wqe_ops.iw_set_fragment(wqe, 0,
+ op_info->lo_sg_list,
+ qp->swqe_polarity);
+ i = 1;
+ }
+
+ for (byte_off = 32; i < op_info->num_lo_sges; i++) {
+ qp->wqe_ops.iw_set_fragment(wqe, byte_off,
+ &op_info->lo_sg_list[i],
+ qp->swqe_polarity);
+ byte_off += 16;
+ }
+
+ /* if not an odd number set valid bit in next fragment */
+ if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) &&
+ frag_cnt) {
+ qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
+ qp->swqe_polarity);
+ if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
+ ++addl_frag_cnt;
+ }
+
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+
+ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
+
+ set_64bit_val(wqe, 24, hdr);
+ if (info->push_wqe) {
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ } else {
+ if (post_sq)
+ irdma_qp_post_wr(qp);
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_rdma_read - rdma read command
+ * @qp: hw qp ptr
+ * @info: post sq information
+ * @inv_stag: flag for inv_stag
+ * @post_sq: flag to post sq
+ */
+static enum irdma_status_code irdma_rdma_read(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool inv_stag, bool post_sq)
+{
+ struct irdma_rdma_read *op_info;
+ enum irdma_status_code ret_code;
+ __u32 i, byte_off, total_size = 0;
+ bool local_fence = false;
+ __u32 addl_frag_cnt;
+ __le64 *wqe;
+ __u32 wqe_idx;
+ __u16 quanta;
+ __u64 hdr;
+
+ info->push_wqe = qp->push_db ? true : false;
+
+ op_info = &info->op.rdma_read;
+ if (qp->max_sq_frag_cnt < op_info->num_lo_sges)
+ return IRDMA_ERR_INVALID_FRAG_COUNT;
+
+ for (i = 0; i < op_info->num_lo_sges; i++)
+ total_size += op_info->lo_sg_list[i].len;
+
+ ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta);
+ if (ret_code)
+ return ret_code;
+
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
+ info);
+ if (!wqe)
+ return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+ irdma_clr_wqes(qp, wqe_idx);
+
+ addl_frag_cnt = op_info->num_lo_sges > 1 ?
+ (op_info->num_lo_sges - 1) : 0;
+ local_fence |= info->local_fence;
+
+ qp->wqe_ops.iw_set_fragment(wqe, 0, op_info->lo_sg_list,
+ qp->swqe_polarity);
+ for (i = 1, byte_off = 32; i < op_info->num_lo_sges; ++i) {
+ qp->wqe_ops.iw_set_fragment(wqe, byte_off,
+ &op_info->lo_sg_list[i],
+ qp->swqe_polarity);
+ byte_off += 16;
+ }
+
+ /* if not an odd number set valid bit in next fragment */
+ if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 &&
+ !(op_info->num_lo_sges & 0x01) && op_info->num_lo_sges) {
+ qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
+ qp->swqe_polarity);
+ if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
+ ++addl_frag_cnt;
+ }
+ set_64bit_val(wqe, 16,
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE,
+ (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+
+ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
+
+ set_64bit_val(wqe, 24, hdr);
+ if (info->push_wqe) {
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ } else {
+ if (post_sq)
+ irdma_qp_post_wr(qp);
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_send - rdma send command
+ * @qp: hw qp ptr
+ * @info: post sq information
+ * @post_sq: flag to post sq
+ */
+static enum irdma_status_code irdma_send(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_post_send *op_info;
+ __u64 hdr;
+ __u32 i, wqe_idx, total_size = 0, byte_off;
+ enum irdma_status_code ret_code;
+ __u32 frag_cnt, addl_frag_cnt;
+ bool read_fence = false;
+ __u16 quanta;
+
+ info->push_wqe = qp->push_db ? true : false;
+
+ op_info = &info->op.send;
+ if (qp->max_sq_frag_cnt < op_info->num_sges)
+ return IRDMA_ERR_INVALID_FRAG_COUNT;
+
+ for (i = 0; i < op_info->num_sges; i++)
+ total_size += op_info->sg_list[i].len;
+
+ if (info->imm_data_valid)
+ frag_cnt = op_info->num_sges + 1;
+ else
+ frag_cnt = op_info->num_sges;
+ ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, &quanta);
+ if (ret_code)
+ return ret_code;
+
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
+ info);
+ if (!wqe)
+ return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+ irdma_clr_wqes(qp, wqe_idx);
+
+ read_fence |= info->read_fence;
+ addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0;
+ if (info->imm_data_valid) {
+ set_64bit_val(wqe, 0,
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
+ i = 0;
+ } else {
+ qp->wqe_ops.iw_set_fragment(wqe, 0, op_info->sg_list,
+ qp->swqe_polarity);
+ i = 1;
+ }
+
+ for (byte_off = 32; i < op_info->num_sges; i++) {
+ qp->wqe_ops.iw_set_fragment(wqe, byte_off, &op_info->sg_list[i],
+ qp->swqe_polarity);
+ byte_off += 16;
+ }
+
+ /* if not an odd number set valid bit in next fragment */
+ if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) &&
+ frag_cnt) {
+ qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
+ qp->swqe_polarity);
+ if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
+ ++addl_frag_cnt;
+ }
+
+ set_64bit_val(wqe, 16,
+ FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
+ FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
+ FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
+ (info->imm_data_valid ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
+ FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+
+ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
+
+ set_64bit_val(wqe, 24, hdr);
+ if (info->push_wqe) {
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ } else {
+ if (post_sq)
+ irdma_qp_post_wr(qp);
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_set_mw_bind_wqe_gen_1 - set mw bind wqe
+ * @wqe: wqe for setting fragment
+ * @op_info: info for setting bind wqe values
+ */
+static void irdma_set_mw_bind_wqe_gen_1(__le64 *wqe,
+ struct irdma_bind_window *op_info)
+{
+ set_64bit_val(wqe, 0, (uintptr_t)op_info->va);
+ set_64bit_val(wqe, 8,
+ FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) |
+ FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag));
+ set_64bit_val(wqe, 16, op_info->bind_len);
+}
+
+/**
+ * irdma_copy_inline_data_gen_1 - Copy inline data to wqe
+ * @dest: pointer to wqe
+ * @src: pointer to inline data
+ * @len: length of inline data to copy
+ * @polarity: compatibility parameter
+ */
+static void irdma_copy_inline_data_gen_1(__u8 *dest, __u8 *src, __u32 len,
+ __u8 polarity)
+{
+ if (len <= 16) {
+ memcpy(dest, src, len);
+ } else {
+ memcpy(dest, src, 16);
+ src += 16;
+ dest = dest + 32;
+ memcpy(dest, src, len - 16);
+ }
+}
+
+/**
+ * irdma_inline_data_size_to_quanta_gen_1 - based on inline data, quanta
+ * @data_size: data size for inline
+ *
+ * Gets the quanta based on inline and immediate data.
+ */
+static inline __u16 irdma_inline_data_size_to_quanta_gen_1(__u32 data_size)
+{
+ return data_size <= 16 ? IRDMA_QP_WQE_MIN_QUANTA : 2;
+}
+
+/**
+ * irdma_set_mw_bind_wqe - set mw bind in wqe
+ * @wqe: wqe for setting mw bind
+ * @op_info: info for setting wqe values
+ */
+static void irdma_set_mw_bind_wqe(__le64 *wqe,
+ struct irdma_bind_window *op_info)
+{
+ set_64bit_val(wqe, 0, (uintptr_t)op_info->va);
+ set_64bit_val(wqe, 8,
+ FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mr_stag) |
+ FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mw_stag));
+ set_64bit_val(wqe, 16, op_info->bind_len);
+}
+
+/**
+ * irdma_copy_inline_data - Copy inline data to wqe
+ * @dest: pointer to wqe
+ * @src: pointer to inline data
+ * @len: length of inline data to copy
+ * @polarity: polarity of wqe valid bit
+ */
+static void irdma_copy_inline_data(__u8 *dest, __u8 *src, __u32 len, __u8 polarity)
+{
+ __u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
+ __u32 copy_size;
+
+ dest += 8;
+ if (len <= 8) {
+ memcpy(dest, src, len);
+ return;
+ }
+
+ *((__u64 *)dest) = *((__u64 *)src);
+ len -= 8;
+ src += 8;
+ dest += 24; /* point to additional 32 byte quanta */
+
+ while (len) {
+ copy_size = len < 31 ? len : 31;
+ memcpy(dest, src, copy_size);
+ *(dest + 31) = inline_valid;
+ len -= copy_size;
+ dest += 32;
+ src += copy_size;
+ }
+}
+
+/**
+ * irdma_inline_data_size_to_quanta - based on inline data, quanta
+ * @data_size: data size for inline
+ *
+ * Gets the quanta based on inline and immediate data.
+ */
+static __u16 irdma_inline_data_size_to_quanta(__u32 data_size)
+{
+ if (data_size <= 8)
+ return IRDMA_QP_WQE_MIN_QUANTA;
+ else if (data_size <= 39)
+ return 2;
+ else if (data_size <= 70)
+ return 3;
+ else if (data_size <= 101)
+ return 4;
+ else if (data_size <= 132)
+ return 5;
+ else if (data_size <= 163)
+ return 6;
+ else if (data_size <= 194)
+ return 7;
+ else
+ return 8;
+}
+
+/**
+ * irdma_inline_rdma_write - inline rdma write operation
+ * @qp: hw qp ptr
+ * @info: post sq information
+ * @post_sq: flag to post sq
+ */
+static enum irdma_status_code
+irdma_inline_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_inline_rdma_write *op_info;
+ __u64 hdr = 0;
+ __u32 wqe_idx;
+ bool read_fence = false;
+ __u16 quanta;
+
+ info->push_wqe = qp->push_db ? true : false;
+ op_info = &info->op.inline_rdma_write;
+
+ if (op_info->len > qp->max_inline_data)
+ return IRDMA_ERR_INVALID_INLINE_DATA_SIZE;
+
+ quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
+ info);
+ if (!wqe)
+ return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+ irdma_clr_wqes(qp, wqe_idx);
+
+ read_fence |= info->read_fence;
+ set_64bit_val(wqe, 16,
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, op_info->len) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe ? 1 : 0) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+
+ if (info->imm_data_valid)
+ set_64bit_val(wqe, 0,
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
+
+ qp->wqe_ops.iw_copy_inline_data((__u8 *)wqe, op_info->data, op_info->len,
+ qp->swqe_polarity);
+ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
+
+ set_64bit_val(wqe, 24, hdr);
+
+ if (info->push_wqe) {
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ } else {
+ if (post_sq)
+ irdma_qp_post_wr(qp);
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_inline_send - inline send operation
+ * @qp: hw qp ptr
+ * @info: post sq information
+ * @post_sq: flag to post sq
+ */
+static enum irdma_status_code irdma_inline_send(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_post_inline_send *op_info;
+ __u64 hdr;
+ __u32 wqe_idx;
+ bool read_fence = false;
+ __u16 quanta;
+
+ info->push_wqe = qp->push_db ? true : false;
+ op_info = &info->op.inline_send;
+
+ if (op_info->len > qp->max_inline_data)
+ return IRDMA_ERR_INVALID_INLINE_DATA_SIZE;
+
+ quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len);
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len,
+ info);
+ if (!wqe)
+ return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+ irdma_clr_wqes(qp, wqe_idx);
+
+ set_64bit_val(wqe, 16,
+ FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
+ FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
+
+ read_fence |= info->read_fence;
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
+ FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
+ FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, op_info->len) |
+ FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
+ (info->imm_data_valid ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
+ FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+
+ if (info->imm_data_valid)
+ set_64bit_val(wqe, 0,
+ FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
+ qp->wqe_ops.iw_copy_inline_data((__u8 *)wqe, op_info->data, op_info->len,
+ qp->swqe_polarity);
+
+ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
+
+ set_64bit_val(wqe, 24, hdr);
+
+ if (info->push_wqe) {
+ irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
+ } else {
+ if (post_sq)
+ irdma_qp_post_wr(qp);
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_stag_local_invalidate - stag invalidate operation
+ * @qp: hw qp ptr
+ * @info: post sq information
+ * @post_sq: flag to post sq
+ */
+static enum irdma_status_code
+irdma_stag_local_invalidate(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info, bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_inv_local_stag *op_info;
+ __u64 hdr;
+ __u32 wqe_idx;
+ bool local_fence = false;
+ struct irdma_sge sge = {};
+
+ info->push_wqe = qp->push_db ? true : false;
+ op_info = &info->op.inv_local_stag;
+ local_fence = info->local_fence;
+
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
+ 0, info);
+ if (!wqe)
+ return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+ irdma_clr_wqes(qp, wqe_idx);
+
+ sge.stag = op_info->target_stag;
+ qp->wqe_ops.iw_set_fragment(wqe, 0, &sge, 0);
+
+ set_64bit_val(wqe, 16, 0);
+
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_INV_STAG) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+
+ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
+
+ set_64bit_val(wqe, 24, hdr);
+
+ if (info->push_wqe) {
+ irdma_qp_push_wqe(qp, wqe, IRDMA_QP_WQE_MIN_QUANTA, wqe_idx,
+ post_sq);
+ } else {
+ if (post_sq)
+ irdma_qp_post_wr(qp);
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_mw_bind - bind Memory Window
+ * @qp: hw qp ptr
+ * @info: post sq information
+ * @post_sq: flag to post sq
+ */
+static enum irdma_status_code irdma_mw_bind(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool post_sq)
+{
+ __le64 *wqe;
+ struct irdma_bind_window *op_info;
+ __u64 hdr;
+ __u32 wqe_idx;
+ bool local_fence = false;
+
+ info->push_wqe = qp->push_db ? true : false;
+ op_info = &info->op.bind_window;
+ local_fence |= info->local_fence;
+
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
+ 0, info);
+ if (!wqe)
+ return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+ irdma_clr_wqes(qp, wqe_idx);
+
+ qp->wqe_ops.iw_set_mw_bind_wqe(wqe, op_info);
+
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_BIND_MW) |
+ FIELD_PREP(IRDMAQPSQ_STAGRIGHTS,
+ ((op_info->ena_reads << 2) | (op_info->ena_writes << 3))) |
+ FIELD_PREP(IRDMAQPSQ_VABASEDTO,
+ (op_info->addressing_type == IRDMA_ADDR_TYPE_VA_BASED ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_MEMWINDOWTYPE,
+ (op_info->mem_window_type_1 ? 1 : 0)) |
+ FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) |
+ FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+
+ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
+
+ set_64bit_val(wqe, 24, hdr);
+
+ if (info->push_wqe) {
+ irdma_qp_push_wqe(qp, wqe, IRDMA_QP_WQE_MIN_QUANTA, wqe_idx,
+ post_sq);
+ } else {
+ if (post_sq)
+ irdma_qp_post_wr(qp);
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_post_receive - post receive wqe
+ * @qp: hw qp ptr
+ * @info: post rq information
+ */
+static enum irdma_status_code
+irdma_post_receive(struct irdma_qp_uk *qp, struct irdma_post_rq_info *info)
+{
+ __u32 total_size = 0, wqe_idx, i, byte_off;
+ __u32 addl_frag_cnt;
+ __le64 *wqe;
+ __u64 hdr;
+
+ if (qp->max_rq_frag_cnt < info->num_sges)
+ return IRDMA_ERR_INVALID_FRAG_COUNT;
+
+ for (i = 0; i < info->num_sges; i++)
+ total_size += info->sg_list[i].len;
+
+ wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx);
+ if (!wqe)
+ return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+ qp->rq_wrid_array[wqe_idx] = info->wr_id;
+ addl_frag_cnt = info->num_sges > 1 ? (info->num_sges - 1) : 0;
+ qp->wqe_ops.iw_set_fragment(wqe, 0, info->sg_list,
+ qp->rwqe_polarity);
+
+ for (i = 1, byte_off = 32; i < info->num_sges; i++) {
+ qp->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i],
+ qp->rwqe_polarity);
+ byte_off += 16;
+ }
+
+ /* if not an odd number set valid bit in next fragment */
+ if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) &&
+ info->num_sges) {
+ qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
+ qp->rwqe_polarity);
+ if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
+ ++addl_frag_cnt;
+ }
+
+ set_64bit_val(wqe, 16, 0);
+ hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity);
+
+ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
+
+ set_64bit_val(wqe, 24, hdr);
+
+ return 0;
+}
+
+/**
+ * irdma_cq_resize - reset the cq buffer info
+ * @cq: cq to resize
+ * @cq_base: new cq buffer addr
+ * @cq_size: number of cqes
+ */
+static void irdma_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int cq_size)
+{
+ cq->cq_base = cq_base;
+ cq->cq_size = cq_size;
+ IRDMA_RING_INIT(cq->cq_ring, cq->cq_size);
+ cq->polarity = 1;
+}
+
+/**
+ * irdma_cq_set_resized_cnt - record the count of the resized buffers
+ * @cq: cq to resize
+ * @cq_cnt: the count of the resized cq buffers
+ */
+static void irdma_cq_set_resized_cnt(struct irdma_cq_uk *cq, __u16 cq_cnt)
+{
+ __u64 temp_val;
+ __u16 sw_cq_sel;
+ __u8 arm_next_se;
+ __u8 arm_next;
+ __u8 arm_seq_num;
+
+ get_64bit_val(cq->shadow_area, 32, &temp_val);
+
+ sw_cq_sel = (__u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
+ sw_cq_sel += cq_cnt;
+
+ arm_seq_num = (__u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
+ arm_next_se = (__u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
+ arm_next = (__u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT, temp_val);
+
+ temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
+ FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);
+
+ set_64bit_val(cq->shadow_area, 32, temp_val);
+}
+
+/**
+ * irdma_cq_request_notification - cq notification request (door bell)
+ * @cq: hw cq
+ * @cq_notify: notification type
+ */
+static void irdma_cq_request_notification(struct irdma_cq_uk *cq,
+ enum irdma_cmpl_notify cq_notify)
+{
+ __u64 temp_val;
+ __u16 sw_cq_sel;
+ __u8 arm_next_se = 0;
+ __u8 arm_next = 0;
+ __u8 arm_seq_num;
+
+ get_64bit_val(cq->shadow_area, 32, &temp_val);
+ arm_seq_num = (__u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
+ arm_seq_num++;
+ sw_cq_sel = (__u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
+ arm_next_se = (__u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
+ arm_next_se |= 1;
+ if (cq_notify == IRDMA_CQ_COMPL_EVENT)
+ arm_next = 1;
+ temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
+ FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
+ FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);
+
+ set_64bit_val(cq->shadow_area, 32, temp_val);
+
+ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
+
+ db_wr32(cq->cq_id, cq->cqe_alloc_db);
+}
+
+/**
+ * irdma_cq_post_entries - update tail in shadow memory
+ * @cq: hw cq
+ * @count: # of entries processed
+ */
+static enum irdma_status_code irdma_cq_post_entries(struct irdma_cq_uk *cq,
+ __u8 count)
+{
+ IRDMA_RING_MOVE_TAIL_BY_COUNT(cq->cq_ring, count);
+ set_64bit_val(cq->shadow_area, 0,
+ IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
+
+ return 0;
+}
+/**
+ * irdma_cq_poll_cmpl - get cq completion info
+ * @cq: hw cq
+ * @info: cq poll information returned
+ */
+static enum irdma_status_code
+irdma_cq_poll_cmpl(struct irdma_cq_uk *cq, struct irdma_cq_poll_info *info)
+{
+ __u64 comp_ctx, qword0, qword2, qword3;
+ __le64 *cqe;
+ struct irdma_qp_uk *qp;
+ struct irdma_ring *pring = NULL;
+ __u32 wqe_idx, q_type;
+ enum irdma_status_code ret_code;
+ bool move_cq_head = true;
+ __u8 polarity;
+ bool ext_valid;
+ __le64 *ext_cqe;
+
+ if (cq->avoid_mem_cflct)
+ cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(cq);
+ else
+ cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq);
+
+ get_64bit_val(cqe, 24, &qword3);
+ polarity = (__u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
+ if (polarity != cq->polarity)
+ return IRDMA_ERR_Q_EMPTY;
+
+ /* Ensure CQE contents are read after valid bit is checked */
+ udma_from_device_barrier();
+
+ ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
+ if (ext_valid) {
+ __u64 qword6, qword7;
+ __u32 peek_head;
+
+ if (cq->avoid_mem_cflct) {
+ ext_cqe = (__le64 *)((__u8 *)cqe + 32);
+ get_64bit_val(ext_cqe, 24, &qword7);
+ polarity = (__u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
+ } else {
+ peek_head = (cq->cq_ring.head + 1) % cq->cq_ring.size;
+ ext_cqe = cq->cq_base[peek_head].buf;
+ get_64bit_val(ext_cqe, 24, &qword7);
+ polarity = (__u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
+ if (!peek_head)
+ polarity ^= 1;
+ }
+ if (polarity != cq->polarity)
+ return IRDMA_ERR_Q_EMPTY;
+
+ /* Ensure ext CQE contents are read after ext valid bit is checked */
+ udma_from_device_barrier();
+
+ info->imm_valid = (bool)FIELD_GET(IRDMA_CQ_IMMVALID, qword7);
+ if (info->imm_valid) {
+ __u64 qword4;
+
+ get_64bit_val(ext_cqe, 0, &qword4);
+ info->imm_data = (__u32)FIELD_GET(IRDMA_CQ_IMMDATALOW32, qword4);
+ }
+ info->ud_smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);
+ info->ud_vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);
+ if (info->ud_smac_valid || info->ud_vlan_valid) {
+ get_64bit_val(ext_cqe, 16, &qword6);
+ if (info->ud_vlan_valid)
+ info->ud_vlan = (__u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);
+ if (info->ud_smac_valid) {
+ info->ud_smac[5] = qword6 & 0xFF;
+ info->ud_smac[4] = (qword6 >> 8) & 0xFF;
+ info->ud_smac[3] = (qword6 >> 16) & 0xFF;
+ info->ud_smac[2] = (qword6 >> 24) & 0xFF;
+ info->ud_smac[1] = (qword6 >> 32) & 0xFF;
+ info->ud_smac[0] = (qword6 >> 40) & 0xFF;
+ }
+ }
+ } else {
+ info->imm_valid = false;
+ info->ud_smac_valid = false;
+ info->ud_vlan_valid = false;
+ }
+
+ q_type = (__u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
+ info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
+ info->push_dropped = (bool)FIELD_GET(IRDMACQ_PSHDROP, qword3);
+ info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
+ if (info->error) {
+ info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3);
+ info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3);
+ if (info->major_err == IRDMA_FLUSH_MAJOR_ERR) {
+ info->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
+ /* Set the min error to standard flush error code for remaining cqes */
+ if (info->minor_err != FLUSH_GENERAL_ERR) {
+ qword3 &= ~IRDMA_CQ_MINERR;
+ qword3 |= FIELD_PREP(IRDMA_CQ_MINERR, FLUSH_GENERAL_ERR);
+ set_64bit_val(cqe, 24, qword3);
+ }
+ } else {
+ info->comp_status = IRDMA_COMPL_STATUS_UNKNOWN;
+ }
+ } else {
+ info->comp_status = IRDMA_COMPL_STATUS_SUCCESS;
+ }
+
+ get_64bit_val(cqe, 0, &qword0);
+ get_64bit_val(cqe, 16, &qword2);
+
+ info->tcp_seq_num_rtt = (__u32)FIELD_GET(IRDMACQ_TCPSEQNUMRTT, qword0);
+ info->qp_id = (__u32)FIELD_GET(IRDMACQ_QPID, qword2);
+ info->ud_src_qpn = (__u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2);
+
+ get_64bit_val(cqe, 8, &comp_ctx);
+
+ info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3);
+ qp = (struct irdma_qp_uk *)(uintptr_t)comp_ctx;
+ if (!qp || qp->destroy_pending) {
+ ret_code = IRDMA_ERR_Q_DESTROYED;
+ goto exit;
+ }
+ wqe_idx = (__u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
+ info->qp_handle = (irdma_qp_handle)(uintptr_t)qp;
+
+ if (q_type == IRDMA_CQE_QTYPE_RQ) {
+ __u32 array_idx;
+
+ array_idx = wqe_idx / qp->rq_wqe_size_multiplier;
+
+ if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED ||
+ info->comp_status == IRDMA_COMPL_STATUS_UNKNOWN) {
+ if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) {
+ ret_code = IRDMA_ERR_Q_EMPTY;
+ goto exit;
+ }
+
+ info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail];
+ array_idx = qp->rq_ring.tail;
+ } else {
+ info->wr_id = qp->rq_wrid_array[array_idx];
+ }
+
+ info->bytes_xfered = (__u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
+
+ if (info->imm_valid)
+ info->op_type = IRDMA_OP_TYPE_REC_IMM;
+ else
+ info->op_type = IRDMA_OP_TYPE_REC;
+ if (qword3 & IRDMACQ_STAG) {
+ info->stag_invalid_set = true;
+ info->inv_stag = (__u32)FIELD_GET(IRDMACQ_INVSTAG, qword2);
+ } else {
+ info->stag_invalid_set = false;
+ }
+ IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1);
+ if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) {
+ qp->rq_flush_seen = true;
+ if (!IRDMA_RING_MORE_WORK(qp->rq_ring))
+ qp->rq_flush_complete = true;
+ else
+ move_cq_head = false;
+ }
+ pring = &qp->rq_ring;
+ } else { /* q_type is IRDMA_CQE_QTYPE_SQ */
+ if (qp->first_sq_wq) {
+ if (wqe_idx + 1 >= qp->conn_wqes)
+ qp->first_sq_wq = false;
+
+ if (wqe_idx < qp->conn_wqes && qp->sq_ring.head == qp->sq_ring.tail) {
+ IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
+ IRDMA_RING_MOVE_TAIL(cq->cq_ring);
+ set_64bit_val(cq->shadow_area, 0,
+ IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
+ memset(info, 0,
+ sizeof(struct irdma_cq_poll_info));
+ return irdma_cq_poll_cmpl(cq, info);
+ }
+ }
+ /*cease posting push mode on push drop*/
+ if (info->push_dropped) {
+ qp->push_mode = false;
+ qp->push_dropped = true;
+ }
+ if (info->comp_status != IRDMA_COMPL_STATUS_FLUSHED) {
+ info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
+ if (!info->comp_status)
+ info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;
+ info->op_type = (__u8)FIELD_GET(IRDMACQ_OP, qword3);
+ IRDMA_RING_SET_TAIL(qp->sq_ring,
+ wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta);
+ } else {
+ if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) {
+ ret_code = IRDMA_ERR_Q_EMPTY;
+ goto exit;
+ }
+
+ do {
+ __le64 *sw_wqe;
+ __u64 wqe_qword;
+ __u8 op_type;
+ __u32 tail;
+
+ tail = qp->sq_ring.tail;
+ sw_wqe = qp->sq_base[tail].elem;
+ get_64bit_val(sw_wqe, 24,
+ &wqe_qword);
+ op_type = (__u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
+ info->op_type = op_type;
+ IRDMA_RING_SET_TAIL(qp->sq_ring,
+ tail + qp->sq_wrtrk_array[tail].quanta);
+ if (op_type != IRDMAQP_OP_NOP) {
+ info->wr_id = qp->sq_wrtrk_array[tail].wrid;
+ info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
+ break;
+ }
+ } while (1);
+ qp->sq_flush_seen = true;
+ if (!IRDMA_RING_MORE_WORK(qp->sq_ring))
+ qp->sq_flush_complete = true;
+ }
+ pring = &qp->sq_ring;
+ }
+
+ ret_code = 0;
+
+exit:
+ if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED)
+ if (pring && IRDMA_RING_MORE_WORK(*pring))
+ move_cq_head = false;
+
+ if (move_cq_head) {
+ IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
+ if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring))
+ cq->polarity ^= 1;
+
+ if (ext_valid && !cq->avoid_mem_cflct) {
+ IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
+ if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring))
+ cq->polarity ^= 1;
+ }
+
+ IRDMA_RING_MOVE_TAIL(cq->cq_ring);
+ if (!cq->avoid_mem_cflct && ext_valid)
+ IRDMA_RING_MOVE_TAIL(cq->cq_ring);
+ set_64bit_val(cq->shadow_area, 0,
+ IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
+ } else {
+ qword3 &= ~IRDMA_CQ_WQEIDX;
+ qword3 |= FIELD_PREP(IRDMA_CQ_WQEIDX, pring->tail);
+ set_64bit_val(cqe, 24, qword3);
+ }
+
+ return ret_code;
+}
+
+/**
+ * irdma_qp_round_up - return round up qp wq depth
+ * @wqdepth: wq depth in quanta to round up
+ */
+static int irdma_qp_round_up(__u32 wqdepth)
+{
+ int scount = 1;
+
+ for (wqdepth--; scount <= 16; scount *= 2)
+ wqdepth |= wqdepth >> scount;
+
+ return ++wqdepth;
+}
+
+/**
+ * irdma_get_wqe_shift - get shift count for maximum wqe size
+ * @uk_attrs: qp HW attributes
+ * @sge: Maximum Scatter Gather Elements wqe
+ * @inline_data: Maximum inline data size
+ * @shift: Returns the shift needed based on sge
+ *
+ * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size.
+ * For 1 SGE or inline data <= 8, shift = 0 (wqe size of 32
+ * bytes). For 2 or 3 SGEs or inline data <= 39, shift = 1 (wqe
+ * size of 64 bytes).
+ * For 4-7 SGE's and inline <= 101 Shift of 2 otherwise (wqe
+ * size of 256 bytes).
+ */
+void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, __u32 sge,
+ __u32 inline_data, __u8 *shift)
+{
+ *shift = 0;
+ if (uk_attrs->hw_rev >= IRDMA_GEN_2) {
+ if (sge > 1 || inline_data > 8) {
+ if (sge < 4 && inline_data <= 39)
+ *shift = 1;
+ else if (sge < 8 && inline_data <= 101)
+ *shift = 2;
+ else
+ *shift = 3;
+ }
+ } else if (sge > 1 || inline_data > 16) {
+ *shift = (sge < 4 && inline_data <= 48) ? 1 : 2;
+ }
+}
+
+/*
+ * irdma_get_sqdepth - get SQ depth (quanta)
+ * @uk_attrs: qp HW attributes
+ * @sq_size: SQ size
+ * @shift: shift which determines size of WQE
+ * @sqdepth: depth of SQ
+ *
+ */
+enum irdma_status_code irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs,
+ __u32 sq_size, __u8 shift, __u32 *sqdepth)
+{
+ *sqdepth = irdma_qp_round_up((sq_size << shift) + IRDMA_SQ_RSVD);
+
+ if (*sqdepth < (IRDMA_QP_SW_MIN_WQSIZE << shift))
+ *sqdepth = IRDMA_QP_SW_MIN_WQSIZE << shift;
+ else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
+ return IRDMA_ERR_INVALID_SIZE;
+
+ return 0;
+}
+
+/*
+ * irdma_get_rqdepth - get RQ depth (quanta)
+ * @uk_attrs: qp HW attributes
+ * @rq_size: RQ size
+ * @shift: shift which determines size of WQE
+ * @rqdepth: depth of RQ
+ */
+enum irdma_status_code irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs,
+ __u32 rq_size, __u8 shift, __u32 *rqdepth)
+{
+ *rqdepth = irdma_qp_round_up((rq_size << shift) + IRDMA_RQ_RSVD);
+
+ if (*rqdepth < (IRDMA_QP_SW_MIN_WQSIZE << shift))
+ *rqdepth = IRDMA_QP_SW_MIN_WQSIZE << shift;
+ else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
+ return IRDMA_ERR_INVALID_SIZE;
+
+ return 0;
+}
+
+static const struct irdma_qp_uk_ops iw_qp_uk_ops = {
+ .iw_inline_rdma_write = irdma_inline_rdma_write,
+ .iw_inline_send = irdma_inline_send,
+ .iw_mw_bind = irdma_mw_bind,
+ .iw_post_nop = irdma_nop,
+ .iw_post_receive = irdma_post_receive,
+ .iw_qp_post_wr = irdma_qp_post_wr,
+ .iw_qp_ring_push_db = irdma_qp_ring_push_db,
+ .iw_rdma_read = irdma_rdma_read,
+ .iw_rdma_write = irdma_rdma_write,
+ .iw_send = irdma_send,
+ .iw_stag_local_invalidate = irdma_stag_local_invalidate,
+};
+
+static const struct irdma_wqe_uk_ops iw_wqe_uk_ops = {
+ .iw_copy_inline_data = irdma_copy_inline_data,
+ .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta,
+ .iw_set_fragment = irdma_set_fragment,
+ .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe,
+};
+
+static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = {
+ .iw_copy_inline_data = irdma_copy_inline_data_gen_1,
+ .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta_gen_1,
+ .iw_set_fragment = irdma_set_fragment_gen_1,
+ .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe_gen_1,
+};
+
+static const struct irdma_cq_ops iw_cq_ops = {
+ .iw_cq_clean = irdma_clean_cq,
+ .iw_cq_poll_cmpl = irdma_cq_poll_cmpl,
+ .iw_cq_post_entries = irdma_cq_post_entries,
+ .iw_cq_request_notification = irdma_cq_request_notification,
+ .iw_cq_resize = irdma_cq_resize,
+ .iw_cq_set_resized_cnt = irdma_cq_set_resized_cnt,
+};
+
+static const struct irdma_device_uk_ops iw_device_uk_ops = {
+ .iw_cq_uk_init = irdma_cq_uk_init,
+ .iw_qp_uk_init = irdma_qp_uk_init,
+};
+
+/**
+ * irdma_setup_connection_wqes - setup WQEs necessary to complete
+ * connection.
+ * @qp: hw qp (user and kernel)
+ * @info: qp initialization info
+ */
+static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
+ struct irdma_qp_uk_init_info *info)
+{
+ __u16 move_cnt = 1;
+
+ if (!info->legacy_mode &&
+ (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE))
+ move_cnt = 3;
+
+ qp->conn_wqes = move_cnt;
+ IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
+ IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
+ IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt);
+}
+
+/**
+ * irdma_qp_uk_init - initialize shared qp
+ * @qp: hw qp (user and kernel)
+ * @info: qp initialization info
+ *
+ * initializes the vars used in both user and kernel mode.
+ * size of the wqe depends on numbers of max. fragements
+ * allowed. Then size of wqe * the number of wqes should be the
+ * amount of memory allocated for sq and rq.
+ */
+enum irdma_status_code irdma_qp_uk_init(struct irdma_qp_uk *qp,
+ struct irdma_qp_uk_init_info *info)
+{
+ enum irdma_status_code ret_code = 0;
+ __u32 sq_ring_size;
+ __u8 sqshift, rqshift;
+
+ qp->uk_attrs = info->uk_attrs;
+ if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags ||
+ info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags)
+ return IRDMA_ERR_INVALID_FRAG_COUNT;
+
+ irdma_get_wqe_shift(qp->uk_attrs, info->max_rq_frag_cnt, 0, &rqshift);
+ if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) {
+ irdma_get_wqe_shift(qp->uk_attrs, info->max_sq_frag_cnt,
+ info->max_inline_data, &sqshift);
+ if (info->abi_ver > 4)
+ rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
+ } else {
+ irdma_get_wqe_shift(qp->uk_attrs, info->max_sq_frag_cnt + 1,
+ info->max_inline_data, &sqshift);
+ }
+ qp->qp_caps = info->qp_caps;
+ qp->sq_base = info->sq;
+ qp->rq_base = info->rq;
+ qp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP;
+ qp->shadow_area = info->shadow_area;
+ qp->sq_wrtrk_array = info->sq_wrtrk_array;
+
+ qp->rq_wrid_array = info->rq_wrid_array;
+ qp->wqe_alloc_db = info->wqe_alloc_db;
+ qp->qp_id = info->qp_id;
+ qp->sq_size = info->sq_size;
+ qp->push_mode = false;
+ qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
+ sq_ring_size = qp->sq_size << sqshift;
+ IRDMA_RING_INIT(qp->sq_ring, sq_ring_size);
+ IRDMA_RING_INIT(qp->initial_ring, sq_ring_size);
+ if (info->first_sq_wq) {
+ irdma_setup_connection_wqes(qp, info);
+ qp->swqe_polarity = 1;
+ qp->first_sq_wq = true;
+ } else {
+ qp->swqe_polarity = 0;
+ }
+ qp->swqe_polarity_deferred = 1;
+ qp->rwqe_polarity = 0;
+ qp->rq_size = info->rq_size;
+ qp->max_rq_frag_cnt = info->max_rq_frag_cnt;
+ qp->max_inline_data = info->max_inline_data;
+ qp->rq_wqe_size = rqshift;
+ IRDMA_RING_INIT(qp->rq_ring, qp->rq_size);
+ qp->rq_wqe_size_multiplier = 1 << rqshift;
+ qp->qp_ops = iw_qp_uk_ops;
+ if (qp->uk_attrs->hw_rev == IRDMA_GEN_1)
+ qp->wqe_ops = iw_wqe_uk_ops_gen_1;
+ else
+ qp->wqe_ops = iw_wqe_uk_ops;
+ return ret_code;
+}
+
+/**
+ * irdma_cq_uk_init - initialize shared cq (user and kernel)
+ * @cq: hw cq
+ * @info: hw cq initialization info
+ */
+enum irdma_status_code irdma_cq_uk_init(struct irdma_cq_uk *cq,
+ struct irdma_cq_uk_init_info *info)
+{
+ cq->cq_base = info->cq_base;
+ cq->cq_id = info->cq_id;
+ cq->cq_size = info->cq_size;
+ cq->cqe_alloc_db = info->cqe_alloc_db;
+ cq->cq_ack_db = info->cq_ack_db;
+ cq->shadow_area = info->shadow_area;
+ cq->avoid_mem_cflct = info->avoid_mem_cflct;
+ IRDMA_RING_INIT(cq->cq_ring, cq->cq_size);
+ cq->polarity = 1;
+ cq->ops = iw_cq_ops;
+
+ return 0;
+}
+
+/**
+ * irdma_device_init_uk - setup routines for iwarp shared device
+ * @dev: iwarp shared (user and kernel)
+ */
+void irdma_device_init_uk(struct irdma_dev_uk *dev)
+{
+ dev->ops_uk = iw_device_uk_ops;
+}
+
+/**
+ * irdma_clean_cq - clean cq entries
+ * @q: completion context
+ * @cq: cq to clean
+ */
+void irdma_clean_cq(void *q, struct irdma_cq_uk *cq)
+{
+ __le64 *cqe;
+ __u64 qword3, comp_ctx;
+ __u32 cq_head;
+ __u8 polarity, temp;
+
+ cq_head = cq->cq_ring.head;
+ temp = cq->polarity;
+ do {
+ if (cq->avoid_mem_cflct)
+ cqe = ((struct irdma_extended_cqe *)(cq->cq_base))[cq_head].buf;
+ else
+ cqe = cq->cq_base[cq_head].buf;
+ get_64bit_val(cqe, 24, &qword3);
+ polarity = (__u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
+
+ if (polarity != temp)
+ break;
+
+ get_64bit_val(cqe, 8, &comp_ctx);
+ if ((void *)(uintptr_t)comp_ctx == q)
+ set_64bit_val(cqe, 8, 0);
+
+ cq_head = (cq_head + 1) % cq->cq_ring.size;
+ if (!cq_head)
+ temp ^= 1;
+ } while (true);
+}
+
+/**
+ * irdma_nop - post a nop
+ * @qp: hw qp ptr
+ * @wr_id: work request id
+ * @signaled: signaled for completion
+ * @post_sq: ring doorbell
+ */
+enum irdma_status_code irdma_nop(struct irdma_qp_uk *qp, __u64 wr_id,
+ bool signaled, bool post_sq)
+{
+ __le64 *wqe;
+ __u64 hdr;
+ __u32 wqe_idx;
+ struct irdma_post_sq_info info = {};
+
+ info.push_wqe = false;
+ info.wr_id = wr_id;
+ wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
+ 0, &info);
+ if (!wqe)
+ return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
+
+ irdma_clr_wqes(qp, wqe_idx);
+
+ set_64bit_val(wqe, 0, 0);
+ set_64bit_val(wqe, 8, 0);
+ set_64bit_val(wqe, 16, 0);
+
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
+ FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
+
+ udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
+
+ set_64bit_val(wqe, 24, hdr);
+ if (post_sq)
+ irdma_qp_post_wr(qp);
+
+ return 0;
+}
+
+/**
+ * irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ
+ * @frag_cnt: number of fragments
+ * @quanta: quanta for frag_cnt
+ */
+enum irdma_status_code irdma_fragcnt_to_quanta_sq(__u32 frag_cnt, __u16 *quanta)
+{
+ switch (frag_cnt) {
+ case 0:
+ case 1:
+ *quanta = IRDMA_QP_WQE_MIN_QUANTA;
+ break;
+ case 2:
+ case 3:
+ *quanta = 2;
+ break;
+ case 4:
+ case 5:
+ *quanta = 3;
+ break;
+ case 6:
+ case 7:
+ *quanta = 4;
+ break;
+ case 8:
+ case 9:
+ *quanta = 5;
+ break;
+ case 10:
+ case 11:
+ *quanta = 6;
+ break;
+ case 12:
+ case 13:
+ *quanta = 7;
+ break;
+ case 14:
+ case 15: /* when immediate data is present */
+ *quanta = 8;
+ break;
+ default:
+ return IRDMA_ERR_INVALID_FRAG_COUNT;
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ
+ * @frag_cnt: number of fragments
+ * @wqe_size: size in bytes given frag_cnt
+ */
+enum irdma_status_code irdma_fragcnt_to_wqesize_rq(__u32 frag_cnt, __u16 *wqe_size)
+{
+ switch (frag_cnt) {
+ case 0:
+ case 1:
+ *wqe_size = 32;
+ break;
+ case 2:
+ case 3:
+ *wqe_size = 64;
+ break;
+ case 4:
+ case 5:
+ case 6:
+ case 7:
+ *wqe_size = 128;
+ break;
+ case 8:
+ case 9:
+ case 10:
+ case 11:
+ case 12:
+ case 13:
+ case 14:
+ *wqe_size = 256;
+ break;
+ default:
+ return IRDMA_ERR_INVALID_FRAG_COUNT;
+ }
+
+ return 0;
+}
--
1.8.3.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH 4/5] rdma-core/irdma: Add library setup and utility definitions
2021-04-06 21:17 [PATCH 0/5] Add irdma user space provider for Intel Ethernet RDMA Tatyana Nikolova
` (2 preceding siblings ...)
2021-04-06 21:17 ` [PATCH 3/5] rdma-core/irdma: Add user/kernel shared libraries Tatyana Nikolova
@ 2021-04-06 21:17 ` Tatyana Nikolova
2021-04-06 22:59 ` Jason Gunthorpe
2021-04-06 21:17 ` [PATCH 5/5] rdma-core/irdma: Implement device supported verb APIs Tatyana Nikolova
4 siblings, 1 reply; 10+ messages in thread
From: Tatyana Nikolova @ 2021-04-06 21:17 UTC (permalink / raw)
To: jgg, dledford, leon; +Cc: linux-rdma, Tatyana Nikolova
Add definitions to allocate context for user apps,
recognize HW devices and exchange context with device,
set up callback tables and device attributes and add
utility definitions and headers.
Signed-off-by: Tatyana Nikolova <tatyana.e.nikolova@intel.com>
---
providers/irdma/defs.h | 342 +++++++++++++++++++++++++++++++
providers/irdma/i40e_devids.h | 36 ++++
providers/irdma/i40iw_hw.h | 31 +++
providers/irdma/ice_devids.h | 59 ++++++
providers/irdma/osdep.h | 21 ++
providers/irdma/status.h | 71 +++++++
providers/irdma/umain.c | 234 +++++++++++++++++++++
providers/irdma/umain.h | 205 +++++++++++++++++++
providers/irdma/user.h | 466 ++++++++++++++++++++++++++++++++++++++++++
9 files changed, 1465 insertions(+)
create mode 100644 providers/irdma/defs.h
create mode 100644 providers/irdma/i40e_devids.h
create mode 100644 providers/irdma/i40iw_hw.h
create mode 100644 providers/irdma/ice_devids.h
create mode 100644 providers/irdma/osdep.h
create mode 100644 providers/irdma/status.h
create mode 100644 providers/irdma/umain.c
create mode 100644 providers/irdma/umain.h
create mode 100644 providers/irdma/user.h
diff --git a/providers/irdma/defs.h b/providers/irdma/defs.h
new file mode 100644
index 0000000..2f67c7b
--- /dev/null
+++ b/providers/irdma/defs.h
@@ -0,0 +1,342 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2015 - 2021 Intel Corporation */
+#ifndef IRDMA_DEFS_H
+#define IRDMA_DEFS_H
+
+#include "osdep.h"
+
+
+#define IRDMA_QP_TYPE_IWARP 1
+#define IRDMA_QP_TYPE_UDA 2
+#define IRDMA_QP_TYPE_ROCE_RC 3
+#define IRDMA_QP_TYPE_ROCE_UD 4
+
+#define IRDMA_HW_PAGE_SIZE 4096
+#define IRDMA_HW_PAGE_SHIFT 12
+#define IRDMA_CQE_QTYPE_RQ 0
+#define IRDMA_CQE_QTYPE_SQ 1
+
+#define IRDMA_QP_SW_MIN_WQSIZE 8u /* in WRs*/
+#define IRDMA_QP_WQE_MIN_SIZE 32
+#define IRDMA_QP_WQE_MAX_SIZE 256
+#define IRDMA_QP_WQE_MIN_QUANTA 1
+#define IRDMA_MAX_RQ_WQE_SHIFT_GEN1 2
+#define IRDMA_MAX_RQ_WQE_SHIFT_GEN2 3
+
+#define IRDMA_SQ_RSVD 258
+#define IRDMA_RQ_RSVD 1
+
+#define IRDMA_FEATURE_RTS_AE 1ULL
+#define IRDMA_FEATURE_CQ_RESIZE 2ULL
+#define IRDMA_FEATURE_ATOMIC_OPS 32ULL
+#define IRDMA_FEATURE_SRQ 64ULL
+
+#define IRDMAQP_OP_RDMA_WRITE 0x00
+#define IRDMAQP_OP_RDMA_READ 0x01
+#define IRDMAQP_OP_RDMA_SEND 0x03
+#define IRDMAQP_OP_RDMA_SEND_INV 0x04
+#define IRDMAQP_OP_RDMA_SEND_SOL_EVENT 0x05
+#define IRDMAQP_OP_RDMA_SEND_SOL_EVENT_INV 0x06
+#define IRDMAQP_OP_BIND_MW 0x08
+#define IRDMAQP_OP_FAST_REGISTER 0x09
+#define IRDMAQP_OP_LOCAL_INVALIDATE 0x0a
+#define IRDMAQP_OP_RDMA_READ_LOC_INV 0x0b
+#define IRDMAQP_OP_NOP 0x0c
+
+#define LS_64_1(val, bits) ((__u64)(uintptr_t)(val) << (bits))
+#define RS_64_1(val, bits) ((__u64)(uintptr_t)(val) >> (bits))
+#define LS_32_1(val, bits) ((__u32)((val) << (bits)))
+#define RS_32_1(val, bits) ((__u32)((val) >> (bits)))
+
+#define IRDMA_CQPHC_QPCTX GENMASK_ULL(63, 0)
+#define IRDMA_QP_DBSA_HW_SQ_TAIL GENMASK_ULL(14, 0)
+#define IRDMA_CQ_DBSA_CQEIDX GENMASK_ULL(19, 0)
+#define IRDMA_CQ_DBSA_SW_CQ_SELECT GENMASK_ULL(13, 0)
+#define IRDMA_CQ_DBSA_ARM_NEXT BIT_ULL(14)
+#define IRDMA_CQ_DBSA_ARM_NEXT_SE BIT_ULL(15)
+#define IRDMA_CQ_DBSA_ARM_SEQ_NUM GENMASK_ULL(17, 16)
+
+/* CQP and iWARP Completion Queue */
+#define IRDMA_CQ_QPCTX IRDMA_CQPHC_QPCTX
+
+#define IRDMA_CQ_MINERR GENMASK_ULL(15, 0)
+#define IRDMA_CQ_MAJERR GENMASK_ULL(31, 16)
+#define IRDMA_CQ_WQEIDX GENMASK_ULL(46, 32)
+#define IRDMA_CQ_EXTCQE BIT_ULL(50)
+#define IRDMA_OOO_CMPL BIT_ULL(54)
+#define IRDMA_CQ_ERROR BIT_ULL(55)
+#define IRDMA_CQ_SQ BIT_ULL(62)
+
+#define IRDMA_CQ_VALID BIT_ULL(63)
+#define IRDMA_CQ_IMMVALID BIT_ULL(62)
+#define IRDMA_CQ_UDSMACVALID BIT_ULL(61)
+#define IRDMA_CQ_UDVLANVALID BIT_ULL(60)
+#define IRDMA_CQ_UDSMAC GENMASK_ULL(47, 0)
+#define IRDMA_CQ_UDVLAN GENMASK_ULL(63, 48)
+
+#define IRDMA_CQ_IMMDATA_S 0
+#define IRDMA_CQ_IMMDATA_M (0xffffffffffffffffULL << IRDMA_CQ_IMMVALID_S)
+#define IRDMA_CQ_IMMDATALOW32 GENMASK_ULL(31, 0)
+#define IRDMA_CQ_IMMDATAUP32 GENMASK_ULL(63, 32)
+#define IRDMACQ_PAYLDLEN GENMASK_ULL(31, 0)
+#define IRDMACQ_TCPSEQNUMRTT GENMASK_ULL(63, 32)
+#define IRDMACQ_INVSTAG GENMASK_ULL(31, 0)
+#define IRDMACQ_QPID GENMASK_ULL(55, 32)
+
+#define IRDMACQ_UDSRCQPN GENMASK_ULL(31, 0)
+#define IRDMACQ_PSHDROP BIT_ULL(51)
+#define IRDMACQ_STAG BIT_ULL(53)
+#define IRDMACQ_IPV4 BIT_ULL(53)
+#define IRDMACQ_SOEVENT BIT_ULL(54)
+#define IRDMACQ_OP GENMASK_ULL(61, 56)
+
+/* Manage Push Page - MPP */
+#define IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1 0xffff
+#define IRDMA_INVALID_PUSH_PAGE_INDEX 0xffffffff
+
+#define IRDMAQPSQ_OPCODE GENMASK_ULL(37, 32)
+#define IRDMAQPSQ_COPY_HOST_PBL BIT_ULL(43)
+#define IRDMAQPSQ_ADDFRAGCNT GENMASK_ULL(41, 38)
+#define IRDMAQPSQ_PUSHWQE BIT_ULL(56)
+#define IRDMAQPSQ_STREAMMODE BIT_ULL(58)
+#define IRDMAQPSQ_WAITFORRCVPDU BIT_ULL(59)
+#define IRDMAQPSQ_READFENCE BIT_ULL(60)
+#define IRDMAQPSQ_LOCALFENCE BIT_ULL(61)
+#define IRDMAQPSQ_UDPHEADER BIT_ULL(61)
+#define IRDMAQPSQ_L4LEN GENMASK_ULL(45, 42)
+#define IRDMAQPSQ_SIGCOMPL BIT_ULL(62)
+#define IRDMAQPSQ_VALID BIT_ULL(63)
+
+#define IRDMAQPSQ_FRAG_TO IRDMA_CQPHC_QPCTX
+#define IRDMAQPSQ_FRAG_VALID BIT_ULL(63)
+#define IRDMAQPSQ_FRAG_LEN GENMASK_ULL(62, 32)
+#define IRDMAQPSQ_FRAG_STAG GENMASK_ULL(31, 0)
+#define IRDMAQPSQ_GEN1_FRAG_LEN GENMASK_ULL(31, 0)
+#define IRDMAQPSQ_GEN1_FRAG_STAG GENMASK_ULL(63, 32)
+#define IRDMAQPSQ_REMSTAGINV GENMASK_ULL(31, 0)
+#define IRDMAQPSQ_DESTQKEY GENMASK_ULL(31, 0)
+#define IRDMAQPSQ_DESTQPN GENMASK_ULL(55, 32)
+#define IRDMAQPSQ_AHID GENMASK_ULL(16, 0)
+#define IRDMAQPSQ_INLINEDATAFLAG BIT_ULL(57)
+
+#define IRDMA_INLINE_VALID_S 7
+#define IRDMAQPSQ_INLINEDATALEN GENMASK_ULL(55, 48)
+#define IRDMAQPSQ_IMMDATAFLAG BIT_ULL(47)
+#define IRDMAQPSQ_REPORTRTT BIT_ULL(46)
+
+#define IRDMAQPSQ_IMMDATA GENMASK_ULL(63, 0)
+#define IRDMAQPSQ_REMSTAG GENMASK_ULL(31, 0)
+
+#define IRDMAQPSQ_REMTO IRDMA_CQPHC_QPCTX
+
+#define IRDMAQPSQ_STAGRIGHTS GENMASK_ULL(52, 48)
+#define IRDMAQPSQ_VABASEDTO BIT_ULL(53)
+#define IRDMAQPSQ_MEMWINDOWTYPE BIT_ULL(54)
+
+#define IRDMAQPSQ_MWLEN IRDMA_CQPHC_QPCTX
+#define IRDMAQPSQ_PARENTMRSTAG GENMASK_ULL(63, 32)
+#define IRDMAQPSQ_MWSTAG GENMASK_ULL(31, 0)
+
+#define IRDMAQPSQ_BASEVA_TO_FBO IRDMA_CQPHC_QPCTX
+
+#define IRDMAQPSQ_LOCSTAG GENMASK_ULL(31, 0)
+
+/* iwarp QP RQ WQE common fields */
+#define IRDMAQPRQ_ADDFRAGCNT IRDMAQPSQ_ADDFRAGCNT
+#define IRDMAQPRQ_VALID IRDMAQPSQ_VALID
+#define IRDMAQPRQ_COMPLCTX IRDMA_CQPHC_QPCTX
+#define IRDMAQPRQ_FRAG_LEN IRDMAQPSQ_FRAG_LEN
+#define IRDMAQPRQ_STAG IRDMAQPSQ_FRAG_STAG
+#define IRDMAQPRQ_TO IRDMAQPSQ_FRAG_TO
+
+#define IRDMAPFINT_OICR_HMC_ERR_M BIT(26)
+#define IRDMAPFINT_OICR_PE_PUSH_M BIT(27)
+#define IRDMAPFINT_OICR_PE_CRITERR_M BIT(28)
+
+#define IRDMA_CQP_INIT_WQE(wqe) memset(wqe, 0, 64)
+
+#define IRDMA_GET_CURRENT_CQ_ELEM(_cq) \
+ ( \
+ (_cq)->cq_base[IRDMA_RING_CURRENT_HEAD((_cq)->cq_ring)].buf \
+ )
+#define IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(_cq) \
+ ( \
+ ((struct irdma_extended_cqe *) \
+ ((_cq)->cq_base))[IRDMA_RING_CURRENT_HEAD((_cq)->cq_ring)].buf \
+ )
+
+#define IRDMA_RING_INIT(_ring, _size) \
+ { \
+ (_ring).head = 0; \
+ (_ring).tail = 0; \
+ (_ring).size = (_size); \
+ }
+#define IRDMA_RING_SIZE(_ring) ((_ring).size)
+#define IRDMA_RING_CURRENT_HEAD(_ring) ((_ring).head)
+#define IRDMA_RING_CURRENT_TAIL(_ring) ((_ring).tail)
+
+#define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \
+ { \
+ register __u32 size; \
+ size = (_ring).size; \
+ if (!IRDMA_RING_FULL_ERR(_ring)) { \
+ (_ring).head = ((_ring).head + 1) % size; \
+ (_retcode) = 0; \
+ } else { \
+ (_retcode) = IRDMA_ERR_RING_FULL; \
+ } \
+ }
+#define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
+ { \
+ register __u32 size; \
+ size = (_ring).size; \
+ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < size) { \
+ (_ring).head = ((_ring).head + (_count)) % size; \
+ (_retcode) = 0; \
+ } else { \
+ (_retcode) = IRDMA_ERR_RING_FULL; \
+ } \
+ }
+#define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \
+ { \
+ register __u32 size; \
+ size = (_ring).size; \
+ if (!IRDMA_SQ_RING_FULL_ERR(_ring)) { \
+ (_ring).head = ((_ring).head + 1) % size; \
+ (_retcode) = 0; \
+ } else { \
+ (_retcode) = IRDMA_ERR_RING_FULL; \
+ } \
+ }
+#define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
+ { \
+ register __u32 size; \
+ size = (_ring).size; \
+ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < (size - 256)) { \
+ (_ring).head = ((_ring).head + (_count)) % size; \
+ (_retcode) = 0; \
+ } else { \
+ (_retcode) = IRDMA_ERR_RING_FULL; \
+ } \
+ }
+#define IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(_ring, _count) \
+ (_ring).head = ((_ring).head + (_count)) % (_ring).size
+
+#define IRDMA_RING_MOVE_TAIL(_ring) \
+ (_ring).tail = ((_ring).tail + 1) % (_ring).size
+
+#define IRDMA_RING_MOVE_HEAD_NOCHECK(_ring) \
+ (_ring).head = ((_ring).head + 1) % (_ring).size
+
+#define IRDMA_RING_MOVE_TAIL_BY_COUNT(_ring, _count) \
+ (_ring).tail = ((_ring).tail + (_count)) % (_ring).size
+
+#define IRDMA_RING_SET_TAIL(_ring, _pos) \
+ (_ring).tail = (_pos) % (_ring).size
+
+#define IRDMA_RING_FULL_ERR(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 1)) \
+ )
+
+#define IRDMA_ERR_RING_FULL2(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 2)) \
+ )
+
+#define IRDMA_ERR_RING_FULL3(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 3)) \
+ )
+
+#define IRDMA_SQ_RING_FULL_ERR(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 257)) \
+ )
+
+#define IRDMA_ERR_SQ_RING_FULL2(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 258)) \
+ )
+#define IRDMA_ERR_SQ_RING_FULL3(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) == ((_ring).size - 259)) \
+ )
+#define IRDMA_RING_MORE_WORK(_ring) \
+ ( \
+ (IRDMA_RING_USED_QUANTA(_ring) != 0) \
+ )
+
+#define IRDMA_RING_USED_QUANTA(_ring) \
+ ( \
+ (((_ring).head + (_ring).size - (_ring).tail) % (_ring).size) \
+ )
+
+#define IRDMA_RING_FREE_QUANTA(_ring) \
+ ( \
+ ((_ring).size - IRDMA_RING_USED_QUANTA(_ring) - 1) \
+ )
+
+#define IRDMA_SQ_RING_FREE_QUANTA(_ring) \
+ ( \
+ ((_ring).size - IRDMA_RING_USED_QUANTA(_ring) - 257) \
+ )
+
+#define IRDMA_ATOMIC_RING_MOVE_HEAD(_ring, index, _retcode) \
+ { \
+ index = IRDMA_RING_CURRENT_HEAD(_ring); \
+ IRDMA_RING_MOVE_HEAD(_ring, _retcode); \
+ }
+
+enum irdma_qp_wqe_size {
+ IRDMA_WQE_SIZE_32 = 32,
+ IRDMA_WQE_SIZE_64 = 64,
+ IRDMA_WQE_SIZE_96 = 96,
+ IRDMA_WQE_SIZE_128 = 128,
+ IRDMA_WQE_SIZE_256 = 256,
+};
+
+/**
+ * set_64bit_val - set 64 bit value to hw wqe
+ * @wqe_words: wqe addr to write
+ * @byte_index: index in wqe
+ * @val: value to write
+ **/
+static inline void set_64bit_val(__le64 *wqe_words, __u32 byte_index, __u64 val)
+{
+ wqe_words[byte_index >> 3] = htole64(val);
+}
+
+/**
+ * set_32bit_val - set 32 bit value to hw wqe
+ * @wqe_words: wqe addr to write
+ * @byte_index: index in wqe
+ * @val: value to write
+ **/
+static inline void set_32bit_val(__le32 *wqe_words, __u32 byte_index, __u32 val)
+{
+ wqe_words[byte_index >> 2] = htole32(val);
+}
+
+/**
+ * get_64bit_val - read 64 bit value from wqe
+ * @wqe_words: wqe addr
+ * @byte_index: index to read from
+ * @val: read value
+ **/
+static inline void get_64bit_val(__le64 *wqe_words, __u32 byte_index, __u64 *val)
+{
+ *val = le64toh(wqe_words[byte_index >> 3]);
+}
+
+/**
+ * get_32bit_val - read 32 bit value from wqe
+ * @wqe_words: wqe addr
+ * @byte_index: index to reaad from
+ * @val: return 32 bit value
+ **/
+static inline void get_32bit_val(__le32 *wqe_words, __u32 byte_index, __u32 *val)
+{
+ *val = le32toh(wqe_words[byte_index >> 2]);
+}
+#endif /* IRDMA_DEFS_H */
diff --git a/providers/irdma/i40e_devids.h b/providers/irdma/i40e_devids.h
new file mode 100644
index 0000000..2ea4f2d
--- /dev/null
+++ b/providers/irdma/i40e_devids.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2015 - 2019 Intel Corporation */
+#ifndef I40E_DEVIDS_H
+#define I40E_DEVIDS_H
+
+/* Vendor ID */
+#define I40E_INTEL_VENDOR_ID 0x8086
+
+/* Device IDs */
+#define I40E_DEV_ID_SFP_XL710 0x1572
+#define I40E_DEV_ID_QEMU 0x1574
+#define I40E_DEV_ID_KX_B 0x1580
+#define I40E_DEV_ID_KX_C 0x1581
+#define I40E_DEV_ID_QSFP_A 0x1583
+#define I40E_DEV_ID_QSFP_B 0x1584
+#define I40E_DEV_ID_QSFP_C 0x1585
+#define I40E_DEV_ID_10G_BASE_T 0x1586
+#define I40E_DEV_ID_20G_KR2 0x1587
+#define I40E_DEV_ID_20G_KR2_A 0x1588
+#define I40E_DEV_ID_10G_BASE_T4 0x1589
+#define I40E_DEV_ID_25G_B 0x158A
+#define I40E_DEV_ID_25G_SFP28 0x158B
+#define I40E_DEV_ID_VF 0x154C
+#define I40E_DEV_ID_VF_HV 0x1571
+#define I40E_DEV_ID_X722_A0 0x374C
+#define I40E_DEV_ID_X722_A0_VF 0x374D
+#define I40E_DEV_ID_KX_X722 0x37CE
+#define I40E_DEV_ID_QSFP_X722 0x37CF
+#define I40E_DEV_ID_SFP_X722 0x37D0
+#define I40E_DEV_ID_1G_BASE_T_X722 0x37D1
+#define I40E_DEV_ID_10G_BASE_T_X722 0x37D2
+#define I40E_DEV_ID_SFP_I_X722 0x37D3
+#define I40E_DEV_ID_X722_VF 0x37CD
+#define I40E_DEV_ID_X722_VF_HV 0x37D9
+
+#endif /* I40E_DEVIDS_H */
diff --git a/providers/irdma/i40iw_hw.h b/providers/irdma/i40iw_hw.h
new file mode 100644
index 0000000..a0a2848
--- /dev/null
+++ b/providers/irdma/i40iw_hw.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2015 - 2021 Intel Corporation */
+#ifndef I40IW_HW_H
+#define I40IW_HW_H
+
+enum i40iw_device_caps_const {
+ I40IW_MAX_WQ_FRAGMENT_COUNT = 3,
+ I40IW_MAX_SGE_RD = 1,
+ I40IW_MAX_PUSH_PAGE_COUNT = 0,
+ I40IW_MAX_INLINE_DATA_SIZE = 48,
+ I40IW_MAX_IRD_SIZE = 63,
+ I40IW_MAX_ORD_SIZE = 127,
+ I40IW_MAX_WQ_ENTRIES = 2048,
+ I40IW_MAX_WQE_SIZE_RQ = 128,
+ I40IW_MAX_PDS = 32768,
+ I40IW_MAX_STATS_COUNT = 16,
+ I40IW_MAX_CQ_SIZE = 1048575,
+ I40IW_MAX_OUTBOUND_MSG_SIZE = 2147483647,
+ I40IW_MAX_INBOUND_MSG_SIZE = 2147483647,
+};
+
+#define I40IW_QP_WQE_MIN_SIZE 32
+#define I40IW_QP_WQE_MAX_SIZE 128
+#define I40IW_QP_SW_MIN_WQSIZE 4
+#define I40IW_MAX_RQ_WQE_SHIFT 2
+#define I40IW_MAX_QUANTA_PER_WR 2
+
+#define I40IW_QP_SW_MAX_SQ_QUANTA 2048
+#define I40IW_QP_SW_MAX_RQ_QUANTA 16384
+#define I40IW_QP_SW_MAX_WQ_QUANTA 2048
+#endif /* I40IW_HW_H */
diff --git a/providers/irdma/ice_devids.h b/providers/irdma/ice_devids.h
new file mode 100644
index 0000000..655a1d6
--- /dev/null
+++ b/providers/irdma/ice_devids.h
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2019 - 2020 Intel Corporation */
+#ifndef ICE_DEVIDS_H
+#define ICE_DEVIDS_H
+
+#define PCI_VENDOR_ID_INTEL 0x8086
+
+/* Device IDs */
+/* Intel(R) Ethernet Connection E823-L for backplane */
+#define ICE_DEV_ID_E823L_BACKPLANE 0x124C
+/* Intel(R) Ethernet Connection E823-L for SFP */
+#define ICE_DEV_ID_E823L_SFP 0x124D
+/* Intel(R) Ethernet Connection E823-L/X557-AT 10GBASE-T */
+#define ICE_DEV_ID_E823L_10G_BASE_T 0x124E
+/* Intel(R) Ethernet Connection E823-L 1GbE */
+#define ICE_DEV_ID_E823L_1GBE 0x124F
+/* Intel(R) Ethernet Connection E823-L for QSFP */
+#define ICE_DEV_ID_E823L_QSFP 0x151D
+/* Intel(R) Ethernet Controller E810-C for backplane */
+#define ICE_DEV_ID_E810C_BACKPLANE 0x1591
+/* Intel(R) Ethernet Controller E810-C for QSFP */
+#define ICE_DEV_ID_E810C_QSFP 0x1592
+/* Intel(R) Ethernet Controller E810-C for SFP */
+#define ICE_DEV_ID_E810C_SFP 0x1593
+/* Intel(R) Ethernet Controller E810-XXV for backplane */
+#define ICE_DEV_ID_E810_XXV_BACKPLANE 0x1599
+/* Intel(R) Ethernet Controller E810-XXV for QSFP */
+#define ICE_DEV_ID_E810_XXV_QSFP 0x159A
+/* Intel(R) Ethernet Controller E810-XXV for SFP */
+#define ICE_DEV_ID_E810_XXV_SFP 0x159B
+/* Intel(R) Ethernet Connection E823-C for backplane */
+#define ICE_DEV_ID_E823C_BACKPLANE 0x188A
+/* Intel(R) Ethernet Connection E823-C for QSFP */
+#define ICE_DEV_ID_E823C_QSFP 0x188B
+/* Intel(R) Ethernet Connection E823-C for SFP */
+#define ICE_DEV_ID_E823C_SFP 0x188C
+/* Intel(R) Ethernet Connection E823-C/X557-AT 10GBASE-T */
+#define ICE_DEV_ID_E823C_10G_BASE_T 0x188D
+/* Intel(R) Ethernet Connection E823-C 1GbE */
+#define ICE_DEV_ID_E823C_SGMII 0x188E
+/* Intel(R) Ethernet Connection C822N for backplane */
+#define ICE_DEV_ID_C822N_BACKPLANE 0x1890
+/* Intel(R) Ethernet Connection C822N for QSFP */
+#define ICE_DEV_ID_C822N_QSFP 0x1891
+/* Intel(R) Ethernet Connection C822N for SFP */
+#define ICE_DEV_ID_C822N_SFP 0x1892
+/* Intel(R) Ethernet Connection E822-C/X557-AT 10GBASE-T */
+#define ICE_DEV_ID_E822C_10G_BASE_T 0x1893
+/* Intel(R) Ethernet Connection E822-C 1GbE */
+#define ICE_DEV_ID_E822C_SGMII 0x1894
+/* Intel(R) Ethernet Connection E822-L for backplane */
+#define ICE_DEV_ID_E822L_BACKPLANE 0x1897
+/* Intel(R) Ethernet Connection E822-L for SFP */
+#define ICE_DEV_ID_E822L_SFP 0x1898
+/* Intel(R) Ethernet Connection E822-L/X557-AT 10GBASE-T */
+#define ICE_DEV_ID_E822L_10G_BASE_T 0x1899
+/* Intel(R) Ethernet Connection E822-L 1GbE */
+#define ICE_DEV_ID_E822L_SGMII 0x189A
+#endif /* ICE_DEVIDS_H */
diff --git a/providers/irdma/osdep.h b/providers/irdma/osdep.h
new file mode 100644
index 0000000..2abfb19
--- /dev/null
+++ b/providers/irdma/osdep.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2015 - 2021 Intel Corporation */
+#ifndef IRDMA_OSDEP_H
+#define IRDMA_OSDEP_H
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdatomic.h>
+#include <util/udma_barrier.h>
+#include <util/util.h>
+#include <linux/types.h>
+#include <inttypes.h>
+#include <pthread.h>
+#include <endian.h>
+
+static inline void db_wr32(__u32 val, __u32 *wqe_word)
+{
+ *wqe_word = val;
+}
+#endif /* IRDMA_OSDEP_H */
diff --git a/providers/irdma/status.h b/providers/irdma/status.h
new file mode 100644
index 0000000..29f7157
--- /dev/null
+++ b/providers/irdma/status.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2015 - 2020 Intel Corporation */
+#ifndef IRDMA_STATUS_H
+#define IRDMA_STATUS_H
+
+/* Error Codes */
+enum irdma_status_code {
+ IRDMA_SUCCESS = 0,
+ IRDMA_ERR_NVM = -1,
+ IRDMA_ERR_NVM_CHECKSUM = -2,
+ IRDMA_ERR_CFG = -4,
+ IRDMA_ERR_PARAM = -5,
+ IRDMA_ERR_DEVICE_NOT_SUPPORTED = -6,
+ IRDMA_ERR_RESET_FAILED = -7,
+ IRDMA_ERR_SWFW_SYNC = -8,
+ IRDMA_ERR_NO_MEMORY = -9,
+ IRDMA_ERR_BAD_PTR = -10,
+ IRDMA_ERR_INVALID_PD_ID = -11,
+ IRDMA_ERR_INVALID_QP_ID = -12,
+ IRDMA_ERR_INVALID_CQ_ID = -13,
+ IRDMA_ERR_INVALID_CEQ_ID = -14,
+ IRDMA_ERR_INVALID_AEQ_ID = -15,
+ IRDMA_ERR_INVALID_SIZE = -16,
+ IRDMA_ERR_INVALID_ARP_INDEX = -17,
+ IRDMA_ERR_INVALID_FPM_FUNC_ID = -18,
+ IRDMA_ERR_QP_INVALID_MSG_SIZE = -19,
+ IRDMA_ERR_QP_TOOMANY_WRS_POSTED = -20,
+ IRDMA_ERR_INVALID_FRAG_COUNT = -21,
+ IRDMA_ERR_Q_EMPTY = -22,
+ IRDMA_ERR_INVALID_ALIGNMENT = -23,
+ IRDMA_ERR_FLUSHED_Q = -24,
+ IRDMA_ERR_INVALID_PUSH_PAGE_INDEX = -25,
+ IRDMA_ERR_INVALID_INLINE_DATA_SIZE = -26,
+ IRDMA_ERR_TIMEOUT = -27,
+ IRDMA_ERR_OPCODE_MISMATCH = -28,
+ IRDMA_ERR_CQP_COMPL_ERROR = -29,
+ IRDMA_ERR_INVALID_VF_ID = -30,
+ IRDMA_ERR_INVALID_HMCFN_ID = -31,
+ IRDMA_ERR_BACKING_PAGE_ERROR = -32,
+ IRDMA_ERR_NO_PBLCHUNKS_AVAILABLE = -33,
+ IRDMA_ERR_INVALID_PBLE_INDEX = -34,
+ IRDMA_ERR_INVALID_SD_INDEX = -35,
+ IRDMA_ERR_INVALID_PAGE_DESC_INDEX = -36,
+ IRDMA_ERR_INVALID_SD_TYPE = -37,
+ IRDMA_ERR_MEMCPY_FAILED = -38,
+ IRDMA_ERR_INVALID_HMC_OBJ_INDEX = -39,
+ IRDMA_ERR_INVALID_HMC_OBJ_COUNT = -40,
+ IRDMA_ERR_BUF_TOO_SHORT = -43,
+ IRDMA_ERR_BAD_IWARP_CQE = -44,
+ IRDMA_ERR_NVM_BLANK_MODE = -45,
+ IRDMA_ERR_NOT_IMPL = -46,
+ IRDMA_ERR_PE_DOORBELL_NOT_ENA = -47,
+ IRDMA_ERR_NOT_READY = -48,
+ IRDMA_NOT_SUPPORTED = -49,
+ IRDMA_ERR_FIRMWARE_API_VER = -50,
+ IRDMA_ERR_RING_FULL = -51,
+ IRDMA_ERR_MPA_CRC = -61,
+ IRDMA_ERR_NO_TXBUFS = -62,
+ IRDMA_ERR_SEQ_NUM = -63,
+ IRDMA_ERR_LIST_EMPTY = -64,
+ IRDMA_ERR_INVALID_MAC_ADDR = -65,
+ IRDMA_ERR_BAD_STAG = -66,
+ IRDMA_ERR_CQ_COMPL_ERROR = -67,
+ IRDMA_ERR_Q_DESTROYED = -68,
+ IRDMA_ERR_INVALID_FEAT_CNT = -69,
+ IRDMA_ERR_REG_CQ_FULL = -70,
+ IRDMA_ERR_VF_MSG_ERROR = -71,
+ IRDMA_ERR_NO_INTR = -72,
+ IRDMA_ERR_REG_QSET = -73,
+};
+#endif /* IRDMA_STATUS_H */
diff --git a/providers/irdma/umain.c b/providers/irdma/umain.c
new file mode 100644
index 0000000..ac9758f
--- /dev/null
+++ b/providers/irdma/umain.c
@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (C) 2019 - 2020 Intel Corporation */
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "ice_devids.h"
+#include "i40e_devids.h"
+#include "umain.h"
+#include "abi.h"
+
+#define INTEL_HCA(v, d) VERBS_PCI_MATCH(v, d, NULL)
+static const struct verbs_match_ent hca_table[] = {
+ VERBS_DRIVER_ID(RDMA_DRIVER_IRDMA),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E823L_BACKPLANE),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E823L_SFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E823L_10G_BASE_T),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E823L_1GBE),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E823L_QSFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E810C_BACKPLANE),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E810C_QSFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E810C_SFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E810_XXV_BACKPLANE),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E810_XXV_QSFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E810_XXV_SFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E823C_BACKPLANE),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E823C_QSFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E823C_SFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E823C_10G_BASE_T),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E823C_SGMII),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_C822N_BACKPLANE),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_C822N_QSFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_C822N_SFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E822C_10G_BASE_T),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E822C_SGMII),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E822L_BACKPLANE),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E822L_SFP),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E822L_10G_BASE_T),
+ INTEL_HCA(PCI_VENDOR_ID_INTEL, ICE_DEV_ID_E822L_SGMII),
+
+ INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0),
+ INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0_VF),
+ INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_X722),
+ INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_X722),
+ INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722),
+ INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722),
+ INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722),
+ INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_I_X722),
+ INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF),
+ INTEL_HCA(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF_HV),
+
+ {}
+};
+
+/**
+ * irdma_ufree_context - free context that was allocated
+ * @ibctx: context allocated ptr
+ */
+static void irdma_ufree_context(struct ibv_context *ibctx)
+{
+ struct irdma_uvcontext *iwvctx = to_irdma_uctx(ibctx);
+
+ irdma_ufree_pd(&iwvctx->iwupd->ibv_pd);
+ irdma_munmap(iwvctx->db);
+ verbs_uninit_context(&iwvctx->ibv_ctx);
+ free(iwvctx);
+}
+
+static const struct verbs_context_ops irdma_uctx_ops = {
+ .alloc_mw = irdma_ualloc_mw,
+ .alloc_pd = irdma_ualloc_pd,
+ .attach_mcast = irdma_uattach_mcast,
+ .bind_mw = irdma_ubind_mw,
+ .cq_event = irdma_cq_event,
+ .create_ah = irdma_ucreate_ah,
+ .create_cq = irdma_ucreate_cq,
+ .create_cq_ex = irdma_ucreate_cq_ex,
+ .create_qp = irdma_ucreate_qp,
+ .dealloc_mw = irdma_udealloc_mw,
+ .dealloc_pd = irdma_ufree_pd,
+ .dereg_mr = irdma_udereg_mr,
+ .destroy_ah = irdma_udestroy_ah,
+ .destroy_cq = irdma_udestroy_cq,
+ .destroy_qp = irdma_udestroy_qp,
+ .detach_mcast = irdma_udetach_mcast,
+ .modify_qp = irdma_umodify_qp,
+ .poll_cq = irdma_upoll_cq,
+ .post_recv = irdma_upost_recv,
+ .post_send = irdma_upost_send,
+ .query_device_ex = irdma_uquery_device_ex,
+ .query_port = irdma_uquery_port,
+ .query_qp = irdma_uquery_qp,
+ .reg_mr = irdma_ureg_mr,
+ .req_notify_cq = irdma_uarm_cq,
+ .resize_cq = irdma_uresize_cq,
+ .free_context = irdma_ufree_context,
+};
+
+/**
+ * i40iw_set_hw_attrs - set the hw attributes
+ * @attrs: pointer to hw attributes
+ *
+ * Set the device attibutes to allow user mode to work with
+ * driver on older ABI version.
+ */
+static void i40iw_set_hw_attrs(struct irdma_uk_attrs *attrs)
+{
+ attrs->hw_rev = IRDMA_GEN_1;
+ attrs->max_hw_wq_frags = I40IW_MAX_WQ_FRAGMENT_COUNT;
+ attrs->max_hw_read_sges = I40IW_MAX_SGE_RD;
+ attrs->max_hw_inline = I40IW_MAX_INLINE_DATA_SIZE;
+ attrs->max_hw_rq_quanta = I40IW_QP_SW_MAX_RQ_QUANTA;
+ attrs->max_hw_wq_quanta = I40IW_QP_SW_MAX_WQ_QUANTA;
+ attrs->max_hw_sq_chunk = I40IW_MAX_QUANTA_PER_WR;
+ attrs->max_hw_cq_size = I40IW_MAX_CQ_SIZE;
+ attrs->min_hw_cq_size = IRDMA_MIN_CQ_SIZE;
+}
+
+/**
+ * irdma_ualloc_context - allocate context for user app
+ * @ibdev: ib device created during irdma_driver_init
+ * @cmd_fd: save fd for the device
+ * @private_data: device private data
+ *
+ * Returns callback routine table and calls driver for allocating
+ * context and getting back resource information to return as ibv_context.
+ */
+static struct verbs_context *irdma_ualloc_context(struct ibv_device *ibdev,
+ int cmd_fd, void *private_data)
+{
+ struct ibv_pd *ibv_pd;
+ struct irdma_uvcontext *iwvctx;
+ struct irdma_get_context cmd;
+ struct irdma_get_context_resp resp = {};
+ __u64 mmap_key;
+ __u8 user_ver = IRDMA_ABI_VER;
+
+ iwvctx = verbs_init_and_alloc_context(ibdev, cmd_fd, iwvctx, ibv_ctx,
+ RDMA_DRIVER_IRDMA);
+ if (!iwvctx)
+ return NULL;
+
+ cmd.userspace_ver = user_ver;
+ if (ibv_cmd_get_context(&iwvctx->ibv_ctx,
+ (struct ibv_get_context *)&cmd, sizeof(cmd),
+ &resp.ibv_resp, sizeof(resp))) {
+ cmd.userspace_ver = 4;
+ if (ibv_cmd_get_context(&iwvctx->ibv_ctx,
+ (struct ibv_get_context *)&cmd, sizeof(cmd),
+ &resp.ibv_resp, sizeof(resp)))
+ goto err_free;
+ user_ver = cmd.userspace_ver;
+ }
+
+ verbs_set_ops(&iwvctx->ibv_ctx, &irdma_uctx_ops);
+
+ /* Legacy i40iw does not populate hw_rev. The irdma driver always sets it */
+ if (!resp.hw_rev) {
+ i40iw_set_hw_attrs(&iwvctx->uk_attrs);
+ iwvctx->abi_ver = resp.kernel_ver;
+ iwvctx->legacy_mode = true;
+ mmap_key = 0;
+ } else {
+ iwvctx->uk_attrs.feature_flags = resp.feature_flags;
+ iwvctx->uk_attrs.hw_rev = resp.hw_rev;
+ iwvctx->uk_attrs.max_hw_wq_frags = resp.max_hw_wq_frags;
+ iwvctx->uk_attrs.max_hw_read_sges = resp.max_hw_read_sges;
+ iwvctx->uk_attrs.max_hw_inline = resp.max_hw_inline;
+ iwvctx->uk_attrs.max_hw_rq_quanta = resp.max_hw_rq_quanta;
+ iwvctx->uk_attrs.max_hw_wq_quanta = resp.max_hw_wq_quanta;
+ iwvctx->uk_attrs.max_hw_sq_chunk = resp.max_hw_sq_chunk;
+ iwvctx->uk_attrs.max_hw_cq_size = resp.max_hw_cq_size;
+ iwvctx->uk_attrs.min_hw_cq_size = resp.min_hw_cq_size;
+ iwvctx->abi_ver = user_ver;
+ mmap_key = resp.db_mmap_key;
+ }
+
+ iwvctx->db = irdma_mmap(cmd_fd, mmap_key);
+ if (iwvctx->db == MAP_FAILED)
+ goto err_free;
+
+ irdma_device_init_uk(&iwvctx->dev);
+ ibv_pd = irdma_ualloc_pd(&iwvctx->ibv_ctx.context);
+ if (!ibv_pd) {
+ irdma_munmap(iwvctx->db);
+ goto err_free;
+ }
+
+ ibv_pd->context = &iwvctx->ibv_ctx.context;
+ iwvctx->iwupd = to_irdma_upd(ibv_pd);
+ return &iwvctx->ibv_ctx;
+
+err_free:
+ free(iwvctx);
+
+ return NULL;
+}
+
+static void irdma_uninit_device(struct verbs_device *verbs_device)
+{
+ struct irdma_udevice *dev = to_irdma_udev(&verbs_device->device);
+
+ free(dev);
+}
+
+static struct verbs_device *irdma_device_alloc(struct verbs_sysfs_dev *sysfs_dev)
+{
+ struct irdma_udevice *dev;
+
+ dev = calloc(1, sizeof(*dev));
+ if (!dev)
+ return NULL;
+
+ return &dev->ibv_dev;
+}
+
+static const struct verbs_device_ops irdma_udev_ops = {
+ .alloc_context = irdma_ualloc_context,
+ .alloc_device = irdma_device_alloc,
+ .match_max_abi_version = IRDMA_MAX_ABI_VERSION,
+ .match_min_abi_version = IRDMA_MIN_ABI_VERSION,
+ .match_table = hca_table,
+ .name = "irdma",
+ .uninit_device = irdma_uninit_device,
+};
+
+PROVIDER_DRIVER(irdma, irdma_udev_ops);
diff --git a/providers/irdma/umain.h b/providers/irdma/umain.h
new file mode 100644
index 0000000..7df9a92
--- /dev/null
+++ b/providers/irdma/umain.h
@@ -0,0 +1,205 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (C) 2019 - 2020 Intel Corporation */
+#ifndef IRDMA_UMAIN_H
+#define IRDMA_UMAIN_H
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <infiniband/driver.h>
+
+#include "osdep.h"
+#include "irdma.h"
+#include "defs.h"
+#include "i40iw_hw.h"
+#include "status.h"
+#include "user.h"
+
+#define PFX "libirdma-"
+
+#define IRDMA_BASE_PUSH_PAGE 1
+#define IRDMA_U_MINCQ_SIZE 4
+#define IRDMA_DB_SHADOW_AREA_SIZE 64
+#define IRDMA_DB_CQ_OFFSET 64
+
+enum irdma_supported_wc_flags {
+ IRDMA_CQ_SUPPORTED_WC_FLAGS = IBV_WC_EX_WITH_BYTE_LEN
+ | IBV_WC_EX_WITH_IMM
+ | IBV_WC_EX_WITH_QP_NUM
+ | IBV_WC_EX_WITH_SRC_QP
+ | IBV_WC_EX_WITH_SLID
+ | IBV_WC_EX_WITH_SL
+ | IBV_WC_EX_WITH_DLID_PATH_BITS
+ | IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK
+ | IBV_WC_EX_WITH_COMPLETION_TIMESTAMP,
+};
+
+struct irdma_udevice {
+ struct verbs_device ibv_dev;
+};
+
+struct irdma_uah {
+ struct ibv_ah ibv_ah;
+ uint32_t ah_id;
+ struct ibv_global_route grh;
+};
+
+struct irdma_upd {
+ struct ibv_pd ibv_pd;
+ void *arm_cq_page;
+ void *arm_cq;
+ uint32_t pd_id;
+};
+
+struct irdma_uvcontext {
+ struct verbs_context ibv_ctx;
+ struct irdma_upd *iwupd;
+ struct irdma_uk_attrs uk_attrs;
+ struct irdma_dev_uk dev;
+ void *db;
+ int abi_ver;
+ bool legacy_mode;
+};
+
+struct irdma_uqp;
+
+struct irdma_cq_buf {
+ struct list_node list;
+ struct irdma_cq_uk cq;
+ struct verbs_mr vmr;
+};
+
+struct irdma_ucq {
+ struct verbs_cq verbs_cq;
+ struct verbs_mr vmr;
+ struct verbs_mr vmr_shadow_area;
+ pthread_spinlock_t lock;
+ size_t buf_size;
+ bool is_armed;
+ bool skip_arm;
+ bool arm_sol;
+ bool skip_sol;
+ int comp_vector;
+ uint32_t report_rtt;
+ struct irdma_uqp *uqp;
+ struct irdma_cq_uk cq;
+ struct list_head resize_list;
+ /* for extended CQ completion fields */
+ struct irdma_cq_poll_info cur_cqe;
+};
+
+struct irdma_uqp {
+ struct ibv_qp ibv_qp;
+ struct ibv_qp_attr attr;
+ struct irdma_ucq *send_cq;
+ struct irdma_ucq *recv_cq;
+ struct verbs_mr vmr;
+ size_t buf_size;
+ uint32_t irdma_drv_opt;
+ pthread_spinlock_t lock;
+ uint16_t sq_sig_all;
+ uint16_t qperr;
+ uint16_t rsvd;
+ uint32_t pending_rcvs;
+ uint32_t wq_size;
+ struct ibv_recv_wr *pend_rx_wr;
+ struct irdma_qp_uk qp;
+ enum ibv_qp_type qp_type;
+ enum ibv_qp_attr_mask attr_mask;
+ struct irdma_sge *recv_sges;
+};
+
+struct irdma_umr {
+ struct verbs_mr vmr;
+ uint32_t acc_flags;
+};
+
+#define to_irdma_uxxx(xxx, type) \
+ container_of(ib##xxx, struct irdma_u##type, ibv_##xxx)
+
+static inline struct irdma_udevice *to_irdma_udev(struct ibv_device *ibdev)
+{
+ return container_of(ibdev, struct irdma_udevice, ibv_dev.device);
+}
+
+static inline struct irdma_uvcontext *to_irdma_uctx(struct ibv_context *ibctx)
+{
+ return container_of(ibctx, struct irdma_uvcontext, ibv_ctx.context);
+}
+
+static inline struct irdma_uah *to_irdma_uah(struct ibv_ah *ibah)
+{
+ return to_irdma_uxxx(ah, ah);
+}
+
+static inline struct irdma_upd *to_irdma_upd(struct ibv_pd *ibpd)
+{
+ return to_irdma_uxxx(pd, pd);
+}
+
+static inline struct irdma_ucq *to_irdma_ucq(struct ibv_cq *ibcq)
+{
+ return container_of(ibcq, struct irdma_ucq, verbs_cq.cq);
+}
+
+static inline struct irdma_ucq *to_irdma_ucq_ex(struct ibv_cq_ex *ibcq_ex)
+{
+ return container_of(ibcq_ex, struct irdma_ucq, verbs_cq.cq_ex);
+}
+
+static inline struct irdma_uqp *to_irdma_uqp(struct ibv_qp *ibqp)
+{
+ return to_irdma_uxxx(qp, qp);
+}
+
+/* irdma_uverbs.c */
+int irdma_uquery_device_ex(struct ibv_context *context,
+ const struct ibv_query_device_ex_input *input,
+ struct ibv_device_attr_ex *attr, size_t attr_size);
+int irdma_uquery_port(struct ibv_context *context, uint8_t port,
+ struct ibv_port_attr *attr);
+struct ibv_pd *irdma_ualloc_pd(struct ibv_context *context);
+int irdma_ufree_pd(struct ibv_pd *pd);
+struct ibv_mr *irdma_ureg_mr(struct ibv_pd *pd, void *addr, size_t length,
+ uint64_t hca_va, int access);
+int irdma_udereg_mr(struct verbs_mr *vmr);
+struct ibv_mw *irdma_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type);
+int irdma_ubind_mw(struct ibv_qp *qp, struct ibv_mw *mw,
+ struct ibv_mw_bind *mw_bind);
+int irdma_udealloc_mw(struct ibv_mw *mw);
+struct ibv_cq *irdma_ucreate_cq(struct ibv_context *context, int cqe,
+ struct ibv_comp_channel *channel,
+ int comp_vector);
+struct ibv_cq_ex *irdma_ucreate_cq_ex(struct ibv_context *context,
+ struct ibv_cq_init_attr_ex *attr_ex);
+void irdma_ibvcq_ex_fill_priv_funcs(struct irdma_ucq *iwucq,
+ struct ibv_cq_init_attr_ex *attr_ex);
+int irdma_uresize_cq(struct ibv_cq *cq, int cqe);
+int irdma_udestroy_cq(struct ibv_cq *cq);
+int irdma_upoll_cq(struct ibv_cq *cq, int entries, struct ibv_wc *entry);
+int irdma_uarm_cq(struct ibv_cq *cq, int solicited);
+void irdma_cq_event(struct ibv_cq *cq);
+struct ibv_qp *irdma_ucreate_qp(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *attr);
+int irdma_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask,
+ struct ibv_qp_init_attr *init_attr);
+int irdma_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
+ int attr_mask);
+int irdma_udestroy_qp(struct ibv_qp *qp);
+int irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
+ struct ibv_send_wr **bad_wr);
+int irdma_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr,
+ struct ibv_recv_wr **bad_wr);
+struct ibv_ah *irdma_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr);
+int irdma_udestroy_ah(struct ibv_ah *ibah);
+int irdma_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid,
+ uint16_t lid);
+int irdma_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid,
+ uint16_t lid);
+void irdma_async_event(struct ibv_context *context,
+ struct ibv_async_event *event);
+void irdma_set_hw_attrs(struct irdma_hw_attrs *attrs);
+void *irdma_mmap(int fd, off_t offset);
+void irdma_munmap(void *map);
+#endif /* IRDMA_UMAIN_H */
diff --git a/providers/irdma/user.h b/providers/irdma/user.h
new file mode 100644
index 0000000..af30ced
--- /dev/null
+++ b/providers/irdma/user.h
@@ -0,0 +1,466 @@
+/* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB */
+/* Copyright (c) 2015 - 2020 Intel Corporation */
+#ifndef IRDMA_USER_H
+#define IRDMA_USER_H
+
+#include "osdep.h"
+
+#define irdma_handle void *
+#define irdma_adapter_handle irdma_handle
+#define irdma_qp_handle irdma_handle
+#define irdma_cq_handle irdma_handle
+#define irdma_pd_id irdma_handle
+#define irdma_stag_handle irdma_handle
+#define irdma_stag_index __u32
+#define irdma_stag __u32
+#define irdma_stag_key __u8
+#define irdma_tagged_offset __u64
+#define irdma_access_privileges __u32
+#define irdma_physical_fragment __u64
+#define irdma_address_list __u64 *
+#define irdma_sgl struct irdma_sge *
+
+#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
+
+#define IRDMA_ACCESS_FLAGS_LOCALREAD 0x01
+#define IRDMA_ACCESS_FLAGS_LOCALWRITE 0x02
+#define IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04
+#define IRDMA_ACCESS_FLAGS_REMOTEREAD 0x05
+#define IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08
+#define IRDMA_ACCESS_FLAGS_REMOTEWRITE 0x0a
+#define IRDMA_ACCESS_FLAGS_BIND_WINDOW 0x10
+#define IRDMA_ACCESS_FLAGS_ZERO_BASED 0x20
+#define IRDMA_ACCESS_FLAGS_ALL 0x3f
+
+#define IRDMA_OP_TYPE_RDMA_WRITE 0x00
+#define IRDMA_OP_TYPE_RDMA_READ 0x01
+#define IRDMA_OP_TYPE_SEND 0x03
+#define IRDMA_OP_TYPE_SEND_INV 0x04
+#define IRDMA_OP_TYPE_SEND_SOL 0x05
+#define IRDMA_OP_TYPE_SEND_SOL_INV 0x06
+#define IRDMA_OP_TYPE_RDMA_WRITE_SOL 0x0d
+#define IRDMA_OP_TYPE_BIND_MW 0x08
+#define IRDMA_OP_TYPE_FAST_REG_NSMR 0x09
+#define IRDMA_OP_TYPE_INV_STAG 0x0a
+#define IRDMA_OP_TYPE_RDMA_READ_INV_STAG 0x0b
+#define IRDMA_OP_TYPE_NOP 0x0c
+#define IRDMA_OP_TYPE_REC 0x3e
+#define IRDMA_OP_TYPE_REC_IMM 0x3f
+
+#define IRDMA_FLUSH_MAJOR_ERR 1
+
+enum irdma_device_caps_const {
+ IRDMA_WQE_SIZE = 4,
+ IRDMA_CQP_WQE_SIZE = 8,
+ IRDMA_CQE_SIZE = 4,
+ IRDMA_EXTENDED_CQE_SIZE = 8,
+ IRDMA_AEQE_SIZE = 2,
+ IRDMA_CEQE_SIZE = 1,
+ IRDMA_CQP_CTX_SIZE = 8,
+ IRDMA_SHADOW_AREA_SIZE = 8,
+ IRDMA_QUERY_FPM_BUF_SIZE = 176,
+ IRDMA_COMMIT_FPM_BUF_SIZE = 176,
+ IRDMA_GATHER_STATS_BUF_SIZE = 1024,
+ IRDMA_MIN_IW_QP_ID = 0,
+ IRDMA_MAX_IW_QP_ID = 262143,
+ IRDMA_MIN_CEQID = 0,
+ IRDMA_MAX_CEQID = 1023,
+ IRDMA_CEQ_MAX_COUNT = IRDMA_MAX_CEQID + 1,
+ IRDMA_MIN_CQID = 0,
+ IRDMA_MAX_CQID = 524287,
+ IRDMA_MIN_AEQ_ENTRIES = 1,
+ IRDMA_MAX_AEQ_ENTRIES = 524287,
+ IRDMA_MIN_CEQ_ENTRIES = 1,
+ IRDMA_MAX_CEQ_ENTRIES = 262143,
+ IRDMA_MIN_CQ_SIZE = 1,
+ IRDMA_MAX_CQ_SIZE = 1048575,
+ IRDMA_DB_ID_ZERO = 0,
+ IRDMA_MAX_WQ_FRAGMENT_COUNT = 13,
+ IRDMA_MAX_SGE_RD = 13,
+ IRDMA_MAX_OUTBOUND_MSG_SIZE = 2147483647,
+ IRDMA_MAX_INBOUND_MSG_SIZE = 2147483647,
+ IRDMA_MAX_PUSH_PAGE_COUNT = 1024,
+ IRDMA_MAX_PE_ENA_VF_COUNT = 32,
+ IRDMA_MAX_VF_FPM_ID = 47,
+ IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
+ IRDMA_MAX_INLINE_DATA_SIZE = 101,
+ IRDMA_MAX_WQ_ENTRIES = 32768,
+ IRDMA_Q2_BUF_SIZE = 256,
+ IRDMA_QP_CTX_SIZE = 256,
+ IRDMA_MAX_PDS = 262144,
+};
+
+enum irdma_addressing_type {
+ IRDMA_ADDR_TYPE_ZERO_BASED = 0,
+ IRDMA_ADDR_TYPE_VA_BASED = 1,
+};
+
+enum irdma_flush_opcode {
+ FLUSH_INVALID = 0,
+ FLUSH_GENERAL_ERR,
+ FLUSH_PROT_ERR,
+ FLUSH_REM_ACCESS_ERR,
+ FLUSH_LOC_QP_OP_ERR,
+ FLUSH_REM_OP_ERR,
+ FLUSH_LOC_LEN_ERR,
+ FLUSH_FATAL_ERR,
+};
+
+enum irdma_cmpl_status {
+ IRDMA_COMPL_STATUS_SUCCESS = 0,
+ IRDMA_COMPL_STATUS_FLUSHED,
+ IRDMA_COMPL_STATUS_INVALID_WQE,
+ IRDMA_COMPL_STATUS_QP_CATASTROPHIC,
+ IRDMA_COMPL_STATUS_REMOTE_TERMINATION,
+ IRDMA_COMPL_STATUS_INVALID_STAG,
+ IRDMA_COMPL_STATUS_BASE_BOUND_VIOLATION,
+ IRDMA_COMPL_STATUS_ACCESS_VIOLATION,
+ IRDMA_COMPL_STATUS_INVALID_PD_ID,
+ IRDMA_COMPL_STATUS_WRAP_ERROR,
+ IRDMA_COMPL_STATUS_STAG_INVALID_PDID,
+ IRDMA_COMPL_STATUS_RDMA_READ_ZERO_ORD,
+ IRDMA_COMPL_STATUS_QP_NOT_PRIVLEDGED,
+ IRDMA_COMPL_STATUS_STAG_NOT_INVALID,
+ IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_SIZE,
+ IRDMA_COMPL_STATUS_INVALID_PHYS_BUF_ENTRY,
+ IRDMA_COMPL_STATUS_INVALID_FBO,
+ IRDMA_COMPL_STATUS_INVALID_LEN,
+ IRDMA_COMPL_STATUS_INVALID_ACCESS,
+ IRDMA_COMPL_STATUS_PHYS_BUF_LIST_TOO_LONG,
+ IRDMA_COMPL_STATUS_INVALID_VIRT_ADDRESS,
+ IRDMA_COMPL_STATUS_INVALID_REGION,
+ IRDMA_COMPL_STATUS_INVALID_WINDOW,
+ IRDMA_COMPL_STATUS_INVALID_TOTAL_LEN,
+ IRDMA_COMPL_STATUS_UNKNOWN,
+};
+
+enum irdma_cmpl_notify {
+ IRDMA_CQ_COMPL_EVENT = 0,
+ IRDMA_CQ_COMPL_SOLICITED = 1,
+};
+
+enum irdma_qp_caps {
+ IRDMA_WRITE_WITH_IMM = 1,
+ IRDMA_SEND_WITH_IMM = 2,
+ IRDMA_ROCE = 4,
+ IRDMA_PUSH_MODE = 8,
+};
+
+struct irdma_qp_uk;
+struct irdma_cq_uk;
+struct irdma_qp_uk_init_info;
+struct irdma_cq_uk_init_info;
+
+struct irdma_sge {
+ irdma_tagged_offset tag_off;
+ __u32 len;
+ irdma_stag stag;
+};
+
+struct irdma_ring {
+ __u32 head;
+ __u32 tail;
+ __u32 size;
+};
+
+struct irdma_cqe {
+ __le64 buf[IRDMA_CQE_SIZE];
+};
+
+struct irdma_extended_cqe {
+ __le64 buf[IRDMA_EXTENDED_CQE_SIZE];
+};
+
+struct irdma_post_send {
+ irdma_sgl sg_list;
+ __u32 num_sges;
+ __u32 qkey;
+ __u32 dest_qp;
+ __u32 ah_id;
+};
+
+struct irdma_post_inline_send {
+ void *data;
+ __u32 len;
+ __u32 qkey;
+ __u32 dest_qp;
+ __u32 ah_id;
+};
+
+struct irdma_post_rq_info {
+ __u64 wr_id;
+ irdma_sgl sg_list;
+ __u32 num_sges;
+};
+
+struct irdma_rdma_write {
+ irdma_sgl lo_sg_list;
+ __u32 num_lo_sges;
+ struct irdma_sge rem_addr;
+};
+
+struct irdma_inline_rdma_write {
+ void *data;
+ __u32 len;
+ struct irdma_sge rem_addr;
+};
+
+struct irdma_rdma_read {
+ irdma_sgl lo_sg_list;
+ __u32 num_lo_sges;
+ struct irdma_sge rem_addr;
+};
+
+struct irdma_bind_window {
+ irdma_stag mr_stag;
+ __u64 bind_len;
+ void *va;
+ enum irdma_addressing_type addressing_type;
+ bool ena_reads:1;
+ bool ena_writes:1;
+ irdma_stag mw_stag;
+ bool mem_window_type_1:1;
+};
+
+struct irdma_inv_local_stag {
+ irdma_stag target_stag;
+};
+
+struct irdma_post_sq_info {
+ __u64 wr_id;
+ __u8 op_type;
+ __u8 l4len;
+ bool signaled:1;
+ bool read_fence:1;
+ bool local_fence:1;
+ bool inline_data:1;
+ bool imm_data_valid:1;
+ bool push_wqe:1;
+ bool report_rtt:1;
+ bool udp_hdr:1;
+ bool defer_flag:1;
+ __u32 imm_data;
+ __u32 stag_to_inv;
+ union {
+ struct irdma_post_send send;
+ struct irdma_rdma_write rdma_write;
+ struct irdma_rdma_read rdma_read;
+ struct irdma_bind_window bind_window;
+ struct irdma_inv_local_stag inv_local_stag;
+ struct irdma_inline_rdma_write inline_rdma_write;
+ struct irdma_post_inline_send inline_send;
+ } op;
+};
+
+struct irdma_cq_poll_info {
+ __u64 wr_id;
+ irdma_qp_handle qp_handle;
+ __u32 bytes_xfered;
+ __u32 tcp_seq_num_rtt;
+ __u32 qp_id;
+ __u32 ud_src_qpn;
+ __u32 imm_data;
+ irdma_stag inv_stag; /* or L_R_Key */
+ enum irdma_cmpl_status comp_status;
+ __u16 major_err;
+ __u16 minor_err;
+ __u16 ud_vlan;
+ __u8 ud_smac[6];
+ __u8 op_type;
+ bool stag_invalid_set:1; /* or L_R_Key set */
+ bool push_dropped:1;
+ bool error:1;
+ bool solicited_event:1;
+ bool ipv4:1;
+ bool ud_vlan_valid:1;
+ bool ud_smac_valid:1;
+ bool imm_valid:1;
+};
+
+struct irdma_qp_uk_ops {
+ enum irdma_status_code (*iw_inline_rdma_write)(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool post_sq);
+ enum irdma_status_code (*iw_inline_send)(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool post_sq);
+ enum irdma_status_code (*iw_mw_bind)(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool post_sq);
+ enum irdma_status_code (*iw_post_nop)(struct irdma_qp_uk *qp, __u64 wr_id,
+ bool signaled, bool post_sq);
+ enum irdma_status_code (*iw_post_receive)(struct irdma_qp_uk *qp,
+ struct irdma_post_rq_info *info);
+ void (*iw_qp_post_wr)(struct irdma_qp_uk *qp);
+ void (*iw_qp_ring_push_db)(struct irdma_qp_uk *qp, __u32 wqe_index);
+ enum irdma_status_code (*iw_rdma_read)(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool inv_stag, bool post_sq);
+ enum irdma_status_code (*iw_rdma_write)(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool post_sq);
+ enum irdma_status_code (*iw_send)(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool post_sq);
+ enum irdma_status_code (*iw_stag_local_invalidate)(struct irdma_qp_uk *qp,
+ struct irdma_post_sq_info *info,
+ bool post_sq);
+};
+
+struct irdma_wqe_uk_ops {
+ void (*iw_copy_inline_data)(__u8 *dest, __u8 *src, __u32 len, __u8 polarity);
+ __u16 (*iw_inline_data_size_to_quanta)(__u32 data_size);
+ void (*iw_set_fragment)(__le64 *wqe, __u32 offset, struct irdma_sge *sge,
+ __u8 valid);
+ void (*iw_set_mw_bind_wqe)(__le64 *wqe,
+ struct irdma_bind_window *op_info);
+};
+
+struct irdma_cq_ops {
+ void (*iw_cq_clean)(void *q, struct irdma_cq_uk *cq);
+ enum irdma_status_code (*iw_cq_poll_cmpl)(struct irdma_cq_uk *cq,
+ struct irdma_cq_poll_info *info);
+ enum irdma_status_code (*iw_cq_post_entries)(struct irdma_cq_uk *cq,
+ __u8 count);
+ void (*iw_cq_request_notification)(struct irdma_cq_uk *cq,
+ enum irdma_cmpl_notify cq_notify);
+ void (*iw_cq_resize)(struct irdma_cq_uk *cq, void *cq_base, int size);
+ void (*iw_cq_set_resized_cnt)(struct irdma_cq_uk *qp, __u16 cnt);
+};
+
+struct irdma_dev_uk;
+
+struct irdma_device_uk_ops {
+ enum irdma_status_code (*iw_cq_uk_init)(struct irdma_cq_uk *cq,
+ struct irdma_cq_uk_init_info *info);
+ enum irdma_status_code (*iw_qp_uk_init)(struct irdma_qp_uk *qp,
+ struct irdma_qp_uk_init_info *info);
+};
+
+struct irdma_dev_uk {
+ struct irdma_device_uk_ops ops_uk;
+};
+
+struct irdma_sq_uk_wr_trk_info {
+ __u64 wrid;
+ __u32 wr_len;
+ __u16 quanta;
+ __u8 reserved[2];
+};
+
+struct irdma_qp_quanta {
+ __le64 elem[IRDMA_WQE_SIZE];
+};
+
+struct irdma_qp_uk {
+ struct irdma_qp_quanta *sq_base;
+ struct irdma_qp_quanta *rq_base;
+ struct irdma_uk_attrs *uk_attrs;
+ __u32 *wqe_alloc_db;
+ struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
+ __u64 *rq_wrid_array;
+ __le64 *shadow_area;
+ __le32 *push_db;
+ __le64 *push_wqe;
+ struct irdma_ring sq_ring;
+ struct irdma_ring rq_ring;
+ struct irdma_ring initial_ring;
+ __u32 qp_id;
+ __u32 qp_caps;
+ __u32 sq_size;
+ __u32 rq_size;
+ __u32 max_sq_frag_cnt;
+ __u32 max_rq_frag_cnt;
+ __u32 max_inline_data;
+ struct irdma_qp_uk_ops qp_ops;
+ struct irdma_wqe_uk_ops wqe_ops;
+ __u16 conn_wqes;
+ __u8 qp_type;
+ __u8 swqe_polarity;
+ __u8 swqe_polarity_deferred;
+ __u8 rwqe_polarity;
+ __u8 rq_wqe_size;
+ __u8 rq_wqe_size_multiplier;
+ bool deferred_flag:1;
+ bool push_mode:1; /* whether the last post wqe was pushed */
+ bool push_dropped:1;
+ bool first_sq_wq:1;
+ bool sq_flush_complete:1; /* Indicates flush was seen and SQ was empty after the flush */
+ bool rq_flush_complete:1; /* Indicates flush was seen and RQ was empty after the flush */
+ bool destroy_pending:1; /* Indicates the QP is being destroyed */
+ void *back_qp;
+ pthread_spinlock_t *lock;
+ __u8 dbg_rq_flushed;
+ __u8 sq_flush_seen;
+ __u8 rq_flush_seen;
+};
+
+struct irdma_cq_uk {
+ struct irdma_cqe *cq_base;
+ __u32 *cqe_alloc_db;
+ __u32 *cq_ack_db;
+ __le64 *shadow_area;
+ __u32 cq_id;
+ __u32 cq_size;
+ struct irdma_ring cq_ring;
+ __u8 polarity;
+ struct irdma_cq_ops ops;
+ bool avoid_mem_cflct:1;
+};
+
+struct irdma_qp_uk_init_info {
+ struct irdma_qp_quanta *sq;
+ struct irdma_qp_quanta *rq;
+ struct irdma_uk_attrs *uk_attrs;
+ __u32 *wqe_alloc_db;
+ __le64 *shadow_area;
+ struct irdma_sq_uk_wr_trk_info *sq_wrtrk_array;
+ __u64 *rq_wrid_array;
+ __u32 qp_id;
+ __u32 qp_caps;
+ __u32 sq_size;
+ __u32 rq_size;
+ __u32 max_sq_frag_cnt;
+ __u32 max_rq_frag_cnt;
+ __u32 max_inline_data;
+ __u8 first_sq_wq;
+ __u8 type;
+ int abi_ver;
+ bool legacy_mode;
+};
+
+struct irdma_cq_uk_init_info {
+ __u32 *cqe_alloc_db;
+ __u32 *cq_ack_db;
+ struct irdma_cqe *cq_base;
+ __le64 *shadow_area;
+ __u32 cq_size;
+ __u32 cq_id;
+ bool avoid_mem_cflct;
+};
+
+void irdma_device_init_uk(struct irdma_dev_uk *dev);
+void irdma_qp_post_wr(struct irdma_qp_uk *qp);
+__le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, __u32 *wqe_idx,
+ __u16 quanta, __u32 total_size,
+ struct irdma_post_sq_info *info);
+__le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, __u32 *wqe_idx);
+enum irdma_status_code irdma_cq_uk_init(struct irdma_cq_uk *cq,
+ struct irdma_cq_uk_init_info *info);
+enum irdma_status_code irdma_qp_uk_init(struct irdma_qp_uk *qp,
+ struct irdma_qp_uk_init_info *info);
+void irdma_clean_cq(void *q, struct irdma_cq_uk *cq);
+enum irdma_status_code irdma_nop(struct irdma_qp_uk *qp, __u64 wr_id,
+ bool signaled, bool post_sq);
+enum irdma_status_code irdma_fragcnt_to_quanta_sq(__u32 frag_cnt, __u16 *quanta);
+enum irdma_status_code irdma_fragcnt_to_wqesize_rq(__u32 frag_cnt, __u16 *wqe_size);
+void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, __u32 sge,
+ __u32 inline_data, __u8 *shift);
+enum irdma_status_code irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs,
+ __u32 sq_size, __u8 shift, __u32 *wqdepth);
+enum irdma_status_code irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs,
+ __u32 rq_size, __u8 shift, __u32 *wqdepth);
+void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, __u16 quanta,
+ __u32 wqe_idx, bool post_sq);
+void irdma_clr_wqes(struct irdma_qp_uk *qp, __u32 qp_wqe_idx);
+#endif /* IRDMA_USER_H */
--
1.8.3.1
^ permalink raw reply related [flat|nested] 10+ messages in thread* Re: [PATCH 4/5] rdma-core/irdma: Add library setup and utility definitions
2021-04-06 21:17 ` [PATCH 4/5] rdma-core/irdma: Add library setup and utility definitions Tatyana Nikolova
@ 2021-04-06 22:59 ` Jason Gunthorpe
0 siblings, 0 replies; 10+ messages in thread
From: Jason Gunthorpe @ 2021-04-06 22:59 UTC (permalink / raw)
To: Tatyana Nikolova; +Cc: dledford, leon, linux-rdma
On Tue, Apr 06, 2021 at 04:17:27PM -0500, Tatyana Nikolova wrote:
> +struct irdma_qp_uk_ops {
> + enum irdma_status_code (*iw_inline_rdma_write)(struct irdma_qp_uk *qp,
> + struct irdma_post_sq_info *info,
> + bool post_sq);
> + enum irdma_status_code (*iw_inline_send)(struct irdma_qp_uk *qp,
> + struct irdma_post_sq_info *info,
> + bool post_sq);
> + enum irdma_status_code (*iw_mw_bind)(struct irdma_qp_uk *qp,
> + struct irdma_post_sq_info *info,
> + bool post_sq);
> + enum irdma_status_code (*iw_post_nop)(struct irdma_qp_uk *qp, __u64 wr_id,
> + bool signaled, bool post_sq);
> + enum irdma_status_code (*iw_post_receive)(struct irdma_qp_uk *qp,
> + struct irdma_post_rq_info *info);
> + void (*iw_qp_post_wr)(struct irdma_qp_uk *qp);
> + void (*iw_qp_ring_push_db)(struct irdma_qp_uk *qp, __u32 wqe_index);
> + enum irdma_status_code (*iw_rdma_read)(struct irdma_qp_uk *qp,
> + struct irdma_post_sq_info *info,
> + bool inv_stag, bool post_sq);
> + enum irdma_status_code (*iw_rdma_write)(struct irdma_qp_uk *qp,
> + struct irdma_post_sq_info *info,
> + bool post_sq);
> + enum irdma_status_code (*iw_send)(struct irdma_qp_uk *qp,
> + struct irdma_post_sq_info *info,
> + bool post_sq);
> + enum irdma_status_code (*iw_stag_local_invalidate)(struct irdma_qp_uk *qp,
> + struct irdma_post_sq_info *info,
> + bool post_sq);
> +};
> +
> +struct irdma_wqe_uk_ops {
> + void (*iw_copy_inline_data)(__u8 *dest, __u8 *src, __u32 len, __u8 polarity);
> + __u16 (*iw_inline_data_size_to_quanta)(__u32 data_size);
> + void (*iw_set_fragment)(__le64 *wqe, __u32 offset, struct irdma_sge *sge,
> + __u8 valid);
> + void (*iw_set_mw_bind_wqe)(__le64 *wqe,
> + struct irdma_bind_window *op_info);
> +};
> +
> +struct irdma_cq_ops {
> + void (*iw_cq_clean)(void *q, struct irdma_cq_uk *cq);
> + enum irdma_status_code (*iw_cq_poll_cmpl)(struct irdma_cq_uk *cq,
> + struct irdma_cq_poll_info *info);
> + enum irdma_status_code (*iw_cq_post_entries)(struct irdma_cq_uk *cq,
> + __u8 count);
> + void (*iw_cq_request_notification)(struct irdma_cq_uk *cq,
> + enum irdma_cmpl_notify cq_notify);
> + void (*iw_cq_resize)(struct irdma_cq_uk *cq, void *cq_base, int size);
> + void (*iw_cq_set_resized_cnt)(struct irdma_cq_uk *qp, __u16 cnt);
> +};
> +
> +struct irdma_dev_uk;
> +
> +struct irdma_device_uk_ops {
> + enum irdma_status_code (*iw_cq_uk_init)(struct irdma_cq_uk *cq,
> + struct irdma_cq_uk_init_info *info);
> + enum irdma_status_code (*iw_qp_uk_init)(struct irdma_qp_uk *qp,
> + struct irdma_qp_uk_init_info *info);
> +};
Is there a reason for all of these internal function pointers when
there is only one implementation? This looks like data path too,
making it even more puzzling
Jason
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 5/5] rdma-core/irdma: Implement device supported verb APIs
2021-04-06 21:17 [PATCH 0/5] Add irdma user space provider for Intel Ethernet RDMA Tatyana Nikolova
` (3 preceding siblings ...)
2021-04-06 21:17 ` [PATCH 4/5] rdma-core/irdma: Add library setup and utility definitions Tatyana Nikolova
@ 2021-04-06 21:17 ` Tatyana Nikolova
4 siblings, 0 replies; 10+ messages in thread
From: Tatyana Nikolova @ 2021-04-06 21:17 UTC (permalink / raw)
To: jgg, dledford, leon; +Cc: linux-rdma, Tatyana Nikolova
Implement device supported userspace verb APIs.
Signed-off-by: Tatyana Nikolova <tatyana.e.nikolova@intel.com>
---
providers/irdma/uverbs.c | 1932 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 1932 insertions(+)
create mode 100644 providers/irdma/uverbs.c
diff --git a/providers/irdma/uverbs.c b/providers/irdma/uverbs.c
new file mode 100644
index 0000000..2fd2e79
--- /dev/null
+++ b/providers/irdma/uverbs.c
@@ -0,0 +1,1932 @@
+// SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
+/* Copyright (C) 2019 - 2020 Intel Corporation */
+#include <config.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <signal.h>
+#include <errno.h>
+#include <sys/param.h>
+#include <sys/mman.h>
+#include <netinet/in.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <malloc.h>
+#include <linux/if_ether.h>
+
+#include "umain.h"
+#include "abi.h"
+
+static inline void print_fw_ver(uint64_t fw_ver, char *str, size_t len)
+{
+ uint16_t major, minor;
+
+ major = fw_ver >> 32 & 0xffff;
+ minor = fw_ver & 0xffff;
+
+ snprintf(str, len, "%d.%d", major, minor);
+}
+
+/**
+ * irdma_uquery_device_ex - query device attributes including extended properties
+ * @context: user context for the device
+ * @input: extensible input struct for ibv_query_device_ex verb
+ * @attr: extended device attribute struct
+ * @attr_size: size of extended device attribute struct
+ **/
+int irdma_uquery_device_ex(struct ibv_context *context,
+ const struct ibv_query_device_ex_input *input,
+ struct ibv_device_attr_ex *attr, size_t attr_size)
+{
+ struct ib_uverbs_ex_query_device_resp resp = {};
+ size_t resp_size = sizeof(resp);
+ int ret;
+
+ ret = ibv_cmd_query_device_any(context, input, attr, attr_size,
+ &resp, &resp_size);
+ if (ret)
+ return ret;
+
+ print_fw_ver(resp.base.fw_ver, attr->orig_attr.fw_ver, sizeof(attr->orig_attr.fw_ver));
+
+ return 0;
+}
+
+/**
+ * irdma_uquery_port - get port attributes (msg size, lnk, mtu...)
+ * @context: user context of the device
+ * @port: port for the attributes
+ * @attr: to return port attributes
+ **/
+int irdma_uquery_port(struct ibv_context *context, uint8_t port,
+ struct ibv_port_attr *attr)
+{
+ struct ibv_query_port cmd;
+
+ return ibv_cmd_query_port(context, port, attr, &cmd, sizeof(cmd));
+}
+
+/**
+ * irdma_ualloc_pd - allocates protection domain and return pd ptr
+ * @context: user context of the device
+ **/
+struct ibv_pd *irdma_ualloc_pd(struct ibv_context *context)
+{
+ struct ibv_alloc_pd cmd;
+ struct irdma_ualloc_pd_resp resp = {};
+ struct irdma_upd *iwupd;
+ int err;
+
+ iwupd = malloc(sizeof(*iwupd));
+ if (!iwupd)
+ return NULL;
+
+ err = ibv_cmd_alloc_pd(context, &iwupd->ibv_pd, &cmd, sizeof(cmd),
+ &resp.ibv_resp, sizeof(resp));
+ if (err)
+ goto err_free;
+
+ iwupd->pd_id = resp.pd_id;
+
+ return &iwupd->ibv_pd;
+
+err_free:
+ free(iwupd);
+ errno = err;
+ return NULL;
+}
+
+/**
+ * irdma_ufree_pd - free pd resources
+ * @pd: pd to free resources
+ */
+int irdma_ufree_pd(struct ibv_pd *pd)
+{
+ struct irdma_upd *iwupd;
+ int ret;
+
+ iwupd = to_irdma_upd(pd);
+ ret = ibv_cmd_dealloc_pd(pd);
+ if (ret)
+ return ret;
+
+ free(iwupd);
+
+ return 0;
+}
+
+/**
+ * irdma_ureg_mr - register user memory region
+ * @pd: pd for the mr
+ * @addr: user address of the memory region
+ * @length: length of the memory
+ * @hca_va: hca_va
+ * @access: access allowed on this mr
+ */
+struct ibv_mr *irdma_ureg_mr(struct ibv_pd *pd, void *addr, size_t length,
+ uint64_t hca_va, int access)
+{
+ struct irdma_umr *umr;
+ struct irdma_ureg_mr cmd;
+ struct ib_uverbs_reg_mr_resp resp;
+ int err;
+
+ umr = malloc(sizeof(*umr));
+ if (!umr)
+ return NULL;
+
+ cmd.reg_type = IW_MEMREG_TYPE_MEM;
+ err = ibv_cmd_reg_mr(pd, addr, length,
+ hca_va, access, &umr->vmr, &cmd.ibv_cmd,
+ sizeof(cmd), &resp, sizeof(resp));
+ if (err) {
+ free(umr);
+ errno = err;
+ return NULL;
+ }
+ umr->acc_flags = access;
+
+ return &umr->vmr.ibv_mr;
+}
+
+/**
+ * irdma_udereg_mr - re-register memory region
+ * @vmr: mr that was allocated
+ */
+int irdma_udereg_mr(struct verbs_mr *vmr)
+{
+ int ret;
+
+ ret = ibv_cmd_dereg_mr(vmr);
+ if (ret)
+ return ret;
+
+ free(vmr);
+
+ return 0;
+}
+
+/**
+ * irdma_ualloc_mw - allocate memory window
+ * @pd: protection domain
+ * @type: memory window type
+ */
+struct ibv_mw *irdma_ualloc_mw(struct ibv_pd *pd, enum ibv_mw_type type)
+{
+ struct ibv_mw *mw;
+ struct ibv_alloc_mw cmd;
+ struct ib_uverbs_alloc_mw_resp resp;
+
+ if (type != IBV_MW_TYPE_1) {
+ errno = ENOTSUP;
+ return NULL;
+ }
+
+ mw = calloc(1, sizeof(*mw));
+ if (!mw)
+ return NULL;
+
+ if (ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp,
+ sizeof(resp))) {
+ free(mw);
+ return NULL;
+ }
+
+ return mw;
+}
+
+/**
+ * irdma_ubind_mw - bind a memory window
+ * @qp: qp to post WR
+ * @mw: memory window to bind
+ * @mw_bind: bind info
+ */
+int irdma_ubind_mw(struct ibv_qp *qp, struct ibv_mw *mw,
+ struct ibv_mw_bind *mw_bind)
+{
+ struct ibv_mw_bind_info *bind_info = &mw_bind->bind_info;
+ struct verbs_mr *vmr = verbs_get_mr(bind_info->mr);
+ struct irdma_umr *umr = container_of(vmr, struct irdma_umr, vmr);
+
+ struct ibv_send_wr wr = {};
+ struct ibv_send_wr *bad_wr;
+ int err;
+
+ if (vmr->mr_type != IBV_MR_TYPE_MR || mw->type != IBV_MW_TYPE_1)
+ return ENOTSUP;
+
+ if (umr->acc_flags & IBV_ACCESS_ZERO_BASED)
+ return EINVAL;
+
+ wr.opcode = IBV_WR_BIND_MW;
+ wr.bind_mw.bind_info = mw_bind->bind_info;
+ wr.bind_mw.mw = mw;
+ wr.bind_mw.rkey = ibv_inc_rkey(mw->rkey);
+
+ wr.wr_id = mw_bind->wr_id;
+ wr.send_flags = mw_bind->send_flags;
+
+ err = irdma_upost_send(qp, &wr, &bad_wr);
+ if (!err)
+ mw->rkey = wr.bind_mw.rkey;
+
+ return err;
+}
+
+/**
+ * irdma_udealloc_mw - deallocate memory window
+ * @mw: memory window to dealloc
+ */
+int irdma_udealloc_mw(struct ibv_mw *mw)
+{
+ int ret;
+
+ ret = ibv_cmd_dealloc_mw(mw);
+ if (ret)
+ return ret;
+ free(mw);
+
+ return 0;
+}
+
+static void *irdma_alloc_hw_buf(size_t size)
+{
+ void *buf;
+
+ buf = memalign(IRDMA_HW_PAGE_SIZE, size);
+
+ if (!buf)
+ return NULL;
+ if (ibv_dontfork_range(buf, size)) {
+ free(buf);
+ return NULL;
+ }
+
+ return buf;
+}
+
+static void irdma_free_hw_buf(void *buf, size_t size)
+{
+ ibv_dofork_range(buf, size);
+ free(buf);
+}
+
+/**
+ * get_cq_size - returns actual cqe needed by HW
+ * @ncqe: minimum cqes requested by application
+ * @hw_rev: HW generation
+ */
+static inline int get_cq_size(int ncqe, __u8 hw_rev)
+{
+ ncqe++;
+
+ /* Completions with immediate require 1 extra entry */
+ if (hw_rev > IRDMA_GEN_1)
+ ncqe *= 2;
+
+ if (ncqe < IRDMA_U_MINCQ_SIZE)
+ ncqe = IRDMA_U_MINCQ_SIZE;
+
+ return ncqe;
+}
+
+static inline size_t get_cq_total_bytes(__u32 cq_size)
+{
+ return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE);
+}
+
+/**
+ * ucreate_cq - irdma util function to create a CQ
+ * @context: ibv context
+ * @attr_ex: CQ init attributes
+ * @ext_cq: flag to create an extendable or normal CQ
+ */
+static struct ibv_cq_ex *ucreate_cq(struct ibv_context *context,
+ struct ibv_cq_init_attr_ex *attr_ex,
+ bool ext_cq)
+{
+ struct irdma_ucq *iwucq;
+ struct irdma_cq_uk_init_info info = {};
+ struct irdma_ureg_mr reg_mr_cmd = {};
+ struct irdma_ucreate_cq_ex cmd = {};
+ struct irdma_ucreate_cq_ex_resp resp = {};
+ struct ib_uverbs_reg_mr_resp reg_mr_resp = {};
+ struct irdma_ureg_mr reg_mr_shadow_cmd = {};
+ struct ib_uverbs_reg_mr_resp reg_mr_shadow_resp = {};
+ struct irdma_uvcontext *iwvctx = to_irdma_uctx(context);
+ struct irdma_uk_attrs *uk_attrs = &iwvctx->uk_attrs;
+ __u8 hw_rev = uk_attrs->hw_rev;
+ size_t total_size;
+ __u32 cq_pages;
+ int ret, ncqe;
+
+ if (ext_cq && hw_rev == IRDMA_GEN_1) {
+ errno = EOPNOTSUPP;
+ return NULL;
+ }
+
+ if (attr_ex->cqe > uk_attrs->max_hw_cq_size) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ /* save the cqe requested by application */
+ ncqe = attr_ex->cqe;
+
+ iwucq = calloc(1, sizeof(*iwucq));
+ if (!iwucq)
+ return NULL;
+
+ if (pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE)) {
+ free(iwucq);
+ return NULL;
+ }
+
+ info.cq_size = get_cq_size(attr_ex->cqe, hw_rev);
+ iwucq->comp_vector = attr_ex->comp_vector;
+ list_head_init(&iwucq->resize_list);
+ total_size = get_cq_total_bytes(info.cq_size);
+ cq_pages = total_size >> IRDMA_HW_PAGE_SHIFT;
+
+ if (!(uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE))
+ total_size = (cq_pages << IRDMA_HW_PAGE_SHIFT) + IRDMA_DB_SHADOW_AREA_SIZE;
+
+ iwucq->buf_size = total_size;
+ info.cq_base = irdma_alloc_hw_buf(total_size);
+ if (!info.cq_base)
+ goto err_cq_base;
+
+ memset(info.cq_base, 0, total_size);
+ reg_mr_cmd.reg_type = IW_MEMREG_TYPE_CQ;
+ reg_mr_cmd.cq_pages = cq_pages;
+
+ ret = ibv_cmd_reg_mr(&iwvctx->iwupd->ibv_pd, info.cq_base,
+ total_size, (uintptr_t)info.cq_base,
+ IBV_ACCESS_LOCAL_WRITE, &iwucq->vmr,
+ ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd),
+ ®_mr_resp, sizeof(reg_mr_resp));
+ if (ret) {
+ errno = ret;
+ goto err_dereg_mr;
+ }
+
+ iwucq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd;
+
+ if (uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE) {
+ info.shadow_area = irdma_alloc_hw_buf(IRDMA_DB_SHADOW_AREA_SIZE);
+ if (!info.shadow_area)
+ goto err_dereg_mr;
+
+ memset(info.shadow_area, 0, IRDMA_DB_SHADOW_AREA_SIZE);
+ reg_mr_shadow_cmd.reg_type = IW_MEMREG_TYPE_CQ;
+ reg_mr_shadow_cmd.cq_pages = 1;
+
+ ret = ibv_cmd_reg_mr(&iwvctx->iwupd->ibv_pd, info.shadow_area,
+ IRDMA_DB_SHADOW_AREA_SIZE, (uintptr_t)info.shadow_area,
+ IBV_ACCESS_LOCAL_WRITE, &iwucq->vmr_shadow_area,
+ ®_mr_shadow_cmd.ibv_cmd, sizeof(reg_mr_shadow_cmd),
+ ®_mr_shadow_resp, sizeof(reg_mr_shadow_resp));
+ if (ret) {
+ errno = ret;
+ goto err_dereg_shadow;
+ }
+
+ iwucq->vmr_shadow_area.ibv_mr.pd = &iwvctx->iwupd->ibv_pd;
+
+ } else {
+ info.shadow_area = (__le64 *)((__u8 *)info.cq_base + (cq_pages << IRDMA_HW_PAGE_SHIFT));
+ }
+
+ attr_ex->cqe = info.cq_size;
+ cmd.user_cq_buf = (__u64)((uintptr_t)info.cq_base);
+ cmd.user_shadow_area = (__u64)((uintptr_t)info.shadow_area);
+
+ ret = ibv_cmd_create_cq_ex(context, attr_ex, &iwucq->verbs_cq,
+ &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp,
+ sizeof(resp), 0);
+ if (ret) {
+ errno = ret;
+ goto err_dereg_shadow;
+ }
+
+ if (ext_cq)
+ irdma_ibvcq_ex_fill_priv_funcs(iwucq, attr_ex);
+ info.cq_id = resp.cq_id;
+ /* Do not report the cqe's burned by HW */
+ iwucq->verbs_cq.cq.cqe = ncqe;
+
+ info.cqe_alloc_db = (__u32 *)((__u8 *)iwvctx->db + IRDMA_DB_CQ_OFFSET);
+ iwvctx->dev.ops_uk.iw_cq_uk_init(&iwucq->cq, &info);
+
+ return &iwucq->verbs_cq.cq_ex;
+
+err_dereg_shadow:
+ ibv_cmd_dereg_mr(&iwucq->vmr);
+ if (iwucq->vmr_shadow_area.ibv_mr.handle) {
+ ibv_cmd_dereg_mr(&iwucq->vmr_shadow_area);
+ irdma_free_hw_buf(info.shadow_area, IRDMA_HW_PAGE_SIZE);
+ }
+err_dereg_mr:
+ irdma_free_hw_buf(info.cq_base, total_size);
+err_cq_base:
+ pthread_spin_destroy(&iwucq->lock);
+
+ free(iwucq);
+
+ return NULL;
+}
+
+struct ibv_cq *irdma_ucreate_cq(struct ibv_context *context, int cqe,
+ struct ibv_comp_channel *channel,
+ int comp_vector)
+{
+ struct ibv_cq_init_attr_ex attr_ex = {
+ .cqe = cqe,
+ .channel = channel,
+ .comp_vector = comp_vector,
+ };
+ struct ibv_cq_ex *ibvcq_ex;
+
+ ibvcq_ex = ucreate_cq(context, &attr_ex, false);
+
+ return ibvcq_ex ? ibv_cq_ex_to_cq(ibvcq_ex) : NULL;
+}
+
+struct ibv_cq_ex *irdma_ucreate_cq_ex(struct ibv_context *context,
+ struct ibv_cq_init_attr_ex *attr_ex)
+{
+ if (attr_ex->wc_flags & ~IRDMA_CQ_SUPPORTED_WC_FLAGS) {
+ errno = EOPNOTSUPP;
+ return NULL;
+ }
+
+ return ucreate_cq(context, attr_ex, true);
+}
+
+/**
+ * irdma_free_cq_buf - free memory for cq buffer
+ * @cq_buf: cq buf to free
+ */
+static void irdma_free_cq_buf(struct irdma_cq_buf *cq_buf)
+{
+ ibv_cmd_dereg_mr(&cq_buf->vmr);
+ irdma_free_hw_buf(cq_buf->cq.cq_base, get_cq_total_bytes(cq_buf->cq.cq_size));
+ free(cq_buf);
+}
+
+/**
+ * irdma_process_resize_list - process the cq list to remove buffers
+ * @iwucq: cq which owns the list
+ * @lcqe_buf: cq buf where the last cqe is found
+ */
+static int irdma_process_resize_list(struct irdma_ucq *iwucq,
+ struct irdma_cq_buf *lcqe_buf)
+{
+ struct irdma_cq_buf *cq_buf, *next;
+ int cq_cnt = 0;
+
+ list_for_each_safe(&iwucq->resize_list, cq_buf, next, list) {
+ if (cq_buf == lcqe_buf)
+ return cq_cnt;
+
+ list_del(&cq_buf->list);
+ irdma_free_cq_buf(cq_buf);
+ cq_cnt++;
+ }
+
+ return cq_cnt;
+}
+
+/**
+ * irdma_udestroy_cq - destroys cq
+ * @cq: ptr to cq to be destroyed
+ */
+int irdma_udestroy_cq(struct ibv_cq *cq)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq(cq);
+ struct irdma_uvcontext *iwvctx = to_irdma_uctx(cq->context);
+ struct irdma_uk_attrs *uk_attrs = &iwvctx->uk_attrs;
+ int ret;
+
+ ret = pthread_spin_destroy(&iwucq->lock);
+ if (ret)
+ goto err;
+
+ irdma_process_resize_list(iwucq, NULL);
+ ret = ibv_cmd_destroy_cq(cq);
+ if (ret)
+ goto err;
+
+ ibv_cmd_dereg_mr(&iwucq->vmr);
+ irdma_free_hw_buf(iwucq->cq.cq_base, iwucq->buf_size);
+
+ if (uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE) {
+ ibv_cmd_dereg_mr(&iwucq->vmr_shadow_area);
+ irdma_free_hw_buf(iwucq->cq.shadow_area, IRDMA_DB_SHADOW_AREA_SIZE);
+ }
+ free(iwucq);
+ return 0;
+
+err:
+ return ret;
+}
+
+static enum ibv_wc_status irdma_flush_err_to_ib_wc_status(enum irdma_flush_opcode opcode)
+{
+ switch (opcode) {
+ case FLUSH_PROT_ERR:
+ return IBV_WC_LOC_PROT_ERR;
+ case FLUSH_REM_ACCESS_ERR:
+ return IBV_WC_REM_ACCESS_ERR;
+ case FLUSH_LOC_QP_OP_ERR:
+ return IBV_WC_LOC_QP_OP_ERR;
+ case FLUSH_REM_OP_ERR:
+ return IBV_WC_REM_OP_ERR;
+ case FLUSH_LOC_LEN_ERR:
+ return IBV_WC_LOC_LEN_ERR;
+ case FLUSH_GENERAL_ERR:
+ return IBV_WC_WR_FLUSH_ERR;
+ case FLUSH_FATAL_ERR:
+ default:
+ return IBV_WC_FATAL_ERR;
+ }
+}
+
+/**
+ * irdma_process_cqe_ext - process current cqe for extended CQ
+ * @cur_cqe - current cqe info
+ */
+static void irdma_process_cqe_ext(struct irdma_cq_poll_info *cur_cqe)
+{
+ struct irdma_ucq *iwucq = container_of(cur_cqe, struct irdma_ucq, cur_cqe);
+ struct ibv_cq_ex *ibvcq_ex = &iwucq->verbs_cq.cq_ex;
+
+ ibvcq_ex->wr_id = cur_cqe->wr_id;
+ if (cur_cqe->error)
+ ibvcq_ex->status = (cur_cqe->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ?
+ irdma_flush_err_to_ib_wc_status(cur_cqe->minor_err) : IBV_WC_GENERAL_ERR;
+ else
+ ibvcq_ex->status = IBV_WC_SUCCESS;
+}
+
+/**
+ * irdma_process_cqe - process current cqe info
+ * @entry - ibv_wc object to fill in for non-extended CQ
+ * @cur_cqe - current cqe info
+ */
+static void irdma_process_cqe(struct ibv_wc *entry, struct irdma_cq_poll_info *cur_cqe)
+{
+ struct irdma_qp_uk *qp;
+ struct ibv_qp *ib_qp;
+
+ entry->wc_flags = 0;
+ entry->wr_id = cur_cqe->wr_id;
+ entry->qp_num = cur_cqe->qp_id;
+ qp = cur_cqe->qp_handle;
+ ib_qp = qp->back_qp;
+
+ if (cur_cqe->error) {
+ if (cur_cqe->comp_status == IRDMA_COMPL_STATUS_FLUSHED)
+ entry->status = (cur_cqe->comp_status == IRDMA_COMPL_STATUS_FLUSHED) ?
+ irdma_flush_err_to_ib_wc_status(cur_cqe->minor_err) : IBV_WC_GENERAL_ERR;
+ entry->vendor_err = cur_cqe->major_err << 16 |
+ cur_cqe->minor_err;
+ } else {
+ entry->status = IBV_WC_SUCCESS;
+ }
+
+ if (cur_cqe->imm_valid) {
+ entry->imm_data = htonl(cur_cqe->imm_data);
+ entry->wc_flags |= IBV_WC_WITH_IMM;
+ }
+
+ switch (cur_cqe->op_type) {
+ case IRDMA_OP_TYPE_RDMA_WRITE:
+ case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
+ entry->opcode = IBV_WC_RDMA_WRITE;
+ break;
+ case IRDMA_OP_TYPE_RDMA_READ:
+ entry->opcode = IBV_WC_RDMA_READ;
+ break;
+ case IRDMA_OP_TYPE_SEND_SOL:
+ case IRDMA_OP_TYPE_SEND_SOL_INV:
+ case IRDMA_OP_TYPE_SEND_INV:
+ case IRDMA_OP_TYPE_SEND:
+ entry->opcode = IBV_WC_SEND;
+ break;
+ case IRDMA_OP_TYPE_BIND_MW:
+ entry->opcode = IBV_WC_BIND_MW;
+ break;
+ case IRDMA_OP_TYPE_REC:
+ entry->opcode = IBV_WC_RECV;
+ if (ib_qp->qp_type != IBV_QPT_UD &&
+ cur_cqe->stag_invalid_set) {
+ entry->invalidated_rkey = cur_cqe->inv_stag;
+ entry->wc_flags |= IBV_WC_WITH_INV;
+ }
+ break;
+ case IRDMA_OP_TYPE_REC_IMM:
+ entry->opcode = IBV_WC_RECV_RDMA_WITH_IMM;
+ if (ib_qp->qp_type != IBV_QPT_UD &&
+ cur_cqe->stag_invalid_set) {
+ entry->invalidated_rkey = cur_cqe->inv_stag;
+ entry->wc_flags |= IBV_WC_WITH_INV;
+ }
+ break;
+ case IRDMA_OP_TYPE_INV_STAG:
+ entry->opcode = IBV_WC_LOCAL_INV;
+ break;
+ default:
+ entry->status = IBV_WC_GENERAL_ERR;
+ return;
+ }
+
+
+ if (ib_qp->qp_type == IBV_QPT_UD) {
+ entry->src_qp = cur_cqe->ud_src_qpn;
+ entry->wc_flags |= IBV_WC_GRH;
+ } else {
+ entry->src_qp = cur_cqe->qp_id;
+ }
+ entry->byte_len = cur_cqe->bytes_xfered;
+}
+
+/**
+ * irdma_poll_one - poll one entry of the CQ
+ * @ukcq: ukcq to poll
+ * @cur_cqe: current CQE info to be filled in
+ * @entry: ibv_wc object to be filled for non-extended CQ or NULL for extended CQ
+ *
+ * Returns the internal irdma device error code or 0 on success
+ */
+static int irdma_poll_one(struct irdma_cq_uk *ukcq, struct irdma_cq_poll_info *cur_cqe,
+ struct ibv_wc *entry)
+{
+ int ret = ukcq->ops.iw_cq_poll_cmpl(ukcq, cur_cqe);
+
+ if (ret)
+ return ret;
+
+ if (!entry)
+ irdma_process_cqe_ext(cur_cqe);
+ else
+ irdma_process_cqe(entry, cur_cqe);
+
+ return 0;
+}
+
+/**
+ * __irdma_upoll_cq - irdma util function to poll device CQ
+ * @iwucq: irdma cq to poll
+ * @num_entries: max cq entries to poll
+ * @entry: pointer to array of ibv_wc objects to be filled in for each completion or NULL if ext CQ
+ *
+ * Returns non-negative value equal to the number of completions
+ * found. On failure, -EINVAL
+ */
+static int __irdma_upoll_cq(struct irdma_ucq *iwucq, int num_entries,
+ struct ibv_wc *entry)
+{
+ struct irdma_cq_buf *cq_buf, *next;
+ struct irdma_cq_buf *last_buf = NULL;
+ struct irdma_cq_poll_info *cur_cqe = &iwucq->cur_cqe;
+ bool cq_new_cqe = false;
+ int resized_bufs = 0;
+ int npolled = 0;
+ int ret;
+
+ /* go through the list of previously resized CQ buffers */
+ list_for_each_safe(&iwucq->resize_list, cq_buf, next, list) {
+ while (npolled < num_entries) {
+ ret = irdma_poll_one(&cq_buf->cq, cur_cqe,
+ entry ? entry + npolled : NULL);
+ if (!ret) {
+ ++npolled;
+ cq_new_cqe = true;
+ continue;
+ }
+ if (ret == IRDMA_ERR_Q_EMPTY)
+ break;
+ /* QP using the CQ is destroyed. Skip reporting this CQE */
+ if (ret == IRDMA_ERR_Q_DESTROYED) {
+ cq_new_cqe = true;
+ continue;
+ }
+ goto error;
+ }
+
+ /* save the resized CQ buffer which received the last cqe */
+ if (cq_new_cqe)
+ last_buf = cq_buf;
+ cq_new_cqe = false;
+ }
+
+ /* check the current CQ for new cqes */
+ while (npolled < num_entries) {
+ ret = irdma_poll_one(&iwucq->cq, cur_cqe,
+ entry ? entry + npolled : NULL);
+ if (!ret) {
+ ++npolled;
+ cq_new_cqe = true;
+ continue;
+ }
+ if (ret == IRDMA_ERR_Q_EMPTY)
+ break;
+ /* QP using the CQ is destroyed. Skip reporting this CQE */
+ if (ret == IRDMA_ERR_Q_DESTROYED) {
+ cq_new_cqe = true;
+ continue;
+ }
+ goto error;
+ }
+
+ if (cq_new_cqe)
+ /* all previous CQ resizes are complete */
+ resized_bufs = irdma_process_resize_list(iwucq, NULL);
+ else if (last_buf)
+ /* only CQ resizes up to the last_buf are complete */
+ resized_bufs = irdma_process_resize_list(iwucq, last_buf);
+ if (resized_bufs)
+ /* report to the HW the number of complete CQ resizes */
+ iwucq->cq.ops.iw_cq_set_resized_cnt(&iwucq->cq, resized_bufs);
+
+ return npolled;
+
+error:
+
+ return -EINVAL;
+}
+
+/**
+ * irdma_upoll_cq - verb API callback to poll device CQ
+ * @cq: ibv_cq to poll
+ * @num_entries: max cq entries to poll
+ * @entry: pointer to array of ibv_wc objects to be filled in for each completion
+ *
+ * Returns non-negative value equal to the number of completions
+ * found and a negative error code on failure
+ */
+int irdma_upoll_cq(struct ibv_cq *cq, int num_entries, struct ibv_wc *entry)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq(cq);
+ int ret;
+
+ ret = pthread_spin_lock(&iwucq->lock);
+ if (ret)
+ return -ret;
+
+ ret = __irdma_upoll_cq(iwucq, num_entries, entry);
+
+ pthread_spin_unlock(&iwucq->lock);
+
+ return ret;
+}
+
+/**
+ * irdma_start_poll - verb_ex API callback to poll batch of WC's
+ * @ibvcq_ex: ibv extended CQ
+ * @attr: attributes (not used)
+ *
+ * Start polling batch of work completions. Return 0 on success, ENONENT when
+ * no completions are available on CQ. And an error code on errors
+ */
+static int irdma_start_poll(struct ibv_cq_ex *ibvcq_ex, struct ibv_poll_cq_attr *attr)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+ int ret;
+
+ ret = pthread_spin_lock(&iwucq->lock);
+ if (ret)
+ return ret;
+
+ ret = __irdma_upoll_cq(iwucq, 1, NULL);
+ if (ret == 1)
+ return 0;
+
+ /* No Completions on CQ */
+ if (!ret)
+ ret = ENOENT;
+
+ pthread_spin_unlock(&iwucq->lock);
+
+ return ret;
+}
+
+/**
+ * irdma_next_poll - verb_ex API callback to get next WC
+ * @ibvcq_ex: ibv extended CQ
+ *
+ * Return 0 on success, ENONENT when no completions are available on CQ.
+ * And an error code on errors
+ */
+static int irdma_next_poll(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+ int ret;
+
+ ret = __irdma_upoll_cq(iwucq, 1, NULL);
+ if (ret == 1)
+ return 0;
+
+ /* No Completions on CQ */
+ if (!ret)
+ ret = ENOENT;
+
+ return ret;
+}
+
+/**
+ * irdma_end_poll - verb_ex API callback to end polling of WC's
+ * @ibvcq_ex: ibv extended CQ
+ */
+static void irdma_end_poll(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+
+ pthread_spin_unlock(&iwucq->lock);
+}
+
+/**
+ * irdma_wc_read_completion_ts - Get completion timestamp
+ * @ibvcq_ex: ibv extended CQ
+ *
+ * Get completion timestamp in HCA clock units
+ */
+static uint64_t irdma_wc_read_completion_ts(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+#define HCA_CORE_CLOCK_800_MHZ 800
+
+ return iwucq->cur_cqe.tcp_seq_num_rtt / HCA_CORE_CLOCK_800_MHZ;
+}
+
+/**
+ * irdma_wc_read_completion_wallclock_ns - Get completion timestamp in ns
+ * @ibvcq_ex: ibv extended CQ
+ *
+ * Get completion timestamp from current completion in wall clock nanoseconds
+ */
+static uint64_t irdma_wc_read_completion_wallclock_ns(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+
+ /* RTT is in usec */
+ return iwucq->cur_cqe.tcp_seq_num_rtt * 1000;
+}
+
+static enum ibv_wc_opcode irdma_wc_read_opcode(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+
+ switch (iwucq->cur_cqe.op_type) {
+ case IRDMA_OP_TYPE_RDMA_WRITE:
+ case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
+ return IBV_WC_RDMA_WRITE;
+ case IRDMA_OP_TYPE_RDMA_READ:
+ return IBV_WC_RDMA_READ;
+ case IRDMA_OP_TYPE_SEND_SOL:
+ case IRDMA_OP_TYPE_SEND_SOL_INV:
+ case IRDMA_OP_TYPE_SEND_INV:
+ case IRDMA_OP_TYPE_SEND:
+ return IBV_WC_SEND;
+ case IRDMA_OP_TYPE_BIND_MW:
+ return IBV_WC_BIND_MW;
+ case IRDMA_OP_TYPE_REC:
+ return IBV_WC_RECV;
+ case IRDMA_OP_TYPE_REC_IMM:
+ return IBV_WC_RECV_RDMA_WITH_IMM;
+ case IRDMA_OP_TYPE_INV_STAG:
+ return IBV_WC_LOCAL_INV;
+ }
+
+ return 0;
+}
+
+static uint32_t irdma_wc_read_vendor_err(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+ struct irdma_cq_poll_info *cur_cqe = &iwucq->cur_cqe;
+
+ return cur_cqe->error ? cur_cqe->major_err << 16 | cur_cqe->minor_err : 0;
+}
+
+static unsigned int irdma_wc_read_wc_flags(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+ struct irdma_cq_poll_info *cur_cqe = &iwucq->cur_cqe;
+ struct irdma_qp_uk *qp = cur_cqe->qp_handle;
+ struct ibv_qp *ib_qp = qp->back_qp;
+ unsigned int wc_flags = 0;
+
+ if (cur_cqe->imm_valid)
+ wc_flags |= IBV_WC_WITH_IMM;
+
+ if (ib_qp->qp_type == IBV_QPT_UD) {
+ wc_flags |= IBV_WC_GRH;
+ } else {
+ if (cur_cqe->stag_invalid_set) {
+ switch (cur_cqe->op_type) {
+ case IRDMA_OP_TYPE_REC:
+ wc_flags |= IBV_WC_WITH_INV;
+ break;
+ case IRDMA_OP_TYPE_REC_IMM:
+ wc_flags |= IBV_WC_WITH_INV;
+ break;
+ }
+ }
+ }
+
+ return wc_flags;
+}
+
+static uint32_t irdma_wc_read_byte_len(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+
+ return iwucq->cur_cqe.bytes_xfered;
+}
+
+static __be32 irdma_wc_read_imm_data(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+ struct irdma_cq_poll_info *cur_cqe = &iwucq->cur_cqe;
+
+ return cur_cqe->imm_valid ? htonl(cur_cqe->imm_data) : 0;
+}
+
+static uint32_t irdma_wc_read_qp_num(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+
+ return iwucq->cur_cqe.qp_id;
+}
+
+static uint32_t irdma_wc_read_src_qp(struct ibv_cq_ex *ibvcq_ex)
+{
+ struct irdma_ucq *iwucq = to_irdma_ucq_ex(ibvcq_ex);
+ struct irdma_cq_poll_info *cur_cqe = &iwucq->cur_cqe;
+ struct irdma_qp_uk *qp = cur_cqe->qp_handle;
+ struct ibv_qp *ib_qp = qp->back_qp;
+
+ return ib_qp->qp_type == IBV_QPT_UD ? cur_cqe->ud_src_qpn : cur_cqe->qp_id;
+}
+
+static uint32_t irdma_wc_read_slid(struct ibv_cq_ex *ibvcq_ex)
+{
+ return 0;
+}
+
+static uint8_t irdma_wc_read_sl(struct ibv_cq_ex *ibvcq_ex)
+{
+ return 0;
+}
+
+static uint8_t irdma_wc_read_dlid_path_bits(struct ibv_cq_ex *ibvcq_ex)
+{
+ return 0;
+}
+
+void irdma_ibvcq_ex_fill_priv_funcs(struct irdma_ucq *iwucq,
+ struct ibv_cq_init_attr_ex *attr_ex)
+{
+ struct ibv_cq_ex *ibvcq_ex = &iwucq->verbs_cq.cq_ex;
+
+ ibvcq_ex->start_poll = irdma_start_poll;
+ ibvcq_ex->end_poll = irdma_end_poll;
+ ibvcq_ex->next_poll = irdma_next_poll;
+
+ if (attr_ex->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP) {
+ ibvcq_ex->read_completion_ts = irdma_wc_read_completion_ts;
+ iwucq->report_rtt = true;
+ }
+ if (attr_ex->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK) {
+ ibvcq_ex->read_completion_wallclock_ns = irdma_wc_read_completion_wallclock_ns;
+ iwucq->report_rtt = true;
+ }
+
+ ibvcq_ex->read_opcode = irdma_wc_read_opcode;
+ ibvcq_ex->read_vendor_err = irdma_wc_read_vendor_err;
+ ibvcq_ex->read_wc_flags = irdma_wc_read_wc_flags;
+
+ if (attr_ex->wc_flags & IBV_WC_EX_WITH_BYTE_LEN)
+ ibvcq_ex->read_byte_len = irdma_wc_read_byte_len;
+ if (attr_ex->wc_flags & IBV_WC_EX_WITH_IMM)
+ ibvcq_ex->read_imm_data = irdma_wc_read_imm_data;
+ if (attr_ex->wc_flags & IBV_WC_EX_WITH_QP_NUM)
+ ibvcq_ex->read_qp_num = irdma_wc_read_qp_num;
+ if (attr_ex->wc_flags & IBV_WC_EX_WITH_SRC_QP)
+ ibvcq_ex->read_src_qp = irdma_wc_read_src_qp;
+ if (attr_ex->wc_flags & IBV_WC_EX_WITH_SLID)
+ ibvcq_ex->read_slid = irdma_wc_read_slid;
+ if (attr_ex->wc_flags & IBV_WC_EX_WITH_SL)
+ ibvcq_ex->read_sl = irdma_wc_read_sl;
+ if (attr_ex->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS)
+ ibvcq_ex->read_dlid_path_bits = irdma_wc_read_dlid_path_bits;
+}
+
+/**
+ * irdma_arm_cq - arm of cq
+ * @iwucq: cq to which arm
+ * @cq_notify: notification params
+ */
+static void irdma_arm_cq(struct irdma_ucq *iwucq,
+ enum irdma_cmpl_notify cq_notify)
+{
+ iwucq->is_armed = true;
+ iwucq->arm_sol = true;
+ iwucq->skip_arm = false;
+ iwucq->skip_sol = true;
+ iwucq->cq.ops.iw_cq_request_notification(&iwucq->cq, cq_notify);
+}
+
+/**
+ * irdma_uarm_cq - callback for arm of cq
+ * @cq: cq to arm
+ * @solicited: to get notify params
+ */
+int irdma_uarm_cq(struct ibv_cq *cq, int solicited)
+{
+ struct irdma_ucq *iwucq;
+ enum irdma_cmpl_notify cq_notify = IRDMA_CQ_COMPL_EVENT;
+ int ret;
+
+ iwucq = to_irdma_ucq(cq);
+ if (solicited)
+ cq_notify = IRDMA_CQ_COMPL_SOLICITED;
+
+ ret = pthread_spin_lock(&iwucq->lock);
+ if (ret)
+ return ret;
+
+ if (iwucq->is_armed) {
+ if (iwucq->arm_sol && !solicited) {
+ irdma_arm_cq(iwucq, cq_notify);
+ } else {
+ iwucq->skip_arm = true;
+ iwucq->skip_sol = solicited ? true : false;
+ }
+ } else {
+ irdma_arm_cq(iwucq, cq_notify);
+ }
+
+ pthread_spin_unlock(&iwucq->lock);
+
+ return 0;
+}
+
+/**
+ * irdma_cq_event - cq to do completion event
+ * @cq: cq to arm
+ */
+void irdma_cq_event(struct ibv_cq *cq)
+{
+ struct irdma_ucq *iwucq;
+
+ iwucq = to_irdma_ucq(cq);
+ if (pthread_spin_lock(&iwucq->lock))
+ return;
+
+ if (iwucq->skip_arm)
+ irdma_arm_cq(iwucq, IRDMA_CQ_COMPL_EVENT);
+ else
+ iwucq->is_armed = false;
+
+ pthread_spin_unlock(&iwucq->lock);
+}
+
+void *irdma_mmap(int fd, off_t offset)
+{
+ void *map;
+
+ map = mmap(NULL, IRDMA_HW_PAGE_SIZE, PROT_WRITE | PROT_READ, MAP_SHARED,
+ fd, offset);
+ if (map == MAP_FAILED)
+ return map;
+
+ if (ibv_dontfork_range(map, IRDMA_HW_PAGE_SIZE)) {
+ munmap(map, IRDMA_HW_PAGE_SIZE);
+ return MAP_FAILED;
+ }
+
+ return map;
+}
+
+void irdma_munmap(void *map)
+{
+ ibv_dofork_range(map, IRDMA_HW_PAGE_SIZE);
+ munmap(map, IRDMA_HW_PAGE_SIZE);
+}
+
+/**
+ * irdma_destroy_vmapped_qp - destroy resources for qp
+ * @iwuqp: qp struct for resources
+ */
+static int irdma_destroy_vmapped_qp(struct irdma_uqp *iwuqp)
+{
+ int ret;
+
+ ret = ibv_cmd_destroy_qp(&iwuqp->ibv_qp);
+ if (ret)
+ return ret;
+
+ if (iwuqp->qp.push_db)
+ irdma_munmap(iwuqp->qp.push_db);
+ if (iwuqp->qp.push_wqe)
+ irdma_munmap(iwuqp->qp.push_wqe);
+
+ ibv_cmd_dereg_mr(&iwuqp->vmr);
+
+ return 0;
+}
+
+/**
+ * irdma_vmapped_qp - create resources for qp
+ * @iwuqp: qp struct for resources
+ * @pd: pd for the qp
+ * @attr: attributes of qp passed
+ * @resp: response back from create qp
+ * @sqdepth: depth of sq
+ * @rqdepth: depth of rq
+ * @info: info for initializing user level qp
+ * @abi_ver: abi version of the create qp command
+ */
+static int irdma_vmapped_qp(struct irdma_uqp *iwuqp, struct ibv_pd *pd,
+ struct ibv_qp_init_attr *attr, int sqdepth,
+ int rqdepth, struct irdma_qp_uk_init_info *info,
+ bool legacy_mode)
+{
+ struct irdma_ucreate_qp cmd = {};
+ size_t sqsize, rqsize, totalqpsize;
+ struct irdma_ucreate_qp_resp resp = {};
+ struct irdma_ureg_mr reg_mr_cmd = {};
+ struct ib_uverbs_reg_mr_resp reg_mr_resp = {};
+ int ret;
+
+ sqsize = roundup(sqdepth * IRDMA_QP_WQE_MIN_SIZE, IRDMA_HW_PAGE_SIZE);
+ rqsize = roundup(rqdepth * IRDMA_QP_WQE_MIN_SIZE, IRDMA_HW_PAGE_SIZE);
+ totalqpsize = rqsize + sqsize + IRDMA_DB_SHADOW_AREA_SIZE;
+ info->sq = irdma_alloc_hw_buf(totalqpsize);
+ iwuqp->buf_size = totalqpsize;
+
+ if (!info->sq)
+ return ENOMEM;
+
+ memset(info->sq, 0, totalqpsize);
+ info->rq = &info->sq[sqsize / IRDMA_QP_WQE_MIN_SIZE];
+ info->shadow_area = info->rq[rqsize / IRDMA_QP_WQE_MIN_SIZE].elem;
+
+ reg_mr_cmd.reg_type = IW_MEMREG_TYPE_QP;
+ reg_mr_cmd.sq_pages = sqsize >> IRDMA_HW_PAGE_SHIFT;
+ reg_mr_cmd.rq_pages = rqsize >> IRDMA_HW_PAGE_SHIFT;
+
+ ret = ibv_cmd_reg_mr(pd, info->sq, totalqpsize,
+ (uintptr_t)info->sq, IBV_ACCESS_LOCAL_WRITE,
+ &iwuqp->vmr, ®_mr_cmd.ibv_cmd,
+ sizeof(reg_mr_cmd), ®_mr_resp,
+ sizeof(reg_mr_resp));
+ if (ret)
+ goto err_dereg_mr;
+
+ cmd.user_wqe_bufs = (__u64)((uintptr_t)info->sq);
+ cmd.user_compl_ctx = (__u64)(uintptr_t)&iwuqp->qp;
+ ret = ibv_cmd_create_qp(pd, &iwuqp->ibv_qp, attr, &cmd.ibv_cmd,
+ sizeof(cmd), &resp.ibv_resp,
+ sizeof(struct irdma_ucreate_qp_resp));
+ if (ret)
+ goto err_qp;
+
+ info->sq_size = resp.actual_sq_size;
+ info->rq_size = resp.actual_rq_size;
+ info->first_sq_wq = legacy_mode ? 1 : resp.lsmm;
+ info->qp_caps = resp.qp_caps;
+ info->qp_id = resp.qp_id;
+ iwuqp->irdma_drv_opt = resp.irdma_drv_opt;
+ iwuqp->ibv_qp.qp_num = resp.qp_id;
+
+ iwuqp->send_cq = to_irdma_ucq(attr->send_cq);
+ iwuqp->recv_cq = to_irdma_ucq(attr->recv_cq);
+ iwuqp->send_cq->uqp = iwuqp;
+ iwuqp->recv_cq->uqp = iwuqp;
+
+ return 0;
+err_qp:
+ ibv_cmd_dereg_mr(&iwuqp->vmr);
+err_dereg_mr:
+ irdma_free_hw_buf(info->sq, iwuqp->buf_size);
+ return ret;
+}
+
+/**
+ * irdma_ucreate_qp - create qp on user app
+ * @pd: pd for the qp
+ * @attr: attributes of the qp to be created (sizes, sge, cq)
+ */
+struct ibv_qp *irdma_ucreate_qp(struct ibv_pd *pd,
+ struct ibv_qp_init_attr *attr)
+{
+ struct irdma_uvcontext *iwvctx = to_irdma_uctx(pd->context);
+ struct irdma_uk_attrs *uk_attrs = &iwvctx->uk_attrs;
+ struct irdma_qp_uk_init_info info = {};
+ struct irdma_uqp *iwuqp;
+ __u32 sqdepth, rqdepth;
+ __u8 sqshift, rqshift;
+ int status;
+
+ if (attr->qp_type != IBV_QPT_RC && attr->qp_type != IBV_QPT_UD) {
+ errno = EOPNOTSUPP;
+ return NULL;
+ }
+
+ if (attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags ||
+ attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags ||
+ attr->cap.max_inline_data > uk_attrs->max_hw_inline) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ irdma_get_wqe_shift(uk_attrs,
+ uk_attrs->hw_rev > IRDMA_GEN_1 ? attr->cap.max_send_sge + 1 :
+ attr->cap.max_send_sge,
+ attr->cap.max_inline_data, &sqshift);
+ status = irdma_get_sqdepth(uk_attrs, attr->cap.max_send_wr, sqshift,
+ &sqdepth);
+ if (status) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ if (uk_attrs->hw_rev == IRDMA_GEN_1 && iwvctx->abi_ver > 4)
+ rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
+ else
+ irdma_get_wqe_shift(uk_attrs, attr->cap.max_recv_sge, 0,
+ &rqshift);
+
+ status = irdma_get_rqdepth(uk_attrs, attr->cap.max_recv_wr, rqshift,
+ &rqdepth);
+ if (status) {
+ errno = EINVAL;
+ return NULL;
+ }
+
+ iwuqp = memalign(1024, sizeof(*iwuqp));
+ if (!iwuqp)
+ return NULL;
+
+ memset(iwuqp, 0, sizeof(*iwuqp));
+
+ if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE))
+ goto err_free_qp;
+
+ info.sq_size = sqdepth >> sqshift;
+ info.rq_size = rqdepth >> rqshift;
+ attr->cap.max_send_wr = info.sq_size;
+ attr->cap.max_recv_wr = info.rq_size;
+
+ info.uk_attrs = uk_attrs;
+ info.max_sq_frag_cnt = attr->cap.max_send_sge;
+ info.max_rq_frag_cnt = attr->cap.max_recv_sge;
+ iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges));
+ if (!iwuqp->recv_sges)
+ goto err_destroy_lock;
+
+ info.wqe_alloc_db = (__u32 *)iwvctx->db;
+ info.abi_ver = iwvctx->abi_ver;
+ info.legacy_mode = iwvctx->legacy_mode;
+ info.sq_wrtrk_array = calloc(sqdepth, sizeof(*info.sq_wrtrk_array));
+ if (!info.sq_wrtrk_array)
+ goto err_free_rsges;
+
+ info.rq_wrid_array = calloc(rqdepth, sizeof(*info.rq_wrid_array));
+ if (!info.rq_wrid_array)
+ goto err_free_sq_wrtrk;
+
+ iwuqp->sq_sig_all = attr->sq_sig_all;
+ iwuqp->qp_type = attr->qp_type;
+ status = irdma_vmapped_qp(iwuqp, pd, attr, sqdepth, rqdepth, &info, iwvctx->legacy_mode);
+ if (status) {
+ errno = status;
+ goto err_free_rq_wrid;
+ }
+
+ iwuqp->qp.back_qp = iwuqp;
+ iwuqp->qp.lock = &iwuqp->lock;
+
+ info.max_sq_frag_cnt = attr->cap.max_send_sge;
+ info.max_rq_frag_cnt = attr->cap.max_recv_sge;
+ info.max_inline_data = attr->cap.max_inline_data;
+ status = iwvctx->dev.ops_uk.iw_qp_uk_init(&iwuqp->qp, &info);
+ if (status) {
+ errno = EINVAL;
+ goto err_free_vmap_qp;
+ }
+
+ attr->cap.max_send_wr = (sqdepth - IRDMA_SQ_RSVD) >> sqshift;
+ attr->cap.max_recv_wr = (rqdepth - IRDMA_RQ_RSVD) >> rqshift;
+ return &iwuqp->ibv_qp;
+
+err_free_vmap_qp:
+ irdma_destroy_vmapped_qp(iwuqp);
+ irdma_free_hw_buf(info.sq, iwuqp->buf_size);
+err_free_rq_wrid:
+ free(info.rq_wrid_array);
+err_free_sq_wrtrk:
+ free(info.sq_wrtrk_array);
+err_free_rsges:
+ free(iwuqp->recv_sges);
+err_destroy_lock:
+ pthread_spin_destroy(&iwuqp->lock);
+err_free_qp:
+ free(iwuqp);
+
+ return NULL;
+}
+
+/**
+ * irdma_uquery_qp - query qp for some attribute
+ * @qp: qp for the attributes query
+ * @attr: to return the attributes
+ * @attr_mask: mask of what is query for
+ * @init_attr: initial attributes during create_qp
+ */
+int irdma_uquery_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask,
+ struct ibv_qp_init_attr *init_attr)
+{
+ struct ibv_query_qp cmd;
+
+ return ibv_cmd_query_qp(qp, attr, attr_mask, init_attr, &cmd,
+ sizeof(cmd));
+}
+
+/**
+ * irdma_umodify_qp - send qp modify to driver
+ * @qp: qp to modify
+ * @attr: attribute to modify
+ * @attr_mask: mask of the attribute
+ */
+int irdma_umodify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
+{
+ struct irdma_umodify_qp_resp resp = {};
+ struct ibv_modify_qp cmd = {};
+ struct irdma_modify_qp_cmd cmd_ex = {};
+ struct irdma_uqp *iwuqp = to_irdma_uqp(qp);
+ struct irdma_uvcontext *iwctx = to_irdma_uctx(qp->context);
+
+ iwuqp->attr_mask = attr_mask;
+ memcpy(&iwuqp->attr, attr, sizeof(iwuqp->attr));
+
+ if (iwuqp->qp.qp_caps & IRDMA_PUSH_MODE && attr_mask & IBV_QP_STATE && iwctx->uk_attrs.hw_rev > IRDMA_GEN_1) {
+ __u64 offset;
+ void *map;
+ int ret;
+
+ ret = ibv_cmd_modify_qp_ex(qp, attr, attr_mask, &cmd_ex.ibv_cmd,
+ sizeof(cmd_ex), &resp.ibv_resp, sizeof(resp));
+ if (ret || !resp.push_valid)
+ return ret;
+
+ if (iwuqp->qp.push_wqe)
+ return ret;
+
+ offset = resp.push_wqe_mmap_key;
+ map = irdma_mmap(qp->context->cmd_fd, offset);
+ if (map == MAP_FAILED)
+ return ret;
+
+ iwuqp->qp.push_wqe = map;
+
+ offset = resp.push_db_mmap_key;
+ map = irdma_mmap(qp->context->cmd_fd, offset);
+ if (map == MAP_FAILED) {
+ irdma_munmap(iwuqp->qp.push_wqe);
+ iwuqp->qp.push_wqe = NULL;
+ return ret;
+ }
+ iwuqp->qp.push_wqe += resp.push_offset;
+ iwuqp->qp.push_db = map + resp.push_offset;
+
+ return ret;
+ } else {
+ return ibv_cmd_modify_qp(qp, attr, attr_mask, &cmd, sizeof(cmd));
+ }
+}
+
+static void irdma_issue_flush(struct ibv_qp *qp, bool sq_flush, bool rq_flush)
+{
+ struct ib_uverbs_ex_modify_qp_resp resp = {};
+ struct irdma_modify_qp_cmd cmd_ex = {};
+ struct irdma_uqp *iwuqp = to_irdma_uqp(qp);
+
+ cmd_ex.sq_flush = sq_flush;
+ cmd_ex.rq_flush = rq_flush;
+
+ ibv_cmd_modify_qp_ex(qp, &iwuqp->attr, iwuqp->attr_mask,
+ &cmd_ex.ibv_cmd, sizeof(cmd_ex),
+ &resp, sizeof(resp));
+}
+
+/**
+ * irdma_clean_cqes - clean cq entries for qp
+ * @qp: qp for which completions are cleaned
+ * @iwcq: cq to be cleaned
+ */
+static void irdma_clean_cqes(struct irdma_qp_uk *qp, struct irdma_ucq *iwucq)
+{
+ struct irdma_cq_uk *ukcq = &iwucq->cq;
+ int ret;
+
+ ret = pthread_spin_lock(&iwucq->lock);
+ if (ret)
+ return;
+
+ ukcq->ops.iw_cq_clean(qp, ukcq);
+ pthread_spin_unlock(&iwucq->lock);
+}
+
+/**
+ * irdma_udestroy_qp - destroy qp
+ * @qp: qp to destroy
+ */
+int irdma_udestroy_qp(struct ibv_qp *qp)
+{
+ struct irdma_uqp *iwuqp = to_irdma_uqp(qp);
+ int ret;
+
+ ret = pthread_spin_destroy(&iwuqp->lock);
+ if (ret)
+ goto err;
+
+ ret = irdma_destroy_vmapped_qp(iwuqp);
+ if (ret)
+ goto err;
+
+ /* Clean any pending completions from the cq(s) */
+ if (iwuqp->send_cq)
+ irdma_clean_cqes(&iwuqp->qp, iwuqp->send_cq);
+
+ if (iwuqp->recv_cq && iwuqp->recv_cq != iwuqp->send_cq)
+ irdma_clean_cqes(&iwuqp->qp, iwuqp->recv_cq);
+
+ if (iwuqp->qp.sq_wrtrk_array)
+ free(iwuqp->qp.sq_wrtrk_array);
+ if (iwuqp->qp.rq_wrid_array)
+ free(iwuqp->qp.rq_wrid_array);
+
+ irdma_free_hw_buf(iwuqp->qp.sq_base, iwuqp->buf_size);
+ free(iwuqp->recv_sges);
+ free(iwuqp);
+ return 0;
+
+err:
+ return ret;
+}
+
+/**
+ * irdma_copy_sg_list - copy sg list for qp
+ * @sg_list: copied into sg_list
+ * @sgl: copy from sgl
+ * @num_sges: count of sg entries
+ * @max_sges: count of max supported sg entries
+ */
+static void irdma_copy_sg_list(struct irdma_sge *sg_list, struct ibv_sge *sgl,
+ int num_sges)
+{
+ int i;
+
+ for (i = 0; i < num_sges; i++) {
+ sg_list[i].tag_off = sgl[i].addr;
+ sg_list[i].len = sgl[i].length;
+ sg_list[i].stag = sgl[i].lkey;
+ }
+}
+
+/**
+ * irdma_post_send - post send wr for user application
+ * @ib_qp: qp to post wr
+ * @ib_wr: work request ptr
+ * @bad_wr: return of bad wr if err
+ */
+int irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
+ struct ibv_send_wr **bad_wr)
+{
+ struct irdma_uqp *iwuqp;
+ struct irdma_post_sq_info info;
+ struct irdma_uvcontext *iwvctx = to_irdma_uctx(ib_qp->context);
+ struct irdma_uk_attrs *uk_attrs = &iwvctx->uk_attrs;
+ enum irdma_status_code ret = 0;
+ bool reflush = false;
+ int err = 0;
+
+ iwuqp = (struct irdma_uqp *)ib_qp;
+
+ err = pthread_spin_lock(&iwuqp->lock);
+ if (err)
+ return err;
+
+ if (!IRDMA_RING_MORE_WORK(iwuqp->qp.sq_ring) &&
+ ib_qp->state == IBV_QPS_ERR)
+ reflush = true;
+
+ while (ib_wr) {
+ memset(&info, 0, sizeof(info));
+ info.wr_id = (__u64)(ib_wr->wr_id);
+ if ((ib_wr->send_flags & IBV_SEND_SIGNALED) ||
+ iwuqp->sq_sig_all)
+ info.signaled = true;
+ if (ib_wr->send_flags & IBV_SEND_FENCE)
+ info.read_fence = true;
+ if (iwuqp->send_cq->report_rtt)
+ info.report_rtt = true;
+
+ switch (ib_wr->opcode) {
+ case IBV_WR_SEND_WITH_IMM:
+ if (iwuqp->qp.qp_caps & IRDMA_SEND_WITH_IMM) {
+ info.imm_data_valid = true;
+ info.imm_data = ntohl(ib_wr->imm_data);
+ } else {
+ err = EINVAL;
+ break;
+ }
+ SWITCH_FALLTHROUGH;
+ case IBV_WR_SEND:
+ case IBV_WR_SEND_WITH_INV:
+ if (ib_wr->opcode == IBV_WR_SEND ||
+ ib_wr->opcode == IBV_WR_SEND_WITH_IMM) {
+ if (ib_wr->send_flags & IBV_SEND_SOLICITED)
+ info.op_type = IRDMA_OP_TYPE_SEND_SOL;
+ else
+ info.op_type = IRDMA_OP_TYPE_SEND;
+ } else {
+ if (ib_wr->send_flags & IBV_SEND_SOLICITED)
+ info.op_type = IRDMA_OP_TYPE_SEND_SOL_INV;
+ else
+ info.op_type = IRDMA_OP_TYPE_SEND_INV;
+ info.stag_to_inv = ib_wr->invalidate_rkey;
+ }
+ if (ib_wr->send_flags & IBV_SEND_INLINE) {
+ info.op.inline_send.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr;
+ info.op.inline_send.len = ib_wr->sg_list[0].length;
+ if (ib_qp->qp_type == IBV_QPT_UD) {
+ struct irdma_uah *ah = to_irdma_uah(ib_wr->wr.ud.ah);
+
+ info.op.inline_send.ah_id = ah->ah_id;
+ info.op.inline_send.qkey = ib_wr->wr.ud.remote_qkey;
+ info.op.inline_send.dest_qp = ib_wr->wr.ud.remote_qpn;
+ ret = iwuqp->qp.qp_ops.iw_inline_send(&iwuqp->qp, &info, false);
+ } else {
+ ret = iwuqp->qp.qp_ops.iw_inline_send(
+ &iwuqp->qp, &info, false);
+ }
+ } else {
+ info.op.send.num_sges = ib_wr->num_sge;
+ info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list;
+ if (ib_qp->qp_type == IBV_QPT_UD) {
+ struct irdma_uah *ah = to_irdma_uah(ib_wr->wr.ud.ah);
+
+ info.op.inline_send.ah_id = ah->ah_id;
+ info.op.inline_send.qkey = ib_wr->wr.ud.remote_qkey;
+ info.op.inline_send.dest_qp = ib_wr->wr.ud.remote_qpn;
+ ret = iwuqp->qp.qp_ops.iw_send(&iwuqp->qp, &info, false);
+ } else {
+ ret = iwuqp->qp.qp_ops.iw_send(
+ &iwuqp->qp, &info, false);
+ }
+ }
+ if (ret)
+ err = (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) ? ENOMEM : EINVAL;
+ break;
+ case IBV_WR_RDMA_WRITE_WITH_IMM:
+ if (iwuqp->qp.qp_caps & IRDMA_WRITE_WITH_IMM) {
+ info.imm_data_valid = true;
+ info.imm_data = ntohl(ib_wr->imm_data);
+ } else {
+ err = EINVAL;
+ break;
+ }
+ SWITCH_FALLTHROUGH;
+ case IBV_WR_RDMA_WRITE:
+ if (ib_wr->send_flags & IBV_SEND_SOLICITED)
+ info.op_type = IRDMA_OP_TYPE_RDMA_WRITE_SOL;
+ else
+ info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
+
+ if (ib_wr->send_flags & IBV_SEND_INLINE) {
+ info.op.inline_rdma_write.data = (void *)(uintptr_t)ib_wr->sg_list[0].addr;
+ info.op.inline_rdma_write.len = ib_wr->sg_list[0].length;
+ info.op.inline_rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
+ info.op.inline_rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
+ ret = iwuqp->qp.qp_ops.iw_inline_rdma_write(&iwuqp->qp, &info, false);
+ } else {
+ info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
+ info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
+ info.op.rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
+ info.op.rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
+ ret = iwuqp->qp.qp_ops.iw_rdma_write(&iwuqp->qp, &info, false);
+ }
+ if (ret)
+ err = (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) ? ENOMEM : EINVAL;
+ break;
+ case IBV_WR_RDMA_READ:
+ if (ib_wr->num_sge > uk_attrs->max_hw_read_sges) {
+ err = EINVAL;
+ break;
+ }
+ info.op_type = IRDMA_OP_TYPE_RDMA_READ;
+ info.op.rdma_read.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
+ info.op.rdma_read.rem_addr.stag = ib_wr->wr.rdma.rkey;
+
+ info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list;
+ info.op.rdma_read.num_lo_sges = ib_wr->num_sge;
+ ret = iwuqp->qp.qp_ops.iw_rdma_read(&iwuqp->qp, &info,
+ false, false);
+ if (ret)
+ err = (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) ? ENOMEM : EINVAL;
+ break;
+ case IBV_WR_BIND_MW:
+ if (ib_qp->qp_type != IBV_QPT_RC) {
+ err = EINVAL;
+ break;
+ }
+ info.op_type = IRDMA_OP_TYPE_BIND_MW;
+ info.op.bind_window.mr_stag = ib_wr->bind_mw.bind_info.mr->rkey;
+ info.op.bind_window.mem_window_type_1 = true;
+ info.op.bind_window.mw_stag = ib_wr->bind_mw.rkey;
+
+ if (ib_wr->bind_mw.bind_info.mw_access_flags & IBV_ACCESS_ZERO_BASED) {
+ info.op.bind_window.addressing_type = IRDMA_ADDR_TYPE_ZERO_BASED;
+ info.op.bind_window.va = NULL;
+ } else {
+ info.op.bind_window.addressing_type = IRDMA_ADDR_TYPE_VA_BASED;
+ info.op.bind_window.va = (void *)(uintptr_t)ib_wr->bind_mw.bind_info.addr;
+ }
+ info.op.bind_window.bind_len = ib_wr->bind_mw.bind_info.length;
+ info.op.bind_window.ena_reads =
+ (ib_wr->bind_mw.bind_info.mw_access_flags & IBV_ACCESS_REMOTE_READ) ? 1 : 0;
+ info.op.bind_window.ena_writes =
+ (ib_wr->bind_mw.bind_info.mw_access_flags & IBV_ACCESS_REMOTE_WRITE) ? 1 : 0;
+
+ ret = iwuqp->qp.qp_ops.iw_mw_bind(&iwuqp->qp, &info, false);
+ if (ret)
+ err = (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) ? ENOMEM : EINVAL;
+ break;
+ case IBV_WR_LOCAL_INV:
+ info.op_type = IRDMA_OP_TYPE_INV_STAG;
+ info.op.inv_local_stag.target_stag = ib_wr->invalidate_rkey;
+ ret = iwuqp->qp.qp_ops.iw_stag_local_invalidate(&iwuqp->qp, &info, true);
+ if (ret)
+ err = (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) ? ENOMEM : EINVAL;
+ break;
+ default:
+ /* error */
+ err = EINVAL;
+ break;
+ }
+ if (err)
+ break;
+
+ ib_wr = ib_wr->next;
+ }
+
+ if (err)
+ *bad_wr = ib_wr;
+
+ iwuqp->qp.qp_ops.iw_qp_post_wr(&iwuqp->qp);
+ if (reflush)
+ irdma_issue_flush(ib_qp, 1, 0);
+
+ pthread_spin_unlock(&iwuqp->lock);
+
+ return err;
+}
+
+/**
+ * irdma_post_recv - post receive wr for user application
+ * @ib_wr: work request for receive
+ * @bad_wr: bad wr caused an error
+ */
+int irdma_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr,
+ struct ibv_recv_wr **bad_wr)
+{
+ struct irdma_uqp *iwuqp = to_irdma_uqp(ib_qp);
+ enum irdma_status_code ret = 0;
+ struct irdma_post_rq_info post_recv = {};
+ struct irdma_sge *sg_list;
+ bool reflush = false;
+ int err = 0;
+
+ sg_list = iwuqp->recv_sges;
+
+ err = pthread_spin_lock(&iwuqp->lock);
+ if (err)
+ return err;
+
+ if (!IRDMA_RING_MORE_WORK(iwuqp->qp.rq_ring) &&
+ ib_qp->state == IBV_QPS_ERR)
+ reflush = true;
+
+ while (ib_wr) {
+ if (ib_wr->num_sge > iwuqp->qp.max_rq_frag_cnt) {
+ *bad_wr = ib_wr;
+ err = EINVAL;
+ goto error;
+ }
+ post_recv.num_sges = ib_wr->num_sge;
+ post_recv.wr_id = ib_wr->wr_id;
+ irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
+ post_recv.sg_list = sg_list;
+ ret = iwuqp->qp.qp_ops.iw_post_receive(&iwuqp->qp, &post_recv);
+ if (ret) {
+ err = (ret == IRDMA_ERR_QP_TOOMANY_WRS_POSTED) ? ENOMEM : EINVAL;
+ *bad_wr = ib_wr;
+ goto error;
+ }
+
+ if (reflush)
+ irdma_issue_flush(ib_qp, 0, 1);
+
+ ib_wr = ib_wr->next;
+ }
+error:
+ pthread_spin_unlock(&iwuqp->lock);
+
+ return err;
+}
+
+/**
+ * irdma_ucreate_ah - create address handle associated with a pd
+ * @ibpd: pd for the address handle
+ * @attr: attributes of address handle
+ */
+struct ibv_ah *irdma_ucreate_ah(struct ibv_pd *ibpd, struct ibv_ah_attr *attr)
+{
+ struct irdma_uah *ah;
+ union ibv_gid sgid;
+ struct irdma_ucreate_ah_resp resp;
+ int err;
+
+ err = ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index,
+ &sgid);
+ if (err) {
+ errno = err;
+ return NULL;
+ }
+
+ ah = calloc(1, sizeof(*ah));
+ if (!ah)
+ return NULL;
+
+ err = ibv_cmd_create_ah(ibpd, &ah->ibv_ah, attr, &resp.ibv_resp,
+ sizeof(resp));
+ if (err) {
+ free(ah);
+ errno = err;
+ return NULL;
+ }
+
+ ah->ah_id = resp.ah_id;
+
+ return &ah->ibv_ah;
+}
+
+/**
+ * irdma_udestroy_ah - destroy the address handle
+ * @ibah: address handle
+ */
+int irdma_udestroy_ah(struct ibv_ah *ibah)
+{
+ struct irdma_uah *ah;
+ int ret;
+
+ ah = to_irdma_uah(ibah);
+
+ ret = ibv_cmd_destroy_ah(ibah);
+ if (ret)
+ return ret;
+
+ free(ah);
+
+ return 0;
+}
+
+/**
+ * irdma_uattach_mcast - Attach qp to multicast group implemented
+ * @qp: The queue pair
+ * @gid:The Global ID for multicast group
+ * @lid: The Local ID
+ */
+int irdma_uattach_mcast(struct ibv_qp *qp, const union ibv_gid *gid,
+ uint16_t lid)
+{
+ return ibv_cmd_attach_mcast(qp, gid, lid);
+}
+
+/**
+ * irdma_udetach_mcast - Detach qp from multicast group
+ * @qp: The queue pair
+ * @gid:The Global ID for multicast group
+ * @lid: The Local ID
+ */
+int irdma_udetach_mcast(struct ibv_qp *qp, const union ibv_gid *gid,
+ uint16_t lid)
+{
+ return ibv_cmd_detach_mcast(qp, gid, lid);
+}
+
+/**
+ * irdma_uresize_cq - resizes a cq
+ * @cq: cq to resize
+ * @cqe: the number of cqes of the new cq
+ */
+int irdma_uresize_cq(struct ibv_cq *cq, int cqe)
+{
+ struct irdma_uvcontext *iwvctx;
+ struct irdma_uk_attrs *uk_attrs;
+ struct irdma_uresize_cq cmd = {};
+ struct ib_uverbs_resize_cq_resp resp = {};
+ struct irdma_ureg_mr reg_mr_cmd = {};
+ struct ib_uverbs_reg_mr_resp reg_mr_resp = {};
+ struct irdma_ucq *iwucq = to_irdma_ucq(cq);
+ struct irdma_cq_buf *cq_buf = NULL;
+ struct irdma_cqe *cq_base = NULL;
+ struct verbs_mr new_mr = {};
+ size_t cq_size;
+ __u32 cq_pages;
+ int cqe_needed;
+ int ret = 0;
+
+ iwvctx = to_irdma_uctx(cq->context);
+ uk_attrs = &iwvctx->uk_attrs;
+
+ if (!(uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE))
+ return EOPNOTSUPP;
+
+ if (cqe > IRDMA_MAX_CQ_SIZE)
+ return EINVAL;
+
+ cqe_needed = cqe + 1;
+ if (uk_attrs->hw_rev > IRDMA_GEN_1)
+ cqe_needed *= 2;
+
+ if (cqe_needed < IRDMA_U_MINCQ_SIZE)
+ cqe_needed = IRDMA_U_MINCQ_SIZE;
+
+ if (cqe_needed == iwucq->cq.cq_size)
+ return 0;
+
+ cq_size = get_cq_total_bytes(cqe_needed);
+ cq_pages = cq_size >> IRDMA_HW_PAGE_SHIFT;
+ cq_base = irdma_alloc_hw_buf(cq_size);
+ if (!cq_base)
+ return ENOMEM;
+
+ memset(cq_base, 0, cq_size);
+
+ cq_buf = malloc(sizeof(*cq_buf));
+ if (!cq_buf) {
+ ret = ENOMEM;
+ goto err_buf;
+ }
+
+ new_mr.ibv_mr.pd = iwucq->vmr.ibv_mr.pd;
+ reg_mr_cmd.reg_type = IW_MEMREG_TYPE_CQ;
+ reg_mr_cmd.cq_pages = cq_pages;
+
+ ret = ibv_cmd_reg_mr(new_mr.ibv_mr.pd, cq_base, cq_size,
+ (uintptr_t)cq_base, IBV_ACCESS_LOCAL_WRITE,
+ &new_mr, ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd),
+ ®_mr_resp, sizeof(reg_mr_resp));
+ if (ret)
+ goto err_dereg_mr;
+
+ ret = pthread_spin_lock(&iwucq->lock);
+ if (ret)
+ goto err_lock;
+
+ cmd.user_cq_buffer = (__u64)((uintptr_t)cq_base);
+ ret = ibv_cmd_resize_cq(&iwucq->verbs_cq.cq, cqe_needed, &cmd.ibv_cmd,
+ sizeof(cmd), &resp, sizeof(resp));
+ if (ret)
+ goto err_resize;
+
+ memcpy(&cq_buf->cq, &iwucq->cq, sizeof(cq_buf->cq));
+ cq_buf->vmr = iwucq->vmr;
+ iwucq->vmr = new_mr;
+ iwucq->cq.ops.iw_cq_resize(&iwucq->cq, cq_base,
+ cqe_needed);
+ iwucq->verbs_cq.cq.cqe = cqe;
+ list_add_tail(&iwucq->resize_list, &cq_buf->list);
+
+ pthread_spin_unlock(&iwucq->lock);
+
+ return ret;
+
+err_resize:
+ pthread_spin_unlock(&iwucq->lock);
+err_lock:
+ ibv_cmd_dereg_mr(&new_mr);
+err_dereg_mr:
+ free(cq_buf);
+err_buf:
+ irdma_free_hw_buf(cq_base, cq_size);
+ return ret;
+}
--
1.8.3.1
^ permalink raw reply related [flat|nested] 10+ messages in thread