From mboxrd@z Thu Jan 1 00:00:00 1970 From: Shachar Beiser Subject: [PATCH v3 1/2] net/mlx5: load libmlx5 and libibverbs in run-time Date: Thu, 23 Nov 2017 15:24:02 +0000 Message-ID: <37757d5cfe32610ef1c7a17d72ea6bc1466023de.1511450393.git.shacharbe@mellanox.com> References: <16cfbe93b923f728d1863153a20d61a253791bf8.1511442287.git.shacharbe@mellanox.com> Cc: Shachar Beiser , Adrien Mazarguil , Nelio Laranjeiro To: dev@dpdk.org Return-path: Received: from mellanox.co.il (mail-il-dmz.mellanox.com [193.47.165.129]) by dpdk.org (Postfix) with ESMTP id 8D9A32BD0 for ; Thu, 23 Nov 2017 16:24:09 +0100 (CET) In-Reply-To: <16cfbe93b923f728d1863153a20d61a253791bf8.1511442287.git.shacharbe@mellanox.com> List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" MLX5 PMD loads libraries: libibverbs and libmlx5. MLX5 PMD is not linked to external libraries. Signed-off-by: Shachar Beiser --- v1: load external libraries in run-time v2 * fix checkpatch warnings v3: * fix checkpatch warnings --- config/common_base | 1 + drivers/net/mlx5/Makefile | 27 +- drivers/net/mlx5/lib/mlx5_dll.c | 756 +++++++++++++++++++++++++++++++++++++++ drivers/net/mlx5/lib/mlx5_dll.h | 94 +++++ drivers/net/mlx5/mlx5.c | 17 +- drivers/net/mlx5/mlx5.h | 4 + drivers/net/mlx5/mlx5_flow.c | 4 + drivers/net/mlx5/mlx5_mac.c | 4 + drivers/net/mlx5/mlx5_mr.c | 4 + drivers/net/mlx5/mlx5_rss.c | 4 + drivers/net/mlx5/mlx5_rxmode.c | 4 + drivers/net/mlx5/mlx5_rxq.c | 4 + drivers/net/mlx5/mlx5_rxtx.c | 4 + drivers/net/mlx5/mlx5_rxtx.h | 6 +- drivers/net/mlx5/mlx5_rxtx_vec.c | 4 + drivers/net/mlx5/mlx5_txq.c | 4 + mk/rte.app.mk | 8 +- 17 files changed, 936 insertions(+), 13 deletions(-) create mode 100644 drivers/net/mlx5/lib/mlx5_dll.c create mode 100644 drivers/net/mlx5/lib/mlx5_dll.h diff --git a/config/common_base b/config/common_base index e74febe..3708de4 100644 --- a/config/common_base +++ b/config/common_base @@ -237,6 +237,7 @@ CONFIG_RTE_LIBRTE_MLX4_TX_MP_CACHE=8 # Compile burst-oriented Mellanox ConnectX-4 & ConnectX-5 (MLX5) PMD # CONFIG_RTE_LIBRTE_MLX5_PMD=n +CONFIG_RTE_LIBRTE_MLX5_DLL=y CONFIG_RTE_LIBRTE_MLX5_DEBUG=n CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE=8 diff --git a/drivers/net/mlx5/Makefile b/drivers/net/mlx5/Makefile index a3984eb..1dc0a05 100644 --- a/drivers/net/mlx5/Makefile +++ b/drivers/net/mlx5/Makefile @@ -53,7 +53,9 @@ SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_rss.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_mr.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_flow.c SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5_socket.c - +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) +SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += lib/mlx5_dll.c +endif # Basic CFLAGS. CFLAGS += -O3 CFLAGS += -std=c11 -Wall -Wextra @@ -64,7 +66,11 @@ CFLAGS += -D_DEFAULT_SOURCE CFLAGS += -D_XOPEN_SOURCE=600 CFLAGS += $(WERROR_FLAGS) CFLAGS += -Wno-strict-prototypes +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) +LDLIBS += -ldl +else LDLIBS += -libverbs -lmlx5 +endif LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring LDLIBS += -lrte_ethdev -lrte_net -lrte_kvargs LDLIBS += -lrte_bus_pci @@ -91,7 +97,11 @@ endif ifdef CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE CFLAGS += -DMLX5_PMD_TX_MP_CACHE=$(CONFIG_RTE_LIBRTE_MLX5_TX_MP_CACHE) endif - +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) +CFLAGS += -DMLX5_PMD_DLL +else +CFLAGS += -UMLX5_PMD_DLL +endif include $(RTE_SDK)/mk/rte.lib.mk # Generate and clean-up mlx5_autoconf.h. @@ -105,26 +115,28 @@ endif mlx5_autoconf.h.new: FORCE +VERBS_H := infiniband/verbs.h +MLX5DV_H := infiniband/mlx5dv.h mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh $Q $(RM) -f -- '$@' $Q sh -- '$<' '$@' \ HAVE_IBV_DEVICE_VXLAN_SUPPORT \ - infiniband/verbs.h \ + $(VERBS_H) \ enum IBV_DEVICE_VXLAN_SUPPORT \ $(AUTOCONF_OUTPUT) $Q sh -- '$<' '$@' \ HAVE_IBV_WQ_FLAG_RX_END_PADDING \ - infiniband/verbs.h \ + $(VERBS_H) \ enum IBV_WQ_FLAG_RX_END_PADDING \ $(AUTOCONF_OUTPUT) $Q sh -- '$<' '$@' \ HAVE_IBV_MLX5_MOD_MPW \ - infiniband/mlx5dv.h \ + $(MLX5DV_H) \ enum MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED \ $(AUTOCONF_OUTPUT) $Q sh -- '$<' '$@' \ HAVE_IBV_MLX5_MOD_CQE_128B_COMP \ - infiniband/mlx5dv.h \ + $(MLX5DV_H) \ enum MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP \ $(AUTOCONF_OUTPUT) $Q sh -- '$<' '$@' \ @@ -144,10 +156,9 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh $(AUTOCONF_OUTPUT) $Q sh -- '$<' '$@' \ HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \ - infiniband/verbs.h \ + $(VERBS_H) \ enum IBV_FLOW_SPEC_ACTION_COUNT \ $(AUTOCONF_OUTPUT) - # Create mlx5_autoconf.h or update it in case it differs from the new one. mlx5_autoconf.h: mlx5_autoconf.h.new diff --git a/drivers/net/mlx5/lib/mlx5_dll.c b/drivers/net/mlx5/lib/mlx5_dll.c new file mode 100644 index 0000000..2c4ac74 --- /dev/null +++ b/drivers/net/mlx5/lib/mlx5_dll.c @@ -0,0 +1,756 @@ +/*- + * BSD LICENSE + * + * Copyright 2015 6WIND S.A. + * Copyright 2015 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif +#include +#include +#include +#include +#include +#include "mlx5_dll.h" + +#define VERBS_LIB_DIR "/usr/lib64/libibverbs" +#define MLX5_LIB_DIR "/usr/lib64/libmlx5" +#define DIR_LENGTH 25 +/** + * Load a libibverbs and libmlx5 symbols table. + * + * @return + * 0 on success. + */ +int mlx5_load_libs(void) +{ + void *dlhandle; + int ret; + + dlhandle = mlx5_lib_load("ibverbs.so"); + if (dlhandle == NULL) { + ERROR("cannot load ibverbs.so"); + return -1; + } + ret = mlx5_lverbs_function_register(dlhandle); + if (ret == -1) { + ERROR("cannot register a function in libverbs.so "); + return ret; + } + dlhandle = mlx5_lib_load("mlx5.so"); + if (dlhandle == NULL) { + ERROR("cannot load mlx5.so"); + return -1; + } + ret = mlx5_lmlx5_function_register(dlhandle); + if (ret == -1) { + ERROR("cannot register a function in lmlx5.so "); + return ret; + } + return 0; +} +/** + * Load a libibverbs or libmlx5 symbols table. + * + * @param name[in] + * The library name. + * @return + * dlhandle on success. + */ +void *mlx5_lib_load(const char *name) +{ + char *so_name; + void *dlhandle; + char lib[DIR_LENGTH]; + + if (strcmp(name, "ibverbs.so")) + strcpy(lib, VERBS_LIB_DIR); + else + strcpy(lib, MLX5_LIB_DIR); + /* If the name is an absolute path then open that path after appending + * the trailer suffix + */ + if (name[0] == '/') { + if (asprintf(&so_name, "%s", name) < 0) + goto out_asprintf; + dlhandle = dlopen(so_name, RTLD_LAZY); + if (!dlhandle) + goto out_dlopen; + free(so_name); + return dlhandle; + } + /* If configured with a provider plugin path then try that next */ + if (strlen(lib) > 1) { + if (asprintf(&so_name, "%s/lib%s", lib, name) < 0) + goto out_asprintf; + dlhandle = dlopen(so_name, RTLD_LAZY); + free(so_name); + if (dlhandle) + return dlhandle; + } + /* Otherwise use the system library search path. This is the historical + * behavior of libibverbs + */ + if (asprintf(&so_name, "lib%s", name) < 0) + goto out_asprintf; + dlhandle = dlopen(so_name, RTLD_LAZY); + if (!dlhandle) + goto out_dlopen; + free(so_name); + return dlhandle; +out_asprintf: + ERROR("Warning: couldn't load driver '%s'.\n", name); + return NULL; +out_dlopen: + ERROR("Warning: couldn't load driver '%s': %s\n", so_name, dlerror()); + free(so_name); + return NULL; +} + +/** + * Register libibverbs functions apis . + * + * @param handle[in] + * The library handle. + * @return + * 0 on success. + */ +int mlx5_lverbs_function_register(void *handle) +{ + char *error; + int ret = 0; + + *(void **)(&mlx5_libverbs.ibv_destroy_qp) = + dlsym(handle, "ibv_destroy_qp"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_destroy_cq) = + dlsym(handle, "ibv_destroy_cq"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_dealloc_pd) = + dlsym(handle, "ibv_dealloc_pd"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_get_device_list) = + dlsym(handle, "ibv_get_device_list"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_open_device) = + dlsym(handle, "ibv_open_device"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_query_device) = + dlsym(handle, "ibv_query_device"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_close_device) = + dlsym(handle, "ibv_close_device"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_free_device_list) = + dlsym(handle, "ibv_free_device_list"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_alloc_pd) = + dlsym(handle, "ibv_alloc_pd"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_get_device_name) = + dlsym(handle, "ibv_get_device_name"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_fork_init) = + dlsym(handle, "ibv_fork_init"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_destroy_comp_channel) = + dlsym(handle, "ibv_destroy_comp_channel"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_dereg_mr) = + dlsym(handle, "ibv_dereg_mr"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_create_comp_channel) = + dlsym(handle, "ibv_create_comp_channel"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_get_cq_event) = + dlsym(handle, "ibv_get_cq_event"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_create_qp) = + dlsym(handle, "ibv_create_qp"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_ack_async_event) = + dlsym(handle, "ibv_ack_async_event"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_get_async_event) = + dlsym(handle, "ibv_get_async_event"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_reg_mr) = + dlsym(handle, "ibv_reg_mr"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_create_cq) = + dlsym(handle, "ibv_create_cq"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_modify_qp) = + dlsym(handle, "ibv_modify_qp"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_libverbs.ibv_query_port) = + dlsym(handle, "ibv_query_port"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + return 0; +exit: + return -ret; +} + +/** + * Register libmlx5 functions apis . + * + * @param handle[in] + * The library handle. + * @return + * 0 on success. + */ +int mlx5_lmlx5_function_register(void *handle) +{ + char *error; + int ret = 0; + + *(void **)(&mlx5_lmlx5.mlx5dv_create_cq) = + dlsym(handle, "mlx5dv_create_cq"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_lmlx5.mlx5dv_init_obj) = + dlsym(handle, "mlx5dv_init_obj"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_lmlx5.mlx5dv_query_device) = + dlsym(handle, "mlx5dv_query_device"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + *(void **)(&mlx5_lmlx5.mlx5dv_set_context_attr) = + dlsym(handle, "mlx5dv_set_context_attr"); + error = dlerror(); + if ((error) != NULL) { + ERROR("%s\n", error); + ret = EINVAL; + goto exit; + } + return 0; +exit: + return -ret; +} + +/** + * Function pointer calls mlx5 API . + * + * @param context[in] + * The ibv context. + * @param cq_attr[in] + * The completion queue attribute. + * @param mlx5_cq_attr[in] + * The mlx5 completion queue attribute. + * @return + * a pointer on success. + */ +struct ibv_cq_ex *mlx5dv_create_cq(struct ibv_context *context, + struct ibv_cq_init_attr_ex *cq_attr, + struct mlx5dv_cq_init_attr *mlx5_cq_attr) +{ + return mlx5_lmlx5.mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr); +} +/** + * Function pointer calls mlx5 API . + * + * @param mlx5dv_obj[in] + * The mlx5 object. + * @param obj_type[in] + * The object type. + * @return + * a pointer on success. + */ +int mlx5dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type) +{ + return mlx5_lmlx5.mlx5dv_init_obj(obj, obj_type); +} +/** + * Function pointer calls mlx5 API . + * + * @param ctx_in[in] + * The ibverbs context. + * @param attrs_out[out] + * The mlx5 context. + * @return + * 0 on success. + */ +int mlx5dv_query_device(struct ibv_context *ctx_in, + struct mlx5dv_context *attrs_out) +{ + return mlx5_lmlx5.mlx5dv_query_device(ctx_in, attrs_out); +} +/** + * Function pointer calls mlx5 API . + * + * @param context[in] + * The ibverbs context. + * @param type[in] + * The mlx5 context attribute type. + * @param attr[in] + * The mlx5 context attribute. + * @return + * 0 on success. + */ +int mlx5dv_set_context_attr(struct ibv_context *context, + enum mlx5dv_set_ctx_attr_type type, void *attr) +{ + return mlx5_lmlx5.mlx5dv_set_context_attr(context, type, attr); +} +/** + * Function pointer calls libibverbs API . + * + * @param qp[in] + * Queue pair pointer. + * @return + * 0 on success. + */ +int ibv_destroy_qp(struct ibv_qp *qp) +{ + return mlx5_libverbs.ibv_destroy_qp(qp); +} +/** + * Function pointer calls libibverbs API . + * + * @param cq[in] + * Completion queue pointer. + * @return + * 0 on success. + */ +int ibv_destroy_cq(struct ibv_cq *cq) +{ + return mlx5_libverbs.ibv_destroy_cq(cq); +} +/** + * Function pointer calls libibverbs API . + * + * @param pd[in] + * Protection domain. + * @return + * 0 on success. + */ +int ibv_dealloc_pd(struct ibv_pd *pd) +{ + return mlx5_libverbs.ibv_dealloc_pd(pd); +} +/** + * Function pointer calls libibverbs API . + * + * @param num_devices[in] + * The number of devices. + * @return + * 0 on success. + */ +struct ibv_device **ibv_get_device_list(int *num_devices) +{ + return mlx5_libverbs.ibv_get_device_list(num_devices); +} +/** + * Function pointer calls libibverbs API . + * + * @param context[in] + * The ibverbs context. + * @param device_attr[in] + * The device attribute. + * @return + * 0 on success. + */ +int ibv_query_device(struct ibv_context *context, + struct ibv_device_attr *device_attr) +{ + return mlx5_libverbs.ibv_query_device(context, device_attr); +} +/** + * Function pointer calls libibverbs API . + * + * @param context[in] + * The ibverbs context. + * @return + * 0 on success. + */ +int ibv_close_device(struct ibv_context *context) +{ + return mlx5_libverbs.ibv_close_device(context); +} +/** + * Function pointer calls libibverbs API . + * + * @param list[in] + * The list of devices. + */ +void ibv_free_device_list(struct ibv_device **list) +{ + return mlx5_libverbs.ibv_free_device_list(list); +} +/** + * Function pointer calls libibverbs API . + * + * @param device[in] + * A device pointer. + * @return + * The context on success. + */ +struct ibv_context *ibv_open_device(struct ibv_device *device) +{ + return mlx5_libverbs.ibv_open_device(device); +} +/** + * Function pointer calls libibverbs API . + * + * @param context[in] + * An ibverbs context. + * @return + * protection domain pointer on success. + */ +struct ibv_pd *ibv_alloc_pd(struct ibv_context *context) +{ + return mlx5_libverbs.ibv_alloc_pd(context); +} +/** + * Function pointer calls libibverbs API . + * + * @param device[in] + * An ibverbs device pointer. + * @return + * device name on success. + */ +const char *ibv_get_device_name(struct ibv_device *device) +{ + return mlx5_libverbs.ibv_get_device_name(device); +} +/** + * Function pointer calls libibverbs API . + * + * @return + * 0 on success. + */ +int ibv_fork_init(void) +{ + return mlx5_libverbs.ibv_fork_init(); +} +/** + * Function pointer calls libibverbs API . + * + * @param channel[in] + * A completion channel. + * @return + * 0 on success. + */ +int ibv_destroy_comp_channel(struct ibv_comp_channel *channel) +{ + return mlx5_libverbs.ibv_destroy_comp_channel(channel); +} +/** + * Function pointer calls libibverbs API . + * + * @param mr[in] + * A pointer to memory region. + * @return + * 0 on success. + */ +int ibv_dereg_mr(struct ibv_mr *mr) +{ + return mlx5_libverbs.ibv_dereg_mr(mr); +} +/** + * Function pointer calls libibverbs API . + * + * @param context[in] + * A pointer to ibverbs context. + * @return + * 0 on success. + */ +struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context) +{ + return mlx5_libverbs.ibv_create_comp_channel(context); +} +/** + * Function pointer calls libibverbs API . + * + * @param channel[in] + * A pointer to completion channel. + * @param cq[in] + * A pointer to completion queue. + * @param cq_context[in] + * A pointer to completion queue context. + * @return + * 0 on success. + */ +int ibv_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq, + void **cq_context) +{ + return mlx5_libverbs.ibv_get_cq_event(channel, cq, cq_context); +} +/** + * Function pointer calls libibverbs API . + * + * @param cq[in] + * A pointer to completion queue. + * @param nevents[in] + * The number of events. + * @return + * 0 on success. + */ +void ibv_ack_cq_events(struct ibv_cq *cq, unsigned int nevents) +{ + return mlx5_libverbs.ibv_ack_cq_events(cq, nevents); +} +/** + * Function pointer calls libibverbs API . + * + * @param pd[in] + * The protection domain pointer. + * @param qp_init_attr[in] + * The queue pair attributes. + * @return + * A pointer to queue pair on success. + */ +struct ibv_qp *ibv_create_qp(struct ibv_pd *pd, + struct ibv_qp_init_attr *qp_init_attr) +{ + return mlx5_libverbs.ibv_create_qp(pd, qp_init_attr); +} +/** + * Function pointer calls libibverbs API . + * + * @param event[in] + * A pointer to a-synchronic event. + */ +void ibv_ack_async_event(struct ibv_async_event *event) +{ + return mlx5_libverbs.ibv_ack_async_event(event); +} +/** + * Function pointer calls libibverbs API . + * + * @param context[in] + * A ibverbs context. + * @param event[in] + * A pointer to a-synchronic event. + */ +int ibv_get_async_event(struct ibv_context *context, + struct ibv_async_event *event) +{ + return mlx5_libverbs.ibv_get_async_event(context, event); +} +/** + * Function pointer calls libibverbs API . + * + * @param pd[in] + * The protection domain pointer. + * @param addr[in] + * The address. + * @param length[in] + * The length. + * @param access[in] + * The access type. + * @return + * A pointer to memory region on success. + */ +struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr, size_t length, + int access) +{ + return mlx5_libverbs.ibv_reg_mr(pd, addr, length, access); +} +/** + * Function pointer calls libibverbs API . + * + * @param context[in] + * Ibverbs context. + * @param cqe[in] + * A completion queue entry. + * @param cq_context[in] + * Completion queue context. + * @param channel[in] + * The completion channel. + * @param comp_vector[in] + * Completion vector. + * @return + * 0 on success. + */ +struct ibv_cq *ibv_create_cq(struct ibv_context *context, int cqe, + void *cq_context, struct ibv_comp_channel *channel, int comp_vector) +{ + return mlx5_libverbs.ibv_create_cq(context, cqe, cq_context, channel, + comp_vector); +} +/** + * Function pointer calls libibverbs API . + * + * @param qp[in] + * Queue pair. + * @param attr[in] + * The queue pair attribute. + * @param attr_mask[in] + * Attribute mask. + * @return + * 0 on success. + */ +int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask) +{ + return mlx5_libverbs.ibv_modify_qp(qp, attr, attr_mask); +} +/** + * Function pointer calls libibverbs API . + * + * @param context[in] + * An ibverbs context. + * @param port_num[in] + * The port number. + * @param port_attr[in] + * Port attribute. + * @return + * 0 on success. + */ +int ibv_query_port(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr) +{ + port_attr->link_layer = IBV_LINK_LAYER_UNSPECIFIED; + port_attr->reserved = 0; + return mlx5_libverbs.ibv_query_port(context, port_num, port_attr); +} diff --git a/drivers/net/mlx5/lib/mlx5_dll.h b/drivers/net/mlx5/lib/mlx5_dll.h new file mode 100644 index 0000000..120083d --- /dev/null +++ b/drivers/net/mlx5/lib/mlx5_dll.h @@ -0,0 +1,94 @@ +/*- + * BSD LICENSE + * + * Copyright 2015 6WIND S.A. + * Copyright 2015 Mellanox. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of 6WIND S.A. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef RTE_PMD_MLX5_IBVERBS_H_ +#define RTE_PMD_MLX5_IBVERBS_H_ +#include +#include + +void *mlx5_lib_load(const char *name); +int mlx5_lverbs_function_register(void *handle); +int mlx5_lmlx5_function_register(void *handle); +int mlx5_load_libs(void); +#undef ibv_query_port + +struct { + int size; + int version; + int (*ibv_destroy_qp)(struct ibv_qp *qp); + int (*ibv_destroy_cq)(struct ibv_cq *cq); + int (*ibv_dealloc_pd)(struct ibv_pd *pd); + struct ibv_device **(*ibv_get_device_list)(int *num_devices); + int (*ibv_query_device)(struct ibv_context *context, + struct ibv_device_attr *device_attr); + int (*ibv_close_device)(struct ibv_context *context); + void (*ibv_free_device_list)(struct ibv_device **list); + struct ibv_context *(*ibv_open_device)(struct ibv_device *device); + struct ibv_pd *(*ibv_alloc_pd)(struct ibv_context *context); + const char *(*ibv_get_device_name)(struct ibv_device *device); + int (*ibv_fork_init)(void); + int (*ibv_destroy_comp_channel)(struct ibv_comp_channel *channel); + int (*ibv_dereg_mr)(struct ibv_mr *mr); + struct ibv_comp_channel *(*ibv_create_comp_channel)( + struct ibv_context *context); + int (*ibv_get_cq_event)(struct ibv_comp_channel *channel, + struct ibv_cq **cq, void **cq_context); + void (*ibv_ack_cq_events)(struct ibv_cq *cq, unsigned int nevents); + struct ibv_qp *(*ibv_create_qp)(struct ibv_pd *pd, + struct ibv_qp_init_attr *qp_init_attr); + void (*ibv_ack_async_event)(struct ibv_async_event *event); + int (*ibv_get_async_event)(struct ibv_context *context, + struct ibv_async_event *event); + struct ibv_mr *(*ibv_reg_mr)(struct ibv_pd *pd, void *addr, + size_t length, int access); + struct ibv_cq *(*ibv_create_cq)(struct ibv_context *context, int cqe, + void *cq_context, struct ibv_comp_channel *channel, + int comp_vector); + int (*ibv_modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, + int attr_mask); + int (*ibv_query_port)(struct ibv_context *context, uint8_t port_num, + struct ibv_port_attr *port_attr); +} mlx5_libverbs; + +struct { + int size; + int version; + struct ibv_cq_ex *(*mlx5dv_create_cq)(struct ibv_context *context, + struct ibv_cq_init_attr_ex *cq_attr, + struct mlx5dv_cq_init_attr *mlx5_cq_attr); + int (*mlx5dv_init_obj)(struct mlx5dv_obj *obj, uint64_t obj_type); + int (*mlx5dv_query_device)(struct ibv_context *ctx_in, + struct mlx5dv_context *attrs_out); + int (*mlx5dv_set_context_attr)(struct ibv_context *context, + enum mlx5dv_set_ctx_attr_type type, void *attr); +} mlx5_lmlx5; +#endif /* RTE_PMD_MLX5_IBVERBS_H_ */ diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index 0548d17..323ced7 100644 --- a/drivers/net/mlx5/mlx5.c +++ b/drivers/net/mlx5/mlx5.c @@ -30,7 +30,8 @@ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ - +#define _GNU_SOURCE +#include #include #include #include @@ -39,13 +40,17 @@ #include #include #include - +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif @@ -1029,6 +1034,9 @@ struct mlx5_args { rte_mlx5_pmd_init(void) { /* Build the static table for ptype conversion. */ +#ifdef MLX5_PMD_DLL + int ret; +#endif mlx5_set_ptype_table(); /* * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use @@ -1040,6 +1048,11 @@ struct mlx5_args { /* Match the size of Rx completion entry to the size of a cacheline. */ if (RTE_CACHE_LINE_SIZE == 128) setenv("MLX5_CQE_SIZE", "128", 0); +#ifdef MLX5_PMD_DLL + ret = mlx5_load_libs(); + if (ret != 0) + return; +#endif ibv_fork_init(); rte_pci_register(&mlx5_driver); } diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index e6a69b8..1dde985 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -46,7 +46,11 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif diff --git a/drivers/net/mlx5/mlx5_flow.c b/drivers/net/mlx5/mlx5_flow.c index f32dfdd..ffd419c 100644 --- a/drivers/net/mlx5/mlx5_flow.c +++ b/drivers/net/mlx5/mlx5_flow.c @@ -39,7 +39,11 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif diff --git a/drivers/net/mlx5/mlx5_mac.c b/drivers/net/mlx5/mlx5_mac.c index d17b991..9e8902b 100644 --- a/drivers/net/mlx5/mlx5_mac.c +++ b/drivers/net/mlx5/mlx5_mac.c @@ -46,7 +46,11 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif diff --git a/drivers/net/mlx5/mlx5_mr.c b/drivers/net/mlx5/mlx5_mr.c index 6b29eed..c91fcdf 100644 --- a/drivers/net/mlx5/mlx5_mr.c +++ b/drivers/net/mlx5/mlx5_mr.c @@ -36,7 +36,11 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif diff --git a/drivers/net/mlx5/mlx5_rss.c b/drivers/net/mlx5/mlx5_rss.c index f3de46d..9f58147 100644 --- a/drivers/net/mlx5/mlx5_rss.c +++ b/drivers/net/mlx5/mlx5_rss.c @@ -42,7 +42,11 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif diff --git a/drivers/net/mlx5/mlx5_rxmode.c b/drivers/net/mlx5/mlx5_rxmode.c index 0ef2cdf..f9f7d6f 100644 --- a/drivers/net/mlx5/mlx5_rxmode.c +++ b/drivers/net/mlx5/mlx5_rxmode.c @@ -40,7 +40,11 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 85399ef..dfd6400 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -44,8 +44,12 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c index 2d30c50..e2215aa 100644 --- a/drivers/net/mlx5/mlx5_rxtx.c +++ b/drivers/net/mlx5/mlx5_rxtx.c @@ -41,8 +41,12 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h index d34f3cc..7419f2c 100644 --- a/drivers/net/mlx5/mlx5_rxtx.h +++ b/drivers/net/mlx5/mlx5_rxtx.h @@ -43,12 +43,14 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include -#include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif - #include #include #include diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.c b/drivers/net/mlx5/mlx5_rxtx_vec.c index ba6c8ce..fc7f701 100644 --- a/drivers/net/mlx5/mlx5_rxtx_vec.c +++ b/drivers/net/mlx5/mlx5_rxtx_vec.c @@ -41,8 +41,12 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c index 9c5860f..8f52525 100644 --- a/drivers/net/mlx5/mlx5_txq.c +++ b/drivers/net/mlx5/mlx5_txq.c @@ -44,7 +44,11 @@ #ifdef PEDANTIC #pragma GCC diagnostic ignored "-Wpedantic" #endif +#ifdef MLX5_PMD_DLL +#include "lib/mlx5_dll.h" +#else #include +#endif #ifdef PEDANTIC #pragma GCC diagnostic error "-Wpedantic" #endif diff --git a/mk/rte.app.mk b/mk/rte.app.mk index 6a6a745..0542c9a 100644 --- a/mk/rte.app.mk +++ b/mk/rte.app.mk @@ -51,7 +51,9 @@ endif # Link only the libraries used in the application LDFLAGS += --as-needed - +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) +LDFLAGS += -ldl +endif # default path for libs _LDLIBS-y += -L$(RTE_SDK_BIN)/lib @@ -142,7 +144,11 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_KNI) += -lrte_pmd_kni endif _LDLIBS-$(CONFIG_RTE_LIBRTE_LIO_PMD) += -lrte_pmd_lio _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX4_PMD) += -lrte_pmd_mlx4 -libverbs -lmlx4 +ifeq ($(CONFIG_RTE_LIBRTE_MLX5_DLL),y) +_LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 +else _LDLIBS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += -lrte_pmd_mlx5 -libverbs -lmlx5 +endif _LDLIBS-$(CONFIG_RTE_LIBRTE_MRVL_PMD) += -lrte_pmd_mrvl -L$(LIBMUSDK_PATH)/lib -lmusdk _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD) += -lrte_pmd_nfp _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL) += -lrte_pmd_null -- 1.8.3.1