[PATCH v3 02/20] thunderx/nicvf: add pmd skeleton

[Jerin Jacob <jerin.jacob@caviumnetworks.com>]

Introduce driver initialization and enable build infrastructure for
nicvf pmd driver.

By default, It is enabled only for defconfig_arm64-thunderx-*
config as it is an inbuilt NIC device.

Signed-off-by: Jerin Jacob <jerin.jacob@caviumnetworks.com>
Signed-off-by: Maciej Czekaj <maciej.czekaj@caviumnetworks.com>
Signed-off-by: Kamil Rytarowski <Kamil.Rytarowski@caviumnetworks.com>
Signed-off-by: Zyta Szpak <zyta.szpak@semihalf.com>
Signed-off-by: Slawomir Rosek <slawomir.rosek@semihalf.com>
Signed-off-by: Radoslaw Biernacki <rad@semihalf.com>
---
 config/common_base                                 |  10 +
 config/defconfig_arm64-thunderx-linuxapp-gcc       |  10 +
 drivers/net/Makefile                               |   1 +
 drivers/net/thunderx/Makefile                      |  63 ++++++
 drivers/net/thunderx/nicvf_ethdev.c                | 251 +++++++++++++++++++++
 drivers/net/thunderx/nicvf_ethdev.h                |  48 ++++
 drivers/net/thunderx/nicvf_logs.h                  |  83 +++++++
 drivers/net/thunderx/nicvf_struct.h                | 124 ++++++++++
 .../thunderx/rte_pmd_thunderx_nicvf_version.map    |   4 +
 mk/rte.app.mk                                      |   2 +
 10 files changed, 596 insertions(+)
 create mode 100644 drivers/net/thunderx/Makefile
 create mode 100644 drivers/net/thunderx/nicvf_ethdev.c
 create mode 100644 drivers/net/thunderx/nicvf_ethdev.h
 create mode 100644 drivers/net/thunderx/nicvf_logs.h
 create mode 100644 drivers/net/thunderx/nicvf_struct.h
 create mode 100644 drivers/net/thunderx/rte_pmd_thunderx_nicvf_version.map

diff --git a/config/common_base b/config/common_base
index 47c26f6..ad5686b 100644
--- a/config/common_base
+++ b/config/common_base
@@ -259,6 +259,16 @@ CONFIG_RTE_LIBRTE_PMD_SZEDATA2=n
 CONFIG_RTE_LIBRTE_PMD_SZEDATA2_AS=0
 
 #
+# Compile burst-oriented Cavium Thunderx NICVF PMD driver
+#
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD=n
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_INIT=n
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_RX=n
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_TX=n
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_DRIVER=n
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_MBOX=n
+
+#
 # Compile burst-oriented VIRTIO PMD driver
 #
 CONFIG_RTE_LIBRTE_VIRTIO_PMD=y
diff --git a/config/defconfig_arm64-thunderx-linuxapp-gcc b/config/defconfig_arm64-thunderx-linuxapp-gcc
index fe5e987..7940bbd 100644
--- a/config/defconfig_arm64-thunderx-linuxapp-gcc
+++ b/config/defconfig_arm64-thunderx-linuxapp-gcc
@@ -34,3 +34,13 @@
 CONFIG_RTE_MACHINE="thunderx"
 
 CONFIG_RTE_CACHE_LINE_SIZE=128
+
+#
+# Compile Cavium Thunderx NICVF PMD driver
+#
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD=y
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_INIT=n
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_RX=n
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_TX=n
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_DRIVER=n
+CONFIG_RTE_LIBRTE_THUNDERX_NICVF_DEBUG_MBOX=n
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index 6ba7658..0e29a33 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -50,6 +50,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_PMD_PCAP) += pcap
 DIRS-$(CONFIG_RTE_LIBRTE_QEDE_PMD) += qede
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_RING) += ring
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_SZEDATA2) += szedata2
+DIRS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += thunderx
 DIRS-$(CONFIG_RTE_LIBRTE_VIRTIO_PMD) += virtio
 DIRS-$(CONFIG_RTE_LIBRTE_VMXNET3_PMD) += vmxnet3
 DIRS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT) += xenvirt
diff --git a/drivers/net/thunderx/Makefile b/drivers/net/thunderx/Makefile
new file mode 100644
index 0000000..eb9f100
--- /dev/null
+++ b/drivers/net/thunderx/Makefile
@@ -0,0 +1,63 @@
+#   BSD LICENSE
+#
+#   Copyright(c) 2016 Cavium Networks. All rights reserved.
+#   All rights reserved.
+#
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in
+#       the documentation and/or other materials provided with the
+#       distribution.
+#     * Neither the name of Cavium Networks nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+#
+# library name
+#
+LIB = librte_pmd_thunderx_nicvf.a
+
+CFLAGS += $(WERROR_FLAGS)
+
+EXPORT_MAP := rte_pmd_thunderx_nicvf_version.map
+
+LIBABIVER := 1
+
+OBJS_BASE_DRIVER=$(patsubst %.c,%.o,$(notdir $(wildcard $(SRCDIR)/base/*.c)))
+$(foreach obj, $(OBJS_BASE_DRIVER), $(eval CFLAGS_$(obj)+=$(CFLAGS_BASE_DRIVER)))
+
+VPATH += $(SRCDIR)/base
+
+#
+# all source are stored in SRCS-y
+#
+SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_hw.c
+SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_mbox.c
+SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_ethdev.c
+
+
+# this lib depends upon:
+DEPDIRS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += lib/librte_eal lib/librte_ether
+DEPDIRS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += lib/librte_mempool lib/librte_mbuf
+
+include $(RTE_SDK)/mk/rte.lib.mk
diff --git a/drivers/net/thunderx/nicvf_ethdev.c b/drivers/net/thunderx/nicvf_ethdev.c
new file mode 100644
index 0000000..45bfc13
--- /dev/null
+++ b/drivers/net/thunderx/nicvf_ethdev.c
@@ -0,0 +1,251 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Cavium networks Ltd. 2016.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium networks nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdarg.h>
+#include <inttypes.h>
+#include <netinet/in.h>
+#include <sys/queue.h>
+#include <sys/timerfd.h>
+
+#include <rte_common.h>
+#include <rte_byteorder.h>
+#include <rte_cycles.h>
+#include <rte_interrupts.h>
+#include <rte_log.h>
+#include <rte_debug.h>
+#include <rte_pci.h>
+#include <rte_atomic.h>
+#include <rte_branch_prediction.h>
+#include <rte_memory.h>
+#include <rte_memzone.h>
+#include <rte_tailq.h>
+#include <rte_eal.h>
+#include <rte_alarm.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_malloc.h>
+#include <rte_random.h>
+#include <rte_dev.h>
+
+#include "base/nicvf_plat.h"
+
+#include "nicvf_ethdev.h"
+
+#include "nicvf_logs.h"
+
+static void
+nicvf_interrupt(void *arg)
+{
+	struct nicvf *nic = (struct nicvf *)arg;
+
+	nicvf_reg_poll_interrupts(nic);
+
+	rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
+				nicvf_interrupt, nic);
+}
+
+static int
+nicvf_periodic_alarm_start(struct nicvf *nic)
+{
+	return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
+					nicvf_interrupt, nic);
+}
+
+static int
+nicvf_periodic_alarm_stop(struct nicvf *nic)
+{
+	return rte_eal_alarm_cancel(nicvf_interrupt, nic);
+}
+
+/* Initialize and register driver with DPDK Application */
+static const struct eth_dev_ops nicvf_eth_dev_ops = {
+};
+
+static int
+nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
+{
+	int ret;
+	struct rte_pci_device *pci_dev;
+	struct nicvf *nic = nicvf_pmd_priv(eth_dev);
+
+	PMD_INIT_FUNC_TRACE();
+
+	eth_dev->dev_ops = &nicvf_eth_dev_ops;
+
+	pci_dev = eth_dev->pci_dev;
+	rte_eth_copy_pci_info(eth_dev, pci_dev);
+
+	nic->device_id = pci_dev->id.device_id;
+	nic->vendor_id = pci_dev->id.vendor_id;
+	nic->subsystem_device_id = pci_dev->id.subsystem_device_id;
+	nic->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id;
+	nic->eth_dev = eth_dev;
+
+	PMD_INIT_LOG(DEBUG, "nicvf: device (%x:%x) %u:%u:%u:%u",
+			pci_dev->id.vendor_id, pci_dev->id.device_id,
+			pci_dev->addr.domain, pci_dev->addr.bus,
+			pci_dev->addr.devid, pci_dev->addr.function);
+
+	nic->reg_base = (uintptr_t)pci_dev->mem_resource[0].addr;
+	if (!nic->reg_base) {
+		PMD_INIT_LOG(ERR, "Failed to map BAR0");
+		ret = -ENODEV;
+		goto fail;
+	}
+
+	nicvf_disable_all_interrupts(nic);
+
+	ret = nicvf_periodic_alarm_start(nic);
+	if (ret) {
+		PMD_INIT_LOG(ERR, "Failed to start period alarm");
+		goto fail;
+	}
+
+	ret = nicvf_mbox_check_pf_ready(nic);
+	if (ret) {
+		PMD_INIT_LOG(ERR, "Failed to get ready message from PF");
+		goto alarm_fail;
+	} else {
+		PMD_INIT_LOG(INFO,
+			"node=%d vf=%d mode=%s sqs=%s loopback_supported=%s",
+			nic->node, nic->vf_id,
+			nic->tns_mode == NIC_TNS_MODE ? "tns" : "tns-bypass",
+			nic->sqs_mode ? "true" : "false",
+			nic->loopback_supported ? "true" : "false"
+			);
+	}
+
+	if (nic->sqs_mode) {
+		PMD_INIT_LOG(INFO, "Unsupported SQS VF detected, Detaching...");
+		/* Detach port by returning Postive error number */
+		ret = ENOTSUP;
+		goto alarm_fail;
+	}
+
+	eth_dev->data->mac_addrs = rte_zmalloc("mac_addr", ETHER_ADDR_LEN, 0);
+	if (eth_dev->data->mac_addrs == NULL) {
+		PMD_INIT_LOG(ERR, "Failed to allocate memory for mac addr");
+		ret = -ENOMEM;
+		goto alarm_fail;
+	}
+	if (is_zero_ether_addr((struct ether_addr *)nic->mac_addr))
+		eth_random_addr(&nic->mac_addr[0]);
+
+	ether_addr_copy((struct ether_addr *)nic->mac_addr,
+			&eth_dev->data->mac_addrs[0]);
+
+	ret = nicvf_mbox_set_mac_addr(nic, nic->mac_addr);
+	if (ret) {
+		PMD_INIT_LOG(ERR, "Failed to set mac addr");
+		goto malloc_fail;
+	}
+
+	ret = nicvf_base_init(nic);
+	if (ret) {
+		PMD_INIT_LOG(ERR, "Failed to execute nicvf_base_init");
+		goto malloc_fail;
+	}
+
+	ret = nicvf_mbox_get_rss_size(nic);
+	if (ret) {
+		PMD_INIT_LOG(ERR, "Failed to get rss table size");
+		goto malloc_fail;
+	}
+
+	PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x",
+		eth_dev->data->port_id, nic->vendor_id, nic->device_id,
+		nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2],
+		nic->mac_addr[3], nic->mac_addr[4], nic->mac_addr[5]);
+
+	return 0;
+
+malloc_fail:
+	rte_free(eth_dev->data->mac_addrs);
+alarm_fail:
+	nicvf_periodic_alarm_stop(nic);
+fail:
+	return ret;
+}
+
+static const struct rte_pci_id pci_id_nicvf_map[] = {
+	{
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVICE_ID_THUNDERX_PASS1_NICVF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS1_NICVF,
+	},
+	{
+		.vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.device_id = PCI_DEVICE_ID_THUNDERX_PASS2_NICVF,
+		.subsystem_vendor_id = PCI_VENDOR_ID_CAVIUM,
+		.subsystem_device_id = PCI_SUB_DEVICE_ID_THUNDERX_PASS2_NICVF,
+	},
+	{
+		.vendor_id = 0,
+	},
+};
+
+static struct eth_driver rte_nicvf_pmd = {
+	.pci_drv = {
+		.name = "rte_nicvf_pmd",
+		.id_table = pci_id_nicvf_map,
+		.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
+	},
+	.eth_dev_init = nicvf_eth_dev_init,
+	.dev_private_size = sizeof(struct nicvf),
+};
+
+static int
+rte_nicvf_pmd_init(const char *name __rte_unused, const char *para __rte_unused)
+{
+	PMD_INIT_FUNC_TRACE();
+	PMD_INIT_LOG(INFO, "librte_pmd_thunderx nicvf version %s",
+			THUNDERX_NICVF_PMD_VERSION);
+
+	rte_eth_driver_register(&rte_nicvf_pmd);
+	return 0;
+}
+
+static struct rte_driver rte_nicvf_driver = {
+	.name = "nicvf_driver",
+	.type = PMD_PDEV,
+	.init = rte_nicvf_pmd_init,
+};
+
+PMD_REGISTER_DRIVER(rte_nicvf_driver);
diff --git a/drivers/net/thunderx/nicvf_ethdev.h b/drivers/net/thunderx/nicvf_ethdev.h
new file mode 100644
index 0000000..d4d2071
--- /dev/null
+++ b/drivers/net/thunderx/nicvf_ethdev.h
@@ -0,0 +1,48 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Cavium networks Ltd. 2016.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium networks nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __THUNDERX_NICVF_ETHDEV_H__
+#define __THUNDERX_NICVF_ETHDEV_H__
+
+#include <rte_ethdev.h>
+
+#define THUNDERX_NICVF_PMD_VERSION      "1.0"
+
+#define NICVF_INTR_POLL_INTERVAL_MS	50
+
+static inline struct nicvf *
+nicvf_pmd_priv(struct rte_eth_dev *eth_dev)
+{
+	return eth_dev->data->dev_private;
+}
+
+#endif /* __THUNDERX_NICVF_ETHDEV_H__  */
diff --git a/drivers/net/thunderx/nicvf_logs.h b/drivers/net/thunderx/nicvf_logs.h
new file mode 100644
index 0000000..0667d46
--- /dev/null
+++ b/drivers/net/thunderx/nicvf_logs.h
@@ -0,0 +1,83 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Cavium networks Ltd. 2016.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium networks nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __THUNDERX_NICVF_LOGS__
+#define __THUNDERX_NICVF_LOGS__
+
+#include <assert.h>
+
+#define PMD_INIT_LOG(level, fmt, args...) \
+	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+
+#ifdef RTE_LIBRTE_THUNDERX_NICVF_DEBUG_INIT
+#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, ">>")
+#else
+#define PMD_INIT_FUNC_TRACE() do { } while (0)
+#endif
+
+#ifdef RTE_LIBRTE_THUNDERX_NICVF_DEBUG_RX
+#define PMD_RX_LOG(level, fmt, args...) \
+	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+#define NICVF_RX_ASSERT(x) assert(x)
+#else
+#define PMD_RX_LOG(level, fmt, args...) do { } while (0)
+#define NICVF_RX_ASSERT(x) do { } while (0)
+#endif
+
+#ifdef RTE_LIBRTE_THUNDERX_NICVF_DEBUG_TX
+#define PMD_TX_LOG(level, fmt, args...) \
+	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+#define NICVF_TX_ASSERT(x) assert(x)
+#else
+#define PMD_TX_LOG(level, fmt, args...) do { } while (0)
+#define NICVF_TX_ASSERT(x) do { } while (0)
+#endif
+
+#ifdef RTE_LIBRTE_THUNDERX_NICVF_DEBUG_DRIVER
+#define PMD_DRV_LOG(level, fmt, args...) \
+	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+#define PMD_DRV_FUNC_TRACE() PMD_DRV_LOG(DEBUG, ">>")
+#else
+#define PMD_DRV_LOG(level, fmt, args...) do { } while (0)
+#define PMD_DRV_FUNC_TRACE() do { } while (0)
+#endif
+
+#ifdef RTE_LIBRTE_THUNDERX_NICVF_DEBUG_MBOX
+#define PMD_MBOX_LOG(level, fmt, args...) \
+	RTE_LOG(level, PMD, "%s(): " fmt "\n", __func__, ## args)
+#define PMD_MBOX_FUNC_TRACE() PMD_DRV_LOG(DEBUG, ">>")
+#else
+#define PMD_MBOX_LOG(level, fmt, args...) do { } while (0)
+#define PMD_MBOX_FUNC_TRACE() do { } while (0)
+#endif
+
+#endif /* __THUNDERX_NICVF_LOGS__ */
diff --git a/drivers/net/thunderx/nicvf_struct.h b/drivers/net/thunderx/nicvf_struct.h
new file mode 100644
index 0000000..c52545d
--- /dev/null
+++ b/drivers/net/thunderx/nicvf_struct.h
@@ -0,0 +1,124 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Cavium networks Ltd. 2016.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium networks nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _THUNDERX_NICVF_STRUCT_H
+#define _THUNDERX_NICVF_STRUCT_H
+
+#include <stdint.h>
+
+#include <rte_spinlock.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_interrupts.h>
+#include <rte_ethdev.h>
+#include <rte_memory.h>
+
+struct nicvf_rbdr {
+	uint64_t rbdr_status;
+	uint64_t rbdr_door;
+	struct rbdr_entry_t *desc;
+	nicvf_phys_addr_t phys;
+	uint32_t buffsz;
+	uint32_t tail;
+	uint32_t next_tail;
+	uint32_t head;
+	uint32_t qlen_mask;
+} __rte_cache_aligned;
+
+struct nicvf_txq {
+	union sq_entry_t *desc;
+	nicvf_phys_addr_t phys;
+	struct rte_mbuf **txbuffs;
+	uint64_t sq_head;
+	uint64_t sq_door;
+	struct rte_mempool *pool;
+	struct nicvf *nic;
+	void (*pool_free)(struct nicvf_txq *sq);
+	uint32_t head;
+	uint32_t tail;
+	int32_t xmit_bufs;
+	uint32_t qlen_mask;
+	uint32_t txq_flags;
+	uint16_t queue_id;
+	uint16_t tx_free_thresh;
+} __rte_cache_aligned;
+
+struct nicvf_rxq {
+	uint64_t mbuf_phys_off;
+	uint64_t cq_status;
+	uint64_t cq_door;
+	nicvf_phys_addr_t phys;
+	union cq_entry_t *desc;
+	struct nicvf_rbdr *shared_rbdr;
+	struct nicvf *nic;
+	struct rte_mempool *pool;
+	uint32_t head;
+	uint32_t qlen_mask;
+	int32_t available_space;
+	int32_t recv_buffers;
+	uint16_t rx_free_thresh;
+	uint16_t queue_id;
+	uint16_t precharge_cnt;
+	uint8_t rx_drop_en;
+	uint8_t  port_id;
+	uint8_t  rbptr_offset;
+} __rte_cache_aligned;
+
+struct nicvf {
+	uint8_t vf_id;
+	uint8_t node;
+	uintptr_t reg_base;
+	bool tns_mode;
+	bool sqs_mode;
+	bool loopback_supported;
+	bool pf_acked:1;
+	bool pf_nacked:1;
+	uint64_t hwcap;
+	uint8_t link_up;
+	uint8_t	duplex;
+	uint32_t speed;
+	uint32_t msg_enable;
+	uint16_t device_id;
+	uint16_t vendor_id;
+	uint16_t subsystem_device_id;
+	uint16_t subsystem_vendor_id;
+	struct nicvf_rbdr *rbdr;
+	struct nicvf_rss_reta_info rss_info;
+	struct rte_eth_dev *eth_dev;
+	struct rte_intr_handle intr_handle;
+	uint8_t cpi_alg;
+	uint16_t mtu;
+	bool vlan_filter_en;
+	uint8_t mac_addr[ETHER_ADDR_LEN];
+} __rte_cache_aligned;
+
+#endif /* _THUNDERX_NICVF_STRUCT_H */
diff --git a/drivers/net/thunderx/rte_pmd_thunderx_nicvf_version.map b/drivers/net/thunderx/rte_pmd_thunderx_nicvf_version.map
new file mode 100644
index 0000000..349c6e1
--- /dev/null
+++ b/drivers/net/thunderx/rte_pmd_thunderx_nicvf_version.map
@@ -0,0 +1,4 @@
+DPDK_16.04 {
+
+	local: *;
+};
diff --git a/mk/rte.app.mk b/mk/rte.app.mk
index b84b56d..1d8d8cd 100644
--- a/mk/rte.app.mk
+++ b/mk/rte.app.mk
@@ -102,6 +102,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_XENVIRT)    += -lxenstore
 _LDLIBS-$(CONFIG_RTE_LIBRTE_MPIPE_PMD)      += -lgxio
 _LDLIBS-$(CONFIG_RTE_LIBRTE_NFP_PMD)        += -lm
 _LDLIBS-$(CONFIG_RTE_LIBRTE_QEDE_PMD)       += -lz
+_LDLIBS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += -lm
 # QAT / AESNI GCM PMDs are dependent on libcrypto (from openssl)
 # for calculating HMAC precomputes
 ifeq ($(CONFIG_RTE_LIBRTE_PMD_QAT),y)
@@ -150,6 +151,7 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_PCAP)       += -lrte_pmd_pcap
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_AF_PACKET)  += -lrte_pmd_af_packet
 _LDLIBS-$(CONFIG_RTE_LIBRTE_QEDE_PMD)       += -lrte_pmd_qede
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_NULL)       += -lrte_pmd_null
+_LDLIBS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += -lrte_pmd_thunderx_nicvf
 
 ifeq ($(CONFIG_RTE_LIBRTE_CRYPTODEV),y)
 _LDLIBS-$(CONFIG_RTE_LIBRTE_PMD_QAT)        += -lrte_pmd_qat
-- 
2.5.5