[dpdk-dev] [PATCH 05/27] common/cnxk: add nix inline device irq API

[Nithin Dabilpuram <ndabilpuram@marvell.com>]

Add API to setup nix inline device IRQ's.

Signed-off-by: Nithin Dabilpuram <ndabilpuram@marvell.com>
---
 drivers/common/cnxk/meson.build           |   1 +
 drivers/common/cnxk/roc_api.h             |   3 +
 drivers/common/cnxk/roc_irq.c             |   7 +-
 drivers/common/cnxk/roc_nix_inl.h         |  10 +
 drivers/common/cnxk/roc_nix_inl_dev_irq.c | 359 ++++++++++++++++++++++++++++++
 drivers/common/cnxk/roc_nix_inl_priv.h    |  57 +++++
 drivers/common/cnxk/roc_platform.h        |   9 +-
 drivers/common/cnxk/roc_priv.h            |   3 +
 8 files changed, 442 insertions(+), 7 deletions(-)
 create mode 100644 drivers/common/cnxk/roc_nix_inl.h
 create mode 100644 drivers/common/cnxk/roc_nix_inl_dev_irq.c
 create mode 100644 drivers/common/cnxk/roc_nix_inl_priv.h

diff --git a/drivers/common/cnxk/meson.build b/drivers/common/cnxk/meson.build
index 8a551d1..207ca00 100644
--- a/drivers/common/cnxk/meson.build
+++ b/drivers/common/cnxk/meson.build
@@ -28,6 +28,7 @@ sources = files(
         'roc_nix_debug.c',
         'roc_nix_fc.c',
         'roc_nix_irq.c',
+        'roc_nix_inl_dev_irq.c',
         'roc_nix_mac.c',
         'roc_nix_mcast.c',
         'roc_nix_npc.c',
diff --git a/drivers/common/cnxk/roc_api.h b/drivers/common/cnxk/roc_api.h
index 7dec845..c1af95e 100644
--- a/drivers/common/cnxk/roc_api.h
+++ b/drivers/common/cnxk/roc_api.h
@@ -129,4 +129,7 @@
 /* HASH computation */
 #include "roc_hash.h"
 
+/* NIX Inline dev */
+#include "roc_nix_inl.h"
+
 #endif /* _ROC_API_H_ */
diff --git a/drivers/common/cnxk/roc_irq.c b/drivers/common/cnxk/roc_irq.c
index 4c2b4c3..28fe691 100644
--- a/drivers/common/cnxk/roc_irq.c
+++ b/drivers/common/cnxk/roc_irq.c
@@ -138,9 +138,10 @@ dev_irq_register(struct plt_intr_handle *intr_handle, plt_intr_callback_fn cb,
 		irq_init(intr_handle);
 	}
 
-	if (vec > intr_handle->max_intr) {
-		plt_err("Vector=%d greater than max_intr=%d", vec,
-			intr_handle->max_intr);
+	if (vec > intr_handle->max_intr || vec >= PLT_DIM(intr_handle->efds)) {
+		plt_err("Vector=%d greater than max_intr=%d or "
+			"max_efd=%" PRIu64,
+			vec, intr_handle->max_intr, PLT_DIM(intr_handle->efds));
 		return -EINVAL;
 	}
 
diff --git a/drivers/common/cnxk/roc_nix_inl.h b/drivers/common/cnxk/roc_nix_inl.h
new file mode 100644
index 0000000..1ec3dda
--- /dev/null
+++ b/drivers/common/cnxk/roc_nix_inl.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+#ifndef _ROC_NIX_INL_H_
+#define _ROC_NIX_INL_H_
+
+/* Inline device SSO Work callback */
+typedef void (*roc_nix_inl_sso_work_cb_t)(uint64_t *gw, void *args);
+
+#endif /* _ROC_NIX_INL_H_ */
diff --git a/drivers/common/cnxk/roc_nix_inl_dev_irq.c b/drivers/common/cnxk/roc_nix_inl_dev_irq.c
new file mode 100644
index 0000000..25ed42f
--- /dev/null
+++ b/drivers/common/cnxk/roc_nix_inl_dev_irq.c
@@ -0,0 +1,359 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include "roc_api.h"
+#include "roc_priv.h"
+
+static void
+nix_inl_sso_work_cb(struct nix_inl_dev *inl_dev)
+{
+	uintptr_t getwrk_op = inl_dev->ssow_base + SSOW_LF_GWS_OP_GET_WORK0;
+	uintptr_t tag_wqe_op = inl_dev->ssow_base + SSOW_LF_GWS_WQE0;
+	uint32_t wdata = BIT(16) | 1;
+	union {
+		__uint128_t get_work;
+		uint64_t u64[2];
+	} gw;
+	uint64_t work;
+
+again:
+	/* Try to do get work */
+	gw.get_work = wdata;
+	plt_write64(gw.u64[0], getwrk_op);
+	do {
+		roc_load_pair(gw.u64[0], gw.u64[1], tag_wqe_op);
+	} while (gw.u64[0] & BIT_ULL(63));
+
+	work = gw.u64[1];
+	/* Do we have any work? */
+	if (work) {
+		if (inl_dev->work_cb)
+			inl_dev->work_cb(gw.u64, inl_dev->cb_args);
+		else
+			plt_warn("Undelivered inl dev work gw0: %p gw1: %p",
+				 (void *)gw.u64[0], (void *)gw.u64[1]);
+		goto again;
+	}
+
+	plt_atomic_thread_fence(__ATOMIC_ACQ_REL);
+}
+
+static int
+nix_inl_nix_reg_dump(struct nix_inl_dev *inl_dev)
+{
+	uintptr_t nix_base = inl_dev->nix_base;
+
+	/* General registers */
+	nix_lf_gen_reg_dump(nix_base, NULL);
+
+	/* Rx, Tx stat registers */
+	nix_lf_stat_reg_dump(nix_base, NULL, inl_dev->lf_tx_stats,
+			     inl_dev->lf_rx_stats);
+
+	/* Intr registers */
+	nix_lf_int_reg_dump(nix_base, NULL, inl_dev->qints, inl_dev->cints);
+
+	return 0;
+}
+
+static void
+nix_inl_sso_hwgrp_irq(void *param)
+{
+	struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
+	uintptr_t sso_base = inl_dev->sso_base;
+	uint64_t intr;
+
+	intr = plt_read64(sso_base + SSO_LF_GGRP_INT);
+	if (intr == 0)
+		return;
+
+	/* Check for work executable interrupt */
+	if (intr & BIT(1))
+		nix_inl_sso_work_cb(inl_dev);
+
+	if (!(intr & BIT(1)))
+		plt_err("GGRP 0 GGRP_INT=0x%" PRIx64 "", intr);
+
+	/* Clear interrupt */
+	plt_write64(intr, sso_base + SSO_LF_GGRP_INT);
+}
+
+static void
+nix_inl_sso_hws_irq(void *param)
+{
+	struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
+	uintptr_t ssow_base = inl_dev->ssow_base;
+	uint64_t intr;
+
+	intr = plt_read64(ssow_base + SSOW_LF_GWS_INT);
+	if (intr == 0)
+		return;
+
+	plt_err("GWS 0 GWS_INT=0x%" PRIx64 "", intr);
+
+	/* Clear interrupt */
+	plt_write64(intr, ssow_base + SSOW_LF_GWS_INT);
+}
+
+int
+nix_inl_sso_register_irqs(struct nix_inl_dev *inl_dev)
+{
+	struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle;
+	uintptr_t ssow_base = inl_dev->ssow_base;
+	uintptr_t sso_base = inl_dev->sso_base;
+	uint16_t sso_msixoff, ssow_msixoff;
+	int rc;
+
+	ssow_msixoff = inl_dev->ssow_msixoff;
+	sso_msixoff = inl_dev->sso_msixoff;
+	if (sso_msixoff == MSIX_VECTOR_INVALID ||
+	    ssow_msixoff == MSIX_VECTOR_INVALID) {
+		plt_err("Invalid SSO/SSOW MSIX offsets (0x%x, 0x%x)",
+			sso_msixoff, ssow_msixoff);
+		return -EINVAL;
+	}
+
+	/*
+	 * Setup SSOW interrupt
+	 */
+
+	/* Clear SSOW interrupt enable */
+	plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1C);
+	/* Register interrupt with vfio */
+	rc = dev_irq_register(handle, nix_inl_sso_hws_irq, inl_dev,
+			      ssow_msixoff + SSOW_LF_INT_VEC_IOP);
+	/* Set SSOW interrupt enable */
+	plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1S);
+
+	/*
+	 * Setup SSO/HWGRP interrupt
+	 */
+
+	/* Clear SSO interrupt enable */
+	plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1C);
+	/* Register IRQ */
+	rc |= dev_irq_register(handle, nix_inl_sso_hwgrp_irq, (void *)inl_dev,
+			       sso_msixoff + SSO_LF_INT_VEC_GRP);
+	/* Enable hw interrupt */
+	plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1S);
+
+	/* Setup threshold for work exec interrupt to 1 wqe in IAQ */
+	plt_write64(0x1ull, sso_base + SSO_LF_GGRP_INT_THR);
+
+	return rc;
+}
+
+void
+nix_inl_sso_unregister_irqs(struct nix_inl_dev *inl_dev)
+{
+	struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle;
+	uintptr_t ssow_base = inl_dev->ssow_base;
+	uintptr_t sso_base = inl_dev->sso_base;
+	uint16_t sso_msixoff, ssow_msixoff;
+
+	ssow_msixoff = inl_dev->ssow_msixoff;
+	sso_msixoff = inl_dev->sso_msixoff;
+
+	/* Clear SSOW interrupt enable */
+	plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1C);
+	/* Clear SSO/HWGRP interrupt enable */
+	plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1C);
+	/* Clear SSO threshold */
+	plt_write64(0, sso_base + SSO_LF_GGRP_INT_THR);
+
+	/* Unregister IRQ */
+	dev_irq_unregister(handle, nix_inl_sso_hws_irq, (void *)inl_dev,
+			   ssow_msixoff + SSOW_LF_INT_VEC_IOP);
+	dev_irq_unregister(handle, nix_inl_sso_hwgrp_irq, (void *)inl_dev,
+			   sso_msixoff + SSO_LF_INT_VEC_GRP);
+}
+
+static void
+nix_inl_nix_q_irq(void *param)
+{
+	struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
+	uintptr_t nix_base = inl_dev->nix_base;
+	struct dev *dev = &inl_dev->dev;
+	volatile void *ctx;
+	uint64_t reg, intr;
+	uint8_t irq;
+	int rc;
+
+	intr = plt_read64(nix_base + NIX_LF_QINTX_INT(0));
+	if (intr == 0)
+		return;
+
+	plt_err("Queue_intr=0x%" PRIx64 " qintx 0 pf=%d, vf=%d", intr, dev->pf,
+		dev->vf);
+
+	/* Get and clear RQ0 interrupt */
+	reg = roc_atomic64_add_nosync(0,
+				      (int64_t *)(nix_base + NIX_LF_RQ_OP_INT));
+	if (reg & BIT_ULL(42) /* OP_ERR */) {
+		plt_err("Failed to get rq_int");
+		return;
+	}
+	irq = reg & 0xff;
+	plt_write64(0 | irq, nix_base + NIX_LF_RQ_OP_INT);
+
+	if (irq & BIT_ULL(NIX_RQINT_DROP))
+		plt_err("RQ=0 NIX_RQINT_DROP");
+
+	if (irq & BIT_ULL(NIX_RQINT_RED))
+		plt_err("RQ=0 NIX_RQINT_RED");
+
+	/* Clear interrupt */
+	plt_write64(intr, nix_base + NIX_LF_QINTX_INT(0));
+
+	/* Dump registers to std out */
+	nix_inl_nix_reg_dump(inl_dev);
+
+	/* Dump RQ 0 */
+	rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx);
+	if (rc) {
+		plt_err("Failed to get rq context");
+		return;
+	}
+	nix_lf_rq_dump(ctx);
+}
+
+static void
+nix_inl_nix_ras_irq(void *param)
+{
+	struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
+	uintptr_t nix_base = inl_dev->nix_base;
+	struct dev *dev = &inl_dev->dev;
+	volatile void *ctx;
+	uint64_t intr;
+	int rc;
+
+	intr = plt_read64(nix_base + NIX_LF_RAS);
+	if (intr == 0)
+		return;
+
+	plt_err("Ras_intr=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
+	/* Clear interrupt */
+	plt_write64(intr, nix_base + NIX_LF_RAS);
+
+	/* Dump registers to std out */
+	nix_inl_nix_reg_dump(inl_dev);
+
+	/* Dump RQ 0 */
+	rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx);
+	if (rc) {
+		plt_err("Failed to get rq context");
+		return;
+	}
+	nix_lf_rq_dump(ctx);
+}
+
+static void
+nix_inl_nix_err_irq(void *param)
+{
+	struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
+	uintptr_t nix_base = inl_dev->nix_base;
+	struct dev *dev = &inl_dev->dev;
+	volatile void *ctx;
+	uint64_t intr;
+	int rc;
+
+	intr = plt_read64(nix_base + NIX_LF_ERR_INT);
+	if (intr == 0)
+		return;
+
+	plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
+
+	/* Clear interrupt */
+	plt_write64(intr, nix_base + NIX_LF_ERR_INT);
+
+	/* Dump registers to std out */
+	nix_inl_nix_reg_dump(inl_dev);
+
+	/* Dump RQ 0 */
+	rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx);
+	if (rc) {
+		plt_err("Failed to get rq context");
+		return;
+	}
+	nix_lf_rq_dump(ctx);
+}
+
+int
+nix_inl_nix_register_irqs(struct nix_inl_dev *inl_dev)
+{
+	struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle;
+	uintptr_t nix_base = inl_dev->nix_base;
+	uint16_t msixoff;
+	int rc;
+
+	msixoff = inl_dev->nix_msixoff;
+	if (msixoff == MSIX_VECTOR_INVALID) {
+		plt_err("Invalid NIXLF MSIX vector offset: 0x%x", msixoff);
+		return -EINVAL;
+	}
+
+	/* Disable err interrupts */
+	plt_write64(~0ull, nix_base + NIX_LF_ERR_INT_ENA_W1C);
+	/* DIsable RAS interrupts */
+	plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1C);
+
+	/* Register err irq */
+	rc = dev_irq_register(handle, nix_inl_nix_err_irq, inl_dev,
+			      msixoff + NIX_LF_INT_VEC_ERR_INT);
+	rc |= dev_irq_register(handle, nix_inl_nix_ras_irq, inl_dev,
+			       msixoff + NIX_LF_INT_VEC_POISON);
+
+	/* Enable all nix lf error irqs except RQ_DISABLED and CQ_DISABLED */
+	plt_write64(~(BIT_ULL(11) | BIT_ULL(24)),
+		    nix_base + NIX_LF_ERR_INT_ENA_W1S);
+	/* Enable RAS interrupts */
+	plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1S);
+
+	/* Setup queue irq for RQ 0 */
+
+	/* Clear QINT CNT, interrupt */
+	plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0));
+	plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1C(0));
+
+	/* Register queue irq vector */
+	rc |= dev_irq_register(handle, nix_inl_nix_q_irq, inl_dev,
+			       msixoff + NIX_LF_INT_VEC_QINT_START);
+
+	plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0));
+	plt_write64(0, nix_base + NIX_LF_QINTX_INT(0));
+	/* Enable QINT interrupt */
+	plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1S(0));
+
+	return rc;
+}
+
+void
+nix_inl_nix_unregister_irqs(struct nix_inl_dev *inl_dev)
+{
+	struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle;
+	uintptr_t nix_base = inl_dev->nix_base;
+	uint16_t msixoff;
+
+	msixoff = inl_dev->nix_msixoff;
+	/* Disable err interrupts */
+	plt_write64(~0ull, nix_base + NIX_LF_ERR_INT_ENA_W1C);
+	/* DIsable RAS interrupts */
+	plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1C);
+
+	dev_irq_unregister(handle, nix_inl_nix_err_irq, inl_dev,
+			   msixoff + NIX_LF_INT_VEC_ERR_INT);
+	dev_irq_unregister(handle, nix_inl_nix_ras_irq, inl_dev,
+			   msixoff + NIX_LF_INT_VEC_POISON);
+
+	/* Clear QINT CNT */
+	plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0));
+	plt_write64(0, nix_base + NIX_LF_QINTX_INT(0));
+
+	/* Disable QINT interrupt */
+	plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1C(0));
+
+	/* Unregister queue irq vector */
+	dev_irq_unregister(handle, nix_inl_nix_q_irq, inl_dev,
+			   msixoff + NIX_LF_INT_VEC_QINT_START);
+}
diff --git a/drivers/common/cnxk/roc_nix_inl_priv.h b/drivers/common/cnxk/roc_nix_inl_priv.h
new file mode 100644
index 0000000..f424009
--- /dev/null
+++ b/drivers/common/cnxk/roc_nix_inl_priv.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+#ifndef _ROC_NIX_INL_PRIV_H_
+#define _ROC_NIX_INL_PRIV_H_
+
+struct nix_inl_dev {
+	/* Base device object */
+	struct dev dev;
+
+	/* PCI device */
+	struct plt_pci_device *pci_dev;
+
+	/* LF specific BAR2 regions */
+	uintptr_t nix_base;
+	uintptr_t ssow_base;
+	uintptr_t sso_base;
+
+	/* MSIX vector offsets */
+	uint16_t nix_msixoff;
+	uint16_t ssow_msixoff;
+	uint16_t sso_msixoff;
+
+	/* SSO data */
+	uint32_t xaq_buf_size;
+	uint32_t xae_waes;
+	uint32_t iue;
+	uint64_t xaq_aura;
+	void *xaq_mem;
+	roc_nix_inl_sso_work_cb_t work_cb;
+	void *cb_args;
+
+	/* NIX data */
+	uint8_t lf_tx_stats;
+	uint8_t lf_rx_stats;
+	uint16_t cints;
+	uint16_t qints;
+	struct roc_nix_rq rq;
+	uint16_t rq_refs;
+	bool is_nix1;
+
+	/* NIX/CPT data */
+	void *inb_sa_base;
+	uint16_t inb_sa_sz;
+
+	/* Device arguments */
+	uint8_t selftest;
+	uint16_t ipsec_in_max_spi;
+};
+
+int nix_inl_sso_register_irqs(struct nix_inl_dev *inl_dev);
+void nix_inl_sso_unregister_irqs(struct nix_inl_dev *inl_dev);
+
+int nix_inl_nix_register_irqs(struct nix_inl_dev *inl_dev);
+void nix_inl_nix_unregister_irqs(struct nix_inl_dev *inl_dev);
+
+#endif /* _ROC_NIX_INL_PRIV_H_ */
diff --git a/drivers/common/cnxk/roc_platform.h b/drivers/common/cnxk/roc_platform.h
index 285b24b..177db3d 100644
--- a/drivers/common/cnxk/roc_platform.h
+++ b/drivers/common/cnxk/roc_platform.h
@@ -113,10 +113,11 @@
 #define plt_write64(val, addr)                                                 \
 	rte_write64_relaxed((val), (volatile void *)(addr))
 
-#define plt_wmb() rte_wmb()
-#define plt_rmb() rte_rmb()
-#define plt_io_wmb() rte_io_wmb()
-#define plt_io_rmb() rte_io_rmb()
+#define plt_wmb()		rte_wmb()
+#define plt_rmb()		rte_rmb()
+#define plt_io_wmb()		rte_io_wmb()
+#define plt_io_rmb()		rte_io_rmb()
+#define plt_atomic_thread_fence rte_atomic_thread_fence
 
 #define plt_mmap       mmap
 #define PLT_PROT_READ  PROT_READ
diff --git a/drivers/common/cnxk/roc_priv.h b/drivers/common/cnxk/roc_priv.h
index 7494b8d..f72bbd5 100644
--- a/drivers/common/cnxk/roc_priv.h
+++ b/drivers/common/cnxk/roc_priv.h
@@ -38,4 +38,7 @@
 /* CPT */
 #include "roc_cpt_priv.h"
 
+/* NIX Inline dev */
+#include "roc_nix_inl_priv.h"
+
 #endif /* _ROC_PRIV_H_ */
-- 
2.8.4