* [PATCH 1/2] event/cnxk: add additional checks in OP_RELEASE @ 2022-04-26 21:47 Pavan Nikhilesh 2022-04-26 21:47 ` [PATCH 2/2] event/cnxk: move post-processing to separate function Pavan Nikhilesh 2022-04-27 10:58 ` [PATCH v2 1/2] event/cnxk: add additional checks in OP_RELEASE Pavan Nikhilesh 0 siblings, 2 replies; 5+ messages in thread From: Pavan Nikhilesh @ 2022-04-26 21:47 UTC (permalink / raw) To: jerinj, Pavan Nikhilesh, Shijith Thotton; +Cc: dev Add additional checks while performing RTE_EVENT_OP_RELEASE to ensure that there are no pending SWTAGs and FLUSHEs in flight. Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com> --- drivers/event/cnxk/cn10k_eventdev.c | 4 +--- drivers/event/cnxk/cn10k_worker.c | 8 ++++++-- drivers/event/cnxk/cn9k_eventdev.c | 4 +--- drivers/event/cnxk/cn9k_worker.c | 16 ++++++++++++---- drivers/event/cnxk/cn9k_worker.h | 3 +-- drivers/event/cnxk/cnxk_worker.h | 17 ++++++++++++++--- 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/drivers/event/cnxk/cn10k_eventdev.c b/drivers/event/cnxk/cn10k_eventdev.c index 9b4d2895ec..2fa2cd31c2 100644 --- a/drivers/event/cnxk/cn10k_eventdev.c +++ b/drivers/event/cnxk/cn10k_eventdev.c @@ -137,9 +137,7 @@ cn10k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, if (fn != NULL && ev.u64 != 0) fn(arg, ev); if (ev.sched_type != SSO_TT_EMPTY) - cnxk_sso_hws_swtag_flush( - ws->base + SSOW_LF_GWS_WQE0, - ws->base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + cnxk_sso_hws_swtag_flush(ws->base); do { val = plt_read64(ws->base + SSOW_LF_GWS_PENDSTATE); } while (val & BIT_ULL(56)); diff --git a/drivers/event/cnxk/cn10k_worker.c b/drivers/event/cnxk/cn10k_worker.c index 975a22336a..0d99b4c5e5 100644 --- a/drivers/event/cnxk/cn10k_worker.c +++ b/drivers/event/cnxk/cn10k_worker.c @@ -18,8 +18,12 @@ cn10k_sso_hws_enq(void *port, const struct rte_event *ev) cn10k_sso_hws_forward_event(ws, ev); break; case RTE_EVENT_OP_RELEASE: - cnxk_sso_hws_swtag_flush(ws->base + SSOW_LF_GWS_WQE0, - ws->base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + if (ws->swtag_req) { + cnxk_sso_hws_desched(ev->u64, ws->base); + ws->swtag_req = 0; + break; + } + cnxk_sso_hws_swtag_flush(ws->base); break; default: return 0; diff --git a/drivers/event/cnxk/cn9k_eventdev.c b/drivers/event/cnxk/cn9k_eventdev.c index 4bba477dd1..41bbe3cb22 100644 --- a/drivers/event/cnxk/cn9k_eventdev.c +++ b/drivers/event/cnxk/cn9k_eventdev.c @@ -156,9 +156,7 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, if (fn != NULL && ev.u64 != 0) fn(arg, ev); if (ev.sched_type != SSO_TT_EMPTY) - cnxk_sso_hws_swtag_flush( - ws_base + SSOW_LF_GWS_TAG, - ws_base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + cnxk_sso_hws_swtag_flush(ws_base); do { val = plt_read64(ws_base + SSOW_LF_GWS_PENDSTATE); } while (val & BIT_ULL(56)); diff --git a/drivers/event/cnxk/cn9k_worker.c b/drivers/event/cnxk/cn9k_worker.c index a981bc986f..41dbe6cafb 100644 --- a/drivers/event/cnxk/cn9k_worker.c +++ b/drivers/event/cnxk/cn9k_worker.c @@ -19,8 +19,12 @@ cn9k_sso_hws_enq(void *port, const struct rte_event *ev) cn9k_sso_hws_forward_event(ws, ev); break; case RTE_EVENT_OP_RELEASE: - cnxk_sso_hws_swtag_flush(ws->base + SSOW_LF_GWS_TAG, - ws->base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + if (ws->swtag_req) { + cnxk_sso_hws_desched(ev->u64, ws->base); + ws->swtag_req = 0; + break; + } + cnxk_sso_hws_swtag_flush(ws->base); break; default: return 0; @@ -78,8 +82,12 @@ cn9k_sso_hws_dual_enq(void *port, const struct rte_event *ev) cn9k_sso_hws_dual_forward_event(dws, base, ev); break; case RTE_EVENT_OP_RELEASE: - cnxk_sso_hws_swtag_flush(base + SSOW_LF_GWS_TAG, - base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + if (dws->swtag_req) { + cnxk_sso_hws_desched(ev->u64, base); + dws->swtag_req = 0; + break; + } + cnxk_sso_hws_swtag_flush(base); break; default: return 0; diff --git a/drivers/event/cnxk/cn9k_worker.h b/drivers/event/cnxk/cn9k_worker.h index 917d1e0b40..88eb4e9cf9 100644 --- a/drivers/event/cnxk/cn9k_worker.h +++ b/drivers/event/cnxk/cn9k_worker.h @@ -841,8 +841,7 @@ cn9k_sso_hws_event_tx(uint64_t base, struct rte_event *ev, uint64_t *cmd, return 1; } - cnxk_sso_hws_swtag_flush(base + SSOW_LF_GWS_TAG, - base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + cnxk_sso_hws_swtag_flush(base); return 1; } diff --git a/drivers/event/cnxk/cnxk_worker.h b/drivers/event/cnxk/cnxk_worker.h index 7de03f3fbb..5e5e96b1ec 100644 --- a/drivers/event/cnxk/cnxk_worker.h +++ b/drivers/event/cnxk/cnxk_worker.h @@ -45,11 +45,15 @@ cnxk_sso_hws_swtag_untag(uintptr_t swtag_untag_op) } static __rte_always_inline void -cnxk_sso_hws_swtag_flush(uint64_t tag_op, uint64_t flush_op) +cnxk_sso_hws_swtag_flush(uint64_t base) { - if (CNXK_TT_FROM_TAG(plt_read64(tag_op)) == SSO_TT_EMPTY) + /* Ensure that there is no previous flush is pending. */ + while (plt_read64(base + SSOW_LF_GWS_PENDSTATE) & BIT_ULL(56)) + ; + if (CNXK_TT_FROM_TAG(plt_read64(base + SSOW_LF_GWS_TAG)) == + SSO_TT_EMPTY) return; - plt_write64(0, flush_op); + plt_write64(0, base + SSOW_LF_GWS_OP_SWTAG_FLUSH); } static __rte_always_inline uint64_t @@ -78,4 +82,11 @@ cnxk_sso_hws_swtag_wait(uintptr_t tag_op) return swtp; } +static __rte_always_inline void +cnxk_sso_hws_desched(uint64_t u64, uint64_t base) +{ + plt_write64(u64, base + SSOW_LF_GWS_OP_UPD_WQP_GRP1); + plt_write64(0, base + SSOW_LF_GWS_OP_DESCHED); +} + #endif -- 2.35.1 ^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH 2/2] event/cnxk: move post-processing to separate function 2022-04-26 21:47 [PATCH 1/2] event/cnxk: add additional checks in OP_RELEASE Pavan Nikhilesh @ 2022-04-26 21:47 ` Pavan Nikhilesh 2022-05-16 15:21 ` Jerin Jacob 2022-04-27 10:58 ` [PATCH v2 1/2] event/cnxk: add additional checks in OP_RELEASE Pavan Nikhilesh 1 sibling, 1 reply; 5+ messages in thread From: Pavan Nikhilesh @ 2022-04-26 21:47 UTC (permalink / raw) To: jerinj, Pavan Nikhilesh, Shijith Thotton; +Cc: dev Move event post-processing to a separate function. Do complete event post-processing in tear-down functions to prevent incorrect memory free. Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com> --- drivers/event/cnxk/cn10k_eventdev.c | 5 +- drivers/event/cnxk/cn10k_worker.h | 190 +++++++++++++--------------- drivers/event/cnxk/cn9k_eventdev.c | 9 +- drivers/event/cnxk/cn9k_worker.h | 114 ++++++----------- 4 files changed, 138 insertions(+), 180 deletions(-) diff --git a/drivers/event/cnxk/cn10k_eventdev.c b/drivers/event/cnxk/cn10k_eventdev.c index 2fa2cd31c2..94829e789c 100644 --- a/drivers/event/cnxk/cn10k_eventdev.c +++ b/drivers/event/cnxk/cn10k_eventdev.c @@ -133,7 +133,10 @@ cn10k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, while (aq_cnt || cq_ds_cnt || ds_cnt) { plt_write64(req, ws->base + SSOW_LF_GWS_OP_GET_WORK0); - cn10k_sso_hws_get_work_empty(ws, &ev); + cn10k_sso_hws_get_work_empty( + ws, &ev, + (NIX_RX_OFFLOAD_MAX - 1) | NIX_RX_REAS_F | + NIX_RX_MULTI_SEG_F | CPT_RX_WQE_F); if (fn != NULL && ev.u64 != 0) fn(arg, ev); if (ev.sched_type != SSO_TT_EMPTY) diff --git a/drivers/event/cnxk/cn10k_worker.h b/drivers/event/cnxk/cn10k_worker.h index c96048f47d..03bae4bd53 100644 --- a/drivers/event/cnxk/cn10k_worker.h +++ b/drivers/event/cnxk/cn10k_worker.h @@ -196,15 +196,88 @@ cn10k_process_vwqe(uintptr_t vwqe, uint16_t port_id, const uint32_t flags, } } +static __rte_always_inline void +cn10k_sso_hws_post_process(struct cn10k_sso_hws *ws, uint64_t *u64, + const uint32_t flags) +{ + uint64_t tstamp_ptr; + + u64[0] = (u64[0] & (0x3ull << 32)) << 6 | + (u64[0] & (0x3FFull << 36)) << 4 | (u64[0] & 0xffffffff); + if ((flags & CPT_RX_WQE_F) && + (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_CRYPTODEV)) { + u64[1] = cn10k_cpt_crypto_adapter_dequeue(u64[1]); + } else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_ETHDEV) { + uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]); + uint64_t mbuf; + + mbuf = u64[1] - sizeof(struct rte_mbuf); + rte_prefetch0((void *)mbuf); + if (flags & NIX_RX_OFFLOAD_SECURITY_F) { + const uint64_t mbuf_init = + 0x100010000ULL | RTE_PKTMBUF_HEADROOM | + (flags & NIX_RX_OFFLOAD_TSTAMP_F ? 8 : 0); + struct rte_mbuf *m; + uintptr_t sa_base; + uint64_t iova = 0; + uint8_t loff = 0; + uint16_t d_off; + uint64_t cq_w1; + uint64_t cq_w5; + + m = (struct rte_mbuf *)mbuf; + d_off = (uintptr_t)(m->buf_addr) - (uintptr_t)m; + d_off += RTE_PKTMBUF_HEADROOM; + + cq_w1 = *(uint64_t *)(u64[1] + 8); + cq_w5 = *(uint64_t *)(u64[1] + 40); + + sa_base = cnxk_nix_sa_base_get(port, ws->lookup_mem); + sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1); + + mbuf = (uint64_t)nix_sec_meta_to_mbuf_sc( + cq_w1, cq_w5, sa_base, (uintptr_t)&iova, &loff, + (struct rte_mbuf *)mbuf, d_off, flags, + mbuf_init | ((uint64_t)port) << 48); + if (loff) + roc_npa_aura_op_free(m->pool->pool_id, 0, iova); + } + + u64[0] = CNXK_CLR_SUB_EVENT(u64[0]); + cn10k_wqe_to_mbuf(u64[1], mbuf, port, u64[0] & 0xFFFFF, flags, + ws->lookup_mem); + /* Extracting tstamp, if PTP enabled*/ + tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)u64[1]) + + CNXK_SSO_WQE_SG_PTR); + cn10k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp, + flags & NIX_RX_OFFLOAD_TSTAMP_F, + (uint64_t *)tstamp_ptr); + u64[1] = mbuf; + } else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == + RTE_EVENT_TYPE_ETHDEV_VECTOR) { + uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]); + __uint128_t vwqe_hdr = *(__uint128_t *)u64[1]; + + vwqe_hdr = ((vwqe_hdr >> 64) & 0xFFF) | BIT_ULL(31) | + ((vwqe_hdr & 0xFFFF) << 48) | ((uint64_t)port << 32); + *(uint64_t *)u64[1] = (uint64_t)vwqe_hdr; + cn10k_process_vwqe(u64[1], port, flags, ws->lookup_mem, + ws->tstamp, ws->lmt_base); + /* Mark vector mempool object as get */ + RTE_MEMPOOL_CHECK_COOKIES( + rte_mempool_from_obj((void *)gw.u64[1]), + (void **)&u64[1], 1, 1); + } +} + static __rte_always_inline uint16_t cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev, - const uint32_t flags, void *lookup_mem) + const uint32_t flags) { union { __uint128_t get_work; uint64_t u64[2]; } gw; - uint64_t tstamp_ptr; gw.get_work = ws->gw_wdata; #if defined(RTE_ARCH_ARM64) && !defined(__clang__) @@ -222,83 +295,8 @@ cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev, } while (gw.u64[0] & BIT_ULL(63)); #endif ws->gw_rdata = gw.u64[0]; - if (gw.u64[1]) { - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | - (gw.u64[0] & (0x3FFull << 36)) << 4 | - (gw.u64[0] & 0xffffffff); - if ((flags & CPT_RX_WQE_F) && - (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_CRYPTODEV)) { - gw.u64[1] = cn10k_cpt_crypto_adapter_dequeue(gw.u64[1]); - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - uint64_t mbuf; - - mbuf = gw.u64[1] - sizeof(struct rte_mbuf); - rte_prefetch0((void *)mbuf); - if (flags & NIX_RX_OFFLOAD_SECURITY_F) { - const uint64_t mbuf_init = 0x100010000ULL | - RTE_PKTMBUF_HEADROOM | - (flags & NIX_RX_OFFLOAD_TSTAMP_F ? 8 : 0); - struct rte_mbuf *m; - uintptr_t sa_base; - uint64_t iova = 0; - uint8_t loff = 0; - uint16_t d_off; - uint64_t cq_w1; - uint64_t cq_w5; - - m = (struct rte_mbuf *)mbuf; - d_off = (uintptr_t)(m->buf_addr) - (uintptr_t)m; - d_off += RTE_PKTMBUF_HEADROOM; - - cq_w1 = *(uint64_t *)(gw.u64[1] + 8); - cq_w5 = *(uint64_t *)(gw.u64[1] + 40); - - sa_base = - cnxk_nix_sa_base_get(port, lookup_mem); - sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1); - - mbuf = (uint64_t)nix_sec_meta_to_mbuf_sc( - cq_w1, cq_w5, sa_base, (uintptr_t)&iova, &loff, - (struct rte_mbuf *)mbuf, d_off, flags, - mbuf_init | ((uint64_t)port) << 48); - if (loff) - roc_npa_aura_op_free(m->pool->pool_id, - 0, iova); - } - - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); - cn10k_wqe_to_mbuf(gw.u64[1], mbuf, port, - gw.u64[0] & 0xFFFFF, flags, - lookup_mem); - /* Extracting tstamp, if PTP enabled*/ - tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *) - gw.u64[1]) + - CNXK_SSO_WQE_SG_PTR); - cn10k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, - ws->tstamp, - flags & NIX_RX_OFFLOAD_TSTAMP_F, - (uint64_t *)tstamp_ptr); - gw.u64[1] = mbuf; - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV_VECTOR) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - __uint128_t vwqe_hdr = *(__uint128_t *)gw.u64[1]; - - vwqe_hdr = ((vwqe_hdr >> 64) & 0xFFF) | BIT_ULL(31) | - ((vwqe_hdr & 0xFFFF) << 48) | - ((uint64_t)port << 32); - *(uint64_t *)gw.u64[1] = (uint64_t)vwqe_hdr; - cn10k_process_vwqe(gw.u64[1], port, flags, lookup_mem, - ws->tstamp, ws->lmt_base); - /* Mark vector mempool object as get */ - RTE_MEMPOOL_CHECK_COOKIES( - rte_mempool_from_obj((void *)gw.u64[1]), - (void **)&gw.u64[1], 1, 1); - } - } + if (gw.u64[1]) + cn10k_sso_hws_post_process(ws, gw.u64, flags); ev->event = gw.u64[0]; ev->u64 = gw.u64[1]; @@ -308,13 +306,13 @@ cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev, /* Used in cleaning up workslot. */ static __rte_always_inline uint16_t -cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev) +cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev, + const uint32_t flags) { union { __uint128_t get_work; uint64_t u64[2]; } gw; - uint64_t mbuf; #ifdef RTE_ARCH_ARM64 asm volatile(PLT_CPU_FEATURE_PREAMBLE @@ -325,9 +323,7 @@ cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev) " ldp %[tag], %[wqp], [%[tag_loc]] \n" " tbnz %[tag], 63, rty%= \n" "done%=: dmb ld \n" - " sub %[mbuf], %[wqp], #0x80 \n" - : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1]), - [mbuf] "=&r"(mbuf) + : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1]) : [tag_loc] "r"(ws->base + SSOW_LF_GWS_WQE0) : "memory"); #else @@ -335,24 +331,11 @@ cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev) roc_load_pair(gw.u64[0], gw.u64[1], ws->base + SSOW_LF_GWS_WQE0); } while (gw.u64[0] & BIT_ULL(63)); - mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); #endif - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | - (gw.u64[0] & (0x3FFull << 36)) << 4 | - (gw.u64[0] & 0xffffffff); - - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { - if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); - cn10k_wqe_to_mbuf(gw.u64[1], mbuf, port, - gw.u64[0] & 0xFFFFF, 0, NULL); - gw.u64[1] = mbuf; - } - } + ws->gw_rdata = gw.u64[0]; + if (gw.u64[1]) + cn10k_sso_hws_post_process(ws, gw.u64, flags); ev->event = gw.u64[0]; ev->u64 = gw.u64[1]; @@ -471,7 +454,7 @@ NIX_RX_FASTPATH_MODES ws->base + SSOW_LF_GWS_WQE0); \ return 1; \ } \ - return cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \ + return cn10k_sso_hws_get_work(ws, ev, flags); \ } #define SSO_DEQ_SEG(fn, flags) SSO_DEQ(fn, flags | NIX_RX_MULTI_SEG_F) @@ -491,10 +474,9 @@ NIX_RX_FASTPATH_MODES ws->base + SSOW_LF_GWS_WQE0); \ return ret; \ } \ - ret = cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \ + ret = cn10k_sso_hws_get_work(ws, ev, flags); \ for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) \ - ret = cn10k_sso_hws_get_work(ws, ev, flags, \ - ws->lookup_mem); \ + ret = cn10k_sso_hws_get_work(ws, ev, flags); \ return ret; \ } diff --git a/drivers/event/cnxk/cn9k_eventdev.c b/drivers/event/cnxk/cn9k_eventdev.c index 41bbe3cb22..987888d3db 100644 --- a/drivers/event/cnxk/cn9k_eventdev.c +++ b/drivers/event/cnxk/cn9k_eventdev.c @@ -122,6 +122,7 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, cnxk_handle_event_t fn, void *arg) { struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(arg); + struct cnxk_timesync_info *tstamp; struct cn9k_sso_hws_dual *dws; struct cn9k_sso_hws *ws; uint64_t cq_ds_cnt = 1; @@ -130,6 +131,7 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, struct rte_event ev; uintptr_t ws_base; uint64_t val, req; + void *lookup_mem; plt_write64(0, base + SSO_LF_GGRP_QCTL); @@ -145,14 +147,19 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, if (dev->dual_ws) { dws = hws; ws_base = dws->base[0]; + lookup_mem = dws->lookup_mem; + tstamp = dws->tstamp; } else { ws = hws; ws_base = ws->base; + lookup_mem = ws->lookup_mem; + tstamp = ws->tstamp; } while (aq_cnt || cq_ds_cnt || ds_cnt) { plt_write64(req, ws_base + SSOW_LF_GWS_OP_GET_WORK0); - cn9k_sso_hws_get_work_empty(ws_base, &ev); + cn9k_sso_hws_get_work_empty(ws_base, &ev, dev->rx_offloads, + lookup_mem, tstamp); if (fn != NULL && ev.u64 != 0) fn(arg, ev); if (ev.sched_type != SSO_TT_EMPTY) diff --git a/drivers/event/cnxk/cn9k_worker.h b/drivers/event/cnxk/cn9k_worker.h index 88eb4e9cf9..64e97e321a 100644 --- a/drivers/event/cnxk/cn9k_worker.h +++ b/drivers/event/cnxk/cn9k_worker.h @@ -169,6 +169,34 @@ cn9k_wqe_to_mbuf(uint64_t wqe, const uint64_t mbuf, uint8_t port_id, mbuf_init | ((uint64_t)port_id) << 48, flags); } +static __rte_always_inline void +cn9k_sso_hws_post_process(uint64_t *u64, uint64_t mbuf, const uint32_t flags, + const void *const lookup_mem, + struct cnxk_timesync_info *tstamp) +{ + uint64_t tstamp_ptr; + + u64[0] = (u64[0] & (0x3ull << 32)) << 6 | + (u64[0] & (0x3FFull << 36)) << 4 | (u64[0] & 0xffffffff); + if ((flags & CPT_RX_WQE_F) && + (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_CRYPTODEV)) { + u64[1] = cn9k_cpt_crypto_adapter_dequeue(u64[1]); + } else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_ETHDEV) { + uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]); + + u64[0] = CNXK_CLR_SUB_EVENT(u64[0]); + cn9k_wqe_to_mbuf(u64[1], mbuf, port, u64[0] & 0xFFFFF, flags, + lookup_mem); + /* Extracting tstamp, if PTP enabled*/ + tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)u64[1]) + + CNXK_SSO_WQE_SG_PTR); + cn9k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, tstamp, + flags & NIX_RX_OFFLOAD_TSTAMP_F, + (uint64_t *)tstamp_ptr); + u64[1] = mbuf; + } +} + static __rte_always_inline uint16_t cn9k_sso_hws_dual_get_work(uint64_t base, uint64_t pair_base, struct rte_event *ev, const uint32_t flags, @@ -178,7 +206,6 @@ cn9k_sso_hws_dual_get_work(uint64_t base, uint64_t pair_base, __uint128_t get_work; uint64_t u64[2]; } gw; - uint64_t tstamp_ptr; uint64_t mbuf; if (flags & NIX_RX_OFFLOAD_PTYPE_F) @@ -207,34 +234,9 @@ cn9k_sso_hws_dual_get_work(uint64_t base, uint64_t pair_base, mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); #endif - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | - (gw.u64[0] & (0x3FFull << 36)) << 4 | - (gw.u64[0] & 0xffffffff); - - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { - if ((flags & CPT_RX_WQE_F) && - (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_CRYPTODEV)) { - gw.u64[1] = cn9k_cpt_crypto_adapter_dequeue(gw.u64[1]); - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); - cn9k_wqe_to_mbuf(gw.u64[1], mbuf, port, - gw.u64[0] & 0xFFFFF, flags, - dws->lookup_mem); - /* Extracting tstamp, if PTP enabled*/ - tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *) - gw.u64[1]) + - CNXK_SSO_WQE_SG_PTR); - cn9k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, - dws->tstamp, - flags & NIX_RX_OFFLOAD_TSTAMP_F, - (uint64_t *)tstamp_ptr); - gw.u64[1] = mbuf; - } - } + if (gw.u64[1]) + cn9k_sso_hws_post_process(gw.u64, mbuf, flags, dws->lookup_mem, + dws->tstamp); ev->event = gw.u64[0]; ev->u64 = gw.u64[1]; @@ -250,7 +252,6 @@ cn9k_sso_hws_get_work(struct cn9k_sso_hws *ws, struct rte_event *ev, __uint128_t get_work; uint64_t u64[2]; } gw; - uint64_t tstamp_ptr; uint64_t mbuf; plt_write64(ws->gw_wdata, ws->base + SSOW_LF_GWS_OP_GET_WORK0); @@ -283,34 +284,9 @@ cn9k_sso_hws_get_work(struct cn9k_sso_hws *ws, struct rte_event *ev, mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); #endif - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | - (gw.u64[0] & (0x3FFull << 36)) << 4 | - (gw.u64[0] & 0xffffffff); - - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { - if ((flags & CPT_RX_WQE_F) && - (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_CRYPTODEV)) { - gw.u64[1] = cn9k_cpt_crypto_adapter_dequeue(gw.u64[1]); - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); - cn9k_wqe_to_mbuf(gw.u64[1], mbuf, port, - gw.u64[0] & 0xFFFFF, flags, - lookup_mem); - /* Extracting tstamp, if PTP enabled*/ - tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *) - gw.u64[1]) + - CNXK_SSO_WQE_SG_PTR); - cn9k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, - ws->tstamp, - flags & NIX_RX_OFFLOAD_TSTAMP_F, - (uint64_t *)tstamp_ptr); - gw.u64[1] = mbuf; - } - } + if (gw.u64[1]) + cn9k_sso_hws_post_process(gw.u64, mbuf, flags, lookup_mem, + ws->tstamp); ev->event = gw.u64[0]; ev->u64 = gw.u64[1]; @@ -320,7 +296,9 @@ cn9k_sso_hws_get_work(struct cn9k_sso_hws *ws, struct rte_event *ev, /* Used in cleaning up workslot. */ static __rte_always_inline uint16_t -cn9k_sso_hws_get_work_empty(uint64_t base, struct rte_event *ev) +cn9k_sso_hws_get_work_empty(uint64_t base, struct rte_event *ev, + const uint32_t flags, void *lookup_mem, + struct cnxk_timesync_info *tstamp) { union { __uint128_t get_work; @@ -353,21 +331,9 @@ cn9k_sso_hws_get_work_empty(uint64_t base, struct rte_event *ev) mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); #endif - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | - (gw.u64[0] & (0x3FFull << 36)) << 4 | - (gw.u64[0] & 0xffffffff); - - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { - if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); - cn9k_wqe_to_mbuf(gw.u64[1], mbuf, port, - gw.u64[0] & 0xFFFFF, 0, NULL); - gw.u64[1] = mbuf; - } - } + if (gw.u64[1]) + cn9k_sso_hws_post_process(gw.u64, mbuf, flags, lookup_mem, + tstamp); ev->event = gw.u64[0]; ev->u64 = gw.u64[1]; -- 2.35.1 ^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [PATCH 2/2] event/cnxk: move post-processing to separate function 2022-04-26 21:47 ` [PATCH 2/2] event/cnxk: move post-processing to separate function Pavan Nikhilesh @ 2022-05-16 15:21 ` Jerin Jacob 0 siblings, 0 replies; 5+ messages in thread From: Jerin Jacob @ 2022-05-16 15:21 UTC (permalink / raw) To: Pavan Nikhilesh; +Cc: Jerin Jacob, Shijith Thotton, dpdk-dev On Wed, Apr 27, 2022 at 3:17 AM Pavan Nikhilesh <pbhagavatula@marvell.com> wrote: > > Move event post-processing to a separate function. > Do complete event post-processing in tear-down functions to prevent > incorrect memory free. > > Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com> > --- > drivers/event/cnxk/cn10k_eventdev.c | 5 +- > drivers/event/cnxk/cn10k_worker.h | 190 +++++++++++++--------------- > drivers/event/cnxk/cn9k_eventdev.c | 9 +- > drivers/event/cnxk/cn9k_worker.h | 114 ++++++----------- > 4 files changed, 138 insertions(+), 180 deletions(-) Series applied to dpdk-next-net-eventdev/for-main. Thanks > diff --git a/drivers/event/cnxk/cn10k_eventdev.c b/drivers/event/cnxk/cn10k_eventdev.c > index 2fa2cd31c2..94829e789c 100644 > --- a/drivers/event/cnxk/cn10k_eventdev.c > +++ b/drivers/event/cnxk/cn10k_eventdev.c > @@ -133,7 +133,10 @@ cn10k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, > > while (aq_cnt || cq_ds_cnt || ds_cnt) { > plt_write64(req, ws->base + SSOW_LF_GWS_OP_GET_WORK0); > - cn10k_sso_hws_get_work_empty(ws, &ev); > + cn10k_sso_hws_get_work_empty( > + ws, &ev, > + (NIX_RX_OFFLOAD_MAX - 1) | NIX_RX_REAS_F | > + NIX_RX_MULTI_SEG_F | CPT_RX_WQE_F); > if (fn != NULL && ev.u64 != 0) > fn(arg, ev); > if (ev.sched_type != SSO_TT_EMPTY) > diff --git a/drivers/event/cnxk/cn10k_worker.h b/drivers/event/cnxk/cn10k_worker.h > index c96048f47d..03bae4bd53 100644 > --- a/drivers/event/cnxk/cn10k_worker.h > +++ b/drivers/event/cnxk/cn10k_worker.h > @@ -196,15 +196,88 @@ cn10k_process_vwqe(uintptr_t vwqe, uint16_t port_id, const uint32_t flags, > } > } > > +static __rte_always_inline void > +cn10k_sso_hws_post_process(struct cn10k_sso_hws *ws, uint64_t *u64, > + const uint32_t flags) > +{ > + uint64_t tstamp_ptr; > + > + u64[0] = (u64[0] & (0x3ull << 32)) << 6 | > + (u64[0] & (0x3FFull << 36)) << 4 | (u64[0] & 0xffffffff); > + if ((flags & CPT_RX_WQE_F) && > + (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_CRYPTODEV)) { > + u64[1] = cn10k_cpt_crypto_adapter_dequeue(u64[1]); > + } else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_ETHDEV) { > + uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]); > + uint64_t mbuf; > + > + mbuf = u64[1] - sizeof(struct rte_mbuf); > + rte_prefetch0((void *)mbuf); > + if (flags & NIX_RX_OFFLOAD_SECURITY_F) { > + const uint64_t mbuf_init = > + 0x100010000ULL | RTE_PKTMBUF_HEADROOM | > + (flags & NIX_RX_OFFLOAD_TSTAMP_F ? 8 : 0); > + struct rte_mbuf *m; > + uintptr_t sa_base; > + uint64_t iova = 0; > + uint8_t loff = 0; > + uint16_t d_off; > + uint64_t cq_w1; > + uint64_t cq_w5; > + > + m = (struct rte_mbuf *)mbuf; > + d_off = (uintptr_t)(m->buf_addr) - (uintptr_t)m; > + d_off += RTE_PKTMBUF_HEADROOM; > + > + cq_w1 = *(uint64_t *)(u64[1] + 8); > + cq_w5 = *(uint64_t *)(u64[1] + 40); > + > + sa_base = cnxk_nix_sa_base_get(port, ws->lookup_mem); > + sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1); > + > + mbuf = (uint64_t)nix_sec_meta_to_mbuf_sc( > + cq_w1, cq_w5, sa_base, (uintptr_t)&iova, &loff, > + (struct rte_mbuf *)mbuf, d_off, flags, > + mbuf_init | ((uint64_t)port) << 48); > + if (loff) > + roc_npa_aura_op_free(m->pool->pool_id, 0, iova); > + } > + > + u64[0] = CNXK_CLR_SUB_EVENT(u64[0]); > + cn10k_wqe_to_mbuf(u64[1], mbuf, port, u64[0] & 0xFFFFF, flags, > + ws->lookup_mem); > + /* Extracting tstamp, if PTP enabled*/ > + tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)u64[1]) + > + CNXK_SSO_WQE_SG_PTR); > + cn10k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp, > + flags & NIX_RX_OFFLOAD_TSTAMP_F, > + (uint64_t *)tstamp_ptr); > + u64[1] = mbuf; > + } else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == > + RTE_EVENT_TYPE_ETHDEV_VECTOR) { > + uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]); > + __uint128_t vwqe_hdr = *(__uint128_t *)u64[1]; > + > + vwqe_hdr = ((vwqe_hdr >> 64) & 0xFFF) | BIT_ULL(31) | > + ((vwqe_hdr & 0xFFFF) << 48) | ((uint64_t)port << 32); > + *(uint64_t *)u64[1] = (uint64_t)vwqe_hdr; > + cn10k_process_vwqe(u64[1], port, flags, ws->lookup_mem, > + ws->tstamp, ws->lmt_base); > + /* Mark vector mempool object as get */ > + RTE_MEMPOOL_CHECK_COOKIES( > + rte_mempool_from_obj((void *)gw.u64[1]), > + (void **)&u64[1], 1, 1); > + } > +} > + > static __rte_always_inline uint16_t > cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev, > - const uint32_t flags, void *lookup_mem) > + const uint32_t flags) > { > union { > __uint128_t get_work; > uint64_t u64[2]; > } gw; > - uint64_t tstamp_ptr; > > gw.get_work = ws->gw_wdata; > #if defined(RTE_ARCH_ARM64) && !defined(__clang__) > @@ -222,83 +295,8 @@ cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev, > } while (gw.u64[0] & BIT_ULL(63)); > #endif > ws->gw_rdata = gw.u64[0]; > - if (gw.u64[1]) { > - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | > - (gw.u64[0] & (0x3FFull << 36)) << 4 | > - (gw.u64[0] & 0xffffffff); > - if ((flags & CPT_RX_WQE_F) && > - (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == > - RTE_EVENT_TYPE_CRYPTODEV)) { > - gw.u64[1] = cn10k_cpt_crypto_adapter_dequeue(gw.u64[1]); > - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == > - RTE_EVENT_TYPE_ETHDEV) { > - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); > - uint64_t mbuf; > - > - mbuf = gw.u64[1] - sizeof(struct rte_mbuf); > - rte_prefetch0((void *)mbuf); > - if (flags & NIX_RX_OFFLOAD_SECURITY_F) { > - const uint64_t mbuf_init = 0x100010000ULL | > - RTE_PKTMBUF_HEADROOM | > - (flags & NIX_RX_OFFLOAD_TSTAMP_F ? 8 : 0); > - struct rte_mbuf *m; > - uintptr_t sa_base; > - uint64_t iova = 0; > - uint8_t loff = 0; > - uint16_t d_off; > - uint64_t cq_w1; > - uint64_t cq_w5; > - > - m = (struct rte_mbuf *)mbuf; > - d_off = (uintptr_t)(m->buf_addr) - (uintptr_t)m; > - d_off += RTE_PKTMBUF_HEADROOM; > - > - cq_w1 = *(uint64_t *)(gw.u64[1] + 8); > - cq_w5 = *(uint64_t *)(gw.u64[1] + 40); > - > - sa_base = > - cnxk_nix_sa_base_get(port, lookup_mem); > - sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1); > - > - mbuf = (uint64_t)nix_sec_meta_to_mbuf_sc( > - cq_w1, cq_w5, sa_base, (uintptr_t)&iova, &loff, > - (struct rte_mbuf *)mbuf, d_off, flags, > - mbuf_init | ((uint64_t)port) << 48); > - if (loff) > - roc_npa_aura_op_free(m->pool->pool_id, > - 0, iova); > - } > - > - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); > - cn10k_wqe_to_mbuf(gw.u64[1], mbuf, port, > - gw.u64[0] & 0xFFFFF, flags, > - lookup_mem); > - /* Extracting tstamp, if PTP enabled*/ > - tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *) > - gw.u64[1]) + > - CNXK_SSO_WQE_SG_PTR); > - cn10k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, > - ws->tstamp, > - flags & NIX_RX_OFFLOAD_TSTAMP_F, > - (uint64_t *)tstamp_ptr); > - gw.u64[1] = mbuf; > - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == > - RTE_EVENT_TYPE_ETHDEV_VECTOR) { > - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); > - __uint128_t vwqe_hdr = *(__uint128_t *)gw.u64[1]; > - > - vwqe_hdr = ((vwqe_hdr >> 64) & 0xFFF) | BIT_ULL(31) | > - ((vwqe_hdr & 0xFFFF) << 48) | > - ((uint64_t)port << 32); > - *(uint64_t *)gw.u64[1] = (uint64_t)vwqe_hdr; > - cn10k_process_vwqe(gw.u64[1], port, flags, lookup_mem, > - ws->tstamp, ws->lmt_base); > - /* Mark vector mempool object as get */ > - RTE_MEMPOOL_CHECK_COOKIES( > - rte_mempool_from_obj((void *)gw.u64[1]), > - (void **)&gw.u64[1], 1, 1); > - } > - } > + if (gw.u64[1]) > + cn10k_sso_hws_post_process(ws, gw.u64, flags); > > ev->event = gw.u64[0]; > ev->u64 = gw.u64[1]; > @@ -308,13 +306,13 @@ cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev, > > /* Used in cleaning up workslot. */ > static __rte_always_inline uint16_t > -cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev) > +cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev, > + const uint32_t flags) > { > union { > __uint128_t get_work; > uint64_t u64[2]; > } gw; > - uint64_t mbuf; > > #ifdef RTE_ARCH_ARM64 > asm volatile(PLT_CPU_FEATURE_PREAMBLE > @@ -325,9 +323,7 @@ cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev) > " ldp %[tag], %[wqp], [%[tag_loc]] \n" > " tbnz %[tag], 63, rty%= \n" > "done%=: dmb ld \n" > - " sub %[mbuf], %[wqp], #0x80 \n" > - : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1]), > - [mbuf] "=&r"(mbuf) > + : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1]) > : [tag_loc] "r"(ws->base + SSOW_LF_GWS_WQE0) > : "memory"); > #else > @@ -335,24 +331,11 @@ cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev) > roc_load_pair(gw.u64[0], gw.u64[1], > ws->base + SSOW_LF_GWS_WQE0); > } while (gw.u64[0] & BIT_ULL(63)); > - mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); > #endif > > - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | > - (gw.u64[0] & (0x3FFull << 36)) << 4 | > - (gw.u64[0] & 0xffffffff); > - > - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { > - if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == > - RTE_EVENT_TYPE_ETHDEV) { > - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); > - > - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); > - cn10k_wqe_to_mbuf(gw.u64[1], mbuf, port, > - gw.u64[0] & 0xFFFFF, 0, NULL); > - gw.u64[1] = mbuf; > - } > - } > + ws->gw_rdata = gw.u64[0]; > + if (gw.u64[1]) > + cn10k_sso_hws_post_process(ws, gw.u64, flags); > > ev->event = gw.u64[0]; > ev->u64 = gw.u64[1]; > @@ -471,7 +454,7 @@ NIX_RX_FASTPATH_MODES > ws->base + SSOW_LF_GWS_WQE0); \ > return 1; \ > } \ > - return cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \ > + return cn10k_sso_hws_get_work(ws, ev, flags); \ > } > > #define SSO_DEQ_SEG(fn, flags) SSO_DEQ(fn, flags | NIX_RX_MULTI_SEG_F) > @@ -491,10 +474,9 @@ NIX_RX_FASTPATH_MODES > ws->base + SSOW_LF_GWS_WQE0); \ > return ret; \ > } \ > - ret = cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \ > + ret = cn10k_sso_hws_get_work(ws, ev, flags); \ > for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) \ > - ret = cn10k_sso_hws_get_work(ws, ev, flags, \ > - ws->lookup_mem); \ > + ret = cn10k_sso_hws_get_work(ws, ev, flags); \ > return ret; \ > } > > diff --git a/drivers/event/cnxk/cn9k_eventdev.c b/drivers/event/cnxk/cn9k_eventdev.c > index 41bbe3cb22..987888d3db 100644 > --- a/drivers/event/cnxk/cn9k_eventdev.c > +++ b/drivers/event/cnxk/cn9k_eventdev.c > @@ -122,6 +122,7 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, > cnxk_handle_event_t fn, void *arg) > { > struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(arg); > + struct cnxk_timesync_info *tstamp; > struct cn9k_sso_hws_dual *dws; > struct cn9k_sso_hws *ws; > uint64_t cq_ds_cnt = 1; > @@ -130,6 +131,7 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, > struct rte_event ev; > uintptr_t ws_base; > uint64_t val, req; > + void *lookup_mem; > > plt_write64(0, base + SSO_LF_GGRP_QCTL); > > @@ -145,14 +147,19 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, > if (dev->dual_ws) { > dws = hws; > ws_base = dws->base[0]; > + lookup_mem = dws->lookup_mem; > + tstamp = dws->tstamp; > } else { > ws = hws; > ws_base = ws->base; > + lookup_mem = ws->lookup_mem; > + tstamp = ws->tstamp; > } > > while (aq_cnt || cq_ds_cnt || ds_cnt) { > plt_write64(req, ws_base + SSOW_LF_GWS_OP_GET_WORK0); > - cn9k_sso_hws_get_work_empty(ws_base, &ev); > + cn9k_sso_hws_get_work_empty(ws_base, &ev, dev->rx_offloads, > + lookup_mem, tstamp); > if (fn != NULL && ev.u64 != 0) > fn(arg, ev); > if (ev.sched_type != SSO_TT_EMPTY) > diff --git a/drivers/event/cnxk/cn9k_worker.h b/drivers/event/cnxk/cn9k_worker.h > index 88eb4e9cf9..64e97e321a 100644 > --- a/drivers/event/cnxk/cn9k_worker.h > +++ b/drivers/event/cnxk/cn9k_worker.h > @@ -169,6 +169,34 @@ cn9k_wqe_to_mbuf(uint64_t wqe, const uint64_t mbuf, uint8_t port_id, > mbuf_init | ((uint64_t)port_id) << 48, flags); > } > > +static __rte_always_inline void > +cn9k_sso_hws_post_process(uint64_t *u64, uint64_t mbuf, const uint32_t flags, > + const void *const lookup_mem, > + struct cnxk_timesync_info *tstamp) > +{ > + uint64_t tstamp_ptr; > + > + u64[0] = (u64[0] & (0x3ull << 32)) << 6 | > + (u64[0] & (0x3FFull << 36)) << 4 | (u64[0] & 0xffffffff); > + if ((flags & CPT_RX_WQE_F) && > + (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_CRYPTODEV)) { > + u64[1] = cn9k_cpt_crypto_adapter_dequeue(u64[1]); > + } else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_ETHDEV) { > + uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]); > + > + u64[0] = CNXK_CLR_SUB_EVENT(u64[0]); > + cn9k_wqe_to_mbuf(u64[1], mbuf, port, u64[0] & 0xFFFFF, flags, > + lookup_mem); > + /* Extracting tstamp, if PTP enabled*/ > + tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)u64[1]) + > + CNXK_SSO_WQE_SG_PTR); > + cn9k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, tstamp, > + flags & NIX_RX_OFFLOAD_TSTAMP_F, > + (uint64_t *)tstamp_ptr); > + u64[1] = mbuf; > + } > +} > + > static __rte_always_inline uint16_t > cn9k_sso_hws_dual_get_work(uint64_t base, uint64_t pair_base, > struct rte_event *ev, const uint32_t flags, > @@ -178,7 +206,6 @@ cn9k_sso_hws_dual_get_work(uint64_t base, uint64_t pair_base, > __uint128_t get_work; > uint64_t u64[2]; > } gw; > - uint64_t tstamp_ptr; > uint64_t mbuf; > > if (flags & NIX_RX_OFFLOAD_PTYPE_F) > @@ -207,34 +234,9 @@ cn9k_sso_hws_dual_get_work(uint64_t base, uint64_t pair_base, > mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); > #endif > > - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | > - (gw.u64[0] & (0x3FFull << 36)) << 4 | > - (gw.u64[0] & 0xffffffff); > - > - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { > - if ((flags & CPT_RX_WQE_F) && > - (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == > - RTE_EVENT_TYPE_CRYPTODEV)) { > - gw.u64[1] = cn9k_cpt_crypto_adapter_dequeue(gw.u64[1]); > - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == > - RTE_EVENT_TYPE_ETHDEV) { > - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); > - > - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); > - cn9k_wqe_to_mbuf(gw.u64[1], mbuf, port, > - gw.u64[0] & 0xFFFFF, flags, > - dws->lookup_mem); > - /* Extracting tstamp, if PTP enabled*/ > - tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *) > - gw.u64[1]) + > - CNXK_SSO_WQE_SG_PTR); > - cn9k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, > - dws->tstamp, > - flags & NIX_RX_OFFLOAD_TSTAMP_F, > - (uint64_t *)tstamp_ptr); > - gw.u64[1] = mbuf; > - } > - } > + if (gw.u64[1]) > + cn9k_sso_hws_post_process(gw.u64, mbuf, flags, dws->lookup_mem, > + dws->tstamp); > > ev->event = gw.u64[0]; > ev->u64 = gw.u64[1]; > @@ -250,7 +252,6 @@ cn9k_sso_hws_get_work(struct cn9k_sso_hws *ws, struct rte_event *ev, > __uint128_t get_work; > uint64_t u64[2]; > } gw; > - uint64_t tstamp_ptr; > uint64_t mbuf; > > plt_write64(ws->gw_wdata, ws->base + SSOW_LF_GWS_OP_GET_WORK0); > @@ -283,34 +284,9 @@ cn9k_sso_hws_get_work(struct cn9k_sso_hws *ws, struct rte_event *ev, > mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); > #endif > > - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | > - (gw.u64[0] & (0x3FFull << 36)) << 4 | > - (gw.u64[0] & 0xffffffff); > - > - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { > - if ((flags & CPT_RX_WQE_F) && > - (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == > - RTE_EVENT_TYPE_CRYPTODEV)) { > - gw.u64[1] = cn9k_cpt_crypto_adapter_dequeue(gw.u64[1]); > - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == > - RTE_EVENT_TYPE_ETHDEV) { > - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); > - > - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); > - cn9k_wqe_to_mbuf(gw.u64[1], mbuf, port, > - gw.u64[0] & 0xFFFFF, flags, > - lookup_mem); > - /* Extracting tstamp, if PTP enabled*/ > - tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *) > - gw.u64[1]) + > - CNXK_SSO_WQE_SG_PTR); > - cn9k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, > - ws->tstamp, > - flags & NIX_RX_OFFLOAD_TSTAMP_F, > - (uint64_t *)tstamp_ptr); > - gw.u64[1] = mbuf; > - } > - } > + if (gw.u64[1]) > + cn9k_sso_hws_post_process(gw.u64, mbuf, flags, lookup_mem, > + ws->tstamp); > > ev->event = gw.u64[0]; > ev->u64 = gw.u64[1]; > @@ -320,7 +296,9 @@ cn9k_sso_hws_get_work(struct cn9k_sso_hws *ws, struct rte_event *ev, > > /* Used in cleaning up workslot. */ > static __rte_always_inline uint16_t > -cn9k_sso_hws_get_work_empty(uint64_t base, struct rte_event *ev) > +cn9k_sso_hws_get_work_empty(uint64_t base, struct rte_event *ev, > + const uint32_t flags, void *lookup_mem, > + struct cnxk_timesync_info *tstamp) > { > union { > __uint128_t get_work; > @@ -353,21 +331,9 @@ cn9k_sso_hws_get_work_empty(uint64_t base, struct rte_event *ev) > mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); > #endif > > - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | > - (gw.u64[0] & (0x3FFull << 36)) << 4 | > - (gw.u64[0] & 0xffffffff); > - > - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { > - if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == > - RTE_EVENT_TYPE_ETHDEV) { > - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); > - > - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); > - cn9k_wqe_to_mbuf(gw.u64[1], mbuf, port, > - gw.u64[0] & 0xFFFFF, 0, NULL); > - gw.u64[1] = mbuf; > - } > - } > + if (gw.u64[1]) > + cn9k_sso_hws_post_process(gw.u64, mbuf, flags, lookup_mem, > + tstamp); > > ev->event = gw.u64[0]; > ev->u64 = gw.u64[1]; > -- > 2.35.1 > ^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v2 1/2] event/cnxk: add additional checks in OP_RELEASE 2022-04-26 21:47 [PATCH 1/2] event/cnxk: add additional checks in OP_RELEASE Pavan Nikhilesh 2022-04-26 21:47 ` [PATCH 2/2] event/cnxk: move post-processing to separate function Pavan Nikhilesh @ 2022-04-27 10:58 ` Pavan Nikhilesh 2022-04-27 10:58 ` [PATCH v2 2/2] event/cnxk: move post-processing to separate function Pavan Nikhilesh 1 sibling, 1 reply; 5+ messages in thread From: Pavan Nikhilesh @ 2022-04-27 10:58 UTC (permalink / raw) To: jerinj, Pavan Nikhilesh, Shijith Thotton; +Cc: dev Add additional checks while performing RTE_EVENT_OP_RELEASE to ensure that there are no pending SWTAGs and FLUSHEs in flight. Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com> --- v2 Changes: - Fix compilation with RTE_LIBRTE_MEMPOOL_DEBUG enabled. drivers/event/cnxk/cn10k_eventdev.c | 4 +--- drivers/event/cnxk/cn10k_worker.c | 8 ++++++-- drivers/event/cnxk/cn9k_eventdev.c | 4 +--- drivers/event/cnxk/cn9k_worker.c | 16 ++++++++++++---- drivers/event/cnxk/cn9k_worker.h | 3 +-- drivers/event/cnxk/cnxk_worker.h | 17 ++++++++++++++--- 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/drivers/event/cnxk/cn10k_eventdev.c b/drivers/event/cnxk/cn10k_eventdev.c index 9b4d2895ec..2fa2cd31c2 100644 --- a/drivers/event/cnxk/cn10k_eventdev.c +++ b/drivers/event/cnxk/cn10k_eventdev.c @@ -137,9 +137,7 @@ cn10k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, if (fn != NULL && ev.u64 != 0) fn(arg, ev); if (ev.sched_type != SSO_TT_EMPTY) - cnxk_sso_hws_swtag_flush( - ws->base + SSOW_LF_GWS_WQE0, - ws->base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + cnxk_sso_hws_swtag_flush(ws->base); do { val = plt_read64(ws->base + SSOW_LF_GWS_PENDSTATE); } while (val & BIT_ULL(56)); diff --git a/drivers/event/cnxk/cn10k_worker.c b/drivers/event/cnxk/cn10k_worker.c index 975a22336a..0d99b4c5e5 100644 --- a/drivers/event/cnxk/cn10k_worker.c +++ b/drivers/event/cnxk/cn10k_worker.c @@ -18,8 +18,12 @@ cn10k_sso_hws_enq(void *port, const struct rte_event *ev) cn10k_sso_hws_forward_event(ws, ev); break; case RTE_EVENT_OP_RELEASE: - cnxk_sso_hws_swtag_flush(ws->base + SSOW_LF_GWS_WQE0, - ws->base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + if (ws->swtag_req) { + cnxk_sso_hws_desched(ev->u64, ws->base); + ws->swtag_req = 0; + break; + } + cnxk_sso_hws_swtag_flush(ws->base); break; default: return 0; diff --git a/drivers/event/cnxk/cn9k_eventdev.c b/drivers/event/cnxk/cn9k_eventdev.c index 4bba477dd1..41bbe3cb22 100644 --- a/drivers/event/cnxk/cn9k_eventdev.c +++ b/drivers/event/cnxk/cn9k_eventdev.c @@ -156,9 +156,7 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, if (fn != NULL && ev.u64 != 0) fn(arg, ev); if (ev.sched_type != SSO_TT_EMPTY) - cnxk_sso_hws_swtag_flush( - ws_base + SSOW_LF_GWS_TAG, - ws_base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + cnxk_sso_hws_swtag_flush(ws_base); do { val = plt_read64(ws_base + SSOW_LF_GWS_PENDSTATE); } while (val & BIT_ULL(56)); diff --git a/drivers/event/cnxk/cn9k_worker.c b/drivers/event/cnxk/cn9k_worker.c index a981bc986f..41dbe6cafb 100644 --- a/drivers/event/cnxk/cn9k_worker.c +++ b/drivers/event/cnxk/cn9k_worker.c @@ -19,8 +19,12 @@ cn9k_sso_hws_enq(void *port, const struct rte_event *ev) cn9k_sso_hws_forward_event(ws, ev); break; case RTE_EVENT_OP_RELEASE: - cnxk_sso_hws_swtag_flush(ws->base + SSOW_LF_GWS_TAG, - ws->base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + if (ws->swtag_req) { + cnxk_sso_hws_desched(ev->u64, ws->base); + ws->swtag_req = 0; + break; + } + cnxk_sso_hws_swtag_flush(ws->base); break; default: return 0; @@ -78,8 +82,12 @@ cn9k_sso_hws_dual_enq(void *port, const struct rte_event *ev) cn9k_sso_hws_dual_forward_event(dws, base, ev); break; case RTE_EVENT_OP_RELEASE: - cnxk_sso_hws_swtag_flush(base + SSOW_LF_GWS_TAG, - base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + if (dws->swtag_req) { + cnxk_sso_hws_desched(ev->u64, base); + dws->swtag_req = 0; + break; + } + cnxk_sso_hws_swtag_flush(base); break; default: return 0; diff --git a/drivers/event/cnxk/cn9k_worker.h b/drivers/event/cnxk/cn9k_worker.h index 917d1e0b40..88eb4e9cf9 100644 --- a/drivers/event/cnxk/cn9k_worker.h +++ b/drivers/event/cnxk/cn9k_worker.h @@ -841,8 +841,7 @@ cn9k_sso_hws_event_tx(uint64_t base, struct rte_event *ev, uint64_t *cmd, return 1; } - cnxk_sso_hws_swtag_flush(base + SSOW_LF_GWS_TAG, - base + SSOW_LF_GWS_OP_SWTAG_FLUSH); + cnxk_sso_hws_swtag_flush(base); return 1; } diff --git a/drivers/event/cnxk/cnxk_worker.h b/drivers/event/cnxk/cnxk_worker.h index 7de03f3fbb..5e5e96b1ec 100644 --- a/drivers/event/cnxk/cnxk_worker.h +++ b/drivers/event/cnxk/cnxk_worker.h @@ -45,11 +45,15 @@ cnxk_sso_hws_swtag_untag(uintptr_t swtag_untag_op) } static __rte_always_inline void -cnxk_sso_hws_swtag_flush(uint64_t tag_op, uint64_t flush_op) +cnxk_sso_hws_swtag_flush(uint64_t base) { - if (CNXK_TT_FROM_TAG(plt_read64(tag_op)) == SSO_TT_EMPTY) + /* Ensure that there is no previous flush is pending. */ + while (plt_read64(base + SSOW_LF_GWS_PENDSTATE) & BIT_ULL(56)) + ; + if (CNXK_TT_FROM_TAG(plt_read64(base + SSOW_LF_GWS_TAG)) == + SSO_TT_EMPTY) return; - plt_write64(0, flush_op); + plt_write64(0, base + SSOW_LF_GWS_OP_SWTAG_FLUSH); } static __rte_always_inline uint64_t @@ -78,4 +82,11 @@ cnxk_sso_hws_swtag_wait(uintptr_t tag_op) return swtp; } +static __rte_always_inline void +cnxk_sso_hws_desched(uint64_t u64, uint64_t base) +{ + plt_write64(u64, base + SSOW_LF_GWS_OP_UPD_WQP_GRP1); + plt_write64(0, base + SSOW_LF_GWS_OP_DESCHED); +} + #endif -- 2.25.1 ^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH v2 2/2] event/cnxk: move post-processing to separate function 2022-04-27 10:58 ` [PATCH v2 1/2] event/cnxk: add additional checks in OP_RELEASE Pavan Nikhilesh @ 2022-04-27 10:58 ` Pavan Nikhilesh 0 siblings, 0 replies; 5+ messages in thread From: Pavan Nikhilesh @ 2022-04-27 10:58 UTC (permalink / raw) To: jerinj, Pavan Nikhilesh, Shijith Thotton; +Cc: dev Move event post-processing to a separate function. Do complete event post-processing in tear-down functions to prevent incorrect memory free. Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com> --- drivers/event/cnxk/cn10k_eventdev.c | 5 +- drivers/event/cnxk/cn10k_worker.h | 189 +++++++++++++--------------- drivers/event/cnxk/cn9k_eventdev.c | 9 +- drivers/event/cnxk/cn9k_worker.h | 114 ++++++----------- 4 files changed, 137 insertions(+), 180 deletions(-) diff --git a/drivers/event/cnxk/cn10k_eventdev.c b/drivers/event/cnxk/cn10k_eventdev.c index 2fa2cd31c2..94829e789c 100644 --- a/drivers/event/cnxk/cn10k_eventdev.c +++ b/drivers/event/cnxk/cn10k_eventdev.c @@ -133,7 +133,10 @@ cn10k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, while (aq_cnt || cq_ds_cnt || ds_cnt) { plt_write64(req, ws->base + SSOW_LF_GWS_OP_GET_WORK0); - cn10k_sso_hws_get_work_empty(ws, &ev); + cn10k_sso_hws_get_work_empty( + ws, &ev, + (NIX_RX_OFFLOAD_MAX - 1) | NIX_RX_REAS_F | + NIX_RX_MULTI_SEG_F | CPT_RX_WQE_F); if (fn != NULL && ev.u64 != 0) fn(arg, ev); if (ev.sched_type != SSO_TT_EMPTY) diff --git a/drivers/event/cnxk/cn10k_worker.h b/drivers/event/cnxk/cn10k_worker.h index c96048f47d..034f508dd8 100644 --- a/drivers/event/cnxk/cn10k_worker.h +++ b/drivers/event/cnxk/cn10k_worker.h @@ -196,15 +196,87 @@ cn10k_process_vwqe(uintptr_t vwqe, uint16_t port_id, const uint32_t flags, } } +static __rte_always_inline void +cn10k_sso_hws_post_process(struct cn10k_sso_hws *ws, uint64_t *u64, + const uint32_t flags) +{ + uint64_t tstamp_ptr; + + u64[0] = (u64[0] & (0x3ull << 32)) << 6 | + (u64[0] & (0x3FFull << 36)) << 4 | (u64[0] & 0xffffffff); + if ((flags & CPT_RX_WQE_F) && + (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_CRYPTODEV)) { + u64[1] = cn10k_cpt_crypto_adapter_dequeue(u64[1]); + } else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_ETHDEV) { + uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]); + uint64_t mbuf; + + mbuf = u64[1] - sizeof(struct rte_mbuf); + rte_prefetch0((void *)mbuf); + if (flags & NIX_RX_OFFLOAD_SECURITY_F) { + const uint64_t mbuf_init = + 0x100010000ULL | RTE_PKTMBUF_HEADROOM | + (flags & NIX_RX_OFFLOAD_TSTAMP_F ? 8 : 0); + struct rte_mbuf *m; + uintptr_t sa_base; + uint64_t iova = 0; + uint8_t loff = 0; + uint16_t d_off; + uint64_t cq_w1; + uint64_t cq_w5; + + m = (struct rte_mbuf *)mbuf; + d_off = (uintptr_t)(m->buf_addr) - (uintptr_t)m; + d_off += RTE_PKTMBUF_HEADROOM; + + cq_w1 = *(uint64_t *)(u64[1] + 8); + cq_w5 = *(uint64_t *)(u64[1] + 40); + + sa_base = cnxk_nix_sa_base_get(port, ws->lookup_mem); + sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1); + + mbuf = (uint64_t)nix_sec_meta_to_mbuf_sc( + cq_w1, cq_w5, sa_base, (uintptr_t)&iova, &loff, + (struct rte_mbuf *)mbuf, d_off, flags, + mbuf_init | ((uint64_t)port) << 48); + if (loff) + roc_npa_aura_op_free(m->pool->pool_id, 0, iova); + } + + u64[0] = CNXK_CLR_SUB_EVENT(u64[0]); + cn10k_wqe_to_mbuf(u64[1], mbuf, port, u64[0] & 0xFFFFF, flags, + ws->lookup_mem); + /* Extracting tstamp, if PTP enabled*/ + tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)u64[1]) + + CNXK_SSO_WQE_SG_PTR); + cn10k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp, + flags & NIX_RX_OFFLOAD_TSTAMP_F, + (uint64_t *)tstamp_ptr); + u64[1] = mbuf; + } else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == + RTE_EVENT_TYPE_ETHDEV_VECTOR) { + uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]); + __uint128_t vwqe_hdr = *(__uint128_t *)u64[1]; + + vwqe_hdr = ((vwqe_hdr >> 64) & 0xFFF) | BIT_ULL(31) | + ((vwqe_hdr & 0xFFFF) << 48) | ((uint64_t)port << 32); + *(uint64_t *)u64[1] = (uint64_t)vwqe_hdr; + cn10k_process_vwqe(u64[1], port, flags, ws->lookup_mem, + ws->tstamp, ws->lmt_base); + /* Mark vector mempool object as get */ + RTE_MEMPOOL_CHECK_COOKIES(rte_mempool_from_obj((void *)u64[1]), + (void **)&u64[1], 1, 1); + } +} + static __rte_always_inline uint16_t cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev, - const uint32_t flags, void *lookup_mem) + const uint32_t flags) { union { __uint128_t get_work; uint64_t u64[2]; } gw; - uint64_t tstamp_ptr; gw.get_work = ws->gw_wdata; #if defined(RTE_ARCH_ARM64) && !defined(__clang__) @@ -222,83 +294,8 @@ cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev, } while (gw.u64[0] & BIT_ULL(63)); #endif ws->gw_rdata = gw.u64[0]; - if (gw.u64[1]) { - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | - (gw.u64[0] & (0x3FFull << 36)) << 4 | - (gw.u64[0] & 0xffffffff); - if ((flags & CPT_RX_WQE_F) && - (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_CRYPTODEV)) { - gw.u64[1] = cn10k_cpt_crypto_adapter_dequeue(gw.u64[1]); - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - uint64_t mbuf; - - mbuf = gw.u64[1] - sizeof(struct rte_mbuf); - rte_prefetch0((void *)mbuf); - if (flags & NIX_RX_OFFLOAD_SECURITY_F) { - const uint64_t mbuf_init = 0x100010000ULL | - RTE_PKTMBUF_HEADROOM | - (flags & NIX_RX_OFFLOAD_TSTAMP_F ? 8 : 0); - struct rte_mbuf *m; - uintptr_t sa_base; - uint64_t iova = 0; - uint8_t loff = 0; - uint16_t d_off; - uint64_t cq_w1; - uint64_t cq_w5; - - m = (struct rte_mbuf *)mbuf; - d_off = (uintptr_t)(m->buf_addr) - (uintptr_t)m; - d_off += RTE_PKTMBUF_HEADROOM; - - cq_w1 = *(uint64_t *)(gw.u64[1] + 8); - cq_w5 = *(uint64_t *)(gw.u64[1] + 40); - - sa_base = - cnxk_nix_sa_base_get(port, lookup_mem); - sa_base &= ~(ROC_NIX_INL_SA_BASE_ALIGN - 1); - - mbuf = (uint64_t)nix_sec_meta_to_mbuf_sc( - cq_w1, cq_w5, sa_base, (uintptr_t)&iova, &loff, - (struct rte_mbuf *)mbuf, d_off, flags, - mbuf_init | ((uint64_t)port) << 48); - if (loff) - roc_npa_aura_op_free(m->pool->pool_id, - 0, iova); - } - - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); - cn10k_wqe_to_mbuf(gw.u64[1], mbuf, port, - gw.u64[0] & 0xFFFFF, flags, - lookup_mem); - /* Extracting tstamp, if PTP enabled*/ - tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *) - gw.u64[1]) + - CNXK_SSO_WQE_SG_PTR); - cn10k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, - ws->tstamp, - flags & NIX_RX_OFFLOAD_TSTAMP_F, - (uint64_t *)tstamp_ptr); - gw.u64[1] = mbuf; - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV_VECTOR) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - __uint128_t vwqe_hdr = *(__uint128_t *)gw.u64[1]; - - vwqe_hdr = ((vwqe_hdr >> 64) & 0xFFF) | BIT_ULL(31) | - ((vwqe_hdr & 0xFFFF) << 48) | - ((uint64_t)port << 32); - *(uint64_t *)gw.u64[1] = (uint64_t)vwqe_hdr; - cn10k_process_vwqe(gw.u64[1], port, flags, lookup_mem, - ws->tstamp, ws->lmt_base); - /* Mark vector mempool object as get */ - RTE_MEMPOOL_CHECK_COOKIES( - rte_mempool_from_obj((void *)gw.u64[1]), - (void **)&gw.u64[1], 1, 1); - } - } + if (gw.u64[1]) + cn10k_sso_hws_post_process(ws, gw.u64, flags); ev->event = gw.u64[0]; ev->u64 = gw.u64[1]; @@ -308,13 +305,13 @@ cn10k_sso_hws_get_work(struct cn10k_sso_hws *ws, struct rte_event *ev, /* Used in cleaning up workslot. */ static __rte_always_inline uint16_t -cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev) +cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev, + const uint32_t flags) { union { __uint128_t get_work; uint64_t u64[2]; } gw; - uint64_t mbuf; #ifdef RTE_ARCH_ARM64 asm volatile(PLT_CPU_FEATURE_PREAMBLE @@ -325,9 +322,7 @@ cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev) " ldp %[tag], %[wqp], [%[tag_loc]] \n" " tbnz %[tag], 63, rty%= \n" "done%=: dmb ld \n" - " sub %[mbuf], %[wqp], #0x80 \n" - : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1]), - [mbuf] "=&r"(mbuf) + : [tag] "=&r"(gw.u64[0]), [wqp] "=&r"(gw.u64[1]) : [tag_loc] "r"(ws->base + SSOW_LF_GWS_WQE0) : "memory"); #else @@ -335,24 +330,11 @@ cn10k_sso_hws_get_work_empty(struct cn10k_sso_hws *ws, struct rte_event *ev) roc_load_pair(gw.u64[0], gw.u64[1], ws->base + SSOW_LF_GWS_WQE0); } while (gw.u64[0] & BIT_ULL(63)); - mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); #endif - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | - (gw.u64[0] & (0x3FFull << 36)) << 4 | - (gw.u64[0] & 0xffffffff); - - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { - if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); - cn10k_wqe_to_mbuf(gw.u64[1], mbuf, port, - gw.u64[0] & 0xFFFFF, 0, NULL); - gw.u64[1] = mbuf; - } - } + ws->gw_rdata = gw.u64[0]; + if (gw.u64[1]) + cn10k_sso_hws_post_process(ws, gw.u64, flags); ev->event = gw.u64[0]; ev->u64 = gw.u64[1]; @@ -471,7 +453,7 @@ NIX_RX_FASTPATH_MODES ws->base + SSOW_LF_GWS_WQE0); \ return 1; \ } \ - return cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \ + return cn10k_sso_hws_get_work(ws, ev, flags); \ } #define SSO_DEQ_SEG(fn, flags) SSO_DEQ(fn, flags | NIX_RX_MULTI_SEG_F) @@ -491,10 +473,9 @@ NIX_RX_FASTPATH_MODES ws->base + SSOW_LF_GWS_WQE0); \ return ret; \ } \ - ret = cn10k_sso_hws_get_work(ws, ev, flags, ws->lookup_mem); \ + ret = cn10k_sso_hws_get_work(ws, ev, flags); \ for (iter = 1; iter < timeout_ticks && (ret == 0); iter++) \ - ret = cn10k_sso_hws_get_work(ws, ev, flags, \ - ws->lookup_mem); \ + ret = cn10k_sso_hws_get_work(ws, ev, flags); \ return ret; \ } diff --git a/drivers/event/cnxk/cn9k_eventdev.c b/drivers/event/cnxk/cn9k_eventdev.c index 41bbe3cb22..987888d3db 100644 --- a/drivers/event/cnxk/cn9k_eventdev.c +++ b/drivers/event/cnxk/cn9k_eventdev.c @@ -122,6 +122,7 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, cnxk_handle_event_t fn, void *arg) { struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(arg); + struct cnxk_timesync_info *tstamp; struct cn9k_sso_hws_dual *dws; struct cn9k_sso_hws *ws; uint64_t cq_ds_cnt = 1; @@ -130,6 +131,7 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, struct rte_event ev; uintptr_t ws_base; uint64_t val, req; + void *lookup_mem; plt_write64(0, base + SSO_LF_GGRP_QCTL); @@ -145,14 +147,19 @@ cn9k_sso_hws_flush_events(void *hws, uint8_t queue_id, uintptr_t base, if (dev->dual_ws) { dws = hws; ws_base = dws->base[0]; + lookup_mem = dws->lookup_mem; + tstamp = dws->tstamp; } else { ws = hws; ws_base = ws->base; + lookup_mem = ws->lookup_mem; + tstamp = ws->tstamp; } while (aq_cnt || cq_ds_cnt || ds_cnt) { plt_write64(req, ws_base + SSOW_LF_GWS_OP_GET_WORK0); - cn9k_sso_hws_get_work_empty(ws_base, &ev); + cn9k_sso_hws_get_work_empty(ws_base, &ev, dev->rx_offloads, + lookup_mem, tstamp); if (fn != NULL && ev.u64 != 0) fn(arg, ev); if (ev.sched_type != SSO_TT_EMPTY) diff --git a/drivers/event/cnxk/cn9k_worker.h b/drivers/event/cnxk/cn9k_worker.h index 88eb4e9cf9..64e97e321a 100644 --- a/drivers/event/cnxk/cn9k_worker.h +++ b/drivers/event/cnxk/cn9k_worker.h @@ -169,6 +169,34 @@ cn9k_wqe_to_mbuf(uint64_t wqe, const uint64_t mbuf, uint8_t port_id, mbuf_init | ((uint64_t)port_id) << 48, flags); } +static __rte_always_inline void +cn9k_sso_hws_post_process(uint64_t *u64, uint64_t mbuf, const uint32_t flags, + const void *const lookup_mem, + struct cnxk_timesync_info *tstamp) +{ + uint64_t tstamp_ptr; + + u64[0] = (u64[0] & (0x3ull << 32)) << 6 | + (u64[0] & (0x3FFull << 36)) << 4 | (u64[0] & 0xffffffff); + if ((flags & CPT_RX_WQE_F) && + (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_CRYPTODEV)) { + u64[1] = cn9k_cpt_crypto_adapter_dequeue(u64[1]); + } else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_ETHDEV) { + uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]); + + u64[0] = CNXK_CLR_SUB_EVENT(u64[0]); + cn9k_wqe_to_mbuf(u64[1], mbuf, port, u64[0] & 0xFFFFF, flags, + lookup_mem); + /* Extracting tstamp, if PTP enabled*/ + tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)u64[1]) + + CNXK_SSO_WQE_SG_PTR); + cn9k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, tstamp, + flags & NIX_RX_OFFLOAD_TSTAMP_F, + (uint64_t *)tstamp_ptr); + u64[1] = mbuf; + } +} + static __rte_always_inline uint16_t cn9k_sso_hws_dual_get_work(uint64_t base, uint64_t pair_base, struct rte_event *ev, const uint32_t flags, @@ -178,7 +206,6 @@ cn9k_sso_hws_dual_get_work(uint64_t base, uint64_t pair_base, __uint128_t get_work; uint64_t u64[2]; } gw; - uint64_t tstamp_ptr; uint64_t mbuf; if (flags & NIX_RX_OFFLOAD_PTYPE_F) @@ -207,34 +234,9 @@ cn9k_sso_hws_dual_get_work(uint64_t base, uint64_t pair_base, mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); #endif - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | - (gw.u64[0] & (0x3FFull << 36)) << 4 | - (gw.u64[0] & 0xffffffff); - - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { - if ((flags & CPT_RX_WQE_F) && - (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_CRYPTODEV)) { - gw.u64[1] = cn9k_cpt_crypto_adapter_dequeue(gw.u64[1]); - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); - cn9k_wqe_to_mbuf(gw.u64[1], mbuf, port, - gw.u64[0] & 0xFFFFF, flags, - dws->lookup_mem); - /* Extracting tstamp, if PTP enabled*/ - tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *) - gw.u64[1]) + - CNXK_SSO_WQE_SG_PTR); - cn9k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, - dws->tstamp, - flags & NIX_RX_OFFLOAD_TSTAMP_F, - (uint64_t *)tstamp_ptr); - gw.u64[1] = mbuf; - } - } + if (gw.u64[1]) + cn9k_sso_hws_post_process(gw.u64, mbuf, flags, dws->lookup_mem, + dws->tstamp); ev->event = gw.u64[0]; ev->u64 = gw.u64[1]; @@ -250,7 +252,6 @@ cn9k_sso_hws_get_work(struct cn9k_sso_hws *ws, struct rte_event *ev, __uint128_t get_work; uint64_t u64[2]; } gw; - uint64_t tstamp_ptr; uint64_t mbuf; plt_write64(ws->gw_wdata, ws->base + SSOW_LF_GWS_OP_GET_WORK0); @@ -283,34 +284,9 @@ cn9k_sso_hws_get_work(struct cn9k_sso_hws *ws, struct rte_event *ev, mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); #endif - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | - (gw.u64[0] & (0x3FFull << 36)) << 4 | - (gw.u64[0] & 0xffffffff); - - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { - if ((flags & CPT_RX_WQE_F) && - (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_CRYPTODEV)) { - gw.u64[1] = cn9k_cpt_crypto_adapter_dequeue(gw.u64[1]); - } else if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); - cn9k_wqe_to_mbuf(gw.u64[1], mbuf, port, - gw.u64[0] & 0xFFFFF, flags, - lookup_mem); - /* Extracting tstamp, if PTP enabled*/ - tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *) - gw.u64[1]) + - CNXK_SSO_WQE_SG_PTR); - cn9k_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, - ws->tstamp, - flags & NIX_RX_OFFLOAD_TSTAMP_F, - (uint64_t *)tstamp_ptr); - gw.u64[1] = mbuf; - } - } + if (gw.u64[1]) + cn9k_sso_hws_post_process(gw.u64, mbuf, flags, lookup_mem, + ws->tstamp); ev->event = gw.u64[0]; ev->u64 = gw.u64[1]; @@ -320,7 +296,9 @@ cn9k_sso_hws_get_work(struct cn9k_sso_hws *ws, struct rte_event *ev, /* Used in cleaning up workslot. */ static __rte_always_inline uint16_t -cn9k_sso_hws_get_work_empty(uint64_t base, struct rte_event *ev) +cn9k_sso_hws_get_work_empty(uint64_t base, struct rte_event *ev, + const uint32_t flags, void *lookup_mem, + struct cnxk_timesync_info *tstamp) { union { __uint128_t get_work; @@ -353,21 +331,9 @@ cn9k_sso_hws_get_work_empty(uint64_t base, struct rte_event *ev) mbuf = (uint64_t)((char *)gw.u64[1] - sizeof(struct rte_mbuf)); #endif - gw.u64[0] = (gw.u64[0] & (0x3ull << 32)) << 6 | - (gw.u64[0] & (0x3FFull << 36)) << 4 | - (gw.u64[0] & 0xffffffff); - - if (CNXK_TT_FROM_EVENT(gw.u64[0]) != SSO_TT_EMPTY) { - if (CNXK_EVENT_TYPE_FROM_TAG(gw.u64[0]) == - RTE_EVENT_TYPE_ETHDEV) { - uint8_t port = CNXK_SUB_EVENT_FROM_TAG(gw.u64[0]); - - gw.u64[0] = CNXK_CLR_SUB_EVENT(gw.u64[0]); - cn9k_wqe_to_mbuf(gw.u64[1], mbuf, port, - gw.u64[0] & 0xFFFFF, 0, NULL); - gw.u64[1] = mbuf; - } - } + if (gw.u64[1]) + cn9k_sso_hws_post_process(gw.u64, mbuf, flags, lookup_mem, + tstamp); ev->event = gw.u64[0]; ev->u64 = gw.u64[1]; -- 2.25.1 ^ permalink raw reply related [flat|nested] 5+ messages in thread
end of thread, other threads:[~2022-05-16 15:22 UTC | newest] Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed) -- links below jump to the message on this page -- 2022-04-26 21:47 [PATCH 1/2] event/cnxk: add additional checks in OP_RELEASE Pavan Nikhilesh 2022-04-26 21:47 ` [PATCH 2/2] event/cnxk: move post-processing to separate function Pavan Nikhilesh 2022-05-16 15:21 ` Jerin Jacob 2022-04-27 10:58 ` [PATCH v2 1/2] event/cnxk: add additional checks in OP_RELEASE Pavan Nikhilesh 2022-04-27 10:58 ` [PATCH v2 2/2] event/cnxk: move post-processing to separate function Pavan Nikhilesh
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.