From: Nagadheeraj Rottela <rnagadheeraj@marvell.com> To: "dev@dpdk.org" <dev@dpdk.org> Cc: Srikanth Jampala <jsrikanth@marvell.com>, Nagadheeraj Rottela <rnagadheeraj@marvell.com> Subject: [dpdk-dev] [PATCH 08/10] crypto/nitrox: add burst enqueue and dequeue operations Date: Wed, 17 Jul 2019 05:29:12 +0000 Message-ID: <20190717052837.647-9-rnagadheeraj@marvell.com> (raw) In-Reply-To: <20190717052837.647-1-rnagadheeraj@marvell.com> Add burst enqueue and dequeue operations along with interface for symmetric request manager. Signed-off-by: Nagadheeraj Rottela <rnagadheeraj@marvell.com> --- drivers/crypto/nitrox/nitrox_qp.h | 55 ++++++++++ drivers/crypto/nitrox/nitrox_sym.c | 123 ++++++++++++++++++++- drivers/crypto/nitrox/nitrox_sym_reqmgr.c | 173 ++++++++++++++++++++++++++++++ drivers/crypto/nitrox/nitrox_sym_reqmgr.h | 10 ++ 4 files changed, 359 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/nitrox/nitrox_qp.h b/drivers/crypto/nitrox/nitrox_qp.h index 0244c4dbf..645fa8925 100644 --- a/drivers/crypto/nitrox/nitrox_qp.h +++ b/drivers/crypto/nitrox/nitrox_qp.h @@ -34,12 +34,67 @@ struct nitrox_qp { rte_atomic16_t pending_count; }; +static inline uint16_t +nitrox_qp_free_count(struct nitrox_qp *qp) +{ + uint16_t pending_count = rte_atomic16_read(&qp->pending_count); + + RTE_ASSERT(qp->count >= pending_count); + return (qp->count - pending_count); +} + static inline bool nitrox_qp_is_empty(struct nitrox_qp *qp) { return (rte_atomic16_read(&qp->pending_count) == 0); } +static inline uint16_t +nitrox_qp_used_count(struct nitrox_qp *qp) +{ + return rte_atomic16_read(&qp->pending_count); +} + +static inline struct nitrox_softreq * +nitrox_qp_get_softreq(struct nitrox_qp *qp) +{ + uint32_t tail = qp->tail % qp->count; + + return qp->ridq[tail].sr; +} + +static inline void +nitrox_ring_dbell(struct nitrox_qp *qp, uint16_t cnt) +{ + struct command_queue *cmdq = &qp->cmdq; + + if (!cnt) + return; + + rte_write64(cnt, cmdq->dbell_csr_addr); +} + +static inline void +nitrox_qp_enqueue(struct nitrox_qp *qp, void *instr, struct nitrox_softreq *sr) +{ + uint32_t head = qp->head % qp->count; + + memcpy(&qp->cmdq.ring[head * qp->cmdq.instr_size], + instr, qp->cmdq.instr_size); + qp->ridq[head].sr = sr; + qp->head++; + rte_atomic16_inc(&qp->pending_count); + rte_wmb(); +} + +static inline void +nitrox_qp_dequeue(struct nitrox_qp *qp) +{ + qp->tail++; + rte_atomic16_dec(&qp->pending_count); + rte_smp_mb(); +} + int nitrox_qp_setup(struct nitrox_qp *qp, uint8_t *bar_addr, const char *dev_name, uint32_t nb_descriptors, uint8_t inst_size, int socket_id); diff --git a/drivers/crypto/nitrox/nitrox_sym.c b/drivers/crypto/nitrox/nitrox_sym.c index 34c62b02e..9ccc28755 100644 --- a/drivers/crypto/nitrox/nitrox_sym.c +++ b/drivers/crypto/nitrox/nitrox_sym.c @@ -521,6 +521,125 @@ nitrox_sym_dev_sess_clear(struct rte_cryptodev *cdev, rte_mempool_put(sess_mp, ctx); } +static struct nitrox_crypto_ctx * +get_crypto_ctx(struct rte_crypto_op *op) +{ + if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { + if (likely(op->sym->session)) + return get_sym_session_private_data(op->sym->session, + nitrox_sym_drv_id); + + } + + return NULL; +} + +static int +nitrox_enq_single_op(struct nitrox_qp *qp, struct rte_crypto_op *op) +{ + struct nitrox_crypto_ctx *ctx; + struct nitrox_softreq *sr; + int err; + + op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; + + ctx = get_crypto_ctx(op); + if (unlikely(!ctx)) { + op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; + return -EINVAL; + } + + if (unlikely(rte_mempool_get(qp->sr_mp, (void **)&sr))) + return -ENOMEM; + + err = nitrox_process_se_req(qp->qno, op, ctx, sr); + if (unlikely(err)) { + rte_mempool_put(qp->sr_mp, sr); + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + return err; + } + + nitrox_qp_enqueue(qp, nitrox_sym_instr_addr(sr), sr); + return 0; +} + +static uint16_t +nitrox_sym_dev_enq_burst(void *queue_pair, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct nitrox_qp *qp = queue_pair; + uint16_t free_slots = 0; + uint16_t cnt = 0; + bool err = false; + + free_slots = nitrox_qp_free_count(qp); + if (nb_ops > free_slots) + nb_ops = free_slots; + + for (cnt = 0; cnt < nb_ops; cnt++) { + if (unlikely(nitrox_enq_single_op(qp, ops[cnt]))) { + err = true; + break; + } + } + + nitrox_ring_dbell(qp, cnt); + qp->stats.enqueued_count += cnt; + if (unlikely(err)) + qp->stats.enqueue_err_count++; + + return cnt; +} + +static int +nitrox_deq_single_op(struct nitrox_qp *qp, struct rte_crypto_op **op_ptr) +{ + struct nitrox_softreq *sr; + int ret; + struct rte_crypto_op *op; + + sr = nitrox_qp_get_softreq(qp); + ret = nitrox_check_se_req(sr, op_ptr); + if (ret < 0) + return -EAGAIN; + + op = *op_ptr; + nitrox_qp_dequeue(qp); + rte_mempool_put(qp->sr_mp, sr); + if (!ret) { + op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; + qp->stats.dequeued_count++; + + return 0; + } + + if (ret == MC_MAC_MISMATCH_ERR_CODE) + op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; + else + op->status = RTE_CRYPTO_OP_STATUS_ERROR; + qp->stats.dequeue_err_count++; + + return 0; +} + +static uint16_t +nitrox_sym_dev_deq_burst(void *queue_pair, struct rte_crypto_op **ops, + uint16_t nb_ops) +{ + struct nitrox_qp *qp = queue_pair; + uint16_t filled_slots = nitrox_qp_used_count(qp); + int cnt = 0; + + if (nb_ops > filled_slots) + nb_ops = filled_slots; + + for (cnt = 0; cnt < nb_ops; cnt++) + if (nitrox_deq_single_op(qp, &ops[cnt])) + break; + + return cnt; +} + static struct rte_cryptodev_ops nitrox_cryptodev_ops = { .dev_configure = nitrox_sym_dev_config, .dev_start = nitrox_sym_dev_start, @@ -565,8 +684,8 @@ nitrox_sym_pmd_create(struct nitrox_device *ndev) ndev->rte_sym_dev.name = cdev->data->name; cdev->driver_id = nitrox_sym_drv_id; cdev->dev_ops = &nitrox_cryptodev_ops; - cdev->enqueue_burst = NULL; - cdev->dequeue_burst = NULL; + cdev->enqueue_burst = nitrox_sym_dev_enq_burst; + cdev->dequeue_burst = nitrox_sym_dev_deq_burst; cdev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | RTE_CRYPTODEV_FF_HW_ACCELERATED | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | diff --git a/drivers/crypto/nitrox/nitrox_sym_reqmgr.c b/drivers/crypto/nitrox/nitrox_sym_reqmgr.c index 42d67317c..87d08a0c1 100644 --- a/drivers/crypto/nitrox/nitrox_sym_reqmgr.c +++ b/drivers/crypto/nitrox/nitrox_sym_reqmgr.c @@ -9,7 +9,107 @@ #include "nitrox_sym_reqmgr.h" #include "nitrox_logs.h" +#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL +#define CMD_TIMEOUT 2 + +union pkt_instr_hdr { + uint64_t value; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t raz_48_63 : 16; + uint64_t g : 1; + uint64_t gsz : 7; + uint64_t ihi : 1; + uint64_t ssz : 7; + uint64_t raz_30_31 : 2; + uint64_t fsz : 6; + uint64_t raz_16_23 : 8; + uint64_t tlen : 16; +#else + uint64_t tlen : 16; + uint64_t raz_16_23 : 8; + uint64_t fsz : 6; + uint64_t raz_30_31 : 2; + uint64_t ssz : 7; + uint64_t ihi : 1; + uint64_t gsz : 7; + uint64_t g : 1; + uint64_t raz_48_63 : 16; +#endif + } s; +}; + +union pkt_hdr { + uint64_t value[2]; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t opcode : 8; + uint64_t arg : 8; + uint64_t ctxc : 2; + uint64_t unca : 1; + uint64_t raz_44 : 1; + uint64_t info : 3; + uint64_t destport : 9; + uint64_t unc : 8; + uint64_t raz_19_23 : 5; + uint64_t grp : 3; + uint64_t raz_15 : 1; + uint64_t ctxl : 7; + uint64_t uddl : 8; +#else + uint64_t uddl : 8; + uint64_t ctxl : 7; + uint64_t raz_15 : 1; + uint64_t grp : 3; + uint64_t raz_19_23 : 5; + uint64_t unc : 8; + uint64_t destport : 9; + uint64_t info : 3; + uint64_t raz_44 : 1; + uint64_t unca : 1; + uint64_t ctxc : 2; + uint64_t arg : 8; + uint64_t opcode : 8; +#endif + uint64_t ctxp; + } s; +}; + +union slc_store_info { + uint64_t value[2]; + struct { +#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN + uint64_t raz_39_63 : 25; + uint64_t ssz : 7; + uint64_t raz_0_31 : 32; +#else + uint64_t raz_0_31 : 32; + uint64_t ssz : 7; + uint64_t raz_39_63 : 25; +#endif + uint64_t rptr; + } s; +}; + +struct nps_pkt_instr { + uint64_t dptr0; + union pkt_instr_hdr ih; + union pkt_hdr irh; + union slc_store_info slc; + uint64_t fdata[2]; +}; + +struct resp_hdr { + uint64_t orh; + uint64_t completion; +}; + struct nitrox_softreq { + struct nitrox_crypto_ctx *ctx; + struct rte_crypto_op *op; + struct nps_pkt_instr instr; + struct resp_hdr resp; + uint64_t timeout; rte_iova_t iova; }; @@ -20,6 +120,79 @@ softreq_init(struct nitrox_softreq *sr, rte_iova_t iova) sr->iova = iova; } +static int +process_cipher_auth_data(struct nitrox_softreq *sr) +{ + RTE_SET_USED(sr); + return 0; +} + +static int +process_softreq(struct nitrox_softreq *sr) +{ + struct nitrox_crypto_ctx *ctx = sr->ctx; + int err = 0; + + switch (ctx->nitrox_chain) { + case NITROX_CHAIN_CIPHER_AUTH: + case NITROX_CHAIN_AUTH_CIPHER: + err = process_cipher_auth_data(sr); + break; + default: + err = -EINVAL; + break; + } + + return err; +} + +int +nitrox_process_se_req(uint16_t qno, struct rte_crypto_op *op, + struct nitrox_crypto_ctx *ctx, + struct nitrox_softreq *sr) +{ + RTE_SET_USED(qno); + softreq_init(sr, sr->iova); + sr->ctx = ctx; + sr->op = op; + process_softreq(sr); + sr->timeout = rte_get_timer_cycles() + CMD_TIMEOUT * rte_get_timer_hz(); + return 0; +} + +int +nitrox_check_se_req(struct nitrox_softreq *sr, struct rte_crypto_op **op) +{ + uint64_t cc; + uint64_t orh; + int err; + + rte_rmb(); + cc = *(volatile uint64_t *)(&sr->resp.completion); + orh = *(volatile uint64_t *)(&sr->resp.orh); + if (cc != PENDING_SIG) + err = 0; + else if ((orh != PENDING_SIG) && (orh & 0xff)) + err = orh & 0xff; + else if (rte_get_timer_cycles() < sr->timeout) + return -EAGAIN; + else + err = 0xff; + + if (unlikely(err)) + NITROX_LOG(ERR, "Request err 0x%x, orh 0x%"PRIx64"\n", err, + sr->resp.orh); + + *op = sr->op; + return err; +} + +void * +nitrox_sym_instr_addr(struct nitrox_softreq *sr) +{ + return &sr->instr; +} + static void req_pool_obj_init(__rte_unused struct rte_mempool *mp, __rte_unused void *opaque, void *obj, diff --git a/drivers/crypto/nitrox/nitrox_sym_reqmgr.h b/drivers/crypto/nitrox/nitrox_sym_reqmgr.h index 5953c958c..fa2637bdb 100644 --- a/drivers/crypto/nitrox/nitrox_sym_reqmgr.h +++ b/drivers/crypto/nitrox/nitrox_sym_reqmgr.h @@ -5,6 +5,16 @@ #ifndef _NITROX_SYM_REQMGR_H_ #define _NITROX_SYM_REQMGR_H_ +#include "nitrox_sym_ctx.h" + +struct nitrox_qp; +struct nitrox_softreq; + +int nitrox_process_se_req(uint16_t qno, struct rte_crypto_op *op, + struct nitrox_crypto_ctx *ctx, + struct nitrox_softreq *sr); +int nitrox_check_se_req(struct nitrox_softreq *sr, struct rte_crypto_op **op); +void *nitrox_sym_instr_addr(struct nitrox_softreq *sr); struct rte_mempool *nitrox_sym_req_pool_create(struct rte_cryptodev *cdev, uint32_t nobjs, uint16_t qp_id, int socket_id); -- 2.13.6
next prev parent reply index Thread overview: 61+ messages / expand[flat|nested] mbox.gz Atom feed top 2019-07-17 5:29 [dpdk-dev] [PATCH 00/10] add Nitrox crypto device support Nagadheeraj Rottela 2019-07-17 5:29 ` [dpdk-dev] [PATCH 01/10] crypto/nitrox: add Nitrox build and doc skeleton Nagadheeraj Rottela 2019-07-17 5:29 ` [dpdk-dev] [PATCH 02/10] crypto/nitrox: add PCI probe and remove routines Nagadheeraj Rottela 2019-07-17 5:29 ` [dpdk-dev] [PATCH 03/10] crypto/nitrox: create Nitrox symmetric cryptodev Nagadheeraj Rottela 2019-07-17 5:29 ` [dpdk-dev] [PATCH 04/10] crypto/nitrox: add basic symmetric cryptodev operations Nagadheeraj Rottela 2019-07-17 5:29 ` [dpdk-dev] [PATCH 05/10] crypto/nitrox: add software queue management functionality Nagadheeraj Rottela 2019-07-17 5:29 ` [dpdk-dev] [PATCH 06/10] crypto/nitrox: add hardware " Nagadheeraj Rottela 2019-07-17 5:29 ` [dpdk-dev] [PATCH 07/10] crypto/nitrox: add session management operations Nagadheeraj Rottela 2019-07-17 5:29 ` Nagadheeraj Rottela [this message] 2019-07-17 14:16 ` [dpdk-dev] [PATCH 08/10] crypto/nitrox: add burst enqueue and dequeue operations Aaron Conole 2019-07-17 5:29 ` [dpdk-dev] [PATCH 09/10] crypto/nitrox: add cipher auth crypto chain processing Nagadheeraj Rottela 2019-07-17 5:29 ` [dpdk-dev] [PATCH 10/10] test/crypto: add tests for Nitrox PMD Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 00/10] add Nitrox crypto device support Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 01/10] crypto/nitrox: add Nitrox build and doc skeleton Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 02/10] crypto/nitrox: add PCI probe and remove routines Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 03/10] crypto/nitrox: create Nitrox symmetric cryptodev Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 04/10] crypto/nitrox: add basic symmetric cryptodev operations Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 05/10] crypto/nitrox: add software queue management functionality Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 06/10] crypto/nitrox: add hardware " Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 07/10] crypto/nitrox: add session management operations Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 08/10] crypto/nitrox: add burst enqueue and dequeue operations Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 09/10] crypto/nitrox: add cipher auth crypto chain processing Nagadheeraj Rottela 2019-07-19 12:33 ` [dpdk-dev] [PATCH v2 10/10] test/crypto: add tests for Nitrox PMD Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 00/11] add Nitrox crypto device support Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 01/11] crypto/nitrox: add Nitrox build and doc skeleton Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 02/11] crypto/nitrox: add PCI probe and remove routines Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 03/11] crypto/nitrox: create Nitrox symmetric cryptodev Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 04/11] crypto/nitrox: add basic symmetric cryptodev operations Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 05/11] crypto/nitrox: add software queue management functionality Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 06/11] crypto/nitrox: add hardware " Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 07/11] crypto/nitrox: add session management operations Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 08/11] crypto/nitrox: add burst enqueue and dequeue operations Nagadheeraj Rottela 2019-08-25 20:55 ` Mattias Rönnblom 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 00/11] add Nitrox crypto device support Nagadheeraj Rottela 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 01/11] crypto/nitrox: add Nitrox build and doc skeleton Nagadheeraj Rottela 2019-09-20 8:56 ` Akhil Goyal 2019-09-20 9:02 ` Akhil Goyal 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 02/11] crypto/nitrox: add PCI probe and remove routines Nagadheeraj Rottela 2019-09-20 9:15 ` Akhil Goyal 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 03/11] crypto/nitrox: create Nitrox symmetric cryptodev Nagadheeraj Rottela 2019-09-20 9:29 ` Akhil Goyal 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 04/11] crypto/nitrox: add basic symmetric cryptodev operations Nagadheeraj Rottela 2019-09-20 9:44 ` Akhil Goyal 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 05/11] crypto/nitrox: add software queue management functionality Nagadheeraj Rottela 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 06/11] crypto/nitrox: add hardware " Nagadheeraj Rottela 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 07/11] crypto/nitrox: add session management operations Nagadheeraj Rottela 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 08/11] crypto/nitrox: add burst enqueue and dequeue operations Nagadheeraj Rottela 2019-09-20 10:15 ` Akhil Goyal 2019-09-20 11:11 ` Nagadheeraj Rottela 2019-09-20 11:13 ` Akhil Goyal 2019-09-20 11:23 ` Nagadheeraj Rottela 2019-09-20 11:25 ` Akhil Goyal 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 09/11] crypto/nitrox: add cipher auth crypto chain processing Nagadheeraj Rottela 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 10/11] test/crypto: add tests for Nitrox PMD Nagadheeraj Rottela 2019-08-26 12:49 ` [dpdk-dev] [PATCH v4 11/11] crypto/nitrox: add SHA224 and SHA256 HMAC algorithms Nagadheeraj Rottela 2019-09-20 8:49 ` Akhil Goyal 2019-09-20 10:16 ` Akhil Goyal 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 09/11] crypto/nitrox: add cipher auth crypto chain processing Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 10/11] test/crypto: add tests for Nitrox PMD Nagadheeraj Rottela 2019-08-23 10:42 ` [dpdk-dev] [PATCH v3 11/11] crypto/nitrox: add SHA224 and SHA256 HMAC algorithms Nagadheeraj Rottela 2019-07-19 7:27 [dpdk-dev] [PATCH 08/10] crypto/nitrox: add burst enqueue and dequeue operations Nagadheeraj Rottela
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=20190717052837.647-9-rnagadheeraj@marvell.com \ --to=rnagadheeraj@marvell.com \ --cc=dev@dpdk.org \ --cc=jsrikanth@marvell.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: link
DPDK-dev Archive on lore.kernel.org Archives are clonable: git clone --mirror https://lore.kernel.org/dpdk-dev/0 dpdk-dev/git/0.git # If you have public-inbox 1.1+ installed, you may # initialize and index your mirror using the following commands: public-inbox-init -V2 dpdk-dev dpdk-dev/ https://lore.kernel.org/dpdk-dev \ dev@dpdk.org public-inbox-index dpdk-dev Example config snippet for mirrors Newsgroup available over NNTP: nntp://nntp.lore.kernel.org/org.dpdk.dev AGPL code for this site: git clone https://public-inbox.org/public-inbox.git