Re: [dpdk-dev] [PATCH v6 08/10] ipsec: add support for SA telemetry

["Ananyev, Konstantin" <konstantin.ananyev@intel.com>]

 
> Signed-off-by: Declan Doherty <declan.doherty@intel.com>
> Signed-off-by: Radu Nicolau <radu.nicolau@intel.com>
> Signed-off-by: Abhijit Sinha <abhijit.sinha@intel.com>
> Signed-off-by: Daniel Martin Buckley <daniel.m.buckley@intel.com>
> Acked-by: Fan Zhang <roy.fan.zhang@intel.com>
> ---
>  lib/ipsec/esp_inb.c   |   1 +
>  lib/ipsec/esp_outb.c  |  12 +-
>  lib/ipsec/meson.build |   2 +-
>  lib/ipsec/rte_ipsec.h |  23 ++++
>  lib/ipsec/sa.c        | 255 +++++++++++++++++++++++++++++++++++++++++-
>  lib/ipsec/sa.h        |  21 ++++
>  lib/ipsec/version.map |   9 ++
>  7 files changed, 317 insertions(+), 6 deletions(-)
> 
> diff --git a/lib/ipsec/esp_inb.c b/lib/ipsec/esp_inb.c
> index a6ab8fbdd5..8cb4c16302 100644
> --- a/lib/ipsec/esp_inb.c
> +++ b/lib/ipsec/esp_inb.c
> @@ -722,6 +722,7 @@ esp_inb_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],
> 
>  	/* process packets, extract seq numbers */
>  	k = process(sa, mb, sqn, dr, num, sqh_len);
> +	sa->statistics.count += k;
> 
>  	/* handle unprocessed mbufs */
>  	if (k != num && k != 0)
> diff --git a/lib/ipsec/esp_outb.c b/lib/ipsec/esp_outb.c
> index 9fc7075796..2c02c3bb12 100644
> --- a/lib/ipsec/esp_outb.c
> +++ b/lib/ipsec/esp_outb.c
> @@ -617,7 +617,7 @@ uint16_t
>  esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
>  	uint16_t num)
>  {
> -	uint32_t i, k, icv_len, *icv;
> +	uint32_t i, k, icv_len, *icv, bytes;
>  	struct rte_mbuf *ml;
>  	struct rte_ipsec_sa *sa;
>  	uint32_t dr[num];
> @@ -626,10 +626,12 @@ esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
> 
>  	k = 0;
>  	icv_len = sa->icv_len;
> +	bytes = 0;
> 
>  	for (i = 0; i != num; i++) {
>  		if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0) {
>  			ml = rte_pktmbuf_lastseg(mb[i]);
> +			bytes += mb[i]->data_len;

Shouldn't it be pkt_len?

>  			/* remove high-order 32 bits of esn from packet len */
>  			mb[i]->pkt_len -= sa->sqh_len;
>  			ml->data_len -= sa->sqh_len;
> @@ -640,6 +642,8 @@ esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
>  		} else
>  			dr[i - k] = i;
>  	}
> +	sa->statistics.count += k;
> +	sa->statistics.bytes += bytes - (sa->hdr_len * k);

I don't think you need to do multiplication here.
It can be postponed for reporting phase (sa->hdr_len is a constant value per sa).

> 
>  	/* handle unprocessed mbufs */
>  	if (k != num) {
> @@ -659,16 +663,19 @@ static inline void
>  inline_outb_mbuf_prepare(const struct rte_ipsec_session *ss,
>  	struct rte_mbuf *mb[], uint16_t num)
>  {
> -	uint32_t i, ol_flags;
> +	uint32_t i, ol_flags, bytes = 0;

Lets keep coding style consistent: please do assignment as separate statement.

> 
>  	ol_flags = ss->security.ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA;
>  	for (i = 0; i != num; i++) {
> 
>  		mb[i]->ol_flags |= PKT_TX_SEC_OFFLOAD;
> +		bytes += mb[i]->data_len;

pkt_len?

>  		if (ol_flags != 0)
>  			rte_security_set_pkt_metadata(ss->security.ctx,
>  				ss->security.ses, mb[i], NULL);
>  	}
> +	ss->sa->statistics.count += num;
> +	ss->sa->statistics.bytes += bytes - (ss->sa->hdr_len * num);
>  }
> 
>  /* check if packet will exceed MSS and segmentation is required */
> @@ -752,6 +759,7 @@ inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss,
>  			sqn += nb_segs[i] - 1;
>  	}
> 
> +

Empty line.

>  	/* copy not processed mbufs beyond good ones */
>  	if (k != num && k != 0)
>  		move_bad_mbufs(mb, dr, num, num - k);
> diff --git a/lib/ipsec/meson.build b/lib/ipsec/meson.build
> index 1497f573bb..f5e44cfe47 100644
> --- a/lib/ipsec/meson.build
> +++ b/lib/ipsec/meson.build
> @@ -6,4 +6,4 @@ sources = files('esp_inb.c', 'esp_outb.c', 'sa.c', 'ses.c', 'ipsec_sad.c')
>  headers = files('rte_ipsec.h', 'rte_ipsec_sa.h', 'rte_ipsec_sad.h')
>  indirect_headers += files('rte_ipsec_group.h')
> 
> -deps += ['mbuf', 'net', 'cryptodev', 'security', 'hash']
> +deps += ['mbuf', 'net', 'cryptodev', 'security', 'hash', 'telemetry']
> diff --git a/lib/ipsec/rte_ipsec.h b/lib/ipsec/rte_ipsec.h
> index dd60d95915..2bb52f4b8f 100644
> --- a/lib/ipsec/rte_ipsec.h
> +++ b/lib/ipsec/rte_ipsec.h
> @@ -158,6 +158,29 @@ rte_ipsec_pkt_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
>  	return ss->pkt_func.process(ss, mb, num);
>  }
> 
> +
> +struct rte_ipsec_telemetry;
> +
> +/**
> + * Initialize IPsec library telemetry.
> + * @return
> + *   0 on success, negative value otherwise.
> + */
> +__rte_experimental
> +int
> +rte_ipsec_telemetry_init(void);
> +
> +/**
> + * Enable per SA telemetry for a specific SA.
> + * @param sa
> + *   Pointer to the *rte_ipsec_sa* object that will have telemetry enabled.
> + * @return
> + *   0 on success, negative value otherwise.
> + */
> +__rte_experimental
> +int
> +rte_ipsec_telemetry_sa_add(struct rte_ipsec_sa *sa);
> +

Why we don't have sa_delete() here?
What user supposed to do when he destroys an sa?
Another question what concurrency model is implied here?

>  #include <rte_ipsec_group.h>
> 
>  #ifdef __cplusplus
> diff --git a/lib/ipsec/sa.c b/lib/ipsec/sa.c
> index 8e369e4618..5b55bbc098 100644
> --- a/lib/ipsec/sa.c
> +++ b/lib/ipsec/sa.c
> @@ -7,7 +7,7 @@
>  #include <rte_ip.h>
>  #include <rte_errno.h>
>  #include <rte_cryptodev.h>
> -
> +#include <rte_telemetry.h>

As a generic one - can we move all telemetry related functions into new .c file
(sa_telemtry or so)? No point to have it here.

>  #include "sa.h"
>  #include "ipsec_sqn.h"
>  #include "crypto.h"
> @@ -25,6 +25,7 @@ struct crypto_xform {
>  	struct rte_crypto_aead_xform *aead;
>  };
> 
> +
>  /*
>   * helper routine, fills internal crypto_xform structure.
>   */
> @@ -532,6 +533,249 @@ rte_ipsec_sa_size(const struct rte_ipsec_sa_prm *prm)
>  	wsz = prm->ipsec_xform.replay_win_sz;
>  	return ipsec_sa_size(type, &wsz, &nb);
>  }
> +struct rte_ipsec_telemetry {
> +	bool initialized;

Why 'initilized' is needed at all?
I think there is a static initializer for list: LIST_HEAD_INITIALIZER

> +	LIST_HEAD(, rte_ipsec_sa) sa_list_head;
> +};
> +
> +#include <rte_malloc.h>
> +
> +static struct rte_ipsec_telemetry rte_ipsec_telemetry_instance = {
> +	.initialized = false };
> +
> +static int
> +handle_telemetry_cmd_ipsec_sa_list(const char *cmd __rte_unused,
> +		const char *params __rte_unused,
> +		struct rte_tel_data *data)
> +{
> +	struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
> +	struct rte_ipsec_sa *sa;
> +
> +	rte_tel_data_start_array(data, RTE_TEL_U64_VAL);
> +
> +	LIST_FOREACH(sa, &telemetry->sa_list_head, telemetry_next) {
> +		rte_tel_data_add_array_u64(data, htonl(sa->spi));

Should be ntohl() I believe.
BTW, why not use rte_be_to_cpu... functions here?

> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * Handle IPsec SA statistics telemetry request
> + *
> + * Return dict of SA's with dict of key/value counters
> + *
> + * {
> + *     "SA_SPI_XX": {"count": 0, "bytes": 0, "errors": 0},
> + *     "SA_SPI_YY": {"count": 0, "bytes": 0, "errors": 0}
> + * }
> + *
> + */
> +static int
> +handle_telemetry_cmd_ipsec_sa_stats(const char *cmd __rte_unused,
> +		const char *params,
> +		struct rte_tel_data *data)
> +{
> +	struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
> +	struct rte_ipsec_sa *sa;
> +	bool user_specified_spi = false;
> +	uint32_t sa_spi;
> +
> +	if (params) {
> +		user_specified_spi = true;
> +		sa_spi = htonl((uint32_t)atoi(params));

strtoul() would be a better choice here.
Another nit - you probably don't need user_specified_spi.
As I remember SPI=0 is a reserved value, so I think It would be enough to:
sa_spi=0; if (params) {sa_spi=..}

> +	}
> +
> +	rte_tel_data_start_dict(data);
> +
> +	LIST_FOREACH(sa, &telemetry->sa_list_head, telemetry_next) {
> +		char sa_name[64];
> +
> +		static const char *name_pkt_cnt = "count";
> +		static const char *name_byte_cnt = "bytes";
> +		static const char *name_error_cnt = "errors";
> +		struct rte_tel_data *sa_data;
> +
> +		/* If user provided SPI only get telemetry for that SA */
> +		if (user_specified_spi && (sa_spi != sa->spi))
> +			continue;
> +
> +		/* allocate telemetry data struct for SA telemetry */
> +		sa_data = rte_tel_data_alloc();
> +		if (!sa_data)
> +			return -ENOMEM;
> +
> +		rte_tel_data_start_dict(sa_data);
> +
> +		/* add telemetry key/values pairs */
> +		rte_tel_data_add_dict_u64(sa_data, name_pkt_cnt,
> +					sa->statistics.count);
> +
> +		rte_tel_data_add_dict_u64(sa_data, name_byte_cnt,
> +					sa->statistics.bytes);
> +
> +		rte_tel_data_add_dict_u64(sa_data, name_error_cnt,
> +					sa->statistics.errors.count);
> +
> +		/* generate telemetry label */
> +		snprintf(sa_name, sizeof(sa_name), "SA_SPI_%i", htonl(sa->spi));

Again - ntohl().

> +
> +		/* add SA telemetry to dictionary container */
> +		rte_tel_data_add_dict_container(data, sa_name, sa_data, 0);
> +	}
> +
> +	return 0;
> +}
> +
> +static int
> +handle_telemetry_cmd_ipsec_sa_configuration(const char *cmd __rte_unused,
> +		const char *params,
> +		struct rte_tel_data *data)
> +{
> +	struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
> +	struct rte_ipsec_sa *sa;
> +	uint32_t sa_spi;
> +
> +	if (params)
> +		sa_spi = htonl((uint32_t)atoi(params));
> +	else
> +		return -EINVAL;
> +
> +	rte_tel_data_start_dict(data);
> +
> +	LIST_FOREACH(sa, &telemetry->sa_list_head, telemetry_next) {
> +		uint64_t mode;
> +
> +		if (sa_spi != sa->spi)
> +			continue;
> +
> +		/* add SA configuration key/values pairs */
> +		rte_tel_data_add_dict_string(data, "Type",
> +			(sa->type & RTE_IPSEC_SATP_PROTO_MASK) ==
> +			RTE_IPSEC_SATP_PROTO_AH ? "AH" : "ESP");
> +
> +		rte_tel_data_add_dict_string(data, "Direction",
> +			(sa->type & RTE_IPSEC_SATP_DIR_MASK) ==
> +			RTE_IPSEC_SATP_DIR_IB ?	"Inbound" : "Outbound");
> +
> +		mode = sa->type & RTE_IPSEC_SATP_MODE_MASK;
> +
> +		if (mode == RTE_IPSEC_SATP_MODE_TRANS) {
> +			rte_tel_data_add_dict_string(data, "Mode", "Transport");
> +		} else {
> +			rte_tel_data_add_dict_string(data, "Mode", "Tunnel");
> +
> +			if ((sa->type & RTE_IPSEC_SATP_NATT_MASK) ==
> +				RTE_IPSEC_SATP_NATT_ENABLE) {
> +				if (sa->type & RTE_IPSEC_SATP_MODE_TUNLV4) {
> +					rte_tel_data_add_dict_string(data,
> +						"Tunnel-Type",
> +						"IPv4-UDP");
> +				} else if (sa->type &
> +						RTE_IPSEC_SATP_MODE_TUNLV6) {
> +					rte_tel_data_add_dict_string(data,
> +						"Tunnel-Type",
> +						"IPv4-UDP");
> +				}
> +			} else {
> +				if (sa->type & RTE_IPSEC_SATP_MODE_TUNLV4) {
> +					rte_tel_data_add_dict_string(data,
> +						"Tunnel-Type",
> +						"IPv4-UDP");
> +				} else if (sa->type &
> +						RTE_IPSEC_SATP_MODE_TUNLV6) {
> +					rte_tel_data_add_dict_string(data,
> +						"Tunnel-Type",
> +						"IPv4-UDP");
> +				}
> +			}
> +		}
> +
> +		rte_tel_data_add_dict_string(data,
> +				"extended-sequence-number",
> +				(sa->type & RTE_IPSEC_SATP_ESN_MASK) ==
> +				 RTE_IPSEC_SATP_ESN_ENABLE ?
> +				"enabled" : "disabled");
> +
> +		if ((sa->type & RTE_IPSEC_SATP_DIR_MASK) ==
> +			RTE_IPSEC_SATP_DIR_IB)
> +
> +			if (sa->sqn.inb.rsn[sa->sqn.inb.rdidx])
> +				rte_tel_data_add_dict_u64(data,
> +				"sequence-number",
> +				sa->sqn.inb.rsn[sa->sqn.inb.rdidx]->sqn);
> +			else
> +				rte_tel_data_add_dict_u64(data,
> +					"sequence-number", 0);
> +		else
> +			rte_tel_data_add_dict_u64(data, "sequence-number",
> +					sa->sqn.outb);
> +
> +		rte_tel_data_add_dict_string(data,
> +				"explicit-congestion-notification",
> +				(sa->type & RTE_IPSEC_SATP_ECN_MASK) ==
> +				RTE_IPSEC_SATP_ECN_ENABLE ?
> +				"enabled" : "disabled");
> +
> +		rte_tel_data_add_dict_string(data,
> +				"copy-DSCP",
> +				(sa->type & RTE_IPSEC_SATP_DSCP_MASK) ==
> +				RTE_IPSEC_SATP_DSCP_ENABLE ?
> +				"enabled" : "disabled");
> +
> +		rte_tel_data_add_dict_string(data, "TSO",
> +				sa->tso.enabled ? "enabled" : "disabled");
> +
> +		if (sa->tso.enabled)
> +			rte_tel_data_add_dict_u64(data, "TSO-MSS", sa->tso.mss);
> +
> +	}
> +
> +	return 0;
> +}
> +int
> +rte_ipsec_telemetry_init(void)
> +{
> +	struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
> +	int rc = 0;
> +
> +	if (telemetry->initialized)
> +		return rc;
> +
> +	LIST_INIT(&telemetry->sa_list_head);
> +
> +	rc = rte_telemetry_register_cmd("/ipsec/sa/list",
> +		handle_telemetry_cmd_ipsec_sa_list,
> +		"Return list of IPsec Security Associations with telemetry enabled.");
> +	if (rc)
> +		return rc;
> +
> +	rc = rte_telemetry_register_cmd("/ipsec/sa/stats",
> +		handle_telemetry_cmd_ipsec_sa_stats,
> +		"Returns IPsec Security Association stastistics. Parameters: int sa_spi");
> +	if (rc)
> +		return rc;
> +
> +	rc = rte_telemetry_register_cmd("/ipsec/sa/details",
> +		handle_telemetry_cmd_ipsec_sa_configuration,
> +		"Returns IPsec Security Association configuration. Parameters: int sa_spi");
> +	if (rc)
> +		return rc;
> +
> +	telemetry->initialized = true;
> +
> +	return rc;
> +}
> +
> +int
> +rte_ipsec_telemetry_sa_add(struct rte_ipsec_sa *sa)
> +{
> +	struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
> +
> +	LIST_INSERT_HEAD(&telemetry->sa_list_head, sa, telemetry_next);
> +
> +	return 0;
> +}
> 
>  int
>  rte_ipsec_sa_init(struct rte_ipsec_sa *sa, const struct rte_ipsec_sa_prm *prm,
> @@ -644,19 +888,24 @@ uint16_t
>  pkt_flag_process(const struct rte_ipsec_session *ss,
>  		struct rte_mbuf *mb[], uint16_t num)
>  {
> -	uint32_t i, k;
> +	uint32_t i, k, bytes = 0;
>  	uint32_t dr[num];
> 
>  	RTE_SET_USED(ss);
> 
>  	k = 0;
>  	for (i = 0; i != num; i++) {
> -		if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0)
> +		if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0) {
>  			k++;
> +			bytes += mb[i]->data_len;
> +		}
>  		else
>  			dr[i - k] = i;
>  	}
> 
> +	ss->sa->statistics.count += k;
> +	ss->sa->statistics.bytes += bytes - (ss->sa->hdr_len * k);
> +
>  	/* handle unprocessed mbufs */
>  	if (k != num) {
>  		rte_errno = EBADMSG;
> diff --git a/lib/ipsec/sa.h b/lib/ipsec/sa.h
> index 3f38921eb3..b9b7ebec5b 100644
> --- a/lib/ipsec/sa.h
> +++ b/lib/ipsec/sa.h
> @@ -122,9 +122,30 @@ struct rte_ipsec_sa {
>  		uint16_t mss;
>  	} tso;
> 
> +	LIST_ENTRY(rte_ipsec_sa) telemetry_next;
> +	/**< list entry for telemetry enabled SA */

I am not really fond of idea to have telemetry list stuff embedded into rte_ipsec_sa structure.
Creates all sort of concurrency problem for adding/removing SA, while reading telemetry data, etc.
Another issue if SA is shared my multiple-processes.
Instead would be much cleaner if telemetry list will contain just a pointer to SA.
Then it would be user responsibility to add del/add sa to the telelmetry list in an appropriate time.
Also MT working model for this new API needs to be documented properly.

> +
> +
> +	RTE_MARKER cachealign_statistics __rte_cache_min_aligned;

What is the reason for all these extra alignments?

> +
> +	/* Statistics */
> +	struct {
> +		uint64_t count;
> +		uint64_t bytes;
> +
> +		struct {
> +			uint64_t count;
> +			uint64_t authentication_failed;
> +		} errors;
> +	} statistics;
> +
> +	RTE_MARKER cachealign_tunnel_header __rte_cache_min_aligned;
> +
>  	/* template for tunnel header */
>  	uint8_t hdr[IPSEC_MAX_HDR_SIZE];
> 
> +
> +	RTE_MARKER cachealign_tunnel_seq_num_replay_win __rte_cache_min_aligned;
>  	/*
>  	 * sqn and replay window
>  	 * In case of SA handled by multiple threads *sqn* cacheline
> diff --git a/lib/ipsec/version.map b/lib/ipsec/version.map
> index ba8753eac4..fed6b6aba1 100644
> --- a/lib/ipsec/version.map
> +++ b/lib/ipsec/version.map
> @@ -19,3 +19,12 @@ DPDK_22 {
> 
>  	local: *;
>  };
> +
> +EXPERIMENTAL {
> +	global:
> +
> +	# added in 21.11
> +	rte_ipsec_telemetry_init;
> +	rte_ipsec_telemetry_sa_add;
> +
> +};
> --
> 2.25.1