Re: [PATCH net-next RFC] Dump SW SQ context as part of tx reporter

[Saeed Mahameed <saeedm@mellanox.com>]

On Mon, 2019-04-29 at 17:17 +0300, Aya Levin wrote:
> TX reporter reports an error on two scenarios:
> - TX timeout on a specific tx queue
> - TX completion error on a specific send queue
> Prior to this patch, no dump data was supported by the tx reporter.
> This
> patch adds support for SW data dump of the related SQ context. The
> dump
> is simply the SQ's raw memory snapshot taken right after the error
> was
> reported, before any recovery procedure was launched. With this
> approach, no maintenance is needed as the driver fetch the actual
> data
> according to the layout on which the SQ was compiled with.  By
> providing
> a SW context, one can easily debug error on a given SQ.
> 
> In order to offline translate the raw memory into a human readable
> format, the user can use some out-of-kernel scripts which receives as
> an
> input the following:
> - Object raw memory
> - Driver object compiled with debug info (can be taken/generated at
> any time from the machine)
> - Object name
> 
> An example of such script output can be seen below.
> Note: the script is not offered as part of this patch as it do not
> belong to the kernel, I just described it in order to grasp the
> general
> idea of how/what can be fetched from SW dump via devlink health.
> 

What was the script ? provided a given raw dump how do you find which
version is it, object name ?

did you use any well known raw debug format, like DWARF ?

> The output of the SW dump can be extracted by devlink health command:
> $ sudo devlink health dump show pci/0000:00:0b.0 reporter tx.
>  mlx5e_txqsq: sqn: 6336
>  memory:
>    00 00 00 00 00 00 00 00
>    01 00 00 00 00 00 00 00
>    00 00 00 00 00 00 00 00
>    45 f4 88 cb 09 00 00 00
>    00 00 00 00 00 00 00 00
>    00 00 00 00 00 00 00 00
>    c0 ff ff ff 1f 00 00 00
>    f8 18 1e 89 81 88 ff ff
>    ...
> 
> script output below, with struct members names and actual values:
> 
> struct  mlx5e_txqsq {
> 	short unsigned int         cc 	 0x5 ;
> 	unsigned int               dma_fifo_cc 	 0x5 ;
> 	struct  net_dim {
> 		unsigned char      state 	 0x1 ;
> 		struct  net_dim_stats {
> 			int        ppms 	 0x0 ;
> 			int        bpms 	 0x0 ;
> 			int        epms 	 0x0 ;
> 		} prev_stats;
> 		struct  net_dim_sample {
> 			long long int time 	 0x90766ef9d ;
> 			unsigned int pkt_ctr 	 0x0 ;
> 			unsigned int byte_ctr 	 0x0 ;
> 			short unsigned int event_ctr 	 0x0 ;
> 		} start_sample;
> 		struct  work_struct {
> 			struct   {
> 				long int counter 	 0x1fffffffc0 ;
> 			} data;
> 			struct  list_head {
> 				struct list_head * next 	 0xffff8881b0
> 8998f8 ;
> 				struct list_head * prev 	 0xffff8881b0
> 8998f8 ;
> 			} entry;
> 			void       (*func)(struct work_struct *) 	 0xff
> ffffffa02d0e30 ;
> 		} work;
> 		unsigned char      profile_ix 	 0x60 ;
> 		unsigned char      mode 	 0x72 ;
> 		unsigned char      tune_state 	 0x35 ;
> 		unsigned char      steps_right 	 0xa0 ;
> 		unsigned char      steps_left 	 0xff ;
> 		unsigned char      tired 	 0xff ;
> 	} dim;
> 	short unsigned int         pc 	 0x0 ;
> 	unsigned int               dma_fifo_pc 	 0x0 ;
> 	struct  mlx5e_cq {
> 		struct  mlx5_cqwq {
> 			struct  mlx5_frag_buf_ctrl {
> 				struct mlx5_buf_list * frags 	 0x50
> 0000005 ;
> 				unsigned int sz_m1 	 0x0 ;
> 				short unsigned int frag_sz_m1 	 0x0
> ;
> 				short unsigned int strides_offset 	
>  0x0 ;
> 				unsigned char log_sz 	 0x0 ;
> 				unsigned char log_stride 	 0x0 ;
> 				unsigned char log_frag_strides 	 0x0
> ;
> 			} fbc;
> 			__be32 *   db 	 0x0 ;
> 			unsigned int cc 	 0x0 ;
> 		} wq;
> 		short unsigned int event_ctr 	 0x0 ;
> 		struct napi_struct * napi 	 0x0 ;
> 		struct  mlx5_core_cq {
> 			unsigned int cqn 	 0x0 ;
> 			int        cqe_sz 	 0x0 ;
> 			__be32 *   set_ci_db 	 0xffff8881b1aa4988 ;
> 			__be32 *   arm_db 	 0x3f000003ff ;
> 			struct mlx5_uars_page * uar 	 0x6060a ;
> 			struct  refcount_struct {
> 				struct   {
> 					int    counter 	 0xa1814500 ;
> 				} refs;
> 			} refcount;
> 			struct  completion {
> 				unsigned int done 	 0x5 ;
> 				struct  wait_queue_head {
> 					struct  spinlock {
> 						union   {
> 							struct  raw_spi
> nlock {
> 								struct 
>  qspinlock {
> 									
> union   {
> 									
> 	struct   {
> 									
> 		int                                                    
> counter 	 0x5 ;
> 									
> 	} val;
> 									
> 	struct   {
> 									
> 		unsigned
> char                                          locked 	 0x5 ;
> 									
> 		unsigned
> char                                          pending 	 0x0 ;
> 									
> 	} ;
> 									
> 	struct   {
> 									
> 		short unsigned
> int                                     locked_pending 	 0x5 ;
> 									
> 		short unsigned
> int                                     tail 	 0x0 ;
> 									
> 	} ;
> 									
> } ;
> 								}
> raw_lock;
> 							} rlock;
> 						} ;
> 					} lock;
> 					struct  list_head {
> 						struct list_head *
> next 	 0xffff8881b089bb88 ;
> 						struct list_head *
> prev 	 0x4000000c0a ;
> 					} head;
> 				} wait;
> 			} free;
> 			unsigned int vector 	 0xa1814500 ;
> 			unsigned int irqn 	 0xffff8881 ;
> 			void       (*comp)(struct mlx5_core_cq *) 	
>  0xffff8881a1814504 ;
> 			void       (*event)(struct mlx5_core_cq *, enum
> mlx5_event) 	 0xffff8881a2cdea08 ;
> 			unsigned int cons_index 	 0x1 ;
> 			unsigned int arm_sn 	 0x0 ;
> 			struct mlx5_rsc_debug * dbg 	 0x0 ;
> 			int        pid 	 0x0 ;
> 			struct   {
> 				struct  list_head {
> 					struct list_head * next 	 0xff
> ffffff ;
> 					struct list_head * prev 	 0xff
> ffffffffffffff ;
> 				} list;
> 				void (*comp)(struct mlx5_core_cq *) 	
>  0xffffffffa0356940 ;
> 				void * priv 	 0x0 ;
> 			} tasklet_ctx;
> 			int        reset_notify_added 	 0x0 ;
> 			struct  list_head {
> 				struct list_head * next 	 0xffffffffa0
> 300700 ;
> 				struct list_head * prev 	 0xd ;
> 			} reset_notify;
> 			struct mlx5_eq_comp * eq 	 0x0 ;
> 			short unsigned int uid 	 0x9a70 ;
> 		} mcq;
> 		struct mlx5e_channel * channel 	 0xffff8881b0899a70 ;
> 		struct mlx5_core_dev * mdev 	 0x4800000001 ;
> 		struct  mlx5_wq_ctrl {
> 			struct mlx5_core_dev * mdev 	 0xffffffffa0
> 2d5350 ;
> 			struct  mlx5_frag_buf {
> 				struct mlx5_buf_list * frags 	 0xff
> ffffffa02d5460 ;
> 				int npages 	 0x0 ;
> 				int size 	 0x5 ;
> 				unsigned char page_shift 	 0x8 ;
> 			} buf;
> 			struct  mlx5_db {
> 				__be32 * db 	 0x1c6 ;
> 				union   {
> 					struct mlx5_db_pgdir * pgdir 	
>  0x0 ;
> 					struct mlx5_ib_user_db_page *
> user_page 	 0x0 ;
> 				} u;
> 				long long unsigned int dma 	 0xff
> ff8881b0899ab0 ;
> 				int index 	 0x0 ;
> 			} db;
> 		} wq_ctrl;
> 	} cq;
> 	struct  mlx5_wq_cyc {
> 		struct  mlx5_frag_buf_ctrl {
> 			struct mlx5_buf_list * frags 	 0xffff8881a7
> 600160 ;
> 			unsigned int sz_m1 	 0xa7600160 ;
> 			short unsigned int frag_sz_m1 	 0x8881 ;
> 			short unsigned int strides_offset 	 0xff
> ff ;
> 			unsigned char log_sz 	 0x88 ;
> 			unsigned char log_stride 	 0x49 ;
> 			unsigned char log_frag_strides 	 0xaa ;
> 		} fbc;
> 		__be32 *           db 	 0x1000000000010 ;
> 		short unsigned int sz 	 0xc ;
> 		short unsigned int wqe_ctr 	 0x0 ;
> 		short unsigned int cur_sz 	 0x0 ;
> 	} wq;
> 	unsigned int               dma_fifo_mask 	 0xa1814500 ;
> 	struct mlx5e_sq_stats *    stats 	 0xffff8881a33a0348 ;
> 	struct   {
> 		struct mlx5e_sq_dma * dma_fifo 	 0x1a1814500 ;
> 		struct mlx5e_tx_wqe_info * wqe_info 	 0x14 ;
> 	} db;
> 	void *                     uar_map 	 0x0 ;
> 	struct netdev_queue *      txq 	 0x0 ;
> 	unsigned int               sqn 	 0x18c0 ;
> 	unsigned char              min_inline_mode 	 0x0 ;
> 	struct device *            pdev 	 0x0 ;
> 	unsigned int               mkey_be 	 0x0 ;
> 	long unsigned int          state 	 0x0 ;
> 	struct hwtstamp_config *   tstamp 	 0x0 ;
> 	struct mlx5_clock *        clock 	 0xffff8881b1aa6f88 ;
> 	struct  mlx5_wq_ctrl {
> 		struct mlx5_core_dev * mdev 	 0x3f000003ff ;
> 		struct  mlx5_frag_buf {
> 			struct mlx5_buf_list * frags 	 0x6060a ;
> 			int        npages 	 0xa1814604 ;
> 			int        size 	 0xffff8881 ;
> 			unsigned char page_shift 	 0x0 ;
> 		} buf;
> 		struct  mlx5_db {
> 			__be32 *   db 	 0xfff ;
> 			union   {
> 				struct mlx5_db_pgdir * pgdir 	 0x0
> ;
> 				struct mlx5_ib_user_db_page *
> user_page 	 0x0 ;
> 			} u;
> 			long long unsigned int dma 	 0xffff888188
> 440000 ;
> 			int        index 	 0x8b074000 ;
> 		} db;
> 	} wq_ctrl;
> 	struct mlx5e_channel *     channel 	 0xffffc9000010d800 ;
> 	int                        txq_ix 	 0xa0020180 ;

txq_ix is too hight to make any sense here.


> 	unsigned int               rate_limit 	 0xffff8881 ;
> 	struct  work_struct {
> 		struct   {
> 			long int   counter 	 0x1000018c0 ;
> 		} data;
> 		struct  list_head {
> 			struct list_head * next 	 0xffff8881c32b68e8 ;
> 			struct list_head * prev 	 0x800 ;
> 		} entry;
> 		void               (*func)(struct work_struct *) 	 0x9
> ;
> 	} recover_work;
> } ;
> 
> Signed-off-by: Aya Levin <ayal@mellanox.com>
> ---
>  .../ethernet/mellanox/mlx5/core/en/reporter_tx.c   | 100
> +++++++++++++++++++++
>  1 file changed, 100 insertions(+)
> 
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
> b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
> index 476dd97f7f2f..8a39f5525e57 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
> @@ -9,6 +9,7 @@
>  
>  struct mlx5e_tx_err_ctx {
>  	int (*recover)(struct mlx5e_txqsq *sq);
> +	int (*dump)(struct mlx5e_txqsq *sq);
>  	struct mlx5e_txqsq *sq;
>  };
>  
> @@ -281,10 +282,109 @@ static int mlx5e_tx_reporter_diagnose(struct
> devlink_health_reporter *reporter,
>  	return err;
>  }
>  
> +static int mlx5e_tx_reporter_sw_dump_from_ctx(struct mlx5e_priv
> *priv,
> +					      struct mlx5e_txqsq *sq,
> +					      struct devlink_fmsg
> *fmsg)
> +{
> +	u64 *ptr = (u64 *)sq;
> +	int copy, err;
> +	int i = 0;
> +
> +	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
> +		return 0;
> +
> +	err = devlink_fmsg_pair_nest_start(fmsg, "mlx5e_txqsq");
> +	if (err)
> +		return err;
> +
> +	err = devlink_fmsg_obj_nest_start(fmsg);
> +	if (err)
> +		return err;
> +
> +	err = devlink_fmsg_arr_pair_nest_start(fmsg, "memory");
> +	if (err)
> +		return err;
> +
> +	while (i < sizeof(struct mlx5e_txqsq)) {
> +		copy = sizeof(u64);
> +
> +		if (i + copy > sizeof(struct mlx5e_txqsq))
> +			copy = sizeof(struct mlx5e_txqsq) - i;
> +
> +		err = devlink_fmsg_binary_put(fmsg, ptr, copy);
> +		if (err)
> +			return err;
> +		ptr++;
> +		i += copy;
> +	}
> +
> +	err = devlink_fmsg_arr_pair_nest_end(fmsg);
> +	if (err)
> +		return err;
> +
> +	err = devlink_fmsg_obj_nest_end(fmsg);
> +	if (err)
> +		return err;
> +
> +	err = devlink_fmsg_pair_nest_end(fmsg);
> +
> +	return err;
> +}
> +
> +static int mlx5e_tx_reporter_sw_dump_all(struct mlx5e_priv *priv,
> +					 struct devlink_fmsg *fmsg)
> +{
> +	int i, err = 0;
> +
> +	mutex_lock(&priv->state_lock);
> +
> +	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
> +		goto unlock;
> +
> +	err = devlink_fmsg_arr_pair_nest_start(fmsg, "SQs");
> +	if (err)
> +		goto unlock;
> +
> +	for (i = 0; i < priv->channels.num * priv-
> >channels.params.num_tc;
> +	     i++) {
> +		err = devlink_fmsg_obj_nest_start(fmsg);
> +		if (err)
> +			goto unlock;
> +
> +		err = mlx5e_tx_reporter_sw_dump_from_ctx(priv, priv-
> >txq2sq[i],
> +							 fmsg);
> +		if (err)
> +			goto unlock;
> +
> +		err = devlink_fmsg_pair_nest_end(fmsg);
> +		if (err)
> +			goto unlock;
> +	}
> +	err = devlink_fmsg_arr_pair_nest_end(fmsg);
> +	if (err)
> +		goto unlock;
> +
> +unlock:
> +	mutex_unlock(&priv->state_lock);
> +	return err;
> +}
> +
> +static int mlx5e_tx_reporter_sw_dump(struct devlink_health_reporter
> *reporter,
> +				     struct devlink_fmsg *fmsg, void
> *context)
> +{
> +	struct mlx5e_priv *priv =
> devlink_health_reporter_priv(reporter);
> +	struct mlx5e_tx_err_ctx *err_ctx = context;
> +
> +	return err_ctx ? mlx5e_tx_reporter_sw_dump_from_ctx(priv,
> err_ctx->sq,
> +							    fmsg) :
> +			 mlx5e_tx_reporter_sw_dump_all(priv, fmsg);
> +}
> +
>  static const struct devlink_health_reporter_ops mlx5_tx_reporter_ops
> = {
>  		.name = "tx",
>  		.recover = mlx5e_tx_reporter_recover,
>  		.diagnose = mlx5e_tx_reporter_diagnose,
> +		.dump = mlx5e_tx_reporter_sw_dump,
>  };
>  
>  #define MLX5_REPORTER_TX_GRACEFUL_PERIOD 500