From: Jack Wang <jinpu.wang@cloud.ionos.com>
To: linux-block@vger.kernel.org, linux-rdma@vger.kernel.org
Cc: axboe@kernel.dk, hch@infradead.org, sagi@grimberg.me,
bvanassche@acm.org, leon@kernel.org, dledford@redhat.com,
jgg@ziepe.ca, danil.kipnis@cloud.ionos.com,
jinpu.wang@cloud.ionos.com, rpenyaev@suse.de,
pankaj.gupta@cloud.ionos.com
Subject: [PATCH v11 05/26] RDMA/rtrs: client: private header with client structs and functions
Date: Fri, 20 Mar 2020 13:16:36 +0100 [thread overview]
Message-ID: <20200320121657.1165-6-jinpu.wang@cloud.ionos.com> (raw)
In-Reply-To: <20200320121657.1165-1-jinpu.wang@cloud.ionos.com>
This header describes main structs and functions used by rtrs-client
module, mainly for managing rtrs sessions, creating/destroying sysfs
entries, accounting statistics on client side.
Signed-off-by: Danil Kipnis <danil.kipnis@cloud.ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
---
drivers/infiniband/ulp/rtrs/rtrs-clt.h | 252 +++++++++++++++++++++++++
1 file changed, 252 insertions(+)
create mode 100644 drivers/infiniband/ulp/rtrs/rtrs-clt.h
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-clt.h b/drivers/infiniband/ulp/rtrs/rtrs-clt.h
new file mode 100644
index 000000000000..d612d00e3d18
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-clt.h
@@ -0,0 +1,252 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#ifndef RTRS_CLT_H
+#define RTRS_CLT_H
+
+#include <linux/device.h>
+#include "rtrs-pri.h"
+
+/**
+ * enum rtrs_clt_state - Client states.
+ */
+enum rtrs_clt_state {
+ RTRS_CLT_CONNECTING,
+ RTRS_CLT_CONNECTING_ERR,
+ RTRS_CLT_RECONNECTING,
+ RTRS_CLT_CONNECTED,
+ RTRS_CLT_CLOSING,
+ RTRS_CLT_CLOSED,
+ RTRS_CLT_DEAD,
+};
+
+enum rtrs_mp_policy {
+ MP_POLICY_RR,
+ MP_POLICY_MIN_INFLIGHT,
+};
+
+/* see Documentation/ABI/testing/sysfs-class-rtrs-client for details */
+struct rtrs_clt_stats_reconnects {
+ int successful_cnt;
+ int fail_cnt;
+};
+
+/* see Documentation/ABI/testing/sysfs-class-rtrs-client for details */
+struct rtrs_clt_stats_cpu_migr {
+ atomic_t from;
+ int to;
+};
+
+/* stats for Read and write operation.
+ * see Documentation/ABI/testing/sysfs-class-rtrs-client for details
+ */
+struct rtrs_clt_stats_rdma {
+ struct {
+ u64 cnt;
+ u64 size_total;
+ } dir[2];
+
+ u64 failover_cnt;
+};
+
+struct rtrs_clt_stats_pcpu {
+ struct rtrs_clt_stats_cpu_migr cpu_migr;
+ struct rtrs_clt_stats_rdma rdma;
+};
+
+struct rtrs_clt_stats {
+ struct rtrs_clt_stats_pcpu __percpu *pcpu_stats;
+ struct rtrs_clt_stats_reconnects reconnects;
+ atomic_t inflight;
+};
+
+struct rtrs_clt_con {
+ struct rtrs_con c;
+ struct rtrs_iu *rsp_ius;
+ u32 queue_size;
+ unsigned int cpu;
+ atomic_t io_cnt;
+ int cm_err;
+};
+
+/**
+ * rtrs_permit - permits the memory allocation for future RDMA operation.
+ * Combine with irq pinning to keep IO on same CPU.
+ */
+struct rtrs_permit {
+ enum rtrs_clt_con_type con_type;
+ unsigned int cpu_id;
+ unsigned int mem_id;
+ unsigned int mem_off;
+};
+
+/**
+ * rtrs_clt_io_req - describes one inflight IO request
+ */
+struct rtrs_clt_io_req {
+ struct list_head list;
+ struct rtrs_iu *iu;
+ struct scatterlist *sglist; /* list holding user data */
+ unsigned int sg_cnt;
+ unsigned int sg_size;
+ unsigned int data_len;
+ unsigned int usr_len;
+ void *priv;
+ bool in_use;
+ struct rtrs_clt_con *con;
+ struct rtrs_sg_desc *desc;
+ struct ib_sge *sge;
+ struct rtrs_permit *permit;
+ enum dma_data_direction dir;
+ void (*conf)(void *priv, int errno);
+ unsigned long start_jiffies;
+
+ struct ib_mr *mr;
+ struct ib_cqe inv_cqe;
+ struct completion inv_comp;
+ int inv_errno;
+ bool need_inv_comp;
+ bool need_inv;
+};
+
+struct rtrs_rbuf {
+ u64 addr;
+ u32 rkey;
+};
+
+struct rtrs_clt_sess {
+ struct rtrs_sess s;
+ struct rtrs_clt *clt;
+ wait_queue_head_t state_wq;
+ enum rtrs_clt_state state;
+ atomic_t connected_cnt;
+ struct mutex init_mutex;
+ struct rtrs_clt_io_req *reqs;
+ struct delayed_work reconnect_dwork;
+ struct work_struct close_work;
+ unsigned int reconnect_attempts;
+ bool established;
+ struct rtrs_rbuf *rbufs;
+ size_t max_io_size;
+ u32 max_hdr_size;
+ u32 chunk_size;
+ size_t queue_depth;
+ u32 max_pages_per_mr;
+ int max_send_sge;
+ u32 flags;
+ struct kobject kobj;
+ struct kobject kobj_stats;
+ struct rtrs_clt_stats stats;
+ /* cache hca_port and hca_name to display in sysfs */
+ u8 hca_port;
+ char hca_name[IB_DEVICE_NAME_MAX];
+ struct list_head __percpu
+ *mp_skip_entry;
+};
+
+struct rtrs_clt {
+ struct list_head paths_list; /* rcu protected list */
+ size_t paths_num;
+ struct rtrs_clt_sess
+ __rcu * __percpu *pcpu_path;
+ uuid_t paths_uuid;
+ int paths_up;
+ struct mutex paths_mutex;
+ struct mutex paths_ev_mutex;
+ char sessname[NAME_MAX];
+ short port;
+ unsigned int max_reconnect_attempts;
+ unsigned int reconnect_delay_sec;
+ unsigned int max_segments;
+ void *permits;
+ unsigned long *permits_map;
+ size_t queue_depth;
+ size_t max_io_size;
+ wait_queue_head_t permits_wait;
+ size_t pdu_sz;
+ void *priv;
+ void (*link_ev)(void *priv,
+ enum rtrs_clt_link_ev ev);
+ struct device dev;
+ struct kobject kobj_paths;
+ enum rtrs_mp_policy mp_policy;
+};
+
+static inline struct rtrs_clt_con *to_clt_con(struct rtrs_con *c)
+{
+ return container_of(c, struct rtrs_clt_con, c);
+}
+
+static inline struct rtrs_clt_sess *to_clt_sess(struct rtrs_sess *s)
+{
+ return container_of(s, struct rtrs_clt_sess, s);
+}
+
+static inline int permit_size(struct rtrs_clt *clt)
+{
+ return sizeof(struct rtrs_permit) + clt->pdu_sz;
+}
+
+static inline struct rtrs_permit *get_permit(struct rtrs_clt *clt, int idx)
+{
+ return (struct rtrs_permit *)(clt->permits + permit_size(clt) * idx);
+}
+
+int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess);
+int rtrs_clt_disconnect_from_sysfs(struct rtrs_clt_sess *sess);
+int rtrs_clt_create_path_from_sysfs(struct rtrs_clt *clt,
+ struct rtrs_addr *addr);
+int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess,
+ const struct attribute *sysfs_self);
+
+void rtrs_clt_set_max_reconnect_attempts(struct rtrs_clt *clt, int value);
+int rtrs_clt_get_max_reconnect_attempts(const struct rtrs_clt *clt);
+
+/* rtrs-clt-stats.c */
+
+int rtrs_clt_init_stats(struct rtrs_clt_stats *stats);
+void rtrs_clt_free_stats(struct rtrs_clt_stats *stats);
+
+void rtrs_clt_inc_failover_cnt(struct rtrs_clt_stats *s);
+
+void rtrs_clt_update_wc_stats(struct rtrs_clt_con *con);
+void rtrs_clt_update_all_stats(struct rtrs_clt_io_req *req, int dir);
+
+int rtrs_clt_reset_rdma_lat_distr_stats(struct rtrs_clt_stats *stats,
+ bool enable);
+ssize_t rtrs_clt_stats_rdma_lat_distr_to_str(struct rtrs_clt_stats *stats,
+ char *page, size_t len);
+int rtrs_clt_reset_cpu_migr_stats(struct rtrs_clt_stats *stats, bool enable);
+int rtrs_clt_stats_migration_cnt_to_str(struct rtrs_clt_stats *stats, char *buf,
+ size_t len);
+int rtrs_clt_reset_reconnects_stat(struct rtrs_clt_stats *stats, bool enable);
+int rtrs_clt_stats_reconnects_to_str(struct rtrs_clt_stats *stats, char *buf,
+ size_t len);
+int rtrs_clt_reset_wc_comp_stats(struct rtrs_clt_stats *stats, bool enable);
+int rtrs_clt_stats_wc_completion_to_str(struct rtrs_clt_stats *stats, char *buf,
+ size_t len);
+int rtrs_clt_reset_rdma_stats(struct rtrs_clt_stats *stats, bool enable);
+ssize_t rtrs_clt_stats_rdma_to_str(struct rtrs_clt_stats *stats,
+ char *page, size_t len);
+int rtrs_clt_reset_all_stats(struct rtrs_clt_stats *stats, bool enable);
+ssize_t rtrs_clt_reset_all_help(struct rtrs_clt_stats *stats,
+ char *page, size_t len);
+
+/* rtrs-clt-sysfs.c */
+
+int rtrs_clt_create_sysfs_root_folders(struct rtrs_clt *clt);
+int rtrs_clt_create_sysfs_root_files(struct rtrs_clt *clt);
+void rtrs_clt_destroy_sysfs_root_folders(struct rtrs_clt *clt);
+void rtrs_clt_destroy_sysfs_root_files(struct rtrs_clt *clt);
+
+int rtrs_clt_create_sess_files(struct rtrs_clt_sess *sess);
+void rtrs_clt_destroy_sess_files(struct rtrs_clt_sess *sess,
+ const struct attribute *sysfs_self);
+
+#endif /* RTRS_CLT_H */
--
2.17.1
next prev parent reply other threads:[~2020-03-20 12:17 UTC|newest]
Thread overview: 71+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-03-20 12:16 [PATCH v11 00/26] RTRS (former IBTRS) RDMA Transport Library and RNBD (former IBNBD) RDMA Network Block Device Jack Wang
2020-03-20 12:16 ` [PATCH v11 01/26] sysfs: export sysfs_remove_file_self() Jack Wang
2020-03-20 12:16 ` [PATCH v11 02/26] RDMA/rtrs: public interface header to establish RDMA connections Jack Wang
2020-03-20 12:16 ` [PATCH v11 03/26] RDMA/rtrs: private headers with rtrs protocol structs and helpers Jack Wang
2020-03-20 12:16 ` [PATCH v11 04/26] RDMA/rtrs: core: lib functions shared between client and server modules Jack Wang
2020-03-28 4:26 ` Bart Van Assche
2020-03-30 10:34 ` Jinpu Wang
2020-03-30 22:25 ` Bart Van Assche
2020-03-31 7:11 ` Jinpu Wang
2020-03-20 12:16 ` Jack Wang [this message]
2020-03-20 12:16 ` [PATCH v11 06/26] RDMA/rtrs: client: main functionality Jack Wang
2020-03-20 12:16 ` [PATCH v11 07/26] RDMA/rtrs: client: statistics functions Jack Wang
2020-03-20 12:16 ` [PATCH v11 08/26] RDMA/rtrs: client: sysfs interface functions Jack Wang
2020-03-30 22:28 ` Bart Van Assche
2020-03-31 7:20 ` Jinpu Wang
2020-03-20 12:16 ` [PATCH v11 09/26] RDMA/rtrs: server: private header with server structs and functions Jack Wang
2020-03-20 12:16 ` [PATCH v11 10/26] RDMA/rtrs: server: main functionality Jack Wang
2020-03-20 12:16 ` [PATCH v11 11/26] RDMA/rtrs: server: statistics functions Jack Wang
2020-03-20 12:16 ` [PATCH v11 12/26] RDMA/rtrs: server: sysfs interface functions Jack Wang
2020-03-30 22:29 ` Bart Van Assche
2020-03-31 7:13 ` Jinpu Wang
2020-03-20 12:16 ` [PATCH v11 13/26] RDMA/rtrs: include client and server modules into kernel compilation Jack Wang
2020-03-20 12:16 ` [PATCH v11 14/26] RDMA/rtrs: a bit of documentation Jack Wang
2020-03-20 12:16 ` [PATCH v11 15/26] block: reexport bio_map_kern Jack Wang
2020-03-28 3:58 ` Bart Van Assche
2020-03-28 8:29 ` Christoph Hellwig
2020-03-28 16:16 ` Bart Van Assche
2020-03-29 15:05 ` Christoph Hellwig
2020-03-29 18:08 ` Chaitanya Kulkarni
2020-03-30 6:28 ` hch
2020-03-30 10:44 ` Jinpu Wang
2020-03-30 18:57 ` Chaitanya Kulkarni
2020-03-20 12:16 ` [PATCH v11 16/26] block/rnbd: private headers with rnbd protocol structs and helpers Jack Wang
2020-03-28 4:58 ` Bart Van Assche
2020-03-31 7:32 ` Jinpu Wang
2020-03-20 12:16 ` [PATCH v11 17/26] block/rnbd: client: private header with client structs and functions Jack Wang
2020-03-28 4:26 ` Bart Van Assche
2020-03-31 9:08 ` Jinpu Wang
2020-03-20 12:16 ` [PATCH v11 18/26] block/rnbd: client: main functionality Jack Wang
2020-03-28 4:45 ` Bart Van Assche
2020-03-31 9:23 ` Jinpu Wang
2020-04-02 16:27 ` Jinpu Wang
2020-03-28 4:59 ` Bart Van Assche
2020-03-31 9:25 ` Jinpu Wang
2020-03-31 14:12 ` Bart Van Assche
2020-03-31 14:20 ` Jinpu Wang
2020-03-20 12:16 ` [PATCH v11 19/26] block/rnbd: client: sysfs interface functions Jack Wang
2020-03-28 4:59 ` Bart Van Assche
2020-03-31 9:26 ` Jinpu Wang
2020-03-20 12:16 ` [PATCH v11 20/26] block/rnbd: server: private header with server structs and functions Jack Wang
2020-03-20 12:16 ` [PATCH v11 21/26] block/rnbd: server: main functionality Jack Wang
2020-03-28 17:40 ` Bart Van Assche
2020-03-31 9:29 ` Jinpu Wang
2020-03-31 15:32 ` Jinpu Wang
2020-03-20 12:16 ` [PATCH v11 22/26] block/rnbd: server: functionality for IO submission to file or block dev Jack Wang
2020-03-28 18:39 ` Bart Van Assche
2020-03-31 10:06 ` Jinpu Wang
2020-03-20 12:16 ` [PATCH v11 23/26] block/rnbd: server: sysfs interface functions Jack Wang
2020-03-28 19:31 ` Bart Van Assche
2020-03-28 23:06 ` Jason Gunthorpe
2020-03-30 13:14 ` Danil Kipnis
2020-03-20 12:16 ` [PATCH v11 24/26] block/rnbd: include client and server modules into kernel compilation Jack Wang
2020-03-28 19:34 ` Bart Van Assche
2020-03-31 7:23 ` Jinpu Wang
2020-03-20 12:16 ` [PATCH v11 25/26] block/rnbd: a bit of documentation Jack Wang
2020-03-28 19:40 ` Bart Van Assche
2020-03-30 10:17 ` Danil Kipnis
2020-03-20 12:16 ` [PATCH v11 26/26] MAINTAINERS: Add maintainers for RNBD/RTRS modules Jack Wang
2020-03-28 19:40 ` Bart Van Assche
2020-03-30 10:12 ` Danil Kipnis
2020-03-26 17:38 ` [PATCH v11 00/26] RTRS (former IBTRS) RDMA Transport Library and RNBD (former IBNBD) RDMA Network Block Device Jinpu Wang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200320121657.1165-6-jinpu.wang@cloud.ionos.com \
--to=jinpu.wang@cloud.ionos.com \
--cc=axboe@kernel.dk \
--cc=bvanassche@acm.org \
--cc=danil.kipnis@cloud.ionos.com \
--cc=dledford@redhat.com \
--cc=hch@infradead.org \
--cc=jgg@ziepe.ca \
--cc=leon@kernel.org \
--cc=linux-block@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=pankaj.gupta@cloud.ionos.com \
--cc=rpenyaev@suse.de \
--cc=sagi@grimberg.me \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).