From: Jack Wang <jinpu.wang@cloud.ionos.com>
To: linux-block@vger.kernel.org, linux-rdma@vger.kernel.org
Cc: axboe@kernel.dk, hch@infradead.org, sagi@grimberg.me,
bvanassche@acm.org, leon@kernel.org, dledford@redhat.com,
jgg@ziepe.ca, danil.kipnis@cloud.ionos.com,
jinpu.wang@cloud.ionos.com, rpenyaev@suse.de,
pankaj.gupta@cloud.ionos.com
Subject: [PATCH v10 09/26] RDMA/rtrs: server: private header with server structs and functions
Date: Wed, 11 Mar 2020 17:12:23 +0100 [thread overview]
Message-ID: <20200311161240.30190-10-jinpu.wang@cloud.ionos.com> (raw)
In-Reply-To: <20200311161240.30190-1-jinpu.wang@cloud.ionos.com>
This header describes main structs and functions used by rtrs-server
module, mainly for accepting rtrs sessions, creating/destroying
sysfs entries, accounting statistics on server side.
Signed-off-by: Danil Kipnis <danil.kipnis@cloud.ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
---
drivers/infiniband/ulp/rtrs/rtrs-srv.h | 149 +++++++++++++++++++++++++
1 file changed, 149 insertions(+)
create mode 100644 drivers/infiniband/ulp/rtrs/rtrs-srv.h
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.h b/drivers/infiniband/ulp/rtrs/rtrs-srv.h
new file mode 100644
index 000000000000..300adcd26a72
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.h
@@ -0,0 +1,149 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#ifndef RTRS_SRV_H
+#define RTRS_SRV_H
+
+#include <linux/device.h>
+#include <linux/refcount.h>
+#include "rtrs-pri.h"
+
+/*
+ * enum rtrs_srv_state - Server states.
+ */
+enum rtrs_srv_state {
+ RTRS_SRV_CONNECTING,
+ RTRS_SRV_CONNECTED,
+ RTRS_SRV_CLOSING,
+ RTRS_SRV_CLOSED,
+};
+
+/* stats for Read and write operation.
+ * see Documentation/ABI/testing/sysfs-class-rtrs-server for details
+ */
+struct rtrs_srv_stats_rdma_stats {
+ struct {
+ atomic64_t cnt;
+ atomic64_t size_total;
+ } dir[2];
+};
+
+struct rtrs_srv_stats {
+ struct rtrs_srv_stats_rdma_stats rdma_stats;
+};
+
+struct rtrs_srv_con {
+ struct rtrs_con c;
+ atomic_t wr_cnt;
+ atomic_t sq_wr_avail;
+ struct list_head rsp_wr_wait_list;
+ spinlock_t rsp_wr_wait_lock;
+};
+
+/* IO context in rtrs_srv, each io has one */
+struct rtrs_srv_op {
+ struct rtrs_srv_con *con;
+ u32 msg_id;
+ u8 dir;
+ struct rtrs_msg_rdma_read *rd_msg;
+ struct ib_rdma_wr *tx_wr;
+ struct ib_sge *tx_sg;
+ struct list_head wait_list;
+ int status;
+ int send_wr_cnt;
+ struct ib_cqe send_cqe;
+};
+
+/*
+ * server side memory region context, when always_invalidate=Y, we need
+ * queue_depth of memory regrion to invalidate each memory region.
+ */
+struct rtrs_srv_mr {
+ struct ib_mr *mr;
+ struct sg_table sgt;
+ struct ib_cqe inv_cqe; /* only for always_invalidate=true */
+ u32 msg_id; /* only for always_invalidate=true */
+ u32 msg_off; /* only for always_invalidate=true */
+ struct rtrs_iu *iu; /* send buffer for new rkey msg */
+};
+
+struct rtrs_srv_sess {
+ struct rtrs_sess s;
+ struct rtrs_srv *srv;
+ struct work_struct close_work;
+ enum rtrs_srv_state state;
+ spinlock_t state_lock;
+ int cur_cq_vector;
+ struct rtrs_srv_op **ops_ids;
+ atomic_t ids_inflight;
+ wait_queue_head_t ids_waitq;
+ struct rtrs_srv_mr *mrs;
+ unsigned int mrs_num;
+ dma_addr_t *dma_addr;
+ bool established;
+ unsigned int mem_bits;
+ struct kobject kobj;
+ struct kobject kobj_stats;
+ struct rtrs_srv_stats stats;
+};
+
+struct rtrs_srv {
+ struct list_head paths_list;
+ int paths_up;
+ struct mutex paths_ev_mutex;
+ size_t paths_num;
+ struct mutex paths_mutex;
+ uuid_t paths_uuid;
+ refcount_t refcount;
+ struct rtrs_srv_ctx *ctx;
+ struct list_head ctx_list;
+ void *priv;
+ size_t queue_depth;
+ struct page **chunks;
+ struct device dev;
+ unsigned int dev_ref;
+ struct kobject kobj_paths;
+};
+
+struct rtrs_srv_ctx {
+ struct rtrs_srv_ops ops;
+ struct rdma_cm_id *cm_id_ip;
+ struct rdma_cm_id *cm_id_ib;
+ struct mutex srv_mutex;
+ struct list_head srv_list;
+};
+
+extern struct class *rtrs_dev_class;
+
+void close_sess(struct rtrs_srv_sess *sess);
+
+static inline void rtrs_srv_update_rdma_stats(struct rtrs_srv_stats *s,
+ size_t size, int d)
+{
+ atomic64_inc(&s->rdma_stats.dir[d].cnt);
+ atomic64_add(size, &s->rdma_stats.dir[d].size_total);
+}
+
+/* functions which are implemented in rtrs-srv-stats.c */
+int rtrs_srv_reset_rdma_stats(struct rtrs_srv_stats *stats, bool enable);
+ssize_t rtrs_srv_stats_rdma_to_str(struct rtrs_srv_stats *stats,
+ char *page, size_t len);
+int rtrs_srv_reset_wc_completion_stats(struct rtrs_srv_stats *stats,
+ bool enable);
+int rtrs_srv_stats_wc_completion_to_str(struct rtrs_srv_stats *stats, char *buf,
+ size_t len);
+int rtrs_srv_reset_all_stats(struct rtrs_srv_stats *stats, bool enable);
+ssize_t rtrs_srv_reset_all_help(struct rtrs_srv_stats *stats,
+ char *page, size_t len);
+
+/* functions which are implemented in rtrs-srv-sysfs.c */
+int rtrs_srv_create_sess_files(struct rtrs_srv_sess *sess);
+void rtrs_srv_destroy_sess_files(struct rtrs_srv_sess *sess);
+
+#endif /* RTRS_SRV_H */
--
2.17.1
next prev parent reply other threads:[~2020-03-11 16:13 UTC|newest]
Thread overview: 39+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-03-11 16:12 [PATCH v10 00/26] RTRS (former IBTRS) RDMA Transport Library and RNBD (former IBNBD) RDMA Network Block Device Jack Wang
2020-03-11 16:12 ` [PATCH v10 01/26] sysfs: export sysfs_remove_file_self() Jack Wang
2020-03-11 16:12 ` [PATCH v10 02/26] RDMA/rtrs: public interface header to establish RDMA connections Jack Wang
2020-03-11 18:45 ` Jason Gunthorpe
2020-03-12 9:43 ` Jinpu Wang
2020-03-11 16:12 ` [PATCH v10 03/26] RDMA/rtrs: private headers with rtrs protocol structs and helpers Jack Wang
2020-03-11 16:12 ` [PATCH v10 04/26] RDMA/rtrs: core: lib functions shared between client and server modules Jack Wang
2020-03-11 16:12 ` [PATCH v10 05/26] RDMA/rtrs: client: private header with client structs and functions Jack Wang
2020-03-11 16:12 ` [PATCH v10 06/26] RDMA/rtrs: client: main functionality Jack Wang
2020-03-11 19:01 ` Jason Gunthorpe
2020-03-12 17:10 ` Danil Kipnis
2020-03-12 17:25 ` Jason Gunthorpe
2020-03-13 12:18 ` Danil Kipnis
2020-03-13 12:25 ` Jason Gunthorpe
2020-03-17 6:46 ` Danil Kipnis
2020-03-18 15:04 ` Jinpu Wang
2020-03-11 16:12 ` [PATCH v10 07/26] RDMA/rtrs: client: statistics functions Jack Wang
2020-03-11 16:12 ` [PATCH v10 08/26] RDMA/rtrs: client: sysfs interface functions Jack Wang
2020-03-11 16:12 ` Jack Wang [this message]
2020-03-11 16:12 ` [PATCH v10 10/26] RDMA/rtrs: server: main functionality Jack Wang
2020-03-11 16:12 ` [PATCH v10 11/26] RDMA/rtrs: server: statistics functions Jack Wang
2020-03-11 16:12 ` [PATCH v10 12/26] RDMA/rtrs: server: sysfs interface functions Jack Wang
2020-03-11 16:12 ` [PATCH v10 13/26] RDMA/rtrs: include client and server modules into kernel compilation Jack Wang
2020-03-11 19:03 ` Jason Gunthorpe
2020-03-12 10:50 ` Danil Kipnis
2020-03-12 12:03 ` Jason Gunthorpe
2020-03-11 16:12 ` [PATCH v10 14/26] RDMA/rtrs: a bit of documentation Jack Wang
2020-03-11 16:12 ` [PATCH v10 15/26] block: reexport bio_map_kern Jack Wang
2020-03-11 16:12 ` [PATCH v10 16/26] block/rnbd: private headers with rnbd protocol structs and helpers Jack Wang
2020-03-11 16:12 ` [PATCH v10 17/26] block/rnbd: client: private header with client structs and functions Jack Wang
2020-03-11 16:12 ` [PATCH v10 18/26] block/rnbd: client: main functionality Jack Wang
2020-03-11 16:12 ` [PATCH v10 19/26] block/rnbd: client: sysfs interface functions Jack Wang
2020-03-11 16:12 ` [PATCH v10 20/26] block/rnbd: server: private header with server structs and functions Jack Wang
2020-03-11 16:12 ` [PATCH v10 21/26] block/rnbd: server: main functionality Jack Wang
2020-03-11 16:12 ` [PATCH v10 22/26] block/rnbd: server: functionality for IO submission to file or block dev Jack Wang
2020-03-11 16:12 ` [PATCH v10 23/26] block/rnbd: server: sysfs interface functions Jack Wang
2020-03-11 16:12 ` [PATCH v10 24/26] block/rnbd: include client and server modules into kernel compilation Jack Wang
2020-03-11 16:12 ` [PATCH v10 25/26] block/rnbd: a bit of documentation Jack Wang
2020-03-11 16:12 ` [PATCH v10 26/26] MAINTAINERS: Add maintainers for RNBD/RTRS modules Jack Wang
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200311161240.30190-10-jinpu.wang@cloud.ionos.com \
--to=jinpu.wang@cloud.ionos.com \
--cc=axboe@kernel.dk \
--cc=bvanassche@acm.org \
--cc=danil.kipnis@cloud.ionos.com \
--cc=dledford@redhat.com \
--cc=hch@infradead.org \
--cc=jgg@ziepe.ca \
--cc=leon@kernel.org \
--cc=linux-block@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=pankaj.gupta@cloud.ionos.com \
--cc=rpenyaev@suse.de \
--cc=sagi@grimberg.me \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.