From: Danil Kipnis <danil.kipnis@cloud.ionos.com>
To: linux-block@vger.kernel.org, linux-rdma@vger.kernel.org
Cc: axboe@kernel.dk, hch@infradead.org, sagi@grimberg.me,
bvanassche@acm.org, leon@kernel.org, dledford@redhat.com,
jgg@ziepe.ca, danil.kipnis@cloud.ionos.com,
jinpu.wang@cloud.ionos.com, pankaj.gupta@cloud.ionos.com
Subject: [PATCH v15 09/25] RDMA/rtrs: server: private header with server structs and functions
Date: Mon, 11 May 2020 15:51:15 +0200 [thread overview]
Message-ID: <20200511135131.27580-10-danil.kipnis@cloud.ionos.com> (raw)
In-Reply-To: <20200511135131.27580-1-danil.kipnis@cloud.ionos.com>
From: Jack Wang <jinpu.wang@cloud.ionos.com>
This header describes main structs and functions used by rtrs-server
module, mainly for accepting rtrs sessions, creating/destroying
sysfs entries, accounting statistics on server side.
Signed-off-by: Danil Kipnis <danil.kipnis@cloud.ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
---
drivers/infiniband/ulp/rtrs/rtrs-srv.h | 148 +++++++++++++++++++++++++
1 file changed, 148 insertions(+)
create mode 100644 drivers/infiniband/ulp/rtrs/rtrs-srv.h
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.h b/drivers/infiniband/ulp/rtrs/rtrs-srv.h
new file mode 100644
index 000000000000..dc95b0932f0d
--- /dev/null
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.h
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * RDMA Transport Layer
+ *
+ * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
+ * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
+ * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
+ */
+
+#ifndef RTRS_SRV_H
+#define RTRS_SRV_H
+
+#include <linux/device.h>
+#include <linux/refcount.h>
+#include "rtrs-pri.h"
+
+/*
+ * enum rtrs_srv_state - Server states.
+ */
+enum rtrs_srv_state {
+ RTRS_SRV_CONNECTING,
+ RTRS_SRV_CONNECTED,
+ RTRS_SRV_CLOSING,
+ RTRS_SRV_CLOSED,
+};
+
+/* stats for Read and write operation.
+ * see Documentation/ABI/testing/sysfs-class-rtrs-server for details
+ */
+struct rtrs_srv_stats_rdma_stats {
+ struct {
+ atomic64_t cnt;
+ atomic64_t size_total;
+ } dir[2];
+};
+
+struct rtrs_srv_stats {
+ struct kobject kobj_stats;
+ struct rtrs_srv_stats_rdma_stats rdma_stats;
+ struct rtrs_srv_sess *sess;
+};
+
+struct rtrs_srv_con {
+ struct rtrs_con c;
+ atomic_t wr_cnt;
+ atomic_t sq_wr_avail;
+ struct list_head rsp_wr_wait_list;
+ spinlock_t rsp_wr_wait_lock;
+};
+
+/* IO context in rtrs_srv, each io has one */
+struct rtrs_srv_op {
+ struct rtrs_srv_con *con;
+ u32 msg_id;
+ u8 dir;
+ struct rtrs_msg_rdma_read *rd_msg;
+ struct ib_rdma_wr tx_wr;
+ struct ib_sge tx_sg;
+ struct list_head wait_list;
+ int status;
+};
+
+/*
+ * server side memory region context, when always_invalidate=Y, we need
+ * queue_depth of memory regrion to invalidate each memory region.
+ */
+struct rtrs_srv_mr {
+ struct ib_mr *mr;
+ struct sg_table sgt;
+ struct ib_cqe inv_cqe; /* only for always_invalidate=true */
+ u32 msg_id; /* only for always_invalidate=true */
+ u32 msg_off; /* only for always_invalidate=true */
+ struct rtrs_iu *iu; /* send buffer for new rkey msg */
+};
+
+struct rtrs_srv_sess {
+ struct rtrs_sess s;
+ struct rtrs_srv *srv;
+ struct work_struct close_work;
+ enum rtrs_srv_state state;
+ spinlock_t state_lock;
+ int cur_cq_vector;
+ struct rtrs_srv_op **ops_ids;
+ atomic_t ids_inflight;
+ wait_queue_head_t ids_waitq;
+ struct rtrs_srv_mr *mrs;
+ unsigned int mrs_num;
+ dma_addr_t *dma_addr;
+ bool established;
+ unsigned int mem_bits;
+ struct kobject kobj;
+ struct rtrs_srv_stats *stats;
+};
+
+struct rtrs_srv {
+ struct list_head paths_list;
+ int paths_up;
+ struct mutex paths_ev_mutex;
+ size_t paths_num;
+ struct mutex paths_mutex;
+ uuid_t paths_uuid;
+ refcount_t refcount;
+ struct rtrs_srv_ctx *ctx;
+ struct list_head ctx_list;
+ void *priv;
+ size_t queue_depth;
+ struct page **chunks;
+ struct device dev;
+ unsigned int dev_ref;
+ struct kobject *kobj_paths;
+};
+
+struct rtrs_srv_ctx {
+ struct rtrs_srv_ops ops;
+ struct rdma_cm_id *cm_id_ip;
+ struct rdma_cm_id *cm_id_ib;
+ struct mutex srv_mutex;
+ struct list_head srv_list;
+};
+
+extern struct class *rtrs_dev_class;
+
+void close_sess(struct rtrs_srv_sess *sess);
+
+static inline void rtrs_srv_update_rdma_stats(struct rtrs_srv_stats *s,
+ size_t size, int d)
+{
+ atomic64_inc(&s->rdma_stats.dir[d].cnt);
+ atomic64_add(size, &s->rdma_stats.dir[d].size_total);
+}
+
+/* functions which are implemented in rtrs-srv-stats.c */
+int rtrs_srv_reset_rdma_stats(struct rtrs_srv_stats *stats, bool enable);
+ssize_t rtrs_srv_stats_rdma_to_str(struct rtrs_srv_stats *stats,
+ char *page, size_t len);
+int rtrs_srv_reset_wc_completion_stats(struct rtrs_srv_stats *stats,
+ bool enable);
+int rtrs_srv_stats_wc_completion_to_str(struct rtrs_srv_stats *stats, char *buf,
+ size_t len);
+int rtrs_srv_reset_all_stats(struct rtrs_srv_stats *stats, bool enable);
+ssize_t rtrs_srv_reset_all_help(struct rtrs_srv_stats *stats,
+ char *page, size_t len);
+
+/* functions which are implemented in rtrs-srv-sysfs.c */
+int rtrs_srv_create_sess_files(struct rtrs_srv_sess *sess);
+void rtrs_srv_destroy_sess_files(struct rtrs_srv_sess *sess);
+
+#endif /* RTRS_SRV_H */
--
2.20.1
next prev parent reply other threads:[~2020-05-11 13:53 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-05-11 13:51 [PATCH v15 00/25] RTRS (former IBTRS) RDMA Transport Library and RNBD (former IBNBD) RDMA Network Block Device Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 01/25] sysfs: export sysfs_remove_file_self() Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 02/25] RDMA/rtrs: public interface header to establish RDMA connections Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 03/25] RDMA/rtrs: private headers with rtrs protocol structs and helpers Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 04/25] RDMA/rtrs: core: lib functions shared between client and server modules Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 05/25] RDMA/rtrs: client: private header with client structs and functions Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 06/25] RDMA/rtrs: client: main functionality Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 07/25] RDMA/rtrs: client: statistics functions Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 08/25] RDMA/rtrs: client: sysfs interface functions Danil Kipnis
2020-05-11 13:51 ` Danil Kipnis [this message]
2020-05-11 13:51 ` [PATCH v15 10/25] RDMA/rtrs: server: main functionality Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 11/25] RDMA/rtrs: server: statistics functions Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 12/25] RDMA/rtrs: server: sysfs interface functions Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 13/25] RDMA/rtrs: include client and server modules into kernel compilation Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 14/25] RDMA/rtrs: a bit of documentation Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 15/25] block/rnbd: private headers with rnbd protocol structs and helpers Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 16/25] block/rnbd: client: private header with client structs and functions Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 17/25] block/rnbd: client: main functionality Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 18/25] block/rnbd: client: sysfs interface functions Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 19/25] block/rnbd: server: private header with server structs and functions Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 20/25] block/rnbd: server: main functionality Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 21/25] block/rnbd: server: functionality for IO submitting to block dev Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 22/25] block/rnbd: server: sysfs interface functions Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 23/25] block/rnbd: include client and server modules into kernel compilation Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 24/25] block/rnbd: a bit of documentation Danil Kipnis
2020-05-11 13:51 ` [PATCH v15 25/25] MAINTAINERS: Add maintainers for RNBD/RTRS modules Danil Kipnis
2020-05-15 10:29 ` [PATCH v15 00/25] RTRS (former IBTRS) RDMA Transport Library and RNBD (former IBNBD) RDMA Network Block Device Danil Kipnis
2020-05-15 13:53 ` Jens Axboe
2020-05-15 14:39 ` Jinpu Wang
2020-05-15 14:42 ` Danil Kipnis
2020-05-17 22:17 ` Jason Gunthorpe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20200511135131.27580-10-danil.kipnis@cloud.ionos.com \
--to=danil.kipnis@cloud.ionos.com \
--cc=axboe@kernel.dk \
--cc=bvanassche@acm.org \
--cc=dledford@redhat.com \
--cc=hch@infradead.org \
--cc=jgg@ziepe.ca \
--cc=jinpu.wang@cloud.ionos.com \
--cc=leon@kernel.org \
--cc=linux-block@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=pankaj.gupta@cloud.ionos.com \
--cc=sagi@grimberg.me \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.