* [PATCH for-next 1/5] block/rnbd: Enable the fault-injection
2021-04-06 7:37 [PATCH for-next 0/5] Enable Fault Injection for RNBD Gioh Kim
@ 2021-04-06 7:37 ` Gioh Kim
2021-04-06 7:37 ` [PATCH for-next 2/5] block/rnbd-srv: Inject a fault at bio processing Gioh Kim
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Gioh Kim @ 2021-04-06 7:37 UTC (permalink / raw)
To: linux-block, linux-doc
Cc: axboe, akinobu.mita, corbet, hch, sagi, bvanassche, haris.iqbal,
jinpu.wang, Gioh Kim, Jack Wang
From: Gioh Kim <gi-oh.kim@cloud.ionos.com>
This patch introduces functions to enable the fault-injection for RNBD.
* rnbd_fault_inject_init/final: initialize the fault-injection
and create a debugfs directory.
* rnbd_fault_inject_add: create a debugfs entry to enable
the fault-injection point.
Signed-off-by: Gioh Kim <gi-oh.kim@cloud.ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
---
drivers/block/rnbd/rnbd-common.c | 44 ++++++++++++++++++++++++++++++++
drivers/block/rnbd/rnbd-proto.h | 14 ++++++++++
2 files changed, 58 insertions(+)
diff --git a/drivers/block/rnbd/rnbd-common.c b/drivers/block/rnbd/rnbd-common.c
index 596c3f732403..84bfbf015f6d 100644
--- a/drivers/block/rnbd/rnbd-common.c
+++ b/drivers/block/rnbd/rnbd-common.c
@@ -21,3 +21,47 @@ const char *rnbd_access_mode_str(enum rnbd_access_mode mode)
return "unknown";
}
}
+
+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+static DECLARE_FAULT_ATTR(fail_default_attr);
+
+void rnbd_fault_inject_init(struct rnbd_fault_inject *fj,
+ const char *dir_name,
+ u32 err_status)
+{
+ struct dentry *dir, *parent;
+ struct fault_attr *attr = &fj->attr;
+
+ /* create debugfs directory and attribute */
+ parent = debugfs_create_dir(dir_name, NULL);
+ if (!parent) {
+ pr_warn("%s: failed to create debugfs directory\n", dir_name);
+ return;
+ }
+
+ *attr = fail_default_attr;
+ dir = fault_create_debugfs_attr("fault_inject", parent, attr);
+ if (IS_ERR(dir)) {
+ pr_warn("%s: failed to create debugfs attr\n", dir_name);
+ debugfs_remove_recursive(parent);
+ return;
+ }
+ fj->parent = parent;
+ fj->dir = dir;
+
+ /* create debugfs for status code */
+ fj->status = err_status;
+ debugfs_create_u32("status", 0600, dir, &fj->status);
+}
+
+void rnbd_fault_inject_add(struct dentry *dir, const char *fname, bool *value)
+{
+ debugfs_create_bool(fname, 0600, dir, value);
+}
+
+void rnbd_fault_inject_final(struct rnbd_fault_inject *fj)
+{
+ /* remove debugfs directories */
+ debugfs_remove_recursive(fj->parent);
+}
+#endif
diff --git a/drivers/block/rnbd/rnbd-proto.h b/drivers/block/rnbd/rnbd-proto.h
index c1bc5c0fef71..d13dc1d3a00e 100644
--- a/drivers/block/rnbd/rnbd-proto.h
+++ b/drivers/block/rnbd/rnbd-proto.h
@@ -15,6 +15,7 @@
#include <linux/inet.h>
#include <linux/in.h>
#include <linux/in6.h>
+#include <linux/fault-inject.h>
#include <rdma/ib.h>
#define RNBD_PROTO_VER_MAJOR 2
@@ -305,6 +306,19 @@ static inline u32 rq_to_rnbd_flags(struct request *rq)
return rnbd_opf;
}
+struct rnbd_fault_inject {
+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+ struct fault_attr attr;
+ struct dentry *parent;
+ struct dentry *dir;
+ u32 status;
+#endif
+};
+
const char *rnbd_access_mode_str(enum rnbd_access_mode mode);
+void rnbd_fault_inject_init(struct rnbd_fault_inject *fj,
+ const char *dev_name, u32 err_status);
+void rnbd_fault_inject_add(struct dentry *dir, const char *fname, bool *value);
+void rnbd_fault_inject_final(struct rnbd_fault_inject *fj);
#endif /* RNBD_PROTO_H */
--
2.25.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH for-next 2/5] block/rnbd-srv: Inject a fault at bio processing
2021-04-06 7:37 [PATCH for-next 0/5] Enable Fault Injection for RNBD Gioh Kim
2021-04-06 7:37 ` [PATCH for-next 1/5] block/rnbd: Enable the fault-injection Gioh Kim
@ 2021-04-06 7:37 ` Gioh Kim
2021-04-06 7:37 ` [PATCH for-next 3/5] block/rnbd-clt: Inject some fault points Gioh Kim
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Gioh Kim @ 2021-04-06 7:37 UTC (permalink / raw)
To: linux-block, linux-doc
Cc: axboe, akinobu.mita, corbet, hch, sagi, bvanassche, haris.iqbal,
jinpu.wang, Gioh Kim, Jack Wang
From: Gioh Kim <gi-oh.kim@cloud.ionos.com>
If the fault is enabled, it sends an error to the client
so that the client thinks the target device on the server has failed.
Signed-off-by: Gioh Kim <gi-oh.kim@cloud.ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
---
drivers/block/rnbd/rnbd-srv-sysfs.c | 37 +++++++++++++++++++++++++++++
drivers/block/rnbd/rnbd-srv.c | 7 ++++++
drivers/block/rnbd/rnbd-srv.h | 13 ++++++++++
3 files changed, 57 insertions(+)
diff --git a/drivers/block/rnbd/rnbd-srv-sysfs.c b/drivers/block/rnbd/rnbd-srv-sysfs.c
index 05ffe488ddc6..03fb26ac435e 100644
--- a/drivers/block/rnbd/rnbd-srv-sysfs.c
+++ b/drivers/block/rnbd/rnbd-srv-sysfs.c
@@ -253,3 +253,40 @@ void rnbd_srv_destroy_sysfs_files(void)
device_destroy(rnbd_dev_class, MKDEV(0, 0));
class_destroy(rnbd_dev_class);
}
+
+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+void rnbd_srv_fault_inject_init(struct rnbd_srv_fault_inject *fault_inject,
+ const char *dev_name)
+{
+ rnbd_fault_inject_init(&fault_inject->fj, dev_name, -EBUSY);
+ /* injection points */
+ rnbd_fault_inject_add(fault_inject->fj.dir,
+ "fail-bio", &fault_inject->fail_bio);
+}
+
+void rnbd_srv_fault_inject_fini(struct rnbd_srv_fault_inject *fault_inject)
+{
+ rnbd_fault_inject_final(&fault_inject->fj);
+}
+
+int rnbd_should_fail_bio(struct rnbd_srv_sess_dev *sess_dev)
+{
+ struct rnbd_srv_fault_inject *fault_inject = &sess_dev->fault_inject;
+
+ if (fault_inject->fail_bio && should_fail(&fault_inject->fj.attr, 1))
+ return fault_inject->fj.status;
+ return 0;
+}
+#else
+void rnbd_srv_fault_inject_init(struct rnbd_srv_fault_inject *fault_inj,
+ const char *dev_name)
+{
+}
+void rnbd_srv_fault_inject_fini(struct rnbd_srv_fault_inject *fault_inject)
+{
+}
+int rnbd_should_fail_bio(struct rnbd_srv_sess_dev *sess_dev)
+{
+ return 0;
+}
+#endif
diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c
index a6a68d44f517..447fb0718525 100644
--- a/drivers/block/rnbd/rnbd-srv.c
+++ b/drivers/block/rnbd/rnbd-srv.c
@@ -88,9 +88,14 @@ void rnbd_endio(void *priv, int error)
{
struct rnbd_io_private *rnbd_priv = priv;
struct rnbd_srv_sess_dev *sess_dev = rnbd_priv->sess_dev;
+ int fail_err = 0;
rnbd_put_sess_dev(sess_dev);
+ fail_err = rnbd_should_fail_bio(sess_dev);
+ if (unlikely(fail_err)) /* over-write error which will be sent to client */
+ error = fail_err;
+
rtrs_srv_resp_rdma(rnbd_priv->id, error);
kfree(priv);
@@ -230,6 +235,7 @@ void rnbd_destroy_sess_dev(struct rnbd_srv_sess_dev *sess_dev, bool keep_id)
rnbd_put_sess_dev(sess_dev);
wait_for_completion(&dc); /* wait for inflights to drop to zero */
+ rnbd_srv_fault_inject_fini(&sess_dev->fault_inject);
rnbd_dev_close(sess_dev->rnbd_dev);
list_del(&sess_dev->sess_list);
mutex_lock(&sess_dev->dev->lock);
@@ -811,6 +817,7 @@ static int process_msg_open(struct rtrs_srv *rtrs,
rnbd_srv_info(srv_sess_dev, "Opened device '%s'\n", srv_dev->id);
kfree(full_path);
+ rnbd_srv_fault_inject_init(&srv_sess_dev->fault_inject, kbasename(srv_sess_dev->pathname));
fill_response:
rnbd_srv_fill_msg_open_rsp(rsp, srv_sess_dev);
diff --git a/drivers/block/rnbd/rnbd-srv.h b/drivers/block/rnbd/rnbd-srv.h
index b157371c25ed..120e6d64cb82 100644
--- a/drivers/block/rnbd/rnbd-srv.h
+++ b/drivers/block/rnbd/rnbd-srv.h
@@ -45,6 +45,13 @@ struct rnbd_srv_dev {
int open_write_cnt;
};
+struct rnbd_srv_fault_inject {
+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+ struct rnbd_fault_inject fj;
+ bool fail_bio;
+#endif
+};
+
/* Structure which binds N devices and N sessions */
struct rnbd_srv_sess_dev {
/* Entry inside rnbd_srv_dev struct */
@@ -62,6 +69,7 @@ struct rnbd_srv_sess_dev {
struct completion *destroy_comp;
char pathname[NAME_MAX];
enum rnbd_access_mode access_mode;
+ struct rnbd_srv_fault_inject fault_inject;
};
void rnbd_srv_sess_dev_force_close(struct rnbd_srv_sess_dev *sess_dev);
@@ -77,4 +85,9 @@ int rnbd_srv_create_sysfs_files(void);
void rnbd_srv_destroy_sysfs_files(void);
void rnbd_destroy_sess_dev(struct rnbd_srv_sess_dev *sess_dev, bool keep_id);
+void rnbd_srv_fault_inject_init(struct rnbd_srv_fault_inject *fault_inj,
+ const char *dev_name);
+void rnbd_srv_fault_inject_fini(struct rnbd_srv_fault_inject *fault_inject);
+int rnbd_should_fail_bio(struct rnbd_srv_sess_dev *sess_dev);
+
#endif /* RNBD_SRV_H */
--
2.25.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH for-next 3/5] block/rnbd-clt: Inject some fault points
2021-04-06 7:37 [PATCH for-next 0/5] Enable Fault Injection for RNBD Gioh Kim
2021-04-06 7:37 ` [PATCH for-next 1/5] block/rnbd: Enable the fault-injection Gioh Kim
2021-04-06 7:37 ` [PATCH for-next 2/5] block/rnbd-srv: Inject a fault at bio processing Gioh Kim
@ 2021-04-06 7:37 ` Gioh Kim
2021-04-06 7:37 ` [PATCH for-next 4/5] docs: fault-injection: Add fault-injection manual of RNBD Gioh Kim
2021-04-06 7:37 ` [PATCH for-next 5/5] docs: Add RTRS/RNBD to the index of fault-injection Gioh Kim
4 siblings, 0 replies; 6+ messages in thread
From: Gioh Kim @ 2021-04-06 7:37 UTC (permalink / raw)
To: linux-block, linux-doc
Cc: axboe, akinobu.mita, corbet, hch, sagi, bvanassche, haris.iqbal,
jinpu.wang, Gioh Kim, Jack Wang
From: Gioh Kim <gi-oh.kim@cloud.ionos.com>
This patch injects two fault points:
1. generate an IO error
2. generate a unmap failure
Signed-off-by: Gioh Kim <gi-oh.kim@cloud.ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
---
drivers/block/rnbd/rnbd-clt-sysfs.c | 53 +++++++++++++++++++++++++++++
drivers/block/rnbd/rnbd-clt.c | 17 +++++++++
drivers/block/rnbd/rnbd-clt.h | 15 ++++++++
3 files changed, 85 insertions(+)
diff --git a/drivers/block/rnbd/rnbd-clt-sysfs.c b/drivers/block/rnbd/rnbd-clt-sysfs.c
index d4aa6bfc9555..d83415875960 100644
--- a/drivers/block/rnbd/rnbd-clt-sysfs.c
+++ b/drivers/block/rnbd/rnbd-clt-sysfs.c
@@ -651,3 +651,56 @@ void rnbd_clt_destroy_sysfs_files(void)
device_destroy(rnbd_dev_class, MKDEV(0, 0));
class_destroy(rnbd_dev_class);
}
+
+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+void rnbd_clt_fault_inject_init(struct rnbd_clt_fault_inject *fault_inject,
+ const char *dev_name)
+{
+ rnbd_fault_inject_init(&fault_inject->fj, dev_name, -EBUSY);
+ /* injection points */
+ rnbd_fault_inject_add(fault_inject->fj.dir,
+ "fail-request", &fault_inject->fail_request);
+ rnbd_fault_inject_add(fault_inject->fj.dir,
+ "fail-unmap", &fault_inject->fail_unmap);
+}
+
+void rnbd_clt_fault_inject_final(struct rnbd_clt_fault_inject *fault_inject)
+{
+ rnbd_fault_inject_final(&fault_inject->fj);
+}
+
+int rnbd_clt_should_fail_request(struct request *req)
+{
+ struct rnbd_clt_dev *dev = req->rq_disk->private_data;
+ struct rnbd_clt_fault_inject *fault_inject = &dev->fault_inject;
+
+ if (fault_inject->fail_request && should_fail(&fault_inject->fj.attr, 1))
+ return fault_inject->fj.status;
+ return 0;
+}
+
+int rnbd_clt_should_fail_unmap(struct rnbd_clt_dev *dev)
+{
+ struct rnbd_clt_fault_inject *fault_inject = &dev->fault_inject;
+
+ if (fault_inject->fail_unmap && should_fail(&fault_inject->fj.attr, 1))
+ return fault_inject->fj.status;
+ return 0;
+}
+#else
+void rnbd_clt_fault_inject_init(struct rnbd_clt_fault_inject *fault_inj,
+ const char *dev_name)
+{
+}
+void rnbd_clt_fault_inject_final(struct rnbd_clt_fault_inject *fault_inject)
+{
+}
+int rnbd_clt_should_fail_request(struct request *req)
+{
+ return 0;
+}
+int rnbd_clt_should_fail_unmap(struct rnbd_clt_dev *dev)
+{
+ return 0;
+}
+#endif
diff --git a/drivers/block/rnbd/rnbd-clt.c b/drivers/block/rnbd/rnbd-clt.c
index 45a470076652..8c9a02c8b8bd 100644
--- a/drivers/block/rnbd/rnbd-clt.c
+++ b/drivers/block/rnbd/rnbd-clt.c
@@ -411,6 +411,11 @@ static void msg_io_conf(void *priv, int errno)
struct rnbd_clt_dev *dev = iu->dev;
struct request *rq = iu->rq;
int rw = rq_data_dir(rq);
+ int fail_err = 0;
+
+ fail_err = rnbd_clt_should_fail_request(rq);
+ if (unlikely(fail_err)) /* over-write error */
+ errno = fail_err;
iu->errno = errno;
@@ -1161,6 +1166,7 @@ static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx,
}
blk_mq_start_request(rq);
+
err = rnbd_client_xfer_request(dev, rq, iu);
if (likely(err == 0))
return BLK_STS_OK;
@@ -1545,6 +1551,8 @@ struct rnbd_clt_dev *rnbd_clt_map_device(const char *sessname,
goto send_close;
}
+ rnbd_clt_fault_inject_init(&dev->fault_inject, dev->gd->disk_name);
+
rnbd_clt_info(dev,
"map_device: Device mapped as %s (nsectors: %zu, logical_block_size: %d, physical_block_size: %d, max_write_same_sectors: %d, max_discard_sectors: %d, discard_granularity: %d, discard_alignment: %d, secure_discard: %d, max_segments: %d, max_hw_sectors: %d, rotational: %d, wc: %d, fua: %d)\n",
dev->gd->disk_name, dev->nsectors,
@@ -1599,8 +1607,16 @@ int rnbd_clt_unmap_device(struct rnbd_clt_dev *dev, bool force,
struct rnbd_clt_session *sess = dev->sess;
int refcount, ret = 0;
bool was_mapped;
+ int fail_err = 0;
mutex_lock(&dev->lock);
+
+ fail_err = rnbd_clt_should_fail_unmap(dev);
+ if (unlikely(fail_err)) {
+ ret = fail_err;
+ goto err;
+ }
+
if (dev->dev_state == DEV_STATE_UNMAPPED) {
rnbd_clt_info(dev, "Device is already being unmapped\n");
ret = -EALREADY;
@@ -1618,6 +1634,7 @@ int rnbd_clt_unmap_device(struct rnbd_clt_dev *dev, bool force,
dev->dev_state = DEV_STATE_UNMAPPED;
mutex_unlock(&dev->lock);
+ rnbd_clt_fault_inject_final(&dev->fault_inject);
delete_dev(dev);
destroy_sysfs(dev, sysfs_self);
destroy_gen_disk(dev);
diff --git a/drivers/block/rnbd/rnbd-clt.h b/drivers/block/rnbd/rnbd-clt.h
index 537d499dad3b..5ecbe8dedf24 100644
--- a/drivers/block/rnbd/rnbd-clt.h
+++ b/drivers/block/rnbd/rnbd-clt.h
@@ -107,6 +107,14 @@ struct rnbd_queue {
struct blk_mq_hw_ctx *hctx;
};
+struct rnbd_clt_fault_inject {
+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
+ struct rnbd_fault_inject fj;
+ bool fail_unmap;
+ bool fail_request;
+#endif
+};
+
struct rnbd_clt_dev {
struct rnbd_clt_session *sess;
struct request_queue *queue;
@@ -139,6 +147,7 @@ struct rnbd_clt_dev {
char *blk_symlink_name;
refcount_t refcount;
struct work_struct unmap_on_rmmod_work;
+ struct rnbd_clt_fault_inject fault_inject;
};
/* rnbd-clt.c */
@@ -163,4 +172,10 @@ void rnbd_clt_destroy_default_group(void);
void rnbd_clt_remove_dev_symlink(struct rnbd_clt_dev *dev);
+void rnbd_clt_fault_inject_init(struct rnbd_clt_fault_inject *fault_inj,
+ const char *dev_name);
+void rnbd_clt_fault_inject_final(struct rnbd_clt_fault_inject *fault_inject);
+int rnbd_clt_should_fail_request(struct request *req);
+int rnbd_clt_should_fail_unmap(struct rnbd_clt_dev *dev);
+int rnbd_clt_should_fail_request_timeout(struct request *req);
#endif /* RNBD_CLT_H */
--
2.25.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH for-next 4/5] docs: fault-injection: Add fault-injection manual of RNBD
2021-04-06 7:37 [PATCH for-next 0/5] Enable Fault Injection for RNBD Gioh Kim
` (2 preceding siblings ...)
2021-04-06 7:37 ` [PATCH for-next 3/5] block/rnbd-clt: Inject some fault points Gioh Kim
@ 2021-04-06 7:37 ` Gioh Kim
2021-04-06 7:37 ` [PATCH for-next 5/5] docs: Add RTRS/RNBD to the index of fault-injection Gioh Kim
4 siblings, 0 replies; 6+ messages in thread
From: Gioh Kim @ 2021-04-06 7:37 UTC (permalink / raw)
To: linux-block, linux-doc
Cc: axboe, akinobu.mita, corbet, hch, sagi, bvanassche, haris.iqbal,
jinpu.wang, Gioh Kim
From: Gioh Kim <gi-oh.kim@cloud.ionos.com>
It describes how to use the fault-injection of RNBD.
Signed-off-by: Gioh Kim <gi-oh.kim@cloud.ionos.com>
---
.../fault-injection/rnbd-fault-injection.rst | 208 ++++++++++++++++++
1 file changed, 208 insertions(+)
create mode 100644 Documentation/fault-injection/rnbd-fault-injection.rst
diff --git a/Documentation/fault-injection/rnbd-fault-injection.rst b/Documentation/fault-injection/rnbd-fault-injection.rst
new file mode 100644
index 000000000000..21594e5e3c91
--- /dev/null
+++ b/Documentation/fault-injection/rnbd-fault-injection.rst
@@ -0,0 +1,208 @@
+RNBD (RDMA Network Block Device) Fault Injection
+================================================
+This document introduces how to enable and use the error injection of RNBD
+via debugfs in the /sys/kernel/debug directory. When enabled, users can
+enable specific error injection point and change the default status code
+via the debugfs.
+
+Following examples show how to inject an error into the RNBD.
+
+First, enable CONFIG_FAULT_INJECTION_DEBUG_FS kernel config,
+recompile the kernel. After booting up the kernel, map a target device.
+
+On client, /sys/kernel/debug/rnbdX directory is created after mapping.
+And /sys/kernel/debug/<mapped-device> directory is created on server.
+
+Example 1: Inject an error into request processing of rnbd-client
+-----------------------------------------------------------------
+
+::
+
+ echo 1 > /sys/kernel/debug/rnbd0/fault_inject/times
+ echo 100 > /sys/kernel/debug/rnbd0/fault_inject/probability
+ echo 1 > /sys/kernel/debug/rnbd0/fault_inject/fail-request
+ dd if=/dev/rnbd0 of=./dd bs=1k count=10
+
+Expected Result::
+
+ dd succeeds but generates an IO error
+
+Message from dmesg::
+
+ FAULT_INJECTION: forcing a failure.
+ name fault_inject, interval 1, probability 100, space 0, times 1
+ CPU: 4 PID: 0 Comm: swapper/4 Tainted: G O 5.4.77-pserver+ #167
+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1ubuntu1.1 04/01/2014
+ Call Trace:
+ <IRQ>
+ dump_stack+0x97/0xe0
+ should_fail.cold+0x5/0x11
+ rnbd_clt_should_fail_request+0x5e/0x80 [rnbd_client]
+ msg_io_conf+0x42/0xb0 [rnbd_client]
+ complete_rdma_req+0x264/0x600 [rtrs_client]
+ rtrs_clt_rdma_done+0x4a2/0x690 [rtrs_client]
+ __ib_process_cq+0x94/0x100 [ib_core]
+ ib_poll_handler+0x3f/0xa0 [ib_core]
+ irq_poll_softirq+0xf8/0x280
+ __do_softirq+0x122/0x550
+ irq_exit+0xfb/0x100
+ do_IRQ+0x8a/0x170
+ common_interrupt+0xf/0xf
+ </IRQ>
+ RIP: 0010:default_idle+0x2b/0x1d0
+ Code: 1f 44 00 00 41 55 41 54 65 44 8b 25 7f fe 0a 5a 55 53 0f 1f 44 00 00 e8 53 65 30 ff e9 07 00 00 00 0f 00 2d b7 59 4b 00 fb f4 <65> 44 8b 25 5d fe 0a 5a 0f 1f 44 00 00 5b 5d 41 5c 41 5d c3 65 8b
+ RSP: 0018:ffff88811963fdc8 EFLAGS: 00000246 ORIG_RAX: ffffffffffffffdd
+ RAX: 0000000000000000 RBX: ffff888119633240 RCX: dffffc0000000000
+ RDX: 0000000000000007 RSI: 0000000000000006 RDI: ffff888119633ad4
+ RBP: 0000000000000004 R08: ffffffffa516d49d R09: 0000000000000000
+ R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000004
+ R13: 0000000000000000 R14: ffff888119633240 R15: 0000000000000000
+ do_idle+0x314/0x370
+ cpu_startup_entry+0x19/0x20
+ start_secondary+0x212/0x280
+ secondary_startup_64+0xa4/0xb0
+ rnbd_client L432: </dev/nullb0@bla> read I/O failed with err: -16
+ blk_update_request: device resource error, dev rnbd0, sector 0 op 0x0:(READ) flags 0x80700 phys_seg 4 prio class 0
+
+Example 2: Inject an error into unmapping of rnbd-client
+--------------------------------------------------------
+
+::
+
+ echo 100 > /sys/kernel/debug/rnbd0/fault_inject/probability
+ echo 1 > /sys/kernel/debug/rnbd0/fault_inject/times
+ echo 1 > /sys/kernel/debug/rnbd0/fault_inject/fail-unmap
+ echo normal > /sys/block/rnbd0/rnbd/unmap_device
+
+Expected Result::
+
+ echo: write error: Device or resource busy
+
+Message from dmesg::
+
+ FAULT_INJECTION: forcing a failure.
+ name fault_inject, interval 1, probability 100, space 0, times 1
+ CPU: 2 PID: 648 Comm: bash Tainted: G O 5.4.77-pserver+ #169
+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1ubuntu1.1 04/01/2014
+ Call Trace:
+ dump_stack+0x97/0xe0
+ should_fail.cold+0x5/0x11
+ rnbd_clt_should_fail_unmap+0x38/0x60 [rnbd_client]
+ rnbd_clt_unmap_device+0x3c/0x1c0 [rnbd_client]
+ rnbd_clt_unmap_dev_store.cold+0xe5/0x13f [rnbd_client]
+ kernfs_fop_write+0x141/0x240
+ vfs_write+0xf2/0x250
+ ksys_write+0xc3/0x160
+ do_syscall_64+0x68/0x260
+ entry_SYSCALL_64_after_hwframe+0x49/0xbe
+ RIP: 0033:0x7ff883091504
+ Code: 00 f7 d8 64 89 02 48 c7 c0 ff ff ff ff eb b3 0f 1f 80 00 00 00 00 48 8d 05 f9 61 0d 00 8b 00 85 c0 75 13 b8 01 00 00 00 0f 05 <48> 3d 00 f0 ff ff 77 54 c3 0f 1f 00 41 54 49 89 d4 55 48 89 f5 53
+ RSP: 002b:00007ffe1bc91458 EFLAGS: 00000246 ORIG_RAX: 0000000000000001
+ RAX: ffffffffffffffda RBX: 0000000000000007 RCX: 00007ff883091504
+ RDX: 0000000000000007 RSI: 000056389b73a180 RDI: 0000000000000001
+ RBP: 000056389b73a180 R08: 000000000000000a R09: 00007ff883121e80
+ R10: 000000000000000a R11: 0000000000000246 R12: 00007ff883163760
+ R13: 0000000000000007 R14: 00007ff88315e760 R15: 0000000000000007
+ rnbd_client L335: </dev/nullb0@bla> unmap_device: -16
+ rnbd_client L321: </dev/nullb0@bla> Unmapping device, option: normal.
+
+Example 3: Inject an error into bio process of rnbd-server
+----------------------------------------------------------
+
+After client maps null0b, you can see /sys/kernel/debug/nullb0 directory on server::
+
+ echo 100 > /sys/kernel/debug/nullb0/fault_inject/probability
+ echo 1 > /sys/kernel/debug/nullb0/fault_inject/times
+ echo 1 > /sys/kernel/debug/nullb0/fault_inject/fail-bio
+
+Then you can generate IO on client::
+
+ dd if=/dev/rnbd0 of=./dd bs=1k count=10
+
+Expected Result on client::
+
+ dd succeeds but generates an IO error
+
+Message from dmesg on client::
+
+ rtrs_client L453: <bla>: IO request failed: error=-16 path=ip:192.168.122.142@ip:192.168.122.130 [mlx4_0:1] notify=1
+ rnbd_client L432: </dev/nullb0@bla> read I/O failed with err: -16
+ blk_update_request: device resource error, dev rnbd0, sector 0 op 0x0:(READ) flags 0x80700 phys_seg 4 prio class 0
+
+Message from dmesg on server::
+
+ FAULT_INJECTION: forcing a failure.
+ name fault_inject, interval 1, probability 100, space 0, times 1
+ CPU: 4 PID: 31 Comm: ksoftirqd/4 Tainted: G O 5.4.77-pserver+ #169
+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1ubuntu1.1 04/01/2014
+ Call Trace:
+ dump_stack+0x97/0xe0
+ should_fail.cold+0x5/0x11
+ rnbd_should_fail_bio+0x38/0x51 [rnbd_server]
+ rnbd_endio+0x41/0x70 [rnbd_server]
+ rnbd_dev_bi_end_io+0x43/0x50 [rnbd_server]
+ blk_update_request+0x1af/0x520
+ blk_mq_end_request+0x2e/0x200
+ blk_done_softirq+0x16e/0x1c0
+ __do_softirq+0x122/0x550
+ run_ksoftirqd+0x24/0x30
+ smpboot_thread_fn+0x1a2/0x2d0
+ kthread+0x191/0x1e0
+ ret_from_fork+0x3a/0x50
+
+Example 4: Change the status code
+---------------------------------
+
+The default status code is -16 (-EBUSY) but you can change it::
+
+ echo 1 > /sys/kernel/debug/rnbd0/fault_inject/times
+ echo 100 > /sys/kernel/debug/rnbd0/fault_inject/probability
+ echo 1 > /sys/kernel/debug/rnbd0/fault_inject/fail-request
+ echo -10 > /sys/kernel/debug/rnbd0/fault_inject/status
+ dd if=/dev/rnbd0 of=./dd bs=1k count=10
+
+Expected Result::
+
+ The error value is -10
+
+Message from dmesg::
+
+ FAULT_INJECTION: forcing a failure.
+ name fault_inject, interval 1, probability 100, space 0, times 1
+ CPU: 4 PID: 0 Comm: swapper/4 Tainted: G O 5.4.77-pserver+ #170
+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1ubuntu1.1 04/01/2014
+ Call Trace:
+ <IRQ>
+ dump_stack+0x97/0xe0
+ should_fail.cold+0x5/0x11
+ rnbd_clt_should_fail_request+0x5e/0x80 [rnbd_client]
+ msg_io_conf+0x42/0xb0 [rnbd_client]
+ complete_rdma_req+0x264/0x600 [rtrs_client]
+ rtrs_clt_rdma_done+0x4a2/0x690 [rtrs_client]
+ __ib_process_cq+0x94/0x100 [ib_core]
+ ib_poll_handler+0x3f/0xa0 [ib_core]
+ irq_poll_softirq+0xf8/0x280
+ __do_softirq+0x122/0x550
+ irq_exit+0xfb/0x100
+ do_IRQ+0x8a/0x170
+ common_interrupt+0xf/0xf
+ </IRQ>
+ RIP: 0010:default_idle+0x2b/0x1d0
+ Code: 1f 44 00 00 41 55 41 54 65 44 8b 25 7f fe 0a 7c 55 53 0f 1f 44 00 00 e8 53 65 30 ff e9 07 00 00 00 0f 00 2d b7 59 4b 00 fb f4 <65> 44 8b 25 5d fe 0a 7c 0f 1f 44 00 00 5b 5d 41 5c 41 5d c3 65 8b
+ RSP: 0018:ffff888114e2fdc8 EFLAGS: 00000246 ORIG_RAX: ffffffffffffffdd
+ RAX: 0000000000000000 RBX: ffff888114e26440 RCX: dffffc0000000000
+ RDX: 0000000000000007 RSI: 0000000000000006 RDI: ffff888114e26cd4
+ RBP: 0000000000000004 R08: ffffffff8316d49d R09: 0000000000000000
+ R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000004
+ R13: 0000000000000000 R14: ffff888114e26440 R15: 0000000000000000
+ ? lockdep_hardirqs_on+0x17d/0x250
+ ? default_idle+0x1d/0x1d0
+ do_idle+0x314/0x370
+ ? arch_cpu_idle_exit+0x40/0x40
+ ? schedule_idle+0x46/0x60
+ cpu_startup_entry+0x19/0x20
+ start_secondary+0x212/0x280
+ ? set_cpu_sibling_map+0xcb0/0xcb0
+ secondary_startup_64+0xa4/0xb0
+ rnbd_client L432: </dev/nullb0@bla> read I/O failed with err: -10
+ blk_update_request: I/O error, dev rnbd0, sector 0 op 0x0:(READ) flags 0x80700 phys_seg 3 prio class 0
--
2.25.1
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH for-next 5/5] docs: Add RTRS/RNBD to the index of fault-injection
2021-04-06 7:37 [PATCH for-next 0/5] Enable Fault Injection for RNBD Gioh Kim
` (3 preceding siblings ...)
2021-04-06 7:37 ` [PATCH for-next 4/5] docs: fault-injection: Add fault-injection manual of RNBD Gioh Kim
@ 2021-04-06 7:37 ` Gioh Kim
4 siblings, 0 replies; 6+ messages in thread
From: Gioh Kim @ 2021-04-06 7:37 UTC (permalink / raw)
To: linux-block, linux-doc
Cc: axboe, akinobu.mita, corbet, hch, sagi, bvanassche, haris.iqbal,
jinpu.wang, Gioh Kim, Jack Wang
From: Gioh Kim <gi-oh.kim@cloud.ionos.com>
Add new documents for RTRS and RNBD to the index file.
Signed-off-by: Gioh Kim <gi-oh.kim@cloud.ionos.com>
Signed-off-by: Jack Wang <jinpu.wang@cloud.ionos.com>
---
Documentation/fault-injection/index.rst | 2 ++
1 file changed, 2 insertions(+)
diff --git a/Documentation/fault-injection/index.rst b/Documentation/fault-injection/index.rst
index 8408a8a91b34..669d146af874 100644
--- a/Documentation/fault-injection/index.rst
+++ b/Documentation/fault-injection/index.rst
@@ -11,6 +11,8 @@ fault-injection
notifier-error-inject
nvme-fault-injection
provoke-crashes
+ rnbd-fault-injection
+ rtrs-fault-injection
.. only:: subproject and html
--
2.25.1
^ permalink raw reply related [flat|nested] 6+ messages in thread