* [PATCH igt 1/3] lib/igt_debugfs: Add helper for writing debugfs files
2021-11-09 18:09 [PATCH igt 0/3] msm: Add tests for gpu fault handling Rob Clark
@ 2021-11-09 18:09 ` Rob Clark
2021-11-10 11:58 ` [igt-dev] " Petri Latvala
2021-11-09 18:09 ` [PATCH igt 2/3] msm: Add helper for cmdstream building and submission Rob Clark
2021-11-09 18:09 ` [PATCH igt 3/3] msm: Add recovery tests Rob Clark
2 siblings, 1 reply; 5+ messages in thread
From: Rob Clark @ 2021-11-09 18:09 UTC (permalink / raw)
To: igt-dev
Cc: freedreno, linux-arm-msm, Jordan Crouse, Akhil P Oommen, Rob Clark
From: Rob Clark <robdclark@chromium.org>
Signed-off-by: Rob Clark <robdclark@chromium.org>
---
lib/igt_debugfs.c | 16 ++++++++++++++++
lib/igt_debugfs.h | 12 ++++++++++++
2 files changed, 28 insertions(+)
diff --git a/lib/igt_debugfs.c b/lib/igt_debugfs.c
index a5bb95ca..39431068 100644
--- a/lib/igt_debugfs.c
+++ b/lib/igt_debugfs.c
@@ -351,6 +351,22 @@ void __igt_debugfs_read(int fd, const char *filename, char *buf, int size)
close(dir);
}
+/**
+ * __igt_debugfs_write:
+ * @filename: file name
+ * @buf: buffer to be written to the debugfs file
+ * @size: size of the buffer
+ *
+ * This function opens the debugfs file, writes it, then closes the file.
+ */
+void __igt_debugfs_write(int fd, const char *filename, const char *buf, int size)
+{
+ int dir = igt_debugfs_dir(fd);
+
+ igt_sysfs_write(dir, filename, buf, size);
+ close(dir);
+}
+
/**
* igt_debugfs_search:
* @filename: file name
diff --git a/lib/igt_debugfs.h b/lib/igt_debugfs.h
index d43ba6c6..249eb56a 100644
--- a/lib/igt_debugfs.h
+++ b/lib/igt_debugfs.h
@@ -40,6 +40,7 @@ int igt_debugfs_pipe_dir(int device, int pipe, int mode);
int igt_debugfs_open(int fd, const char *filename, int mode);
void __igt_debugfs_read(int fd, const char *filename, char *buf, int size);
+void __igt_debugfs_write(int fd, const char *filename, const char *buf, int size);
int igt_debugfs_simple_read(int dir, const char *filename, char *buf, int size);
bool igt_debugfs_search(int fd, const char *filename, const char *substring);
@@ -54,6 +55,17 @@ bool igt_debugfs_search(int fd, const char *filename, const char *substring);
#define igt_debugfs_read(fd, filename, buf) \
__igt_debugfs_read(fd, (filename), (buf), sizeof(buf))
+/**
+ * igt_debugfs_write:
+ * @filename: name of the debugfs file
+ * @buf: buffer to be written to the debugfs file
+ *
+ * This is just a convenience wrapper for __igt_debugfs_read. See its
+ * documentation.
+ */
+#define igt_debugfs_write(fd, filename, buf) \
+ __igt_debugfs_write(fd, (filename), (buf), sizeof(buf))
+
/*
* Pipe CRC
*/
--
2.31.1
^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [igt-dev] [PATCH igt 1/3] lib/igt_debugfs: Add helper for writing debugfs files
2021-11-09 18:09 ` [PATCH igt 1/3] lib/igt_debugfs: Add helper for writing debugfs files Rob Clark
@ 2021-11-10 11:58 ` Petri Latvala
0 siblings, 0 replies; 5+ messages in thread
From: Petri Latvala @ 2021-11-10 11:58 UTC (permalink / raw)
To: Rob Clark
Cc: igt-dev, Rob Clark, linux-arm-msm, freedreno, Akhil P Oommen,
Jordan Crouse
On Tue, Nov 09, 2021 at 10:09:03AM -0800, Rob Clark wrote:
> From: Rob Clark <robdclark@chromium.org>
>
> Signed-off-by: Rob Clark <robdclark@chromium.org>
> ---
> lib/igt_debugfs.c | 16 ++++++++++++++++
> lib/igt_debugfs.h | 12 ++++++++++++
> 2 files changed, 28 insertions(+)
>
> diff --git a/lib/igt_debugfs.c b/lib/igt_debugfs.c
> index a5bb95ca..39431068 100644
> --- a/lib/igt_debugfs.c
> +++ b/lib/igt_debugfs.c
> @@ -351,6 +351,22 @@ void __igt_debugfs_read(int fd, const char *filename, char *buf, int size)
> close(dir);
> }
>
> +/**
> + * __igt_debugfs_write:
> + * @filename: file name
> + * @buf: buffer to be written to the debugfs file
> + * @size: size of the buffer
I see that we have this mistake in __igt_debugfs_read already but no
reason to repeat mistakes: Also document the fd parameter.
> + *
> + * This function opens the debugfs file, writes it, then closes the file.
> + */
> +void __igt_debugfs_write(int fd, const char *filename, const char *buf, int size)
> +{
> + int dir = igt_debugfs_dir(fd);
> +
> + igt_sysfs_write(dir, filename, buf, size);
> + close(dir);
> +}
> +
> /**
> * igt_debugfs_search:
> * @filename: file name
> diff --git a/lib/igt_debugfs.h b/lib/igt_debugfs.h
> index d43ba6c6..249eb56a 100644
> --- a/lib/igt_debugfs.h
> +++ b/lib/igt_debugfs.h
> @@ -40,6 +40,7 @@ int igt_debugfs_pipe_dir(int device, int pipe, int mode);
>
> int igt_debugfs_open(int fd, const char *filename, int mode);
> void __igt_debugfs_read(int fd, const char *filename, char *buf, int size);
> +void __igt_debugfs_write(int fd, const char *filename, const char *buf, int size);
> int igt_debugfs_simple_read(int dir, const char *filename, char *buf, int size);
> bool igt_debugfs_search(int fd, const char *filename, const char *substring);
>
> @@ -54,6 +55,17 @@ bool igt_debugfs_search(int fd, const char *filename, const char *substring);
> #define igt_debugfs_read(fd, filename, buf) \
> __igt_debugfs_read(fd, (filename), (buf), sizeof(buf))
>
> +/**
> + * igt_debugfs_write:
> + * @filename: name of the debugfs file
> + * @buf: buffer to be written to the debugfs file
> + *
> + * This is just a convenience wrapper for __igt_debugfs_read. See its
> + * documentation.
Same here regarding the fd parameter as above. Also copypaste mistake
with s/debugfs_read/debugfs_write/.
--
Petri Latvala
> + */
> +#define igt_debugfs_write(fd, filename, buf) \
> + __igt_debugfs_write(fd, (filename), (buf), sizeof(buf))
> +
> /*
> * Pipe CRC
> */
> --
> 2.31.1
>
^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH igt 2/3] msm: Add helper for cmdstream building and submission
2021-11-09 18:09 [PATCH igt 0/3] msm: Add tests for gpu fault handling Rob Clark
2021-11-09 18:09 ` [PATCH igt 1/3] lib/igt_debugfs: Add helper for writing debugfs files Rob Clark
@ 2021-11-09 18:09 ` Rob Clark
2021-11-09 18:09 ` [PATCH igt 3/3] msm: Add recovery tests Rob Clark
2 siblings, 0 replies; 5+ messages in thread
From: Rob Clark @ 2021-11-09 18:09 UTC (permalink / raw)
To: igt-dev
Cc: freedreno, linux-arm-msm, Jordan Crouse, Akhil P Oommen, Rob Clark
From: Rob Clark <robdclark@chromium.org>
A pretty minimal subset compared to what a full gallium driver would
need, but OTOH for igt tests we should only need to emit fairly basic
command stream.
Signed-off-by: Rob Clark <robdclark@chromium.org>
---
lib/igt_msm.c | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++
lib/igt_msm.h | 48 ++++++++++++++++++++++++
2 files changed, 150 insertions(+)
diff --git a/lib/igt_msm.c b/lib/igt_msm.c
index b9534164..e9cf588f 100644
--- a/lib/igt_msm.c
+++ b/lib/igt_msm.c
@@ -91,6 +91,19 @@ igt_msm_dev_close(struct msm_device *dev)
free(dev);
}
+static uint64_t
+get_iova(struct msm_bo *bo)
+{
+ struct drm_msm_gem_info req = {
+ .handle = bo->handle,
+ .info = MSM_INFO_GET_IOVA,
+ };
+
+ do_ioctl(bo->dev->fd, DRM_IOCTL_MSM_GEM_INFO, &req);
+
+ return req.value;
+}
+
/**
* igt_msm_bo_new:
* @dev: the device to allocate the BO from
@@ -115,6 +128,7 @@ igt_msm_bo_new(struct msm_device *dev, size_t size, uint32_t flags)
do_ioctl(dev->fd, DRM_IOCTL_MSM_GEM_NEW, &req);
bo->handle = req.handle;
+ bo->iova = get_iova(bo);
return bo;
}
@@ -209,3 +223,91 @@ igt_msm_pipe_close(struct msm_pipe *pipe)
do_ioctl(pipe->dev->fd, DRM_IOCTL_MSM_SUBMITQUEUE_CLOSE, &pipe->submitqueue_id);
free(pipe);
}
+
+/**
+ * igt_msm_cmd_new:
+ * @pipe: the submitqueue to submit cmdstream against
+ * @size: the size of requested cmdstream buffer
+ */
+struct msm_cmd *
+igt_msm_cmd_new(struct msm_pipe *pipe, size_t size)
+{
+ struct msm_cmd *cmd = calloc(1, sizeof(*cmd));
+
+ cmd->pipe = pipe;
+ cmd->cmdstream_bo = igt_msm_bo_new(pipe->dev, size, MSM_BO_WC);
+ cmd->cur = igt_msm_bo_map(cmd->cmdstream_bo);
+
+ __igt_msm_append_bo(cmd, cmd->cmdstream_bo);
+
+ return cmd;
+}
+
+static uint32_t
+cmdstream_size(struct msm_cmd *cmd)
+{
+ uint8_t *start = igt_msm_bo_map(cmd->cmdstream_bo);
+ return (uint8_t *)cmd->cur - start;
+}
+
+/**
+ * igt_msm_cmd_submit:
+ * @cmd: the command stream object to submit
+ *
+ * Returns dma-fence fd
+ */
+int
+igt_msm_cmd_submit(struct msm_cmd *cmd)
+{
+ struct drm_msm_gem_submit_bo bos[cmd->nr_bos];
+ struct drm_msm_gem_submit_cmd cmds[] = {
+ [0] = {
+ .type = MSM_SUBMIT_CMD_BUF,
+ .submit_idx = 0,
+ .size = cmdstream_size(cmd),
+ },
+ };
+ struct drm_msm_gem_submit req = {
+ .flags = cmd->pipe->pipe | MSM_SUBMIT_FENCE_FD_OUT,
+ .queueid = cmd->pipe->submitqueue_id,
+ .nr_cmds = ARRAY_SIZE(cmds),
+ .cmds = VOID2U64(cmds),
+ .nr_bos = ARRAY_SIZE(bos),
+ .bos = VOID2U64(bos),
+ };
+
+ for (unsigned i = 0; i < cmd->nr_bos; i++) {
+ bos[i] = (struct drm_msm_gem_submit_bo) {
+ .handle = cmd->bos[i]->handle,
+ .flags = MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE,
+ };
+ }
+
+ do_ioctl(cmd->pipe->dev->fd, DRM_IOCTL_MSM_GEM_SUBMIT, &req);
+
+ return req.fence_fd;
+}
+
+void
+__igt_msm_append_bo(struct msm_cmd *cmd, struct msm_bo *bo)
+{
+ for (unsigned i = 0; i < cmd->nr_bos; i++)
+ if (cmd->bos[i] == bo)
+ return;
+
+ assert((cmd->nr_bos + 1) < ARRAY_SIZE(cmd->bos));
+ cmd->bos[cmd->nr_bos++] = bo;
+}
+
+/**
+ * igt_msm_cmd_free:
+ * @cmd: the command stream object to free
+ *
+ * Free a command stream object
+ */
+void
+igt_msm_cmd_free(struct msm_cmd *cmd)
+{
+ igt_msm_bo_free(cmd->cmdstream_bo);
+ free(cmd);
+}
diff --git a/lib/igt_msm.h b/lib/igt_msm.h
index 99a099c1..1a66c806 100644
--- a/lib/igt_msm.h
+++ b/lib/igt_msm.h
@@ -24,6 +24,8 @@
#ifndef IGT_MSM_H
#define IGT_MSM_H
+#include "ioctl_wrappers.h"
+
#include "msm_drm.h"
/**
@@ -47,6 +49,7 @@ void igt_msm_dev_close(struct msm_device *dev);
* @handle: the BO's GEM handle
* @size: the BO's size
* @map: the BO's memory mapping (if mapped)
+ * @iova: the BO's GPU address
*
* Helper wrapper for a GEM buffer object.
*/
@@ -55,6 +58,7 @@ struct msm_bo {
int handle;
uint32_t size;
void *map;
+ uint64_t iova;
};
struct msm_bo *igt_msm_bo_new(struct msm_device *dev, size_t size, uint32_t flags);
@@ -136,6 +140,50 @@ pm4_pkt7_hdr(uint8_t opcode, uint16_t cnt)
((pm4_odd_parity_bit(opcode) << 23));
}
+/**
+ * msm_cmd:
+ * @pipe: the submitqueue to submit cmdstream against
+ * @cmdstream_bo: the backing cmdstream buffer object
+ * @cur: pointer to current position in cmdstream
+ *
+ * Helper for building cmdstream and cmdstream submission
+ */
+struct msm_cmd {
+ struct msm_pipe *pipe;
+ struct msm_bo *cmdstream_bo;
+ uint32_t *cur;
+ uint32_t nr_bos;
+ struct msm_bo *bos[8];
+};
+
+struct msm_cmd *igt_msm_cmd_new(struct msm_pipe *pipe, size_t size);
+int igt_msm_cmd_submit(struct msm_cmd *cmd);
+void igt_msm_cmd_free(struct msm_cmd *cmd);
+
+static inline void
+msm_cmd_emit(struct msm_cmd *cmd, uint32_t dword)
+{
+ *(cmd->cur++) = dword;
+}
+
+static inline void
+msm_cmd_pkt7(struct msm_cmd *cmd, uint8_t opcode, uint16_t cnt)
+{
+ msm_cmd_emit(cmd, pm4_pkt7_hdr(opcode, cnt));
+}
+
+void __igt_msm_append_bo(struct msm_cmd *cmd, struct msm_bo *bo);
+
+static inline void
+msm_cmd_bo(struct msm_cmd *cmd, struct msm_bo *bo, uint32_t offset)
+{
+ uint64_t addr = bo->iova + offset;
+
+ __igt_msm_append_bo(cmd, bo);
+ msm_cmd_emit(cmd, lower_32_bits(addr));
+ msm_cmd_emit(cmd, upper_32_bits(addr));
+}
+
#define U642VOID(x) ((void *)(uintptr_t)(x))
#define VOID2U64(x) ((uint64_t)(uintptr_t)(x))
--
2.31.1
^ permalink raw reply related [flat|nested] 5+ messages in thread
* [PATCH igt 3/3] msm: Add recovery tests
2021-11-09 18:09 [PATCH igt 0/3] msm: Add tests for gpu fault handling Rob Clark
2021-11-09 18:09 ` [PATCH igt 1/3] lib/igt_debugfs: Add helper for writing debugfs files Rob Clark
2021-11-09 18:09 ` [PATCH igt 2/3] msm: Add helper for cmdstream building and submission Rob Clark
@ 2021-11-09 18:09 ` Rob Clark
2 siblings, 0 replies; 5+ messages in thread
From: Rob Clark @ 2021-11-09 18:09 UTC (permalink / raw)
To: igt-dev
Cc: freedreno, linux-arm-msm, Jordan Crouse, Akhil P Oommen, Rob Clark
From: Rob Clark <robdclark@chromium.org>
Add tests to exercise:
1. sw hangcheck timeout
2. gpu fault (hang) recovery
3. iova fault recovery
Signed-off-by: Rob Clark <robdclark@chromium.org>
---
lib/igt_msm.h | 3 +
tests/meson.build | 1 +
tests/msm_recovery.c | 172 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 176 insertions(+)
create mode 100644 tests/msm_recovery.c
diff --git a/lib/igt_msm.h b/lib/igt_msm.h
index 1a66c806..421d23ed 100644
--- a/lib/igt_msm.h
+++ b/lib/igt_msm.h
@@ -97,6 +97,9 @@ enum adreno_pm4_packet_type {
enum adreno_pm4_type3_packets {
CP_NOP = 16,
+ CP_WAIT_MEM_GTE = 20,
+ CP_WAIT_REG_MEM = 60,
+ CP_MEM_WRITE = 61,
};
static inline unsigned
diff --git a/tests/meson.build b/tests/meson.build
index 0af3e03a..166e3494 100644
--- a/tests/meson.build
+++ b/tests/meson.build
@@ -60,6 +60,7 @@ test_progs = [
'kms_vrr',
'kms_writeback',
'meta_test',
+ 'msm_recovery',
'msm_submit',
'panfrost_get_param',
'panfrost_gem_new',
diff --git a/tests/msm_recovery.c b/tests/msm_recovery.c
new file mode 100644
index 00000000..b71326b8
--- /dev/null
+++ b/tests/msm_recovery.c
@@ -0,0 +1,172 @@
+/*
+ * Copyright © 2021 Google, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include <sys/poll.h>
+
+#include "igt.h"
+#include "igt_msm.h"
+
+static struct msm_device *dev;
+static struct msm_bo *scratch_bo;
+static uint32_t *scratch;
+
+/*
+ * Helpers for cmdstream packet building:
+ */
+
+static void
+wait_mem_gte(struct msm_cmd *cmd, uint32_t offset_dwords, uint32_t ref)
+{
+ msm_cmd_pkt7(cmd, CP_WAIT_MEM_GTE, 4);
+ msm_cmd_emit(cmd, 0); /* RESERVED */
+ msm_cmd_bo (cmd, scratch_bo, offset_dwords * 4); /* POLL_ADDR_LO/HI */
+ msm_cmd_emit(cmd, ref); /* REF */
+}
+
+static void
+mem_write(struct msm_cmd *cmd, uint32_t offset_dwords, uint32_t val)
+{
+ msm_cmd_pkt7(cmd, CP_MEM_WRITE, 3);
+ msm_cmd_bo (cmd, scratch_bo, offset_dwords * 4); /* ADDR_LO/HI */
+ msm_cmd_emit(cmd, val); /* VAL */
+}
+
+/*
+ * Helper to wait on a fence-fd:
+ */
+static void
+wait_and_close(int fence_fd)
+{
+ poll(&(struct pollfd){fence_fd, POLLIN}, 1, -1);
+ close(fence_fd);
+}
+
+/*
+ * Helper for hang tests. Emits multiple submits, with one in the middle
+ * that triggers a fault, and confirms that the submits before and after
+ * the faulting one execute properly, ie. that the driver properly manages
+ * to recover and re-queue the submits after the faulting submit;
+ */
+static void
+do_hang_test(struct msm_pipe *pipe)
+{
+ struct msm_cmd *cmds[16];
+ int fence_fds[ARRAY_SIZE(cmds)];
+
+ memset(scratch, 0, 0x1000);
+
+ for (unsigned i = 0; i < ARRAY_SIZE(cmds); i++) {
+ struct msm_cmd *cmd = igt_msm_cmd_new(pipe, 0x1000);
+
+ cmds[i] = cmd;
+
+ /*
+ * Emit a packet to wait for scratch[0] to be >= 1
+ *
+ * This lets us force the GPU to wait until all the cmdstream is
+ * queued up.
+ */
+ wait_mem_gte(cmd, 0, 1);
+
+ if (i == 10) {
+ msm_cmd_emit(cmd, 0xdeaddead);
+ }
+
+ /* Emit a packet to write scratch[1+i] = 2+i: */
+ mem_write(cmd, 1+i, 2+i);
+ }
+
+ for (unsigned i = 0; i < ARRAY_SIZE(cmds); i++) {
+ fence_fds[i] = igt_msm_cmd_submit(cmds[i]);
+ }
+
+ usleep(10000);
+
+ /* Let the WAIT_MEM_GTE complete: */
+ scratch[0] = 1;
+
+ for (unsigned i = 0; i < ARRAY_SIZE(cmds); i++) {
+ wait_and_close(fence_fds[i]);
+ igt_msm_cmd_free(cmds[i]);
+ if (i == 10)
+ continue;
+ igt_assert_eq(scratch[1+i], 2+i);
+ }
+}
+
+/*
+ * Tests for drm/msm hangcheck, recovery, and fault handling
+ */
+
+igt_main
+{
+ static struct msm_pipe *pipe = NULL;
+
+ igt_fixture {
+ dev = igt_msm_dev_open();
+ pipe = igt_msm_pipe_open(dev, 0);
+ scratch_bo = igt_msm_bo_new(dev, 0x1000, MSM_BO_WC);
+ scratch = igt_msm_bo_map(scratch_bo);
+ }
+
+ igt_describe("Test sw hangcheck handling");
+ igt_subtest("hangcheck") {
+ igt_require(dev->gen >= 6);
+
+ /* Disable hw hang detection to force fallback to sw hangcheck: */
+ igt_debugfs_write(dev->fd, "disable_err_irq", "Y");
+
+ do_hang_test(pipe);
+
+ igt_debugfs_write(dev->fd, "disable_err_irq", "N");
+ }
+
+ igt_describe("Test hw fault handling");
+ igt_subtest("gpu-fault") {
+ igt_require(dev->gen >= 6);
+
+ do_hang_test(pipe);
+ }
+
+ igt_describe("Test iova fault handling");
+ igt_subtest("iova-fault") {
+ struct msm_cmd *cmd;
+
+ igt_require(dev->gen >= 6);
+
+ cmd = igt_msm_cmd_new(pipe, 0x1000);
+
+ msm_cmd_pkt7(cmd, CP_MEM_WRITE, 3);
+ msm_cmd_emit(cmd, 0xdeaddead); /* ADDR_LO */
+ msm_cmd_emit(cmd, 0x1); /* ADDR_HI */
+ msm_cmd_emit(cmd, 0x123); /* VAL */
+
+ wait_and_close(igt_msm_cmd_submit(cmd));
+ }
+
+ igt_fixture {
+ igt_msm_bo_free(scratch_bo);
+ igt_msm_pipe_close(pipe);
+ igt_msm_dev_close(dev);
+ }
+}
--
2.31.1
^ permalink raw reply related [flat|nested] 5+ messages in thread