* [PATCH] drm/amdgpu: Add autodump debugfs node for gpu reset (v2)
@ 2020-04-24 8:49 jianzh
2020-04-24 8:51 ` Zhao, Jiange
2020-04-24 8:58 ` Christian König
0 siblings, 2 replies; 4+ messages in thread
From: jianzh @ 2020-04-24 8:49 UTC (permalink / raw)
To: amd-gfx
Cc: Pierre-eric.Pelloux-prayer, Jiange Zhao, Felix.Kuehling,
Alexander.Deucher, Christian.Koenig, Monk.Liu, Hawking.Zhang
From: Jiange Zhao <Jiange.Zhao@amd.com>
When GPU got timeout, it would notify an interested part
of an opportunity to dump info before actual GPU reset.
A usermode app would open 'autodump' node under debugfs system
and poll() for readable/writable. When a GPU reset is due,
amdgpu would notify usermode app through wait_queue_head and give
it 10 minutes to dump info.
After usermode app has done its work, this 'autodump' node is closed.
On node closure, amdgpu gets to know the dump is done through
the completion that is triggered in release().
There is no write or read callback because necessary info can be
obtained through dmesg and umr. Messages back and forth between
usermode app and amdgpu are unnecessary.
v2: (1) changed 'registered' to 'app_listening'
(2) add a mutex in open() to prevent race condition
Signed-off-by: Jiange Zhao <Jiange.Zhao@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 10 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 92 ++++++++++++++++++++-
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +
4 files changed, 104 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index bc1e0fd71a09..34b8ce9fba47 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -724,6 +724,14 @@ struct amd_powerplay {
const struct amd_pm_funcs *pp_funcs;
};
+struct amdgpu_autodump {
+ bool app_listening;
+ struct completion completed;
+ struct dentry *dentry;
+ struct wait_queue_head gpu_hang_wait;
+ struct mutex mutex;
+};
+
#define AMDGPU_RESET_MAGIC_NUM 64
#define AMDGPU_MAX_DF_PERFMONS 4
struct amdgpu_device {
@@ -990,6 +998,8 @@ struct amdgpu_device {
char product_number[16];
char product_name[32];
char serial[16];
+
+ struct amdgpu_autodump autodump;
};
static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index 1a4894fa3693..693bfcaad312 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -27,7 +27,7 @@
#include <linux/pci.h>
#include <linux/uaccess.h>
#include <linux/pm_runtime.h>
-
+#include <linux/poll.h>
#include <drm/drm_debugfs.h>
#include "amdgpu.h"
@@ -74,8 +74,96 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
return 0;
}
+int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
+{
+#if defined(CONFIG_DEBUG_FS)
+ unsigned long tmo = 600*HZ;
+ int ret;
+
+ if (!adev->autodump.app_listening)
+ return 0;
+
+ wake_up_interruptible(&adev->autodump.gpu_hang_wait);
+
+ ret = wait_for_completion_interruptible_timeout(&adev->autodump.completed, tmo);
+ if (ret == 0) {
+ pr_err("autodump: timeout, move on to gpu recovery\n");
+ return -ETIMEDOUT;
+ }
+#endif
+ return 0;
+}
+
#if defined(CONFIG_DEBUG_FS)
+static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
+{
+ int ret = 0;
+ struct amdgpu_device *adev;
+
+ ret = simple_open(inode, file);
+ if (ret)
+ return ret;
+
+ adev = file->private_data;
+
+ mutex_lock(&adev->autodump.mutex);
+ if (adev->autodump.app_listening == true) {
+ ret = -EBUSY;
+ } else {
+ adev->autodump.app_listening = true;
+ }
+ mutex_unlock(&adev->autodump.mutex);
+
+ return ret;
+}
+
+static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
+{
+ struct amdgpu_device *adev = file->private_data;
+
+ complete(&adev->autodump.completed);
+ adev->autodump.app_listening = false;
+
+ return 0;
+}
+
+unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
+{
+ struct amdgpu_device *adev = file->private_data;
+
+ poll_wait(file, &adev->autodump.gpu_hang_wait, poll_table);
+
+ if (adev->in_gpu_reset)
+ return POLLIN | POLLRDNORM | POLLWRNORM;
+
+ return 0;
+}
+
+static const struct file_operations autodump_debug_fops = {
+ .owner = THIS_MODULE,
+ .open = amdgpu_debugfs_autodump_open,
+ .poll = amdgpu_debugfs_autodump_poll,
+ .release = amdgpu_debugfs_autodump_release,
+};
+
+static int amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
+{
+ struct dentry *entry;
+
+ init_completion(&adev->autodump.completed);
+ init_waitqueue_head(&adev->autodump.gpu_hang_wait);
+ mutex_init(&adev->autodump.mutex);
+ adev->autodump.app_listening = false;
+
+ entry = debugfs_create_file("autodump", 0600,
+ adev->ddev->primary->debugfs_root,
+ adev, &autodump_debug_fops);
+ adev->autodump.dentry = entry;
+
+ return 0;
+}
+
/**
* amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
*
@@ -1434,6 +1522,8 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)
amdgpu_ras_debugfs_create_all(adev);
+ amdgpu_debugfs_autodump_init(adev);
+
return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
ARRAY_SIZE(amdgpu_debugfs_list));
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
index de12d1101526..9428940a696d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
@@ -40,3 +40,4 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
int amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
int amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
+int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 3d601d5dd5af..44e54ea7af0f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3915,6 +3915,8 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
int i, r = 0;
bool need_full_reset = *need_full_reset_arg;
+ amdgpu_debugfs_wait_dump(adev);
+
/* block all schedulers and reset given job's ring */
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
struct amdgpu_ring *ring = adev->rings[i];
--
2.20.1
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH] drm/amdgpu: Add autodump debugfs node for gpu reset (v2)
2020-04-24 8:49 [PATCH] drm/amdgpu: Add autodump debugfs node for gpu reset (v2) jianzh
@ 2020-04-24 8:51 ` Zhao, Jiange
2020-04-24 8:58 ` Christian König
1 sibling, 0 replies; 4+ messages in thread
From: Zhao, Jiange @ 2020-04-24 8:51 UTC (permalink / raw)
To: amd-gfx
Cc: Pelloux-prayer, Pierre-eric, Kuehling, Felix, Deucher, Alexander,
Koenig, Christian, Liu, Monk, Zhang, Hawking
[-- Attachment #1.1: Type: text/plain, Size: 7864 bytes --]
[AMD Official Use Only - Internal Distribution Only]
Hi @Koenig, Christian<mailto:Christian.Koenig@amd.com>,
For the completion restore, I have tested the patch and it is working.
It seems that in release() function, complete() would restore the completion.
Jiange
________________________________
From: Zhao, Jiange <Jiange.Zhao@amd.com>
Sent: Friday, April 24, 2020 4:49 PM
To: amd-gfx@lists.freedesktop.org <amd-gfx@lists.freedesktop.org>
Cc: Koenig, Christian <Christian.Koenig@amd.com>; Kuehling, Felix <Felix.Kuehling@amd.com>; Pelloux-prayer, Pierre-eric <Pierre-eric.Pelloux-prayer@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Zhao, Jiange <Jiange.Zhao@amd.com>
Subject: [PATCH] drm/amdgpu: Add autodump debugfs node for gpu reset (v2)
From: Jiange Zhao <Jiange.Zhao@amd.com>
When GPU got timeout, it would notify an interested part
of an opportunity to dump info before actual GPU reset.
A usermode app would open 'autodump' node under debugfs system
and poll() for readable/writable. When a GPU reset is due,
amdgpu would notify usermode app through wait_queue_head and give
it 10 minutes to dump info.
After usermode app has done its work, this 'autodump' node is closed.
On node closure, amdgpu gets to know the dump is done through
the completion that is triggered in release().
There is no write or read callback because necessary info can be
obtained through dmesg and umr. Messages back and forth between
usermode app and amdgpu are unnecessary.
v2: (1) changed 'registered' to 'app_listening'
(2) add a mutex in open() to prevent race condition
Signed-off-by: Jiange Zhao <Jiange.Zhao@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 10 +++
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 92 ++++++++++++++++++++-
drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +
4 files changed, 104 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index bc1e0fd71a09..34b8ce9fba47 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -724,6 +724,14 @@ struct amd_powerplay {
const struct amd_pm_funcs *pp_funcs;
};
+struct amdgpu_autodump {
+ bool app_listening;
+ struct completion completed;
+ struct dentry *dentry;
+ struct wait_queue_head gpu_hang_wait;
+ struct mutex mutex;
+};
+
#define AMDGPU_RESET_MAGIC_NUM 64
#define AMDGPU_MAX_DF_PERFMONS 4
struct amdgpu_device {
@@ -990,6 +998,8 @@ struct amdgpu_device {
char product_number[16];
char product_name[32];
char serial[16];
+
+ struct amdgpu_autodump autodump;
};
static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index 1a4894fa3693..693bfcaad312 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -27,7 +27,7 @@
#include <linux/pci.h>
#include <linux/uaccess.h>
#include <linux/pm_runtime.h>
-
+#include <linux/poll.h>
#include <drm/drm_debugfs.h>
#include "amdgpu.h"
@@ -74,8 +74,96 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
return 0;
}
+int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
+{
+#if defined(CONFIG_DEBUG_FS)
+ unsigned long tmo = 600*HZ;
+ int ret;
+
+ if (!adev->autodump.app_listening)
+ return 0;
+
+ wake_up_interruptible(&adev->autodump.gpu_hang_wait);
+
+ ret = wait_for_completion_interruptible_timeout(&adev->autodump.completed, tmo);
+ if (ret == 0) {
+ pr_err("autodump: timeout, move on to gpu recovery\n");
+ return -ETIMEDOUT;
+ }
+#endif
+ return 0;
+}
+
#if defined(CONFIG_DEBUG_FS)
+static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
+{
+ int ret = 0;
+ struct amdgpu_device *adev;
+
+ ret = simple_open(inode, file);
+ if (ret)
+ return ret;
+
+ adev = file->private_data;
+
+ mutex_lock(&adev->autodump.mutex);
+ if (adev->autodump.app_listening == true) {
+ ret = -EBUSY;
+ } else {
+ adev->autodump.app_listening = true;
+ }
+ mutex_unlock(&adev->autodump.mutex);
+
+ return ret;
+}
+
+static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
+{
+ struct amdgpu_device *adev = file->private_data;
+
+ complete(&adev->autodump.completed);
+ adev->autodump.app_listening = false;
+
+ return 0;
+}
+
+unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
+{
+ struct amdgpu_device *adev = file->private_data;
+
+ poll_wait(file, &adev->autodump.gpu_hang_wait, poll_table);
+
+ if (adev->in_gpu_reset)
+ return POLLIN | POLLRDNORM | POLLWRNORM;
+
+ return 0;
+}
+
+static const struct file_operations autodump_debug_fops = {
+ .owner = THIS_MODULE,
+ .open = amdgpu_debugfs_autodump_open,
+ .poll = amdgpu_debugfs_autodump_poll,
+ .release = amdgpu_debugfs_autodump_release,
+};
+
+static int amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
+{
+ struct dentry *entry;
+
+ init_completion(&adev->autodump.completed);
+ init_waitqueue_head(&adev->autodump.gpu_hang_wait);
+ mutex_init(&adev->autodump.mutex);
+ adev->autodump.app_listening = false;
+
+ entry = debugfs_create_file("autodump", 0600,
+ adev->ddev->primary->debugfs_root,
+ adev, &autodump_debug_fops);
+ adev->autodump.dentry = entry;
+
+ return 0;
+}
+
/**
* amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
*
@@ -1434,6 +1522,8 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)
amdgpu_ras_debugfs_create_all(adev);
+ amdgpu_debugfs_autodump_init(adev);
+
return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
ARRAY_SIZE(amdgpu_debugfs_list));
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
index de12d1101526..9428940a696d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
@@ -40,3 +40,4 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
int amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
int amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
+int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 3d601d5dd5af..44e54ea7af0f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3915,6 +3915,8 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
int i, r = 0;
bool need_full_reset = *need_full_reset_arg;
+ amdgpu_debugfs_wait_dump(adev);
+
/* block all schedulers and reset given job's ring */
for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
struct amdgpu_ring *ring = adev->rings[i];
--
2.20.1
[-- Attachment #1.2: Type: text/html, Size: 15088 bytes --]
[-- Attachment #2: Type: text/plain, Size: 154 bytes --]
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH] drm/amdgpu: Add autodump debugfs node for gpu reset (v2)
2020-04-24 8:49 [PATCH] drm/amdgpu: Add autodump debugfs node for gpu reset (v2) jianzh
2020-04-24 8:51 ` Zhao, Jiange
@ 2020-04-24 8:58 ` Christian König
2020-04-24 8:59 ` Zhao, Jiange
1 sibling, 1 reply; 4+ messages in thread
From: Christian König @ 2020-04-24 8:58 UTC (permalink / raw)
To: jianzh, amd-gfx
Cc: Pierre-eric.Pelloux-prayer, Jiange Zhao, Felix.Kuehling,
Alexander.Deucher, Christian.Koenig, Monk.Liu, Hawking.Zhang
Am 24.04.20 um 10:49 schrieb jianzh@amd.com:
> From: Jiange Zhao <Jiange.Zhao@amd.com>
>
> When GPU got timeout, it would notify an interested part
> of an opportunity to dump info before actual GPU reset.
>
> A usermode app would open 'autodump' node under debugfs system
> and poll() for readable/writable. When a GPU reset is due,
> amdgpu would notify usermode app through wait_queue_head and give
> it 10 minutes to dump info.
>
> After usermode app has done its work, this 'autodump' node is closed.
> On node closure, amdgpu gets to know the dump is done through
> the completion that is triggered in release().
>
> There is no write or read callback because necessary info can be
> obtained through dmesg and umr. Messages back and forth between
> usermode app and amdgpu are unnecessary.
>
> v2: (1) changed 'registered' to 'app_listening'
> (2) add a mutex in open() to prevent race condition
May I pick this one up and modify it a bit?
There is still a race which needs to be fixed and it is a bit hard to
explain what's going wrong here.
Christian.
>
> Signed-off-by: Jiange Zhao <Jiange.Zhao@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 10 +++
> drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 92 ++++++++++++++++++++-
> drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h | 1 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +
> 4 files changed, 104 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index bc1e0fd71a09..34b8ce9fba47 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -724,6 +724,14 @@ struct amd_powerplay {
> const struct amd_pm_funcs *pp_funcs;
> };
>
> +struct amdgpu_autodump {
> + bool app_listening;
> + struct completion completed;
> + struct dentry *dentry;
> + struct wait_queue_head gpu_hang_wait;
> + struct mutex mutex;
> +};
> +
> #define AMDGPU_RESET_MAGIC_NUM 64
> #define AMDGPU_MAX_DF_PERFMONS 4
> struct amdgpu_device {
> @@ -990,6 +998,8 @@ struct amdgpu_device {
> char product_number[16];
> char product_name[32];
> char serial[16];
> +
> + struct amdgpu_autodump autodump;
> };
>
> static inline struct amdgpu_device *amdgpu_ttm_adev(struct ttm_bo_device *bdev)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
> index 1a4894fa3693..693bfcaad312 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
> @@ -27,7 +27,7 @@
> #include <linux/pci.h>
> #include <linux/uaccess.h>
> #include <linux/pm_runtime.h>
> -
> +#include <linux/poll.h>
> #include <drm/drm_debugfs.h>
>
> #include "amdgpu.h"
> @@ -74,8 +74,96 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
> return 0;
> }
>
> +int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
> +{
> +#if defined(CONFIG_DEBUG_FS)
> + unsigned long tmo = 600*HZ;
> + int ret;
> +
> + if (!adev->autodump.app_listening)
> + return 0;
> +
> + wake_up_interruptible(&adev->autodump.gpu_hang_wait);
> +
> + ret = wait_for_completion_interruptible_timeout(&adev->autodump.completed, tmo);
> + if (ret == 0) {
> + pr_err("autodump: timeout, move on to gpu recovery\n");
> + return -ETIMEDOUT;
> + }
> +#endif
> + return 0;
> +}
> +
> #if defined(CONFIG_DEBUG_FS)
>
> +static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
> +{
> + int ret = 0;
> + struct amdgpu_device *adev;
> +
> + ret = simple_open(inode, file);
> + if (ret)
> + return ret;
> +
> + adev = file->private_data;
> +
> + mutex_lock(&adev->autodump.mutex);
> + if (adev->autodump.app_listening == true) {
> + ret = -EBUSY;
> + } else {
> + adev->autodump.app_listening = true;
> + }
> + mutex_unlock(&adev->autodump.mutex);
> +
> + return ret;
> +}
> +
> +static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
> +{
> + struct amdgpu_device *adev = file->private_data;
> +
> + complete(&adev->autodump.completed);
> + adev->autodump.app_listening = false;
> +
> + return 0;
> +}
> +
> +unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
> +{
> + struct amdgpu_device *adev = file->private_data;
> +
> + poll_wait(file, &adev->autodump.gpu_hang_wait, poll_table);
> +
> + if (adev->in_gpu_reset)
> + return POLLIN | POLLRDNORM | POLLWRNORM;
> +
> + return 0;
> +}
> +
> +static const struct file_operations autodump_debug_fops = {
> + .owner = THIS_MODULE,
> + .open = amdgpu_debugfs_autodump_open,
> + .poll = amdgpu_debugfs_autodump_poll,
> + .release = amdgpu_debugfs_autodump_release,
> +};
> +
> +static int amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
> +{
> + struct dentry *entry;
> +
> + init_completion(&adev->autodump.completed);
> + init_waitqueue_head(&adev->autodump.gpu_hang_wait);
> + mutex_init(&adev->autodump.mutex);
> + adev->autodump.app_listening = false;
> +
> + entry = debugfs_create_file("autodump", 0600,
> + adev->ddev->primary->debugfs_root,
> + adev, &autodump_debug_fops);
> + adev->autodump.dentry = entry;
> +
> + return 0;
> +}
> +
> /**
> * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
> *
> @@ -1434,6 +1522,8 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)
>
> amdgpu_ras_debugfs_create_all(adev);
>
> + amdgpu_debugfs_autodump_init(adev);
> +
> return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
> ARRAY_SIZE(amdgpu_debugfs_list));
> }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
> index de12d1101526..9428940a696d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
> @@ -40,3 +40,4 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
> int amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
> int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
> int amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
> +int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 3d601d5dd5af..44e54ea7af0f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -3915,6 +3915,8 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
> int i, r = 0;
> bool need_full_reset = *need_full_reset_arg;
>
> + amdgpu_debugfs_wait_dump(adev);
> +
> /* block all schedulers and reset given job's ring */
> for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
> struct amdgpu_ring *ring = adev->rings[i];
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 4+ messages in thread
* RE: [PATCH] drm/amdgpu: Add autodump debugfs node for gpu reset (v2)
2020-04-24 8:58 ` Christian König
@ 2020-04-24 8:59 ` Zhao, Jiange
0 siblings, 0 replies; 4+ messages in thread
From: Zhao, Jiange @ 2020-04-24 8:59 UTC (permalink / raw)
To: Koenig, Christian, amd-gfx
Cc: Deucher, Alexander, Pelloux-prayer, Pierre-eric, Kuehling, Felix,
Liu, Monk, Zhang, Hawking
[AMD Official Use Only - Internal Distribution Only]
Sure, go ahead.
-----Original Message-----
From: Christian König <ckoenig.leichtzumerken@gmail.com>
Sent: Friday, April 24, 2020 4:58 PM
To: Zhao, Jiange <Jiange.Zhao@amd.com>; amd-gfx@lists.freedesktop.org
Cc: Pelloux-prayer, Pierre-eric <Pierre-eric.Pelloux-prayer@amd.com>; Zhao, Jiange <Jiange.Zhao@amd.com>; Kuehling, Felix <Felix.Kuehling@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Koenig, Christian <Christian.Koenig@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>
Subject: Re: [PATCH] drm/amdgpu: Add autodump debugfs node for gpu reset (v2)
Am 24.04.20 um 10:49 schrieb jianzh@amd.com:
> From: Jiange Zhao <Jiange.Zhao@amd.com>
>
> When GPU got timeout, it would notify an interested part of an
> opportunity to dump info before actual GPU reset.
>
> A usermode app would open 'autodump' node under debugfs system and
> poll() for readable/writable. When a GPU reset is due, amdgpu would
> notify usermode app through wait_queue_head and give it 10 minutes to
> dump info.
>
> After usermode app has done its work, this 'autodump' node is closed.
> On node closure, amdgpu gets to know the dump is done through the
> completion that is triggered in release().
>
> There is no write or read callback because necessary info can be
> obtained through dmesg and umr. Messages back and forth between
> usermode app and amdgpu are unnecessary.
>
> v2: (1) changed 'registered' to 'app_listening'
> (2) add a mutex in open() to prevent race condition
May I pick this one up and modify it a bit?
There is still a race which needs to be fixed and it is a bit hard to explain what's going wrong here.
Christian.
>
> Signed-off-by: Jiange Zhao <Jiange.Zhao@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 10 +++
> drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 92 ++++++++++++++++++++-
> drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h | 1 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +
> 4 files changed, 104 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index bc1e0fd71a09..34b8ce9fba47 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -724,6 +724,14 @@ struct amd_powerplay {
> const struct amd_pm_funcs *pp_funcs;
> };
>
> +struct amdgpu_autodump {
> + bool app_listening;
> + struct completion completed;
> + struct dentry *dentry;
> + struct wait_queue_head gpu_hang_wait;
> + struct mutex mutex;
> +};
> +
> #define AMDGPU_RESET_MAGIC_NUM 64
> #define AMDGPU_MAX_DF_PERFMONS 4
> struct amdgpu_device {
> @@ -990,6 +998,8 @@ struct amdgpu_device {
> char product_number[16];
> char product_name[32];
> char serial[16];
> +
> + struct amdgpu_autodump autodump;
> };
>
> static inline struct amdgpu_device *amdgpu_ttm_adev(struct
> ttm_bo_device *bdev) diff --git
> a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
> index 1a4894fa3693..693bfcaad312 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
> @@ -27,7 +27,7 @@
> #include <linux/pci.h>
> #include <linux/uaccess.h>
> #include <linux/pm_runtime.h>
> -
> +#include <linux/poll.h>
> #include <drm/drm_debugfs.h>
>
> #include "amdgpu.h"
> @@ -74,8 +74,96 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
> return 0;
> }
>
> +int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev) { #if
> +defined(CONFIG_DEBUG_FS)
> + unsigned long tmo = 600*HZ;
> + int ret;
> +
> + if (!adev->autodump.app_listening)
> + return 0;
> +
> + wake_up_interruptible(&adev->autodump.gpu_hang_wait);
> +
> + ret = wait_for_completion_interruptible_timeout(&adev->autodump.completed, tmo);
> + if (ret == 0) {
> + pr_err("autodump: timeout, move on to gpu recovery\n");
> + return -ETIMEDOUT;
> + }
> +#endif
> + return 0;
> +}
> +
> #if defined(CONFIG_DEBUG_FS)
>
> +static int amdgpu_debugfs_autodump_open(struct inode *inode, struct
> +file *file) {
> + int ret = 0;
> + struct amdgpu_device *adev;
> +
> + ret = simple_open(inode, file);
> + if (ret)
> + return ret;
> +
> + adev = file->private_data;
> +
> + mutex_lock(&adev->autodump.mutex);
> + if (adev->autodump.app_listening == true) {
> + ret = -EBUSY;
> + } else {
> + adev->autodump.app_listening = true;
> + }
> + mutex_unlock(&adev->autodump.mutex);
> +
> + return ret;
> +}
> +
> +static int amdgpu_debugfs_autodump_release(struct inode *inode,
> +struct file *file) {
> + struct amdgpu_device *adev = file->private_data;
> +
> + complete(&adev->autodump.completed);
> + adev->autodump.app_listening = false;
> +
> + return 0;
> +}
> +
> +unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct
> +poll_table_struct *poll_table) {
> + struct amdgpu_device *adev = file->private_data;
> +
> + poll_wait(file, &adev->autodump.gpu_hang_wait, poll_table);
> +
> + if (adev->in_gpu_reset)
> + return POLLIN | POLLRDNORM | POLLWRNORM;
> +
> + return 0;
> +}
> +
> +static const struct file_operations autodump_debug_fops = {
> + .owner = THIS_MODULE,
> + .open = amdgpu_debugfs_autodump_open,
> + .poll = amdgpu_debugfs_autodump_poll,
> + .release = amdgpu_debugfs_autodump_release, };
> +
> +static int amdgpu_debugfs_autodump_init(struct amdgpu_device *adev) {
> + struct dentry *entry;
> +
> + init_completion(&adev->autodump.completed);
> + init_waitqueue_head(&adev->autodump.gpu_hang_wait);
> + mutex_init(&adev->autodump.mutex);
> + adev->autodump.app_listening = false;
> +
> + entry = debugfs_create_file("autodump", 0600,
> + adev->ddev->primary->debugfs_root,
> + adev, &autodump_debug_fops);
> + adev->autodump.dentry = entry;
> +
> + return 0;
> +}
> +
> /**
> * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
> *
> @@ -1434,6 +1522,8 @@ int amdgpu_debugfs_init(struct amdgpu_device
> *adev)
>
> amdgpu_ras_debugfs_create_all(adev);
>
> + amdgpu_debugfs_autodump_init(adev);
> +
> return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
> ARRAY_SIZE(amdgpu_debugfs_list));
> }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
> index de12d1101526..9428940a696d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.h
> @@ -40,3 +40,4 @@ int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
> int amdgpu_debugfs_fence_init(struct amdgpu_device *adev);
> int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
> int amdgpu_debugfs_gem_init(struct amdgpu_device *adev);
> +int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 3d601d5dd5af..44e54ea7af0f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -3915,6 +3915,8 @@ static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
> int i, r = 0;
> bool need_full_reset = *need_full_reset_arg;
>
> + amdgpu_debugfs_wait_dump(adev);
> +
> /* block all schedulers and reset given job's ring */
> for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
> struct amdgpu_ring *ring = adev->rings[i];
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2020-04-24 8:59 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-04-24 8:49 [PATCH] drm/amdgpu: Add autodump debugfs node for gpu reset (v2) jianzh
2020-04-24 8:51 ` Zhao, Jiange
2020-04-24 8:58 ` Christian König
2020-04-24 8:59 ` Zhao, Jiange
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.