* [RFC] drm/msm/a6xx: Serialize GMU communication
@ 2021-09-27 18:03 Rob Clark
2021-10-01 17:39 ` Dmitry Baryshkov
0 siblings, 1 reply; 4+ messages in thread
From: Rob Clark @ 2021-09-27 18:03 UTC (permalink / raw)
To: dri-devel
Cc: linux-arm-msm, freedreno, Jordan Crouse, Rob Clark, Rob Clark,
Sean Paul, David Airlie, Daniel Vetter, Jonathan Marek,
Eric Anholt, Akhil P Oommen, Viresh Kumar, Dmitry Baryshkov,
Sharat Masetty, Douglas Anderson, open list
From: Rob Clark <robdclark@chromium.org>
I've seen some crashes in our crash reporting that *look* like multiple
threads stomping on each other while communicating with GMU. So wrap
all those paths in a lock.
Signed-off-by: Rob Clark <robdclark@chromium.org>
---
Are we allowed to use c99/gnu99 yet?
drivers/gpu/drm/msm/Makefile | 2 +-
drivers/gpu/drm/msm/adreno/a6xx_gmu.c | 6 ++++
drivers/gpu/drm/msm/adreno/a6xx_gmu.h | 9 +++++
drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 50 ++++++++++++++++++++-------
4 files changed, 54 insertions(+), 13 deletions(-)
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 904535eda0c4..57283bbad3f0 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
-ccflags-y := -I $(srctree)/$(src)
+ccflags-y := -I $(srctree)/$(src) -std=gnu99
ccflags-y += -I $(srctree)/$(src)/disp/dpu1
ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(srctree)/$(src)/dsi
ccflags-$(CONFIG_DRM_MSM_DP) += -I $(srctree)/$(src)/dp
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
index a7c58018959f..8b73f70766a4 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -296,6 +296,8 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
u32 val;
int request, ack;
+ WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
+
if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
return -EINVAL;
@@ -337,6 +339,8 @@ void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
{
int bit;
+ WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
+
if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
return;
@@ -1482,6 +1486,8 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
if (!pdev)
return -ENODEV;
+ mutex_init(&gmu->lock);
+
gmu->dev = &pdev->dev;
of_dma_configure(gmu->dev, node, true);
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
index 3c74f64e3126..f05a00c0afd0 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
@@ -44,6 +44,9 @@ struct a6xx_gmu_bo {
struct a6xx_gmu {
struct device *dev;
+ /* For serializing communication with the GMU: */
+ struct mutex lock;
+
struct msm_gem_address_space *aspace;
void * __iomem mmio;
@@ -88,6 +91,12 @@ struct a6xx_gmu {
bool legacy; /* a618 or a630 */
};
+/* Helper macro for serializing GMU access: */
+#define with_gmu_lock(gmu) \
+ for (bool done = ({ mutex_lock(&(gmu)->lock); false; }); \
+ !done; \
+ done = ({ mutex_unlock(&(gmu)->lock); true; }))
+
static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
{
return msm_readl(gmu->mmio + (offset << 2));
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index f6a4dbef796b..5e1ae3df42ba 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -881,7 +881,7 @@ static int a6xx_zap_shader_init(struct msm_gpu *gpu)
A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
-static int a6xx_hw_init(struct msm_gpu *gpu)
+static int hw_init(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
@@ -1135,6 +1135,19 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
return ret;
}
+static int a6xx_hw_init(struct msm_gpu *gpu)
+{
+ struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+ struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+ int ret;
+
+ with_gmu_lock(&a6xx_gpu->gmu) {
+ ret = hw_init(gpu);
+ }
+
+ return ret;
+}
+
static void a6xx_dump(struct msm_gpu *gpu)
{
DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n",
@@ -1509,7 +1522,9 @@ static int a6xx_pm_resume(struct msm_gpu *gpu)
trace_msm_gpu_resume(0);
- ret = a6xx_gmu_resume(a6xx_gpu);
+ with_gmu_lock(&a6xx_gpu->gmu) {
+ ret = a6xx_gmu_resume(a6xx_gpu);
+ }
if (ret)
return ret;
@@ -1532,7 +1547,9 @@ static int a6xx_pm_suspend(struct msm_gpu *gpu)
msm_devfreq_suspend(gpu);
- ret = a6xx_gmu_stop(a6xx_gpu);
+ with_gmu_lock(&a6xx_gpu->gmu) {
+ ret = a6xx_gmu_stop(a6xx_gpu);
+ }
if (ret)
return ret;
@@ -1547,18 +1564,17 @@ static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
- static DEFINE_MUTEX(perfcounter_oob);
- mutex_lock(&perfcounter_oob);
+ with_gmu_lock(&a6xx_gpu->gmu) {
+ /* Force the GPU power on so we can read this register */
+ a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
- /* Force the GPU power on so we can read this register */
- a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
+ *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
+ REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
- *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
- REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
+ a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
+ }
- a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
- mutex_unlock(&perfcounter_oob);
return 0;
}
@@ -1622,6 +1638,16 @@ static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu)
return (unsigned long)busy_time;
}
+void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
+{
+ struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+ struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+
+ with_gmu_lock(&a6xx_gpu->gmu) {
+ a6xx_gmu_set_freq(gpu, opp);
+ }
+}
+
static struct msm_gem_address_space *
a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
{
@@ -1766,7 +1792,7 @@ static const struct adreno_gpu_funcs funcs = {
#endif
.gpu_busy = a6xx_gpu_busy,
.gpu_get_freq = a6xx_gmu_get_freq,
- .gpu_set_freq = a6xx_gmu_set_freq,
+ .gpu_set_freq = a6xx_gpu_set_freq,
#if defined(CONFIG_DRM_MSM_GPU_STATE)
.gpu_state_get = a6xx_gpu_state_get,
.gpu_state_put = a6xx_gpu_state_put,
--
2.31.1
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [RFC] drm/msm/a6xx: Serialize GMU communication
2021-09-27 18:03 [RFC] drm/msm/a6xx: Serialize GMU communication Rob Clark
@ 2021-10-01 17:39 ` Dmitry Baryshkov
2021-10-01 18:05 ` Rob Clark
0 siblings, 1 reply; 4+ messages in thread
From: Dmitry Baryshkov @ 2021-10-01 17:39 UTC (permalink / raw)
To: Rob Clark, dri-devel
Cc: linux-arm-msm, freedreno, Jordan Crouse, Rob Clark, Sean Paul,
David Airlie, Daniel Vetter, Jonathan Marek, Eric Anholt,
Akhil P Oommen, Viresh Kumar, Sharat Masetty, Douglas Anderson,
open list
On 27/09/2021 21:03, Rob Clark wrote:
> From: Rob Clark <robdclark@chromium.org>
>
> I've seen some crashes in our crash reporting that *look* like multiple
> threads stomping on each other while communicating with GMU. So wrap
> all those paths in a lock.
>
> Signed-off-by: Rob Clark <robdclark@chromium.org>
> ---
> Are we allowed to use c99/gnu99 yet?
>
> drivers/gpu/drm/msm/Makefile | 2 +-
> drivers/gpu/drm/msm/adreno/a6xx_gmu.c | 6 ++++
> drivers/gpu/drm/msm/adreno/a6xx_gmu.h | 9 +++++
> drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 50 ++++++++++++++++++++-------
> 4 files changed, 54 insertions(+), 13 deletions(-)
>
> diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
> index 904535eda0c4..57283bbad3f0 100644
> --- a/drivers/gpu/drm/msm/Makefile
> +++ b/drivers/gpu/drm/msm/Makefile
> @@ -1,5 +1,5 @@
> # SPDX-License-Identifier: GPL-2.0
> -ccflags-y := -I $(srctree)/$(src)
> +ccflags-y := -I $(srctree)/$(src) -std=gnu99
> ccflags-y += -I $(srctree)/$(src)/disp/dpu1
> ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(srctree)/$(src)/dsi
> ccflags-$(CONFIG_DRM_MSM_DP) += -I $(srctree)/$(src)/dp
> diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> index a7c58018959f..8b73f70766a4 100644
> --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> @@ -296,6 +296,8 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
> u32 val;
> int request, ack;
>
> + WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
> +
> if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
> return -EINVAL;
>
> @@ -337,6 +339,8 @@ void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
> {
> int bit;
>
> + WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
> +
> if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
> return;
>
> @@ -1482,6 +1486,8 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
> if (!pdev)
> return -ENODEV;
>
> + mutex_init(&gmu->lock);
> +
> gmu->dev = &pdev->dev;
>
> of_dma_configure(gmu->dev, node, true);
> diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
> index 3c74f64e3126..f05a00c0afd0 100644
> --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
> +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
> @@ -44,6 +44,9 @@ struct a6xx_gmu_bo {
> struct a6xx_gmu {
> struct device *dev;
>
> + /* For serializing communication with the GMU: */
> + struct mutex lock;
> +
> struct msm_gem_address_space *aspace;
>
> void * __iomem mmio;
> @@ -88,6 +91,12 @@ struct a6xx_gmu {
> bool legacy; /* a618 or a630 */
> };
>
> +/* Helper macro for serializing GMU access: */
> +#define with_gmu_lock(gmu) \
> + for (bool done = ({ mutex_lock(&(gmu)->lock); false; }); \
> + !done; \
> + done = ({ mutex_unlock(&(gmu)->lock); true; }))
The intent is good, but I'm not sure this kind of syntax sugar would be
a good approach. What about calling lock/unlock explicitly, like we
typically do? Then we won't have to use c99.
> +
> static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
> {
> return msm_readl(gmu->mmio + (offset << 2));
> diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> index f6a4dbef796b..5e1ae3df42ba 100644
> --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> @@ -881,7 +881,7 @@ static int a6xx_zap_shader_init(struct msm_gpu *gpu)
> A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
> A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
>
> -static int a6xx_hw_init(struct msm_gpu *gpu)
> +static int hw_init(struct msm_gpu *gpu)
> {
> struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
> struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
> @@ -1135,6 +1135,19 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
> return ret;
> }
>
> +static int a6xx_hw_init(struct msm_gpu *gpu)
> +{
> + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
> + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
> + int ret;
> +
> + with_gmu_lock(&a6xx_gpu->gmu) {
> + ret = hw_init(gpu);
> + }
> +
> + return ret;
> +}
> +
> static void a6xx_dump(struct msm_gpu *gpu)
> {
> DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n",
> @@ -1509,7 +1522,9 @@ static int a6xx_pm_resume(struct msm_gpu *gpu)
>
> trace_msm_gpu_resume(0);
>
> - ret = a6xx_gmu_resume(a6xx_gpu);
> + with_gmu_lock(&a6xx_gpu->gmu) {
> + ret = a6xx_gmu_resume(a6xx_gpu);
> + }
> if (ret)
> return ret;
>
> @@ -1532,7 +1547,9 @@ static int a6xx_pm_suspend(struct msm_gpu *gpu)
>
> msm_devfreq_suspend(gpu);
>
> - ret = a6xx_gmu_stop(a6xx_gpu);
> + with_gmu_lock(&a6xx_gpu->gmu) {
> + ret = a6xx_gmu_stop(a6xx_gpu);
> + }
> if (ret)
> return ret;
>
> @@ -1547,18 +1564,17 @@ static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
> {
> struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
> struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
> - static DEFINE_MUTEX(perfcounter_oob);
>
> - mutex_lock(&perfcounter_oob);
> + with_gmu_lock(&a6xx_gpu->gmu) {
> + /* Force the GPU power on so we can read this register */
> + a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
>
> - /* Force the GPU power on so we can read this register */
> - a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
> + *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
> + REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
>
> - *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
> - REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
> + a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
> + }
>
> - a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
> - mutex_unlock(&perfcounter_oob);
> return 0;
> }
>
> @@ -1622,6 +1638,16 @@ static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu)
> return (unsigned long)busy_time;
> }
>
> +void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
> +{
> + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
> + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
> +
> + with_gmu_lock(&a6xx_gpu->gmu) {
> + a6xx_gmu_set_freq(gpu, opp);
> + }
> +}
> +
> static struct msm_gem_address_space *
> a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
> {
> @@ -1766,7 +1792,7 @@ static const struct adreno_gpu_funcs funcs = {
> #endif
> .gpu_busy = a6xx_gpu_busy,
> .gpu_get_freq = a6xx_gmu_get_freq,
> - .gpu_set_freq = a6xx_gmu_set_freq,
> + .gpu_set_freq = a6xx_gpu_set_freq,
> #if defined(CONFIG_DRM_MSM_GPU_STATE)
> .gpu_state_get = a6xx_gpu_state_get,
> .gpu_state_put = a6xx_gpu_state_put,
>
--
With best wishes
Dmitry
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [RFC] drm/msm/a6xx: Serialize GMU communication
2021-10-01 18:05 ` Rob Clark
@ 2021-10-01 18:02 ` Dmitry Baryshkov
0 siblings, 0 replies; 4+ messages in thread
From: Dmitry Baryshkov @ 2021-10-01 18:02 UTC (permalink / raw)
To: Rob Clark
Cc: dri-devel, linux-arm-msm, freedreno, Jordan Crouse, Rob Clark,
Sean Paul, David Airlie, Daniel Vetter, Jonathan Marek,
Eric Anholt, Akhil P Oommen, Viresh Kumar, Sharat Masetty,
Douglas Anderson, open list
On 01/10/2021 21:05, Rob Clark wrote:
> On Fri, Oct 1, 2021 at 10:39 AM Dmitry Baryshkov
> <dmitry.baryshkov@linaro.org> wrote:
>>
>> On 27/09/2021 21:03, Rob Clark wrote:
>>> From: Rob Clark <robdclark@chromium.org>
>>>
>>> I've seen some crashes in our crash reporting that *look* like multiple
>>> threads stomping on each other while communicating with GMU. So wrap
>>> all those paths in a lock.
>>>
>>> Signed-off-by: Rob Clark <robdclark@chromium.org>
>>> ---
>>> Are we allowed to use c99/gnu99 yet?
>>>
>>> drivers/gpu/drm/msm/Makefile | 2 +-
>>> drivers/gpu/drm/msm/adreno/a6xx_gmu.c | 6 ++++
>>> drivers/gpu/drm/msm/adreno/a6xx_gmu.h | 9 +++++
>>> drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 50 ++++++++++++++++++++-------
>>> 4 files changed, 54 insertions(+), 13 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
>>> index 904535eda0c4..57283bbad3f0 100644
>>> --- a/drivers/gpu/drm/msm/Makefile
>>> +++ b/drivers/gpu/drm/msm/Makefile
>>> @@ -1,5 +1,5 @@
>>> # SPDX-License-Identifier: GPL-2.0
>>> -ccflags-y := -I $(srctree)/$(src)
>>> +ccflags-y := -I $(srctree)/$(src) -std=gnu99
>>> ccflags-y += -I $(srctree)/$(src)/disp/dpu1
>>> ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(srctree)/$(src)/dsi
>>> ccflags-$(CONFIG_DRM_MSM_DP) += -I $(srctree)/$(src)/dp
>>> diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
>>> index a7c58018959f..8b73f70766a4 100644
>>> --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
>>> +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
>>> @@ -296,6 +296,8 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
>>> u32 val;
>>> int request, ack;
>>>
>>> + WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
>>> +
>>> if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
>>> return -EINVAL;
>>>
>>> @@ -337,6 +339,8 @@ void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
>>> {
>>> int bit;
>>>
>>> + WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
>>> +
>>> if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
>>> return;
>>>
>>> @@ -1482,6 +1486,8 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
>>> if (!pdev)
>>> return -ENODEV;
>>>
>>> + mutex_init(&gmu->lock);
>>> +
>>> gmu->dev = &pdev->dev;
>>>
>>> of_dma_configure(gmu->dev, node, true);
>>> diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
>>> index 3c74f64e3126..f05a00c0afd0 100644
>>> --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
>>> +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
>>> @@ -44,6 +44,9 @@ struct a6xx_gmu_bo {
>>> struct a6xx_gmu {
>>> struct device *dev;
>>>
>>> + /* For serializing communication with the GMU: */
>>> + struct mutex lock;
>>> +
>>> struct msm_gem_address_space *aspace;
>>>
>>> void * __iomem mmio;
>>> @@ -88,6 +91,12 @@ struct a6xx_gmu {
>>> bool legacy; /* a618 or a630 */
>>> };
>>>
>>> +/* Helper macro for serializing GMU access: */
>>> +#define with_gmu_lock(gmu) \
>>> + for (bool done = ({ mutex_lock(&(gmu)->lock); false; }); \
>>> + !done; \
>>> + done = ({ mutex_unlock(&(gmu)->lock); true; }))
>>
>> The intent is good, but I'm not sure this kind of syntax sugar would be
>> a good approach. What about calling lock/unlock explicitly, like we
>> typically do? Then we won't have to use c99.
>
> Yeah, I was planning to resend without the sugar.. but it was a good
> excuse to bring up c99. Ie. I want c99 regardless ;-)
>
> (The sugar was useful initially before I'd sorted thru all the code
> paths and settled on using a mutex vs spinlock)
We can always have GMU_LOCK/GMU_UNLOCK macros.
>
> BR,
> -R
>
>>> +
>>> static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
>>> {
>>> return msm_readl(gmu->mmio + (offset << 2));
>>> diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
>>> index f6a4dbef796b..5e1ae3df42ba 100644
>>> --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
>>> +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
>>> @@ -881,7 +881,7 @@ static int a6xx_zap_shader_init(struct msm_gpu *gpu)
>>> A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
>>> A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
>>>
>>> -static int a6xx_hw_init(struct msm_gpu *gpu)
>>> +static int hw_init(struct msm_gpu *gpu)
>>> {
>>> struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
>>> struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
>>> @@ -1135,6 +1135,19 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
>>> return ret;
>>> }
>>>
>>> +static int a6xx_hw_init(struct msm_gpu *gpu)
>>> +{
>>> + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
>>> + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
>>> + int ret;
>>> +
>>> + with_gmu_lock(&a6xx_gpu->gmu) {
>>> + ret = hw_init(gpu);
>>> + }
>>> +
>>> + return ret;
>>> +}
>>> +
>>> static void a6xx_dump(struct msm_gpu *gpu)
>>> {
>>> DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n",
>>> @@ -1509,7 +1522,9 @@ static int a6xx_pm_resume(struct msm_gpu *gpu)
>>>
>>> trace_msm_gpu_resume(0);
>>>
>>> - ret = a6xx_gmu_resume(a6xx_gpu);
>>> + with_gmu_lock(&a6xx_gpu->gmu) {
>>> + ret = a6xx_gmu_resume(a6xx_gpu);
>>> + }
>>> if (ret)
>>> return ret;
>>>
>>> @@ -1532,7 +1547,9 @@ static int a6xx_pm_suspend(struct msm_gpu *gpu)
>>>
>>> msm_devfreq_suspend(gpu);
>>>
>>> - ret = a6xx_gmu_stop(a6xx_gpu);
>>> + with_gmu_lock(&a6xx_gpu->gmu) {
>>> + ret = a6xx_gmu_stop(a6xx_gpu);
>>> + }
>>> if (ret)
>>> return ret;
>>>
>>> @@ -1547,18 +1564,17 @@ static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
>>> {
>>> struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
>>> struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
>>> - static DEFINE_MUTEX(perfcounter_oob);
>>>
>>> - mutex_lock(&perfcounter_oob);
>>> + with_gmu_lock(&a6xx_gpu->gmu) {
>>> + /* Force the GPU power on so we can read this register */
>>> + a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
>>>
>>> - /* Force the GPU power on so we can read this register */
>>> - a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
>>> + *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
>>> + REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
>>>
>>> - *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
>>> - REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
>>> + a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
>>> + }
>>>
>>> - a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
>>> - mutex_unlock(&perfcounter_oob);
>>> return 0;
>>> }
>>>
>>> @@ -1622,6 +1638,16 @@ static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu)
>>> return (unsigned long)busy_time;
>>> }
>>>
>>> +void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
>>> +{
>>> + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
>>> + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
>>> +
>>> + with_gmu_lock(&a6xx_gpu->gmu) {
>>> + a6xx_gmu_set_freq(gpu, opp);
>>> + }
>>> +}
>>> +
>>> static struct msm_gem_address_space *
>>> a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
>>> {
>>> @@ -1766,7 +1792,7 @@ static const struct adreno_gpu_funcs funcs = {
>>> #endif
>>> .gpu_busy = a6xx_gpu_busy,
>>> .gpu_get_freq = a6xx_gmu_get_freq,
>>> - .gpu_set_freq = a6xx_gmu_set_freq,
>>> + .gpu_set_freq = a6xx_gpu_set_freq,
>>> #if defined(CONFIG_DRM_MSM_GPU_STATE)
>>> .gpu_state_get = a6xx_gpu_state_get,
>>> .gpu_state_put = a6xx_gpu_state_put,
>>>
>>
>>
>> --
>> With best wishes
>> Dmitry
--
With best wishes
Dmitry
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [RFC] drm/msm/a6xx: Serialize GMU communication
2021-10-01 17:39 ` Dmitry Baryshkov
@ 2021-10-01 18:05 ` Rob Clark
2021-10-01 18:02 ` Dmitry Baryshkov
0 siblings, 1 reply; 4+ messages in thread
From: Rob Clark @ 2021-10-01 18:05 UTC (permalink / raw)
To: Dmitry Baryshkov
Cc: dri-devel, linux-arm-msm, freedreno, Jordan Crouse, Rob Clark,
Sean Paul, David Airlie, Daniel Vetter, Jonathan Marek,
Eric Anholt, Akhil P Oommen, Viresh Kumar, Sharat Masetty,
Douglas Anderson, open list
On Fri, Oct 1, 2021 at 10:39 AM Dmitry Baryshkov
<dmitry.baryshkov@linaro.org> wrote:
>
> On 27/09/2021 21:03, Rob Clark wrote:
> > From: Rob Clark <robdclark@chromium.org>
> >
> > I've seen some crashes in our crash reporting that *look* like multiple
> > threads stomping on each other while communicating with GMU. So wrap
> > all those paths in a lock.
> >
> > Signed-off-by: Rob Clark <robdclark@chromium.org>
> > ---
> > Are we allowed to use c99/gnu99 yet?
> >
> > drivers/gpu/drm/msm/Makefile | 2 +-
> > drivers/gpu/drm/msm/adreno/a6xx_gmu.c | 6 ++++
> > drivers/gpu/drm/msm/adreno/a6xx_gmu.h | 9 +++++
> > drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 50 ++++++++++++++++++++-------
> > 4 files changed, 54 insertions(+), 13 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
> > index 904535eda0c4..57283bbad3f0 100644
> > --- a/drivers/gpu/drm/msm/Makefile
> > +++ b/drivers/gpu/drm/msm/Makefile
> > @@ -1,5 +1,5 @@
> > # SPDX-License-Identifier: GPL-2.0
> > -ccflags-y := -I $(srctree)/$(src)
> > +ccflags-y := -I $(srctree)/$(src) -std=gnu99
> > ccflags-y += -I $(srctree)/$(src)/disp/dpu1
> > ccflags-$(CONFIG_DRM_MSM_DSI) += -I $(srctree)/$(src)/dsi
> > ccflags-$(CONFIG_DRM_MSM_DP) += -I $(srctree)/$(src)/dp
> > diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> > index a7c58018959f..8b73f70766a4 100644
> > --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> > +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> > @@ -296,6 +296,8 @@ int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
> > u32 val;
> > int request, ack;
> >
> > + WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
> > +
> > if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
> > return -EINVAL;
> >
> > @@ -337,6 +339,8 @@ void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
> > {
> > int bit;
> >
> > + WARN_ON_ONCE(!mutex_is_locked(&gmu->lock));
> > +
> > if (state >= ARRAY_SIZE(a6xx_gmu_oob_bits))
> > return;
> >
> > @@ -1482,6 +1486,8 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node)
> > if (!pdev)
> > return -ENODEV;
> >
> > + mutex_init(&gmu->lock);
> > +
> > gmu->dev = &pdev->dev;
> >
> > of_dma_configure(gmu->dev, node, true);
> > diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
> > index 3c74f64e3126..f05a00c0afd0 100644
> > --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
> > +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h
> > @@ -44,6 +44,9 @@ struct a6xx_gmu_bo {
> > struct a6xx_gmu {
> > struct device *dev;
> >
> > + /* For serializing communication with the GMU: */
> > + struct mutex lock;
> > +
> > struct msm_gem_address_space *aspace;
> >
> > void * __iomem mmio;
> > @@ -88,6 +91,12 @@ struct a6xx_gmu {
> > bool legacy; /* a618 or a630 */
> > };
> >
> > +/* Helper macro for serializing GMU access: */
> > +#define with_gmu_lock(gmu) \
> > + for (bool done = ({ mutex_lock(&(gmu)->lock); false; }); \
> > + !done; \
> > + done = ({ mutex_unlock(&(gmu)->lock); true; }))
>
> The intent is good, but I'm not sure this kind of syntax sugar would be
> a good approach. What about calling lock/unlock explicitly, like we
> typically do? Then we won't have to use c99.
Yeah, I was planning to resend without the sugar.. but it was a good
excuse to bring up c99. Ie. I want c99 regardless ;-)
(The sugar was useful initially before I'd sorted thru all the code
paths and settled on using a mutex vs spinlock)
BR,
-R
> > +
> > static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset)
> > {
> > return msm_readl(gmu->mmio + (offset << 2));
> > diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> > index f6a4dbef796b..5e1ae3df42ba 100644
> > --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> > +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> > @@ -881,7 +881,7 @@ static int a6xx_zap_shader_init(struct msm_gpu *gpu)
> > A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
> > A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
> >
> > -static int a6xx_hw_init(struct msm_gpu *gpu)
> > +static int hw_init(struct msm_gpu *gpu)
> > {
> > struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
> > struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
> > @@ -1135,6 +1135,19 @@ static int a6xx_hw_init(struct msm_gpu *gpu)
> > return ret;
> > }
> >
> > +static int a6xx_hw_init(struct msm_gpu *gpu)
> > +{
> > + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
> > + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
> > + int ret;
> > +
> > + with_gmu_lock(&a6xx_gpu->gmu) {
> > + ret = hw_init(gpu);
> > + }
> > +
> > + return ret;
> > +}
> > +
> > static void a6xx_dump(struct msm_gpu *gpu)
> > {
> > DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n",
> > @@ -1509,7 +1522,9 @@ static int a6xx_pm_resume(struct msm_gpu *gpu)
> >
> > trace_msm_gpu_resume(0);
> >
> > - ret = a6xx_gmu_resume(a6xx_gpu);
> > + with_gmu_lock(&a6xx_gpu->gmu) {
> > + ret = a6xx_gmu_resume(a6xx_gpu);
> > + }
> > if (ret)
> > return ret;
> >
> > @@ -1532,7 +1547,9 @@ static int a6xx_pm_suspend(struct msm_gpu *gpu)
> >
> > msm_devfreq_suspend(gpu);
> >
> > - ret = a6xx_gmu_stop(a6xx_gpu);
> > + with_gmu_lock(&a6xx_gpu->gmu) {
> > + ret = a6xx_gmu_stop(a6xx_gpu);
> > + }
> > if (ret)
> > return ret;
> >
> > @@ -1547,18 +1564,17 @@ static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
> > {
> > struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
> > struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
> > - static DEFINE_MUTEX(perfcounter_oob);
> >
> > - mutex_lock(&perfcounter_oob);
> > + with_gmu_lock(&a6xx_gpu->gmu) {
> > + /* Force the GPU power on so we can read this register */
> > + a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
> >
> > - /* Force the GPU power on so we can read this register */
> > - a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
> > + *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
> > + REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
> >
> > - *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER_LO,
> > - REG_A6XX_CP_ALWAYS_ON_COUNTER_HI);
> > + a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
> > + }
> >
> > - a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
> > - mutex_unlock(&perfcounter_oob);
> > return 0;
> > }
> >
> > @@ -1622,6 +1638,16 @@ static unsigned long a6xx_gpu_busy(struct msm_gpu *gpu)
> > return (unsigned long)busy_time;
> > }
> >
> > +void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
> > +{
> > + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
> > + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
> > +
> > + with_gmu_lock(&a6xx_gpu->gmu) {
> > + a6xx_gmu_set_freq(gpu, opp);
> > + }
> > +}
> > +
> > static struct msm_gem_address_space *
> > a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
> > {
> > @@ -1766,7 +1792,7 @@ static const struct adreno_gpu_funcs funcs = {
> > #endif
> > .gpu_busy = a6xx_gpu_busy,
> > .gpu_get_freq = a6xx_gmu_get_freq,
> > - .gpu_set_freq = a6xx_gmu_set_freq,
> > + .gpu_set_freq = a6xx_gpu_set_freq,
> > #if defined(CONFIG_DRM_MSM_GPU_STATE)
> > .gpu_state_get = a6xx_gpu_state_get,
> > .gpu_state_put = a6xx_gpu_state_put,
> >
>
>
> --
> With best wishes
> Dmitry
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2021-10-01 18:02 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-09-27 18:03 [RFC] drm/msm/a6xx: Serialize GMU communication Rob Clark
2021-10-01 17:39 ` Dmitry Baryshkov
2021-10-01 18:05 ` Rob Clark
2021-10-01 18:02 ` Dmitry Baryshkov
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).